diff --git a/Documentation/ABI/obsolete/sysfs-selinux-user b/Documentation/ABI/removed/sysfs-selinux-user similarity index 100% rename from Documentation/ABI/obsolete/sysfs-selinux-user rename to Documentation/ABI/removed/sysfs-selinux-user diff --git a/Documentation/admin-guide/mm/damon/lru_sort.rst b/Documentation/admin-guide/mm/damon/lru_sort.rst index 20a8378d5a9460..5f38910c0ba7ef 100644 --- a/Documentation/admin-guide/mm/damon/lru_sort.rst +++ b/Documentation/admin-guide/mm/damon/lru_sort.rst @@ -79,6 +79,10 @@ of parametrs except ``enabled`` again. Once the re-reading is done, this parameter is set as ``N``. If invalid parameters are found while the re-reading, DAMON_LRU_SORT will be disabled. +Once ``Y`` is written to this parameter, the user must not write to any +parameters until reading ``commit_inputs`` again returns ``N``. If users +violate this rule, the kernel may exhibit undefined behavior. + active_mem_bp ------------- @@ -91,8 +95,8 @@ increases and decreases the effective level of the quota aiming the LRU Disabled by default. -Auto-tune monitoring intervals ------------------------------- +autotune_monitoring_intervals +----------------------------- If this parameter is set as ``Y``, DAMON_LRU_SORT automatically tunes DAMON's sampling and aggregation intervals. The auto-tuning aims to capture meaningful diff --git a/Documentation/admin-guide/mm/damon/reclaim.rst b/Documentation/admin-guide/mm/damon/reclaim.rst index 8eba3da8dcee0a..1d68db2aa27ea1 100644 --- a/Documentation/admin-guide/mm/damon/reclaim.rst +++ b/Documentation/admin-guide/mm/damon/reclaim.rst @@ -71,6 +71,10 @@ of parametrs except ``enabled`` again. Once the re-reading is done, this parameter is set as ``N``. If invalid parameters are found while the re-reading, DAMON_RECLAIM will be disabled. +Once ``Y`` is written to this parameter, the user must not write to any +parameters until reading ``commit_inputs`` again returns ``N``. If users +violate this rule, the kernel may exhibit undefined behavior. + min_age ------- diff --git a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml index 38fcee91211e8c..49a007cbcd3aa4 100644 --- a/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml +++ b/Documentation/devicetree/bindings/display/ti/ti,am65x-dss.yaml @@ -36,34 +36,50 @@ properties: reg: description: Addresses to each DSS memory region described in the SoC's TRM. - items: - - description: common DSS register area - - description: VIDL1 light video plane - - description: VID video plane - - description: OVR1 overlay manager for vp1 - - description: OVR2 overlay manager for vp2 - - description: VP1 video port 1 - - description: VP2 video port 2 - - description: common1 DSS register area + oneOf: + - items: + - description: common DSS register area + - description: VIDL1 light video plane + - description: VID video plane + - description: OVR1 overlay manager for vp1 + - description: OVR2 overlay manager for vp2 + - description: VP1 video port 1 + - description: VP2 video port 2 + - description: common1 DSS register area + - items: + - description: common DSS register area + - description: VIDL1 light video plane + - description: OVR1 overlay manager for vp1 + - description: VP1 video port 1 + - description: common1 DSS register area reg-names: - items: - - const: common - - const: vidl1 - - const: vid - - const: ovr1 - - const: ovr2 - - const: vp1 - - const: vp2 - - const: common1 + oneOf: + - items: + - const: common + - const: vidl1 + - const: vid + - const: ovr1 + - const: ovr2 + - const: vp1 + - const: vp2 + - const: common1 + - items: + - const: common + - const: vidl1 + - const: ovr1 + - const: vp1 + - const: common1 clocks: + minItems: 2 items: - description: fck DSS functional clock - description: vp1 Video Port 1 pixel clock - description: vp2 Video Port 2 pixel clock clock-names: + minItems: 2 items: - const: fck - const: vp1 @@ -179,6 +195,24 @@ allOf: ports: properties: port@1: false + reg: + maxItems: 5 + reg-names: + maxItems: 5 + clocks: + maxItems: 2 + clock-names: + maxItems: 2 + else: + properties: + reg: + minItems: 8 + reg-names: + minItems: 8 + clocks: + minItems: 3 + clock-names: + minItems: 3 - if: properties: diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml new file mode 100644 index 00000000000000..0e3780ad9dd79a --- /dev/null +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/dma/apple,sio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SIO Coprocessor + +description: + SIO is a coprocessor on Apple M1 and later chips (and maybe also on earlier + chips). Its role is to offload SPI, UART and DisplayPort audio transfers, + being a pretend DMA controller. + +maintainers: + - Martin Povišer + +allOf: + - $ref: dma-controller.yaml# + +properties: + compatible: + items: + - enum: + - apple,t6000-sio + - apple,t8103-sio + - const: apple,sio + + reg: + maxItems: 1 + + '#dma-cells': + const: 1 + description: + DMA clients specify a single cell that corresponds to the RTKit endpoint + number used for arranging the transfers in question + + dma-channels: + maximum: 128 + + mboxes: + maxItems: 1 + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + memory-region: + minItems: 2 + maxItems: 8 + description: + A number of references to reserved memory regions among which are the DATA/TEXT + sections of coprocessor executable firmware and also auxiliary firmware data + describing the available DMA-enabled peripherals + + apple,sio-firmware-params: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: | + Parameters in the form of opaque key/value pairs that are to be sent to the SIO + coprocesssor once it boots. These parameters can point into the reserved memory + regions (in device address space). + + Note that unlike Apple's firmware, we treat the parameters, and the data they + refer to, as opaque. Apple embed short data blobs into their SIO devicetree node + that describe the DMA-enabled peripherals (presumably with defined semantics). + Their driver processes those blobs and sets up data structure in mapped device + memory, then references this memory in the parameters sent to the SIO. At the + level of description we are opting for in this binding, we assume the job of + constructing those data structures has been done in advance, leaving behind an + opaque list of key/value parameter pairs to be sent by a prospective driver. + + This approach is chosen for two reasons: + + - It means we don't need to try to understand the semantics of Apple's blobs + as long as we know the transformation we need to do from Apple's devicetree + data to SIO data (which can be shoved away into a loader). It also means the + semantics of Apple's blobs (or of something to replace them) need not be part + of the binding and be kept up with Apple's firmware changes in the future. + + - It leaves less work for the driver attaching on this binding. Instead the work + is done upfront in the loader which can be better suited for keeping up with + Apple's firmware changes. + +required: + - compatible + - reg + - '#dma-cells' + - dma-channels + - mboxes + - iommus + - power-domains + +additionalProperties: false + +examples: + - | + sio: dma-controller@36400000 { + compatible = "apple,t8103-sio", "apple,sio"; + reg = <0x36400000 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + memory-region = <&sio_text>, <&sio_data>, + <&sio_auxdata1>, <&sio_auxdata2>; /* Filled by loader */ + apple,sio-firmware-params = <0xb 0x10>, <0xc 0x1b80>, <0xf 0x14>, + <0x10 0x1e000>, <0x30d 0x34>, <0x30e 0x4000>, + <0x1a 0x38>, <0x1b 0x50>; /* Filled by loader */ + }; diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml new file mode 100644 index 00000000000000..2eec317bc4b3e6 --- /dev/null +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/hwmon/apple,smc-hwmon.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SMC Hardware Monitoring + +description: + Apple's System Management Controller (SMC) exposes a vast array of + hardware monitoring sensors, including temperature probes, current and + voltage sense, power meters, and fan speeds. It also provides endpoints + to manually control the speed of each fan individually. Each Apple + Silicon device exposes a different set of endpoints via SMC keys. This + is true even when two machines share an SoC. The CPU core temperature + sensor keys on an M1 Mac mini are different to those on an M1 MacBook + Pro, for example. + +maintainers: + - James Calligeros + +$defs: + sensor: + type: object + + properties: + apple,key-id: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: The SMC FourCC key of the desired sensor. + Must match the node's suffix. + + label: + description: Human-readable name for the sensor + + required: + - apple,key-id + +properties: + compatible: + const: apple,smc-hwmon + +patternProperties: + "^current-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^fan-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + properties: + apple,fan-minimum: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: SMC key containing the fan's minimum speed + + apple,fan-maximum: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: SMC key containing the fan's maximum speed + + apple,fan-target: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: Writeable endpoint for setting desired fan speed + + apple,fan-mode: + $ref: /schemas/types.yaml#/definitions/string + pattern: "^[A-Za-z0-9]{4}$" + description: Writeable key to enable/disable manual fan control + + + "^power-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^temperature-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + + "^voltage-[A-Za-z0-9]{4}$": + $ref: "#/$defs/sensor" + unevaluatedProperties: false + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml index bfd30aae682bf3..360a0643a0b567 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml @@ -50,7 +50,7 @@ properties: The 2nd cell contains the interrupt number for the interrupt type. SPI interrupts are in the range [0-987]. PPI interrupts are in the range [0-15]. Extended SPI interrupts are in the range [0-1023]. - Extended PPI interrupts are in the range [0-127]. + Extended PPI interrupts are in the range [0-63]. The 3rd cell is the flags, encoded as follows: bits[3:0] trigger type and level flags. diff --git a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml index 809fda45b3bd62..42022401d0ffa6 100644 --- a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml +++ b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml @@ -28,16 +28,20 @@ properties: reg: minItems: 1 - items: - - description: The function configuration registers base - - description: The link table configuration registers base - - description: The cache configuration registers base + maxItems: 3 reg-names: - items: - - const: function - - const: link - - const: cache + oneOf: + - items: + - const: link + - const: function + - const: cache + - items: + - const: function + - const: link + - const: cache + deprecated: true + description: Use link,function,cache block order instead. interrupts: maxItems: 1 @@ -123,6 +127,8 @@ allOf: minItems: 5 reset-names: minItems: 5 + required: + - reg-names else: properties: reg: diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index 103dc1f757de7f..d5d30c05384585 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -51,6 +51,9 @@ properties: rtc: $ref: /schemas/rtc/apple,smc-rtc.yaml + hwmon: + $ref: /schemas/hwmon/apple,smc-hwmon.yaml + additionalProperties: false required: @@ -91,5 +94,38 @@ examples: nvmem-cells = <&rtc_offset>; nvmem-cell-names = "rtc_offset"; }; + + hwmon { + compatible = "apple,smc-hwmon"; + + current-ID0R { + apple,key-id = "ID0R"; + label = "AC Input Current"; + }; + + fan-F0Ac { + apple,key-id = "F0Ac"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + apple,fan-target = "F0Tg"; + apple,fan-mode = "F0Md"; + label = "Fan 1"; + }; + + power-PSTR { + apple,key-id = "PSTR"; + label = "Total System Power"; + }; + + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temperature"; + }; + + voltage-VD0R { + apple,key-id = "VD0R"; + label = "AC Input Voltage"; + }; + }; }; }; diff --git a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml index 7e7c55dc244034..5cebe5eb1efb87 100644 --- a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml +++ b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml @@ -50,9 +50,11 @@ properties: maxItems: 1 resets: + minItems: 4 maxItems: 5 reset-names: + minItems: 4 maxItems: 5 rockchip,txclk-tapnum: @@ -146,6 +148,7 @@ allOf: else: properties: resets: + minItems: 5 maxItems: 5 reset-names: items: diff --git a/Documentation/devicetree/bindings/net/dsa/nxp,sja1105.yaml b/Documentation/devicetree/bindings/net/dsa/nxp,sja1105.yaml index 607b7fe8d28ee0..0486489114cd85 100644 --- a/Documentation/devicetree/bindings/net/dsa/nxp,sja1105.yaml +++ b/Documentation/devicetree/bindings/net/dsa/nxp,sja1105.yaml @@ -143,8 +143,6 @@ allOf: else: properties: spi-cpha: false - required: - - spi-cpol unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index 17d623a19d1def..0bed9e126fdaa4 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -85,6 +85,27 @@ properties: power-domains: maxItems: 1 +patternProperties: + "^pci@": + $ref: /schemas/pci/pci-bus.yaml# + type: object + description: A single PCI root port + + properties: + reg: + maxItems: 1 + + pwren-gpios: + description: Optional GPIO to power on the device + maxItems: 1 + + required: + - reset-gpios + - interrupt-controller + - "#interrupt-cells" + - interrupt-map-mask + - interrupt-map + required: - compatible - reg @@ -164,7 +185,7 @@ examples: pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; - pci@0,0 { + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 152 0>; @@ -172,9 +193,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; }; - pci@1,0 { + port01: pci@1,0 { device_type = "pci"; reg = <0x800 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 153 0>; @@ -182,9 +211,17 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; }; - pci@2,0 { + port02: pci@2,0 { device_type = "pci"; reg = <0x1000 0x0 0x0 0x0 0x0>; reset-gpios = <&pinctrl_ap 33 0>; @@ -192,6 +229,14 @@ examples: #address-cells = <3>; #size-cells = <2>; ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; }; }; }; diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-common.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-common.yaml index cddbe21f99f2be..0488c942092d64 100644 --- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-common.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie-common.yaml @@ -17,11 +17,11 @@ description: properties: clocks: minItems: 3 - maxItems: 5 + maxItems: 6 clock-names: minItems: 3 - maxItems: 5 + maxItems: 6 num-lanes: const: 1 diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml index 12a01f7a57443d..21dda80660147e 100644 --- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml @@ -40,7 +40,8 @@ properties: - description: PCIe PHY clock. - description: Additional required clock entry for imx6sx-pcie, imx6sx-pcie-ep, imx8mq-pcie, imx8mq-pcie-ep. - - description: PCIe reference clock. + - description: PCIe internal reference clock. + - description: PCIe additional external reference clock. clock-names: minItems: 3 diff --git a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml index d668782546a235..d1eb92995e2c30 100644 --- a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml +++ b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml @@ -41,22 +41,22 @@ properties: interrupt-names: items: - - description: serr - - description: ser_cor - - description: serr_nonfatal - - description: serr_fatal - - description: axi_err - - description: inta - - description: intb - - description: intc - - description: intd - - description: msi - - description: link_bandwidth - - description: pm_pme - - description: dma - - description: pcie_evt - - description: msg - - description: all + - const: serr + - const: serr_cor + - const: serr_nonfatal + - const: serr_fatal + - const: axi_err + - const: inta + - const: intb + - const: intc + - const: intd + - const: msi + - const: link_bandwidth + - const: pm_pme + - const: dma + - const: pcie_evt + - const: msg + - const: all interrupt-controller: true @@ -67,8 +67,8 @@ properties: clock-names: items: - - description: aclk - - description: pm + - const: aclk + - const: pm resets: items: @@ -82,13 +82,13 @@ properties: reset-names: items: - - description: aresetn - - description: rst_b - - description: rst_gp_b - - description: rst_ps_b - - description: rst_rsm_b - - description: rst_cfg_b - - description: rst_load_b + - const: aresetn + - const: rst_b + - const: rst_gp_b + - const: rst_ps_b + - const: rst_rsm_b + - const: rst_cfg_b + - const: rst_load_b power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml index 4f9013d3687499..727da7fb490cef 100644 --- a/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/marvell,armada3710-xb-pinctrl.yaml @@ -84,11 +84,12 @@ patternProperties: properties: groups: - enum: [ emmc_nb, i2c1, i2c2, jtag, mii_col, onewire, pcie1, - pcie1_clkreq, pcie1_wakeup, pmic0, pmic1, ptp, ptp_clk, - ptp_trig, pwm0, pwm1, pwm2, pwm3, rgmii, sdio0, sdio_sb, smi, - spi_cs1, spi_cs2, spi_cs3, spi_quad, uart1, uart2, - usb2_drvvbus1, usb32_drvvbus0 ] + items: + enum: [ emmc_nb, i2c1, i2c2, jtag, mii_col, onewire, pcie1, + pcie1_clkreq, pcie1_wakeup, pmic0, pmic1, ptp, ptp_clk, + ptp_trig, pwm0, pwm1, pwm2, pwm3, rgmii, sdio0, sdio_sb, + smi, spi_cs1, spi_cs2, spi_cs3, spi_quad, uart1, uart2, + usb2_drvvbus1, usb32_drvvbus0 ] function: enum: [ drvbus, emmc, gpio, i2c, jtag, led, mii, mii_err, onewire, diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 542c46d489a7e1..9e3b394bf64d00 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -83,6 +83,18 @@ properties: minimum: 0 maximum: 15 + apple,force-disable: + description: + Forces this device to be disabled (bus access blocked) when the power + domain is powered down. + type: boolean + + apple,force-reset: + description: + Forces a reset/error recovery of the power control logic when the power + domain is powered down. + type: boolean + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml b/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml index c6585e3bacbec0..00d66b92304750 100644 --- a/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml +++ b/Documentation/devicetree/bindings/regulator/fitipower,fp9931.yaml @@ -66,6 +66,7 @@ properties: required: - compatible - reg + - vin-supply - pg-gpios - enable-gpios diff --git a/Documentation/devicetree/bindings/sound/apple,macaudio.yaml b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml new file mode 100644 index 00000000000000..8fe22dec3015d6 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/apple,macaudio.yaml @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/sound/apple,macaudio.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple Silicon Macs integrated sound peripherals + +description: + This binding represents the overall machine-level integration of sound + peripherals on 'Apple Silicon' machines by Apple. + +maintainers: + - Martin Povišer + +properties: + compatible: + items: + - enum: + - apple,j274-macaudio + - apple,j293-macaudio + - apple,j314-macaudio + - const: apple,macaudio + + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + + model: + description: + Model name for presentation to users + $ref: /schemas/types.yaml#/definitions/string + +patternProperties: + "^dai-link(@[0-9a-f]+)?$": + description: | + Node for each sound peripheral such as the speaker array, headphones jack, + or microphone. + type: object + + additionalProperties: false + + properties: + reg: + maxItems: 1 + + link-name: + description: | + Name for the peripheral, expecting 'Speaker' or 'Speakers' if this is + the speaker array. + $ref: /schemas/types.yaml#/definitions/string + + cpu: + type: object + + properties: + sound-dai: + description: | + DAI list with CPU-side I2S ports involved in this peripheral. + minItems: 1 + maxItems: 2 + + required: + - sound-dai + + codec: + type: object + + properties: + sound-dai: + minItems: 1 + maxItems: 8 + description: | + DAI list with the CODEC-side DAIs connected to the above CPU-side + DAIs and involved in this sound peripheral. + + The list is in left/right order if applicable. If there are more + than one CPU-side DAIs (there can be two), the CODECs must be + listed first those connected to the first CPU, then those + connected to the second. + + In addition, on some machines with many speaker codecs, the CODECs + are listed in this fixed order: + + J293: Left Front, Left Rear, Right Front, Right Rear + J314: Left Woofer 1, Left Tweeter, Left Woofer 2, + Right Woofer 1, Right Tweeter, Right Woofer 2 + + required: + - sound-dai + + required: + - reg + - cpu + - codec + +required: + - compatible + - model + +additionalProperties: false + +examples: + - | + mca: mca@9b600000 { + compatible = "apple,t6000-mca", "apple,mca"; + reg = <0x9b600000 0x10000>, + <0x9b500000 0x20000>; + + clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; + power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, + <&ps_mca2>, <&ps_mca3>; + dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>, + <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>, + <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>, + <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>; + dma-names = "tx0a", "rx0a", "tx0b", "rx0b", + "tx1a", "rx1a", "tx1b", "rx1b", + "tx2a", "rx2a", "tx2b", "rx2b", + "tx3a", "rx3a", "tx3b", "rx3b"; + + #sound-dai-cells = <1>; + }; + + sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314 integrated audio"; + + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Headphones Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml b/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml index c7d17cf4dc42bf..e738153a309c8f 100644 --- a/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml +++ b/Documentation/devicetree/bindings/ufs/rockchip,rk3576-ufshc.yaml @@ -41,7 +41,7 @@ properties: maxItems: 1 resets: - maxItems: 4 + maxItems: 5 reset-names: items: @@ -49,6 +49,7 @@ properties: - const: sys - const: ufs - const: grf + - const: mphy reset-gpios: maxItems: 1 @@ -98,8 +99,8 @@ examples: interrupts = ; power-domains = <&power RK3576_PD_USB>; resets = <&cru SRST_A_UFS_BIU>, <&cru SRST_A_UFS_SYS>, <&cru SRST_A_UFS>, - <&cru SRST_P_UFS_GRF>; - reset-names = "biu", "sys", "ufs", "grf"; + <&cru SRST_P_UFS_GRF>, <&cru SRST_MPHY_INIT>; + reset-names = "biu", "sys", "ufs", "grf", "mphy"; reset-gpios = <&gpio4 RK_PD0 GPIO_ACTIVE_LOW>; }; }; diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 7d2b897d66fa95..017fb155a5bc25 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -464,3 +464,7 @@ SPI WATCHDOG devm_watchdog_register_device() + +WORKQUEUE + devm_alloc_workqueue() + devm_alloc_ordered_workqueue() diff --git a/Documentation/mm/damon/index.rst b/Documentation/mm/damon/index.rst index 82f6c5eea49a78..318f6a7bfea47b 100644 --- a/Documentation/mm/damon/index.rst +++ b/Documentation/mm/damon/index.rst @@ -12,7 +12,7 @@ DAMON is a Linux kernel subsystem for efficient :ref:`data access monitoring - *light-weight* (for production online usages), - *scalable* (in terms of memory size), - *tunable* (for flexible usages), and - - *autoamted* (for production operation without manual tunings). + - *automated* (for production operation without manual tunings). .. toctree:: :maxdepth: 2 diff --git a/Documentation/mm/hugetlbfs_reserv.rst b/Documentation/mm/hugetlbfs_reserv.rst index 4914fbf07966c5..a49115db18c763 100644 --- a/Documentation/mm/hugetlbfs_reserv.rst +++ b/Documentation/mm/hugetlbfs_reserv.rst @@ -155,7 +155,7 @@ are enough free huge pages to accommodate the reservation. If there are, the global reservation count resv_huge_pages is adjusted something like the following:: - if (resv_needed <= (resv_huge_pages - free_huge_pages)) + if (resv_needed <= (free_huge_pages - resv_huge_pages) resv_huge_pages += resv_needed; Note that the global lock hugetlb_lock is held when checking and adjusting diff --git a/Documentation/netlink/specs/psp.yaml b/Documentation/netlink/specs/psp.yaml index f3a57782d2cf4a..49b7563f705f18 100644 --- a/Documentation/netlink/specs/psp.yaml +++ b/Documentation/netlink/specs/psp.yaml @@ -188,6 +188,7 @@ operations: name: dev-set doc: Set the configuration of a PSP device. attribute-set: dev + flags: [admin-perm] do: request: attributes: @@ -207,6 +208,7 @@ operations: name: key-rotate doc: Rotate the device key. attribute-set: dev + flags: [admin-perm] do: request: attributes: diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst index 152289f0bed2fa..2b7e91bd9d3d36 100644 --- a/Documentation/rust/quick-start.rst +++ b/Documentation/rust/quick-start.rst @@ -324,9 +324,9 @@ Configuration ------------- ``Rust support`` (``CONFIG_RUST``) needs to be enabled in the ``General setup`` -menu. The option is only shown if a suitable Rust toolchain is found (see -above), as long as the other requirements are met. In turn, this will make -visible the rest of options that depend on Rust. +menu. In turn, this will make visible the rest of options that depend on Rust. +You can check the value of ``RUST_IS_AVAILABLE`` to determine if your toolchain +is configured correctly. Afterwards, go to:: diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index d74c2c2b9ef391..b4a710484a0b83 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -320,13 +320,15 @@ by a sched_ext scheduler: ops.dispatch(); /* Task is moved to a local DSQ */ } ops.running(); /* Task starts running on its assigned CPU */ - while (task->scx.slice > 0 && task is runnable) - ops.tick(); /* Called every 1/HZ seconds */ - ops.stopping(); /* Task stops running (time slice expires or wait) */ - /* Task's CPU becomes available */ + while task_is_runnable(p) { + while (task->scx.slice > 0 && task_is_runnable(p)) + ops.tick(); /* Called every 1/HZ seconds */ + + ops.dispatch(); /* task->scx.slice can be refilled */ + } - ops.dispatch(); /* task->scx.slice can be refilled */ + ops.stopping(); /* Task stops running (time slice expires or wait) */ } ops.quiescent(); /* Task releases its assigned CPU (wait) */ diff --git a/Documentation/userspace-api/rseq.rst b/Documentation/userspace-api/rseq.rst index 3cd27a3c7c7e5a..8549a6c61531cc 100644 --- a/Documentation/userspace-api/rseq.rst +++ b/Documentation/userspace-api/rseq.rst @@ -24,6 +24,97 @@ Quick access to CPU number, node ID Allows to implement per CPU data efficiently. Documentation is in code and selftests. :( +Optimized RSEQ V2 +----------------- + +On architectures which utilize the generic entry code and generic TIF bits +the kernel supports runtime optimizations for RSEQ, which also enable +enhanced features like scheduler time slice extensions. + +To enable them a task has to register the RSEQ region with at least the +length advertised by getauxval(AT_RSEQ_FEATURE_SIZE). + +If existing binaries register with RSEQ_ORIG_SIZE (32 bytes), the kernel +keeps the legacy low performance mode enabled to fulfil the expectations +of existing users regarding the original RSEQ implementation behaviour. + +The following table documents the ABI and behavioral guarantees of the +legacy and the optimized V2 mode. + +.. list-table:: RSEQ modes + :header-rows: 1 + + * - Nr + - What + + - Legacy + - Optimized V2 + + * - 1 + - The cpu_id_start, cpu_id, node_id and mm_cid fields (User mode read + only) + .. Legacy + - Updated by the kernel unconditionally after each context switch and + before signal delivery + .. Optimized V2 + - Updated by the kernel if and only if they change, i.e. if the task + is migrated or mm_cid changes + + * - 2 + - The rseq_cs critical section field + .. Legacy + - Evaluated and handled unconditionally after each context switch and + before signal delivery + .. Optimized V2 + - Evaluated and handled conditionally only when user space was + interrupted and was scheduled out or before delivering a signal in + the interrupted context. + + * - 3 + - Read only fields + .. Legacy + - No strict enforcement except in debug mode + .. Optimized V2 + - Strict enforcement + + * - 4 + - membarrier(...RSEQ) + .. Legacy + - All running threads of the process are interrupted and the ID fields + are rewritten and eventually active critical sections are aborted + before they return to user space. All threads which are scheduled + out whether voluntary or not are covered by #1/#2 above. + .. Optimized V2 + - All running threads of the process are interrupted and eventually + active critical sections are aborted before these threads return to + user space. The ID fields are only updated if changed as a + consequence of the interrupt. All threads which are scheduled out + whether voluntary or not are covered by #1/#2 above. + + * - 5 + - Time slice extensions + .. Legacy + - Not supported + .. Optimized V2 + - Supported + +The legacy mode is obviously less performant as it does unconditional +updates and critical section checks even if not strictly required by the +ABI contract. That can't be changed anymore as some users depend on that +observed behavior, which in turn enables them to violate the ABI and +overwrite the cpu_id_start field for their own purposes. This is obviously +discouraged as it renders RSEQ incompatible with the intended usage and +breaks the expectation of other libraries in the same application. + +The ABI compliant optimized v2 mode, which respects the read only fields, +does not require unconditional updates and therefore is way more +performant. The kernel validates the read only fields for compliance. If +user space modifies them, the process is killed. Compliant usage allows +multiple libraries in the same application to benefit from the RSEQ +functionality without disturbing each other. The ABI compliant optimized v2 +mode also enables extended RSEQ features like time slice extensions. + + Scheduler time slice extensions ------------------------------- @@ -37,7 +128,8 @@ The prerequisites for this functionality are: * Enabled at boot time (default is enabled) - * A rseq userspace pointer has been registered for the thread + * A rseq userspace pointer has been registered for the thread in + optimized V2 mode The thread has to enable the functionality via prctl(2):: diff --git a/MAINTAINERS b/MAINTAINERS index d1cc0e12fe1f00..eefdd8a1579ac1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1930,6 +1930,13 @@ L: linux-input@vger.kernel.org S: Odd fixes F: drivers/input/mouse/bcm5974.c +APPLE DRM DISPLAY DRIVER +M: Janne Grunau +L: dri-devel@lists.freedesktop.org +S: Maintained +T: git git://anongit.freedesktop.org/drm/drm-misc +F: drivers/gpu/drm/apple/ + APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier L: linux-pci@vger.kernel.org @@ -2480,9 +2487,11 @@ M: Martin Povišer L: asahi@lists.linux.dev L: linux-sound@vger.kernel.org S: Maintained +F: Documentation/devicetree/bindings/dma/apple,sio.yaml F: Documentation/devicetree/bindings/sound/apple,* F: Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml F: Documentation/devicetree/bindings/sound/trivial-codec.yaml +F: drivers/dma/apple-sio.c F: sound/soc/apple/* F: sound/soc/codecs/cs42l83-i2c.c F: sound/soc/codecs/cs42l84.* @@ -2506,6 +2515,7 @@ F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml F: Documentation/devicetree/bindings/dma/apple,admac.yaml F: Documentation/devicetree/bindings/gpio/apple,smc-gpio.yaml F: Documentation/devicetree/bindings/gpu/apple,agx.yaml +F: Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml F: Documentation/devicetree/bindings/i2c/apple,i2c.yaml F: Documentation/devicetree/bindings/input/touchscreen/apple,z2-multitouch.yaml F: Documentation/devicetree/bindings/interrupt-controller/apple,* @@ -2540,6 +2550,7 @@ F: drivers/hwmon/macsmc-hwmon.c F: drivers/pmdomain/apple/ F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c +F: drivers/input/misc/macsmc-input.c F: drivers/input/touchscreen/apple_z2.c F: drivers/iommu/apple-dart.c F: drivers/iommu/io-pgtable-dart.c @@ -2564,6 +2575,8 @@ F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: include/uapi/drm/asahi_drm.h +F: net/bluetooth/brcm.c +F: net/bluetooth/brcm.h ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson diff --git a/Makefile b/Makefile index 36d0a32fbe49be..a95f0b3d26bf57 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 7 PATCHLEVEL = 0 -SUBLEVEL = 0 +SUBLEVEL = 10 EXTRAVERSION = NAME = Baby Opossum Posse @@ -486,6 +486,8 @@ export rust_common_flags := --edition=2021 \ -Wclippy::as_ptr_cast_mut \ -Wclippy::as_underscore \ -Wclippy::cast_lossless \ + -Aclippy::collapsible_if \ + -Aclippy::collapsible_match \ -Wclippy::ignored_unit_patterns \ -Wclippy::mut_mut \ -Wclippy::needless_bitwise_bool \ @@ -495,6 +497,7 @@ export rust_common_flags := --edition=2021 \ -Wclippy::ptr_cast_constness \ -Wclippy::ref_as_ptr \ -Wclippy::undocumented_unsafe_blocks \ + -Aclippy::uninlined_format_args \ -Wclippy::unnecessary_safety_comment \ -Wclippy::unnecessary_safety_doc \ -Wrustdoc::missing_crate_level_docs \ @@ -655,6 +658,8 @@ export RCS_FIND_IGNORE := \( -name SCCS -o -name BitKeeper -o -name .svn -o \ # Basic helpers built in scripts/basic/ PHONY += scripts_basic +scripts_basic: KBUILD_HOSTCFLAGS := $(KBUILD_HOSTCFLAGS) +scripts_basic: KBUILD_HOSTLDFLAGS := $(KBUILD_HOSTLDFLAGS) scripts_basic: $(Q)$(MAKE) $(build)=scripts/basic diff --git a/arch/Kconfig b/arch/Kconfig index 102ddbd4298efa..b5d41bb40672b4 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1128,6 +1128,13 @@ config ARCH_WANTS_MODULES_DATA_IN_VMALLOC For architectures like powerpc/32 which have constraints on module allocation and need to allocate module data outside of module area. +config ARCH_WANTS_MODULES_TEXT_SECTIONS + bool + help + For architectures like 32-bit parisc which require that functions in + modules have to keep code in own text sections (-ffunction-sections) + and to avoid merging all text into one big text section, + config ARCH_WANTS_EXECMEM_LATE bool help diff --git a/arch/arc/net/bpf_jit_arcv2.c b/arch/arc/net/bpf_jit_arcv2.c index 6d989b6d88c69b..7ee50aeae5a456 100644 --- a/arch/arc/net/bpf_jit_arcv2.c +++ b/arch/arc/net/bpf_jit_arcv2.c @@ -2427,7 +2427,7 @@ u8 arc_prologue(u8 *buf, u32 usage, u16 frame_size) #ifdef ARC_BPF_JIT_DEBUG if ((usage & BIT(ARC_R_FP)) && frame_size == 0) { - pr_err("FP is being saved while there is no frame."); + pr_err("FP is being saved while there is no frame.\n"); BUG(); } #endif @@ -2454,7 +2454,7 @@ u8 arc_epilogue(u8 *buf, u32 usage, u16 frame_size) #ifdef ARC_BPF_JIT_DEBUG if ((usage & BIT(ARC_R_FP)) && frame_size == 0) { - pr_err("FP is being saved while there is no frame."); + pr_err("FP is being saved while there is no frame.\n"); BUG(); } #endif @@ -2868,7 +2868,7 @@ u8 gen_jmp_64(u8 *buf, u8 rd, u8 rs, u8 cond, u32 curr_off, u32 targ_off) break; default: #ifdef ARC_BPF_JIT_DEBUG - pr_err("64-bit jump condition is not known."); + pr_err("64-bit jump condition is not known.\n"); BUG(); #endif } @@ -2948,7 +2948,7 @@ u8 gen_jmp_32(u8 *buf, u8 rd, u8 rs, u8 cond, u32 curr_off, u32 targ_off) */ if (cond >= ARC_CC_LAST) { #ifdef ARC_BPF_JIT_DEBUG - pr_err("32-bit jump condition is not known."); + pr_err("32-bit jump condition is not known.\n"); BUG(); #endif return 0; diff --git a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi index d0d5f7e52a9176..46b650abdb904d 100644 --- a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi +++ b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi @@ -479,7 +479,7 @@ }; nand_controller: nand-controller@18028000 { - compatible = "brcm,nand-iproc", "brcm,brcmnand-v6.1", "brcm,brcmnand"; + compatible = "brcm,nand-iproc", "brcm,brcmnand-v6.1"; reg = <0x18028000 0x600>, <0x1811a408 0x600>, <0x18028f00 0x20>; reg-names = "nand", "iproc-idm", "iproc-ext"; interrupts = ; diff --git a/arch/arm/boot/dts/mediatek/mt7623.dtsi b/arch/arm/boot/dts/mediatek/mt7623.dtsi index 4b1685b9398919..71ac2b94c6ba36 100644 --- a/arch/arm/boot/dts/mediatek/mt7623.dtsi +++ b/arch/arm/boot/dts/mediatek/mt7623.dtsi @@ -328,7 +328,7 @@ efuse: efuse@10206000 { compatible = "mediatek,mt7623-efuse", - "mediatek,mt8173-efuse"; + "mediatek,efuse"; reg = <0 0x10206000 0 0x1000>; #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi index c7e92358487826..9f0e65526d5f97 100644 --- a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi +++ b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-cpuimx27.dtsi @@ -106,7 +106,7 @@ compatible = "ns8250"; clocks = <&clk14745600>; fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>; - interrupts = <&gpio2 23 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio2 23 IRQ_TYPE_LEVEL_LOW>; reg = <3 0x200000 0x1000>; reg-shift = <1>; reg-io-width = <1>; @@ -119,7 +119,7 @@ compatible = "ns8250"; clocks = <&clk14745600>; fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>; - interrupts = <&gpio2 22 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio2 22 IRQ_TYPE_LEVEL_LOW>; reg = <3 0x400000 0x1000>; reg-shift = <1>; reg-io-width = <1>; @@ -132,7 +132,7 @@ compatible = "ns8250"; clocks = <&clk14745600>; fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>; - interrupts = <&gpio2 27 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio2 27 IRQ_TYPE_LEVEL_LOW>; reg = <3 0x800000 0x1000>; reg-shift = <1>; reg-io-width = <1>; @@ -145,7 +145,7 @@ compatible = "ns8250"; clocks = <&clk14745600>; fsl,weim-cs-timing = <0x0000d603 0x0d1d0d01 0x00d20000>; - interrupts = <&gpio2 30 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio2 30 IRQ_TYPE_LEVEL_LOW>; reg = <3 0x1000000 0x1000>; reg-shift = <1>; reg-io-width = <1>; diff --git a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts index d78793601306cf..c71f8029833048 100644 --- a/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts +++ b/arch/arm/boot/dts/nxp/imx/imx27-eukrea-mbimxsd27-baseboard.dts @@ -76,7 +76,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_touch>; reg = <0>; - interrupts = <&gpio4 25 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio4 25 IRQ_TYPE_LEVEL_LOW>; spi-cpol; spi-max-frequency = <1500000>; ti,keep-vref-on; diff --git a/arch/arm/mach-omap1/clock_data.c b/arch/arm/mach-omap1/clock_data.c index c58d200e4816b8..5203b047deac81 100644 --- a/arch/arm/mach-omap1/clock_data.c +++ b/arch/arm/mach-omap1/clock_data.c @@ -700,8 +700,8 @@ int __init omap1_clk_init(void) /* Make sure UART clocks are enabled early */ if (cpu_is_omap16xx()) omap_writel(omap_readl(MOD_CONF_CTRL_0) | - CONF_MOD_UART1_CLK_MODE_R | - CONF_MOD_UART3_CLK_MODE_R, MOD_CONF_CTRL_0); + (1 << CONF_MOD_UART1_CLK_MODE_R) | + (1 << CONF_MOD_UART3_CLK_MODE_R), MOD_CONF_CTRL_0); #endif /* USB_REQ_EN will be disabled later if necessary (usb_dc_ck) */ diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 19470d938b2361..4d7ef5cc36b66d 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -304,8 +304,10 @@ void __sync_icache_dcache(pte_t pteval) else mapping = NULL; - if (!test_and_set_bit(PG_dcache_clean, &folio->flags.f)) + if (!test_bit(PG_dcache_clean, &folio->flags.f)) { __flush_dcache_folio(mapping, folio); + set_bit(PG_dcache_clean, &folio->flags.f); + } if (pte_exec(pteval)) __flush_icache_all(); diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index deeb8f292454b2..a900aa97388550 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -1852,6 +1852,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) { u64 val = (u32)imm | (u64)insn[1].imm << 32; + if (insn->src_reg == BPF_PSEUDO_FUNC) + goto notyet; + emit_a32_mov_i64(dst, val, ctx); return 1; @@ -2055,6 +2058,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) const s8 *r5 = bpf2a32[BPF_REG_5]; const u32 func = (u32)__bpf_call_base + (u32)imm; + if (insn->src_reg == BPF_PSEUDO_CALL) + goto notyet; + emit_a32_mov_r64(true, r0, r1, ctx); emit_a32_mov_r64(true, r1, r2, ctx); emit_push_r64(r5, ctx); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9ea19b74b6c3c9..edbaec4b6eead0 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -446,6 +446,9 @@ config KASAN_SHADOW_OFFSET config UNWIND_TABLES bool +config ARM64_ACTLR_STATE + bool + source "arch/arm64/Kconfig.platforms" menu "Kernel Features" @@ -2283,6 +2286,16 @@ config ARM64_DEBUG_PRIORITY_MASKING If unsure, say N endif # ARM64_PSEUDO_NMI +config ARM64_MEMORY_MODEL_CONTROL + bool "Runtime memory model control" + select ARM64_ACTLR_STATE + help + Some ARM64 CPUs support runtime switching of the CPU memory + model, which can be useful to emulate other CPU architectures + which have different memory models. Say Y to enable support + for the PR_SET_MEM_MODEL/PR_GET_MEM_MODEL prctl() calls on + CPUs with this feature. + config RELOCATABLE bool "Build a relocatable kernel image" if EXPERT select ARCH_HAS_RELR diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index cc72491eaf6f52..f1f53fd98ae257 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -72,6 +72,12 @@ compatible = "arm,cortex-a53"; reg = <0x0 0x0>; enable-method = "psci"; + d-cache-line-size = <32>; + d-cache-size = <0x8000>; + d-cache-sets = <32>; + i-cache-line-size = <32>; + i-cache-size = <0x8000>; + i-cache-sets = <32>; next-level-cache = <&l2>; clocks = <&scpi_dvfs 0>; dynamic-power-coefficient = <140>; diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts index 7dffeb5931c9bb..701de57ff0f375 100644 --- a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts +++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-p230.dts @@ -84,7 +84,8 @@ reset-gpios = <&gpio GPIOZ_14 GPIO_ACTIVE_LOW>; interrupt-parent = <&gpio_intc>; - interrupts = <29 IRQ_TYPE_LEVEL_LOW>; + /* MAC_INTR on GPIOZ_15 */ + interrupts = <25 IRQ_TYPE_LEVEL_LOW>; eee-broken-1000t; }; }; diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index 6fc3349a58428f..9fea43f760ec98 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -87,6 +87,13 @@ dtb-$(CONFIG_ARCH_APPLE) += t6021-j416c.dtb dtb-$(CONFIG_ARCH_APPLE) += t6020-j474s.dtb dtb-$(CONFIG_ARCH_APPLE) += t6021-j475c.dtb dtb-$(CONFIG_ARCH_APPLE) += t6022-j475d.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6030-j514s.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6030-j516s.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6031-j514c.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6031-j516c.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6032-j575d.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6034-j514m.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6034-j516m.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j413.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j415.dtb dtb-$(CONFIG_ARCH_APPLE) += t8112-j473.dtb diff --git a/arch/arm64/boot/dts/apple/hwmon-common.dtsi b/arch/arm64/boot/dts/apple/hwmon-common.dtsi new file mode 100644 index 00000000000000..2a74d9a114abb6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-common.dtsi @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors expected on all systems + * + * Copyright The Asahi Linux Contributors + */ + +&smc_hwmon { + power-PSTR { + apple,key-id = "PSTR"; + label = "Total System Power"; + }; + power-PDTR { + apple,key-id = "PDTR"; + label = "AC Input Power"; + }; + power-PMVR { + apple,key-id = "PMVR"; + label = "3.8 V Rail Power"; + }; + temperature-TH0x { + apple,key-id = "TH0x"; + label = "NAND Flash Temperature"; + }; + voltage-VD0R { + apple,key-id = "VD0R"; + label = "AC Input Voltage"; + }; + current-ID0R { + apple,key-id = "ID0R"; + label = "AC Input Current"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi b/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi new file mode 100644 index 00000000000000..61c34692f1cd5a --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-fan-dual.dtsi @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * Fan hwmon sensors for machines with 2 fan. + */ + +#include "hwmon-fan.dtsi" + +&smc_hwmon { + fan-F0Ac { + label = "Fan 1"; + }; + fan-F1Ac { + apple,key-id = "F1Ac"; + label = "Fan 2"; + apple,fan-minimum = "F1Mn"; + apple,fan-maximum = "F1Mx"; + apple,fan-target = "F1Tg"; + apple,fan-mode = "F1Md"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-fan.dtsi b/arch/arm64/boot/dts/apple/hwmon-fan.dtsi new file mode 100644 index 00000000000000..180eb8d7441f44 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-fan.dtsi @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * Fan hwmon sensors for machines with a single fan. + */ + +&smc_hwmon { + fan-F0Ac { + apple,key-id = "F0Ac"; + label = "Fan"; + apple,fan-minimum = "F0Mn"; + apple,fan-maximum = "F0Mx"; + apple,fan-target = "F0Tg"; + apple,fan-mode = "F0Md"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi b/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi new file mode 100644 index 00000000000000..4afb91ee69fe76 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-laptop.dtsi @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors expected on all laptops + * + * Copyright The Asahi Linux Contributors + */ + +&smc_hwmon { + power-PHPC { + apple,key-id = "PHPC"; + label = "Heatpipe Power"; + }; + temperature-TB0T { + apple,key-id = "TB0T"; + label = "Battery Hotspot"; + }; + temperature-TCHP { + apple,key-id = "TCHP"; + label = "Charge Regulator Temp"; + }; + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temp"; + }; + voltage-SBAV { + apple,key-id = "SBAV"; + label = "Battery Voltage"; + }; + voltage-VD0R { + apple,key-id = "VD0R"; + label = "Charger Input Voltage"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/hwmon-mini.dtsi b/arch/arm64/boot/dts/apple/hwmon-mini.dtsi new file mode 100644 index 00000000000000..7fd86e911acfe7 --- /dev/null +++ b/arch/arm64/boot/dts/apple/hwmon-mini.dtsi @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * hwmon sensors common to the Mac mini desktop + * models, but not the Studio or Pro. + * + * Copyright The Asahi Linux Contributors + */ + +#include "hwmon-fan.dtsi" + +&smc_hwmon { + temperature-TW0P { + apple,key-id = "TW0P"; + label = "WiFi/BT Module Temp"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-common.dtsi b/arch/arm64/boot/dts/apple/isp-common.dtsi new file mode 100644 index 00000000000000..739e6e9e66e740 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-common.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Common ISP configuration for Apple silicon platforms. + * + * Copyright The Asahi Linux Contributors + */ + +/ { + aliases { + isp = &isp; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + isp_heap: isp-heap { + compatible = "apple,asc-mem"; + /* Filled in by bootloder */ + reg = <0 0 0 0>; + no-map; + }; + }; +}; + +&isp { + memory-region = <&isp_heap>; + memory-region-names = "heap"; + status = "okay"; +}; + +&isp_dart0 { + status = "okay"; +}; + +&isp_dart1 { + status = "okay"; +}; + +&isp_dart2 { + status = "okay"; +}; + +&ps_isp_sys { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx248.dtsi b/arch/arm64/boot/dts/apple/isp-imx248.dtsi new file mode 100644 index 00000000000000..0a4ac1a0152c2c --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx248.dtsi @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX248 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1280x720 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <1280 720>; + apple,crop = <8 8 1280 720>; + }; + /* 960x720 (4:3) */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <960 720>; + apple,crop = <168 8 960 720>; + }; + /* 960x540 (16:9) */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <960 540>; + apple,crop = <8 8 1280 720>; + }; + /* 640x480 (4:3) */ + preset3 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <640 480>; + apple,crop = <168 8 960 720>; + }; + /* 640x360 (16:9) */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <640 360>; + apple,crop = <8 8 1280 720>; + }; + /* 320x180 (16:9) */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1296 736>; + apple,output-size = <320 180>; + apple,crop = <8 8 1280 720>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx364.dtsi b/arch/arm64/boot/dts/apple/isp-imx364.dtsi new file mode 100644 index 00000000000000..55484d86523657 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx364.dtsi @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX364 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1440x720 (4:3) */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1440 1080>; + apple,crop = <240 0 1440 1080>; + }; + /* 1280x720 (16:9) */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + /* 960x720 (4:3) */ + preset3{ + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <960 720>; + apple,crop = <240 0 1440 1080>; + }; + /* 960x540 (16:9) */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <960 540>; + apple,crop = <0 0 1920 1080>; + }; + /* 640x480 (4:3) */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <640 480>; + apple,crop = <240 0 1440 1080>; + }; + /* 640x360 (16:9) */ + preset6 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <640 360>; + apple,crop = <0 0 1920 1080>; + }; + /* 320x180 (16:9) */ + preset7 { + apple,config-index = <0>; + apple,input-size = <1920 1080>; + apple,output-size = <320 180>; + apple,crop = <0 0 1920 1080>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi b/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi new file mode 100644 index 00000000000000..729b97829cbb7e --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx558-cfg0.dtsi @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX558 sensor in + * config #0 mode. + * + * These platforms enable MLVNR for all configs except + * #0, which we don't support. Config #0 is an uncropped + * square 1920x1920 sensor, with dark corners. + * Therefore, we synthesize common resolutions by using + * crop/scale while always choosing config #0. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1920 1080>; + apple,crop = <0 420 1920 1080>; + }; + /* 1080x1920 */ + preset1 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1080 1920>; + apple,crop = <420 0 1080 1920>; + }; + /* 1920x1440 */ + preset2 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1920 1440>; + apple,crop = <0 240 1920 1440>; + }; + /* 1440x1920 */ + preset3 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1440 1920>; + apple,crop = <240 0 1440 1920>; + }; + /* 1280x720 */ + preset4 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1280 720>; + apple,crop = <0 420 1920 1080>; + }; + /* 720x1280 */ + preset5 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <720 1280>; + apple,crop = <420 0 1080 1920>; + }; + /* 1280x960 */ + preset6 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <1280 960>; + apple,crop = <0 240 1920 1440>; + }; + /* 960x1280 */ + preset7 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <960 1280>; + apple,crop = <240 0 1440 1920>; + }; + /* 640x480 */ + preset8 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <640 480>; + apple,crop = <0 240 1920 1440>; + }; + /* 480x640 */ + preset9 { + apple,config-index = <0>; + apple,input-size = <1920 1920>; + apple,output-size = <480 640>; + apple,crop = <240 0 1440 1920>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/isp-imx558.dtsi b/arch/arm64/boot/dts/apple/isp-imx558.dtsi new file mode 100644 index 00000000000000..d55854c883f5b6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/isp-imx558.dtsi @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * ISP configuration for platforms with IMX558 sensor. + * + * Copyright The Asahi Linux Contributors + */ + +#include "isp-common.dtsi" + +&isp { + apple,temporal-filter = <0>; + + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + /* 1080x1920 */ + preset1 { + apple,config-index = <2>; + apple,input-size = <1080 1920>; + apple,output-size = <1080 1920>; + apple,crop = <0 0 1080 1920>; + }; + /* 1760x1328 */ + preset2 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <1760 1328>; + apple,crop = <0 0 1760 1328>; + }; + /* 1328x1760 */ + preset3 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = < 1328 1760>; + apple,crop = <0 0 1328 1760>; + }; + /* 1152x1152 */ + preset4 { + apple,config-index = <5>; + apple,input-size = <1152 1152>; + apple,output-size = <1152 1152>; + apple,crop = <0 0 1152 1152>; + }; + /* 1280x720 */ + preset5 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1280 720>; + apple,crop = <0 0 1920 1080>; + }; + /* 720x1280 */ + preset6 { + apple,config-index = <2>; + apple,input-size = <1080 1920>; + apple,output-size = <720 1280>; + apple,crop = <0 0 1080 1920>; + }; + /* 1280x960 */ + preset7 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <1280 960>; + apple,crop = <0 4 1760 1320>; + }; + /* 960x1280 */ + preset8 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = <960 1280>; + apple,crop = <4 0 1320 1760>; + }; + /* 640x480 */ + preset9 { + apple,config-index = <3>; + apple,input-size = <1760 1328>; + apple,output-size = <640 480>; + apple,crop = <0 4 1760 1320>; + }; + /* 480x640 */ + preset10 { + apple,config-index = <4>; + apple,input-size = <1328 1760>; + apple,output-size = <480 640>; + apple,crop = <4 0 1320 1760>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j314s.dts b/arch/arm64/boot/dts/apple/t6000-j314s.dts index 1430b91ff1b152..afa86668440f04 100644 --- a/arch/arm64/boot/dts/apple/t6000-j314s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j314s.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J314"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000-j316s.dts b/arch/arm64/boot/dts/apple/t6000-j316s.dts index da0cbe7d96736b..ddfc3c530923c7 100644 --- a/arch/arm64/boot/dts/apple/t6000-j316s.dts +++ b/arch/arm64/boot/dts/apple/t6000-j316s.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J316"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J316"; +}; diff --git a/arch/arm64/boot/dts/apple/t6000.dtsi b/arch/arm64/boot/dts/apple/t6000.dtsi index 0ad77c98073fe6..db7ea6eac5a6f1 100644 --- a/arch/arm64/boot/dts/apple/t6000.dtsi +++ b/arch/arm64/boot/dts/apple/t6000.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6001, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6001.dtsi" / { @@ -16,7 +18,24 @@ }; /delete-node/ &pmgr_south; +/delete-node/ &pmp_report_dispext2; +/delete-node/ &pmp_report_dispext3; +/delete-node/ &pmp_report_venc1; +/delete-node/ &pmp_report_msr1; +/delete-node/ &pmp_report_prores; +/delete-node/ &pmp_report_afnc4_ioa; +/delete-node/ &pmp_report_afnc5_ioa; + +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>; +}; &gpu { - compatible = "apple,agx-g13s"; + compatible = "apple,agx-t6000", "apple,agx-g13x"; }; diff --git a/arch/arm64/boot/dts/apple/t6001-j314c.dts b/arch/arm64/boot/dts/apple/t6001-j314c.dts index c37097dcfdb304..245df6d03ee422 100644 --- a/arch/arm64/boot/dts/apple/t6001-j314c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j314c.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,maldives"; }; + +&panel { + compatible = "apple,panel-j314", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J314"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J314"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j316c.dts b/arch/arm64/boot/dts/apple/t6001-j316c.dts index 3bc6e0c3294cf9..a000d497b705fa 100644 --- a/arch/arm64/boot/dts/apple/t6001-j316c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j316c.dts @@ -24,3 +24,20 @@ &bluetooth0 { brcm,board-type = "apple,madagascar"; }; + +&panel { + compatible = "apple,panel-j316", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J316"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J316"; +}; diff --git a/arch/arm64/boot/dts/apple/t6001-j375c.dts b/arch/arm64/boot/dts/apple/t6001-j375c.dts index 2e7c23714d4d00..f3f98f03800908 100644 --- a/arch/arm64/boot/dts/apple/t6001-j375c.dts +++ b/arch/arm64/boot/dts/apple/t6001-j375c.dts @@ -24,3 +24,39 @@ &bluetooth0 { brcm,board-type = "apple,okinawa"; }; + +&dpaudio0 { + status = "okay"; +}; + +&sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375"; +}; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <230 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <231 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 6dcb71a1d65a8d..73053599d71d81 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -16,6 +16,10 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -28,6 +32,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -64,6 +70,18 @@ }; }; +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x82000000 0x0 0x1000000>, + <0x5 0x2000000 0x0 0x1000000>; +}; + &gpu { - compatible = "apple,agx-g13c", "apple,agx-g13s"; + compatible = "apple,agx-t6001", "apple,agx-g13c", "apple,agx-g13s"; }; diff --git a/arch/arm64/boot/dts/apple/t6002-j375d.dts b/arch/arm64/boot/dts/apple/t6002-j375d.dts index a2a24d028cbbf5..5cf30cd162b679 100644 --- a/arch/arm64/boot/dts/apple/t6002-j375d.dts +++ b/arch/arm64/boot/dts/apple/t6002-j375d.dts @@ -21,6 +21,15 @@ }; }; +&dpaudio0 { + status = "okay"; +}; + +&sound { + compatible = "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J375"; +}; + /* USB Type C */ &i2c0 { /* front-right */ @@ -190,11 +199,13 @@ /delete-node/ &dwc3_2_dart_1_die1; /delete-node/ &dwc3_2_die1; /delete-node/ &atcphy2_die1; +/delete-node/ &atcphy2_xbar_die1; /delete-node/ &dwc3_3_dart_0_die1; /delete-node/ &dwc3_3_dart_1_die1; /delete-node/ &dwc3_3_die1; /delete-node/ &atcphy3_die1; +/delete-node/ &atcphy3_xbar_die1; /* delete unused always-on power-domains on die 1 */ @@ -206,3 +217,14 @@ /delete-node/ &ps_disp0_cpu0_die1; /delete-node/ &ps_disp0_fe_die1; + +&gpu { + apple,avg-power-ki-only = <0.6375>; + apple,avg-power-kp = <0.58>; + apple,avg-power-target-filter-tc = <1>; + apple,perf-base-pstate = <3>; + apple,ppm-ki = <5.8>; + apple,ppm-kp = <0.355>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6002.dtsi b/arch/arm64/boot/dts/apple/t6002.dtsi index a532e5401c4ec4..b5f175d71ee69a 100644 --- a/arch/arm64/boot/dts/apple/t6002.dtsi +++ b/arch/arm64/boot/dts/apple/t6002.dtsi @@ -16,6 +16,10 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif + #include "t600x-common.dtsi" / { @@ -236,6 +240,8 @@ <0x5 0x80000000 0x5 0x80000000 0x1 0x80000000>, <0x7 0x0 0x7 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -247,6 +253,8 @@ ranges = <0x2 0x0 0x22 0x0 0x4 0x0>, <0x7 0x0 0x27 0x0 0xf 0x80000000>; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -297,11 +305,78 @@ }; }; +&dcpext0_die1 { + // TODO: verify + apple,bw-scratch = <&pmgr_dcp 0 4 0x9c0>; +}; + +&dcpext1_die1 { + // TODO: verify + apple,bw-scratch = <&pmgr_dcp 0 4 0x9c8>; +}; + &ps_gfx { // On t6002, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; +&pmp { + apple,pio-ranges = <0x2 0x82000000 0x0 0x1000000>, + <0x3 0x4000000 0x0 0x1000000>, + <0x3 0x83000000 0x0 0x1000000>, + <0x4 0x2000000 0x0 0x1000000>, + <0x2 0x10e70000 0x0 0x90000>, + <0x2 0x11e70000 0x0 0x90000>, + <0x2 0x12e70000 0x0 0x90000>, + <0x4 0x82000000 0x0 0x1000000>, + <0x5 0x2000000 0x0 0x1000000>, + <0x22 0x82000000 0x0 0x1000000>, + <0x23 0x4000000 0x0 0x1000000>, + <0x23 0x83000000 0x0 0x1000000>, + <0x24 0x2000000 0x0 0x1000000>, + <0x24 0x82000000 0x0 0x1000000>, + <0x25 0x2000000 0x0 0x1000000>, + <0x22 0x10e70000 0x0 0x90000>, + <0x22 0x11e70000 0x0 0x90000>, + <0x22 0x12e70000 0x0 0x90000>; +}; + +&pmp_report { + pmp_report_dispext0_die1: report@33 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x33>; + label = "pmp-dispext0_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0_die1>; + }; + + pmp_report_dispext1_die1: report@34 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x34>; + label = "pmp-dispext1_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + }; + + pmp_report_dispext2_die1: report@35 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x35>; + label = "pmp-dispext2_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + status = "disabled"; + }; + + pmp_report_dispext3_die1: report@36 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x36>; + label = "pmp-dispext3_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + status = "disabled"; + }; +}; + &gpu { - compatible = "apple,agx-g13d", "apple,agx-g13s"; + compatible = "apple,agx-t6002", "apple,agx-g13d", "apple,agx-g13s"; }; diff --git a/arch/arm64/boot/dts/apple/t600x-common.dtsi b/arch/arm64/boot/dts/apple/t600x-common.dtsi index e20234ef213538..a7f25092533e7d 100644 --- a/arch/arm64/boot/dts/apple/t600x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-common.dtsi @@ -13,6 +13,9 @@ aliases { gpu = &gpu; + #ifdef APPLE_USE_PMP + pmp = &pmp; + #endif }; cpus { @@ -229,26 +232,31 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <47296>; }; opp02 { opp-hz = /bits/ 64 <972000000>; opp-level = <2>; clock-latency-ns = <23000>; + opp-microwatt = <99715>; }; opp03 { opp-hz = /bits/ 64 <1332000000>; opp-level = <3>; clock-latency-ns = <29000>; + opp-microwatt = <188860>; }; opp04 { opp-hz = /bits/ 64 <1704000000>; opp-level = <4>; clock-latency-ns = <40000>; + opp-microwatt = <288891>; }; opp05 { opp-hz = /bits/ 64 <2064000000>; opp-level = <5>; clock-latency-ns = <50000>; + opp-microwatt = <412979>; }; }; @@ -259,82 +267,139 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <8000>; + opp-microwatt = <290230>; }; opp02 { opp-hz = /bits/ 64 <828000000>; opp-level = <2>; clock-latency-ns = <18000>; + opp-microwatt = <449013>; }; opp03 { opp-hz = /bits/ 64 <1056000000>; opp-level = <3>; clock-latency-ns = <19000>; + opp-microwatt = <647097>; }; opp04 { opp-hz = /bits/ 64 <1296000000>; opp-level = <4>; clock-latency-ns = <23000>; + opp-microwatt = <865620>; }; opp05 { opp-hz = /bits/ 64 <1524000000>; opp-level = <5>; clock-latency-ns = <24000>; + opp-microwatt = <1112838>; }; opp06 { opp-hz = /bits/ 64 <1752000000>; opp-level = <6>; clock-latency-ns = <28000>; + opp-microwatt = <1453271>; }; opp07 { opp-hz = /bits/ 64 <1980000000>; opp-level = <7>; clock-latency-ns = <31000>; + opp-microwatt = <1776667>; }; opp08 { opp-hz = /bits/ 64 <2208000000>; opp-level = <8>; clock-latency-ns = <45000>; + opp-microwatt = <2366690>; }; opp09 { opp-hz = /bits/ 64 <2448000000>; opp-level = <9>; clock-latency-ns = <49000>; + opp-microwatt = <2892193>; }; opp10 { opp-hz = /bits/ 64 <2676000000>; opp-level = <10>; clock-latency-ns = <53000>; + opp-microwatt = <3475417>; }; opp11 { opp-hz = /bits/ 64 <2904000000>; opp-level = <11>; clock-latency-ns = <56000>; + opp-microwatt = <3959410>; }; opp12 { opp-hz = /bits/ 64 <3036000000>; opp-level = <12>; clock-latency-ns = <56000>; + opp-microwatt = <4540620>; }; - /* Not available until CPU deep sleep is implemented opp13 { opp-hz = /bits/ 64 <3132000000>; opp-level = <13>; clock-latency-ns = <56000>; + opp-microwatt = <4745031>; turbo-mode; }; opp14 { opp-hz = /bits/ 64 <3168000000>; opp-level = <14>; clock-latency-ns = <56000>; + opp-microwatt = <4822390>; turbo-mode; }; opp15 { opp-hz = /bits/ 64 <3228000000>; opp-level = <15>; clock-latency-ns = <56000>; + opp-microwatt = <4951324>; turbo-mode; }; - */ + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <388800000>; + opp-microvolt = GPU_REPEAT(634000); + opp-microwatt = <25011450>; + }; + opp02 { + opp-hz = /bits/ 64 <486000000>; + opp-microvolt = GPU_REPEAT(650000); + opp-microwatt = <31681170>; + }; + opp03 { + opp-hz = /bits/ 64 <648000000>; + opp-microvolt = GPU_REPEAT(668000); + opp-microwatt = <41685750>; + }; + opp04 { + opp-hz = /bits/ 64 <777600000>; + opp-microvolt = GPU_REPEAT(715000); + opp-microwatt = <56692620>; + }; + opp05 { + opp-hz = /bits/ 64 <972000000>; + opp-microvolt = GPU_REPEAT(778000); + opp-microwatt = <83371500>; + }; + opp06 { + opp-hz = /bits/ 64 <1296000000>; + opp-microvolt = GPU_REPEAT(903000); + opp-microwatt = <166743000>; + }; }; pmu-e { @@ -373,6 +438,40 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <237333328>; + clock-output-names = "clk_disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + + clk_dispext0_die1: clock-dispext0_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0_die1"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1"; + }; + + clk_dispext1_die1: clock-dispext1_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1_die1"; + }; /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -401,15 +500,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t600x-die0.dtsi b/arch/arm64/boot/dts/apple/t600x-die0.dtsi index f715b19efd1679..6bf9802ed4dc8b 100644 --- a/arch/arm64/boot/dts/apple/t600x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-die0.dtsi @@ -24,6 +24,215 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c800 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + apple,dcs-min-ps = <7>; + }; + + pmp_dart: iommu@28e300000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x8e300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@28e3c0000 { + compatible = "apple,t6000-pmp-v2-report"; + reg = <0x2 0x8e3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_ane_sys: report@a { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@b { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + pmp_report_disp0: report@c { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext0: report@d { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-dispext0"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext1: report@e { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0>; + }; + + pmp_report_venc_sys: report@10 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x10>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@11 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr0: report@12 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + pmp_report_jpg: report@13 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x13>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@14 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x14>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_afnc4_ioa: report@1d { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x1d>; + label = "pmp-afnc4-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_afnc5_ioa: report@1e { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x1e>; + label = "pmp-afnc5-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_dispext2: report@20 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext2"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + status = "disabled"; + }; + + pmp_report_dispext3: report@21 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x21>; + label = "pmp-dispext3"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + status = "disabled"; + }; + + pmp_report_venc1: report@22 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x22>; + label = "pmp-venc1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + status = "disabled"; + }; + + pmp_report_msr1: report@23 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x23>; + label = "pmp-msr1"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + status = "disabled"; + }; + + pmp_report_prores: report@24 { + compatible = "apple,t6000-pmp-v2-report-entry"; + reg = <0x24>; + label = "pmp-prores"; + #power-domain-cells = <0>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + }; + + pmgr_dcp: power-management@28e3d0000 { + reg = <0x2 0x8e3d0000 0x0 0x4000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + + pmp: pmp@28e700000 { + compatible = "apple,t6000-pmp-v2"; + reg = <0x2 0x8e700000 0x0 0x100000>, + <0x2 0x8ec00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_mbox: mbox@28ec08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8ec08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + smc: smc@290400000 { compatible = "apple,t6000-smc", "apple,smc"; reg = <0x2 0x90400000 0x0 0x4000>, @@ -37,12 +246,16 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; }; rtc { @@ -151,22 +364,202 @@ interrupts = ; }; - sio_dart_0: iommu@39b004000 { + aop_mbox: mbox@293408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x93408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@293808000 { compatible = "apple,t6000-dart"; - reg = <0x3 0x9b004000 0x0 0x4000>; + reg = <0x2 0x93808000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@293980000 { + compatible = "apple,t6000-admac", "apple,admac"; + reg = <0x2 0x93980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 0 600 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 7>; + status = "disabled"; + }; + + aop: aop@293c00000 { + compatible = "apple,t6000-aop"; + reg = <0x2 0x93c00000 0x0 0x250000>, + <0x2 0x93400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t6000-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t6000-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t6000-aop-las", "apple,aop-las"; + }; + }; + + disp0_dart: iommu@38b304000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x8b304000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; }; - sio_dart_1: iommu@39b008000 { + dcp_dart: iommu@38b30c000 { compatible = "apple,t6000-dart"; - reg = <0x3 0x9b008000 0x0 0x8000>; + reg = <0x3 0x8b30c000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + }; + + sep_dart: iommu@3952c0000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x952c0000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@396400000 { + compatible = "apple,sep"; + reg = <0x3 0x96400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mbox@396408000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x96408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dpaudio0: audio-controller@39b500000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dcp_mbox: mbox@38bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x8bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@38bc00000 { + compatible = "apple,t6000-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x8bc00000 0x0 0x4000>, + <0x3 0x8a000000 0x0 0x3000000>, + <0x3 0x8b320000 0x0 0x4000>, + <0x3 0x8b344000 0x0 0x4000>, + <0x3 0x8b800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x988>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_disp0>; + #else + power-domains = <&ps_disp0_cpu0>; + #endif + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; }; fpwm0: pwm@39b030000 { @@ -333,10 +726,10 @@ "tx2a", "rx2a", "tx2b", "rx2b", "tx3a", "rx3a", "tx3b", "rx3b"; interrupt-parent = <&aic>; - interrupts = , + interrupts = , + , , - , - ; + ; power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>, <&ps_mca2>, <&ps_mca3>; resets = <&ps_audio_p>; @@ -356,6 +749,43 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <790000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <2.4>; + apple,avg-power-kp = <1.5>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <500.0>; + apple,fast-die0-proportional-gain = <72.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain = <6.3>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <15.75>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <30.0>; + apple,ppm-kp = <1.5>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); }; agx_mbox: mbox@406408000 { @@ -371,6 +801,59 @@ #mbox-cells = <0>; }; + isp_dart0: iommu@3860e8000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart1: iommu@3860f4000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart2: iommu@3860fc000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x860fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp: isp@384000000 { + compatible = "apple,t6000-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x3 0x84000000 0x0 0x2000000>, + <0x3 0x86104000 0x0 0x100>, + <0x3 0x86104170 0x0 0x100>, + <0x3 0x861043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_isp_sys>, <&ps_isp_set0>, + #else + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + #endif + <&ps_isp_set1>, <&ps_isp_fe>, <&ps_isp_set3>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>; + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + pcie0_dart_0: iommu@581008000 { compatible = "apple,t6000-dart"; reg = <0x5 0x81008000 0x0 0x4000>; @@ -448,6 +931,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi index 9676d5127039b7..121d158ff1b3ea 100644 --- a/arch/arm64/boot/dts/apple/t600x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-dieX.dtsi @@ -24,6 +24,168 @@ #performance-domain-cells = <0>; }; + DIE_NODE(dispext0_dart): iommu@289304000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_dart): iommu@28930c000 { + compatible = "apple,t6000-dart"; + reg = <0x2 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_mbox): mbox@289c08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext0): dcp@289c00000 { + compatible = "apple,t6000-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext0_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext0_dart) 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x89c00000 0x0 0x4000>, + <0x2 0x88000000 0x0 0x3000000>, + <0x2 0x89320000 0x0 0x4000>, + <0x2 0x89344000 0x0 0x4000>, + <0x2 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x990>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext0)>; + #else + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + clocks = <&DIE_NODE(clk_dispext0)>; + phandle = <&DIE_NODE(dcpext0)>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext0_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext0_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio1_dcp)>; + }; + }; + }; + }; + + DIE_NODE(dispext1_dart): iommu@28c304000 { + compatible = "apple,t6000-dart", "apple,t8110-dart"; + reg = <0x2 0x8c304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_dart): iommu@28c30c000 { + compatible = "apple,t6000-dart", "apple,t8110-dart"; + reg = <0x2 0x8c30c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x1f0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_mbox): mbox@28cc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8cc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext1): dcp@28cc00000 { + compatible = "apple,t6000-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext1_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext1_dart) 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x8cc00000 0x0 0x4000>, + <0x2 0x8b000000 0x0 0x3000000>, + <0x2 0x8c320000 0x0 0x4000>, + <0x2 0x8c344000 0x0 0x4000>, + <0x2 0x8c800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x998>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext1)>; + #else + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + clocks = <&DIE_NODE(clk_dispext1)>; + phandle = <&DIE_NODE(dcpext1)>; + apple,dcp-index = <2>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext1_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext1_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio2_dcp)>; + }; + }; + }; + }; + DIE_NODE(pmgr): power-management@28e080000 { compatible = "apple,t6000-pmgr", "apple,pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -95,6 +257,24 @@ ; }; + DIE_NODE(sio_dart_0): iommu@39b004000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x9b004000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio_dart_1): iommu@39b008000 { + compatible = "apple,t6000-dart"; + reg = <0x3 0x9b008000 0x0 0x8000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { compatible = "apple,t6000-pinctrl", "apple,pinctrl"; reg = <0x3 0x9b028000 0x0 0x4000>; @@ -120,6 +300,120 @@ #interrupt-cells = <2>; }; + DIE_NODE(sio_mbox): mbox@39bc08000 { + compatible = "apple,t6000-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio): sio@39bc00000 { + compatible = "apple,t6000-sio", "apple,sio"; + reg = <0x3 0x9bc00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&DIE_NODE(sio_mbox)>; + iommus = <&DIE_NODE(sio_dart_0) 0>, <&DIE_NODE(sio_dart_1) 0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + resets = <&DIE_NODE(ps_sio)>; /* TODO: verify reset does something */ + status = "disabled"; + }; + + DIE_NODE(dpaudio1): audio-controller@39b504000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b540000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x66>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa1)>; + reset-domains = <&DIE_NODE(ps_dpa1)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio1_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext0_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio2): audio-controller@39b508000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b580000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x68>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa2)>; + reset-domains = <&DIE_NODE(ps_dpa2)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio2_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext1_audio)>; + }; + }; + }; + }; + + /* + * omit dpaudio3 / 4 as long as the linked dcpext nodes don't exist + * + DIE_NODE(dpaudio3): audio-controller@39b50c000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b5c0000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6a>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa3)>; + reset-domains = <&DIE_NODE(ps_dpa3)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio3_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext2_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio4): audio-controller@39b510000 { + compatible = "apple,t6000-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6c>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa4)>; + reset-domains = <&DIE_NODE(ps_dpa4)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio4_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext3_audio)>; + }; + }; + }; + }; + */ + DIE_NODE(dwc3_0): usb@702280000 { compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; reg = <0x7 0x02280000 0x0 0xcd00>, <0x7 0x0228cd00 0x0 0x3200>; @@ -173,6 +467,14 @@ power-domains = <&DIE_NODE(ps_atc0_usb)>; }; + DIE_NODE(atcphy0_xbar): mux@70304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0x7 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + status = "disabled"; + }; + DIE_NODE(dwc3_1): usb@b02280000 { compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; @@ -226,6 +528,14 @@ power-domains = <&DIE_NODE(ps_atc1_usb)>; }; + DIE_NODE(atcphy1_xbar): mux@b0304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0xb 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + status = "disabled"; + }; + DIE_NODE(dwc3_2): usb@f02280000 { compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; @@ -279,6 +589,14 @@ power-domains = <&DIE_NODE(ps_atc2_usb)>; }; + DIE_NODE(atcphy2_xbar): mux@f0304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0xf 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + status = "disabled"; + }; + DIE_NODE(dwc3_3): usb@1302280000 { compatible = "apple,t6000-dwc3", "apple,t8103-dwc3"; reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; @@ -331,3 +649,11 @@ mode-switch; power-domains = <&DIE_NODE(ps_atc3_usb)>; }; + + DIE_NODE(atcphy3_xbar): mux@130304c000 { + compatible = "apple,t6000-display-crossbar"; + reg = <0x13 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi index fee84f809a9cc3..38808916e29948 100644 --- a/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j314-j316.dtsi @@ -20,7 +20,12 @@ atcphy2 = &atcphy2; atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; + dcp = &dcp; + dcpext0 = &dcpext0; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; + sio = &sio; wifi0 = &wifi0; }; @@ -36,6 +41,9 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + panel = <&panel>; + post-init-providers = <&panel>; + power-domains = <&ps_disp0_cpu0>; }; }; @@ -61,6 +69,76 @@ status = "okay"; }; +&dcp { + panel: panel { + apple,max-brightness = <500>; + }; +}; + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + /* enabled by the loader */ + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_nub 15 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 6 GPIO_ACTIVE_HIGH>; + + phys = <&atcphy3 PHY_TYPE_DP>; + phy-names = "dp-phy"; + mux-controls = <&atcphy3_xbar 0>; + mux-control-names = "dp-xbar"; + mux-index = <0>; + apple,dptx-phy = <3>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@1 { + reg = <1>; + + decpext0_dpout: endpoint { + remote-endpoint = <&atcphy3_dp>; + }; + }; + }; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + +&atcphy3_xbar { + status = "okay"; +}; + +&ps_atc3_common { + apple,always-on; /* Needs to stay on for HDMI resume */ +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { @@ -169,14 +247,138 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + status = "okay"; + + speaker_left_tweet: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0f0>; + }; + + speaker_left_woof2: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + status = "okay"; + + speaker_right_tweet: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f0f>; + }; + + speaker_right_woof2: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; +}; + &nco_clkref { clock-frequency = <1068000000>; }; +#ifndef NO_SPI_TRACKPAD +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 194 0>; + interrupts-extended = <&pinctrl_nub 6 IRQ_TYPE_LEVEL_LOW>; + }; +}; +#endif + /* PCIe devices */ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -196,6 +398,8 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; + status = "okay"; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -208,6 +412,10 @@ status = "okay"; }; +&pcie0_dart_1 { + status = "okay"; +}; + /* USB controllers */ &dwc3_0 { ports { @@ -275,10 +483,11 @@ /* * ps_atc3_usb_aon power-domain is always-on to keep dwc3 working over suspend. * atc3 is used exclusively for the DP-to-HDMI so do not keep this always on. + * On t600x it is required to keep atc DP state over suspend. */ -&ps_atc3_usb_aon { - /delete-property/ apple,always-on; -}; +// &ps_atc3_usb_aon { +// /delete-property/ apple,always-on; +// }; /* ATC3 is used for DisplayPort -> HDMI only */ &dwc3_3_dart_0 { @@ -363,7 +572,80 @@ }; &atcphy3 { - status = "disabled"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@2 { + reg = <2>; + atcphy3_dp: endpoint { + remote-endpoint = <&decpext0_dpout>; + }; + }; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +/ { + sound: sound { + /* compatible is set per machine */ + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; }; #include "spi1-nvram.dtsi" + +#include "isp-imx558.dtsi" + +&isp { + apple,platform-id = <3>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-fan-dual.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-j375.dtsi b/arch/arm64/boot/dts/apple/t600x-j375.dtsi index 8a1494949e4c58..ce339c0855bc98 100644 --- a/arch/arm64/boot/dts/apple/t600x-j375.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-j375.dtsi @@ -18,8 +18,14 @@ atcphy2 = &atcphy2; atcphy3 = &atcphy3; bluetooth0 = &bluetooth0; + #ifndef NO_DCP + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + #endif ethernet0 = ðernet0; serial0 = &serial0; + sio = &sio; wifi0 = &wifi0; }; @@ -35,6 +41,7 @@ reg = <0 0 0 0>; /* To be filled by loader */ /* Format properties will be added by loader */ status = "disabled"; + power-domains = <&ps_disp0_cpu0>; }; }; @@ -48,6 +55,15 @@ status = "okay"; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + /* USB Type C */ &i2c0 { hpm0: usb-pd@38 { @@ -349,14 +365,71 @@ }; }; +/* Audio */ +&i2c1 { + status = "okay"; + + speaker: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 178 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 179 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 4 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 180 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &nco_clkref { clock-frequency = <1068000000>; }; +/ { + sound: sound { + /* compatible is set per machine */ + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + /* PCIe devices */ &port00 { /* WLAN */ bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -376,6 +449,7 @@ &port01 { /* SD card reader */ bus-range = <2 2>; + pwren-gpios = <&smc_gpio 26 GPIO_ACTIVE_HIGH>; sdhci0: mmc@0,0 { compatible = "pci17a0,9755"; reg = <0x20000 0x0 0x0 0x0 0x0>; @@ -398,6 +472,7 @@ &port03 { /* USB xHCI */ bus-range = <4 4>; + pwren-gpios = <&smc_gpio 20 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -411,3 +486,5 @@ }; #include "spi1-nvram.dtsi" + +#include "hwmon-common.dtsi" diff --git a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi index 0bd44753b76a0c..833a59ecf37922 100644 --- a/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t600x-pmgr.dtsi @@ -396,6 +396,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext0_cpu0); power-domains = <&DIE_NODE(ps_dispext0_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_dispext1_cpu0): power-controller@2a8 { @@ -405,6 +406,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext1_cpu0); power-domains = <&DIE_NODE(ps_dispext1_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_ane_sys_cpu): power-controller@2c8 { @@ -424,6 +426,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pmp); + apple,always-on; }; #endif @@ -433,6 +436,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pms_sram); + apple,always-on; }; DIE_NODE(ps_apcie_st_sys): power-controller@2e8 { @@ -824,7 +828,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(sio_cpu); - power-domains = <&DIE_NODE(ps_sio)>; + power-domains = <&DIE_NODE(ps_sio) &DIE_NODE(ps_uart_p) &DIE_NODE(ps_spi_p) &DIE_NODE(ps_audio_p)>; }; DIE_NODE(ps_fpwm0): power-controller@190 { @@ -1113,6 +1117,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca0); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca1): power-controller@290 { @@ -1122,6 +1127,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca1); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca2): power-controller@298 { @@ -1131,6 +1137,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca2); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca3): power-controller@2a0 { @@ -1140,6 +1147,7 @@ #reset-cells = <0>; label = DIE_LABEL(mca3); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_dpa0): power-controller@2a8 { @@ -1293,7 +1301,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_fe); power-domains = <&DIE_NODE(ps_afnc2_lw0)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; DIE_NODE(ps_disp0_cpu0): power-controller@350 { @@ -1303,7 +1310,6 @@ #reset-cells = <0>; label = DIE_LABEL(disp0_cpu0); power-domains = <&DIE_NODE(ps_disp0_fe)>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; @@ -1368,6 +1374,7 @@ #reset-cells = <0>; label = DIE_LABEL(isp_sys); power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + status = "disabled"; }; DIE_NODE(ps_venc_sys): power-controller@3b0 { @@ -1385,12 +1392,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(ans2); - /* - * The ADT makes ps_apcie_st[1]_sys depend on ps_ans2 instead, - * but we'd rather have a single power domain for the downstream - * device to depend on, so use this node as the child. - * This makes more sense anyway (since ANS2 uses APCIE_ST). - */ power-domains = <&DIE_NODE(ps_afnc2_lw0)>; }; @@ -1456,6 +1457,86 @@ label = DIE_LABEL(venc_me1); power-domains = <&DIE_NODE(ps_venc_me0)>; }; + + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + DIE_NODE(ps_isp_set0): power-controller@4000 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set0); + }; + + DIE_NODE(ps_isp_set1): power-controller@4010 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set1); + }; + + DIE_NODE(ps_isp_fe): power-controller@4008 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ps_isp_fe); + }; + + DIE_NODE(ps_isp_set3): power-controller@4028 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set3); + }; + + DIE_NODE(ps_isp_set4): power-controller@4020 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set4); + }; + + DIE_NODE(ps_isp_set5): power-controller@4030 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set5); + }; + + DIE_NODE(ps_isp_set6): power-controller@4018 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set6); + }; + + DIE_NODE(ps_isp_set7): power-controller@4038 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set7); + }; + + DIE_NODE(ps_isp_set8): power-controller@4040 { + compatible = "apple,t6000-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_set8); + }; }; &DIE_NODE(pmgr_south) { @@ -1715,6 +1796,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext2_cpu0); power-domains = <&DIE_NODE(ps_dispext2_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_dispext3_fe): power-controller@210 { @@ -1733,6 +1815,7 @@ #reset-cells = <0>; label = DIE_LABEL(dispext3_cpu0); power-domains = <&DIE_NODE(ps_dispext3_fe)>; + apple,min-state = <4>; }; DIE_NODE(ps_msr1): power-controller@250 { @@ -1881,6 +1964,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(msg); + apple,always-on; /* Core AON device? */ }; DIE_NODE(ps_nub_gpio): power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t6020-j414s.dts b/arch/arm64/boot/dts/apple/t6020-j414s.dts index 631c54c5f03dee..a227069727dd8f 100644 --- a/arch/arm64/boot/dts/apple/t6020-j414s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j414s.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,tokara"; }; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J414"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j414-macaudio", "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J414"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j416s.dts b/arch/arm64/boot/dts/apple/t6020-j416s.dts index c277ed5889a214..3ea2b1d52593e2 100644 --- a/arch/arm64/boot/dts/apple/t6020-j416s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j416s.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,amami"; }; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J416"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j416-macaudio", "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J416"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6020-j474s.dts b/arch/arm64/boot/dts/apple/t6020-j474s.dts index 7c7ad5b8ad189e..12dfe9693502ad 100644 --- a/arch/arm64/boot/dts/apple/t6020-j474s.dts +++ b/arch/arm64/boot/dts/apple/t6020-j474s.dts @@ -45,3 +45,84 @@ <0x200 &pcie0_dart_2 1 1>, <0x300 &pcie0_dart_3 1 1>; }; + +&port02 { + bus-range = <2 2>; +}; + +ðernet0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; +}; + +&port03 { + bus-range = <3 3>; +}; + +&sound { + compatible = "apple,j474-macaudio", "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J474"; +}; + +&lpdptxphy { + status = "okay"; +}; + +#define USE_DCPEXT0 1 + +#if USE_DCPEXT0 +/ { + aliases { + dcpext0 = &dcpext0; + /delete-property/ dcp; + }; +}; + +&framebuffer0 { + power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; +}; + +&dcp { + status = "disabled"; +}; +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext0_dart { + status = "okay"; +}; +&dcpext0_mbox { + status = "okay"; +}; +&dpaudio1 { + status = "okay"; +}; +&dcpext0 { +#else +&dpaudio0 { + status = "okay"; +}; +&dcp { +#endif + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; +}; + +&gpu { + /* Apple does not do this, but they probably should */ + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6020.dtsi b/arch/arm64/boot/dts/apple/t6020.dtsi index bffa66a3ffff3f..500726e697a894 100644 --- a/arch/arm64/boot/dts/apple/t6020.dtsi +++ b/arch/arm64/boot/dts/apple/t6020.dtsi @@ -9,6 +9,8 @@ /* This chip is just a cut down version of t6021, so include it and disable the missing parts */ +#define GPU_REPEAT(x) + #include "t6021.dtsi" / { @@ -16,7 +18,21 @@ }; /delete-node/ &pmgr_south; +/delete-node/ &pmp_report_dispext2; +/delete-node/ &pmp_report_dispext3; +/delete-node/ &pmp_report_venc1; +/delete-node/ &pmp_report_msr1; +/delete-node/ &pmp_report_prores; +/delete-node/ &pmp_report_afnc4_ioa; +/delete-node/ &pmp_report_afnc5_ioa; &gpu { - compatible = "apple,agx-g14s"; + compatible = "apple,agx-t6020", "apple,agx-g14x", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <302>; + apple,avg-power-ki-only = <2.6375>; + apple,avg-power-kp = <0.18>; + apple,fast-die0-integral-gain = <1350.0>; + apple,ppm-filter-time-constant-ms = <32>; + apple,ppm-ki = <28.0>; }; diff --git a/arch/arm64/boot/dts/apple/t6021-j414c.dts b/arch/arm64/boot/dts/apple/t6021-j414c.dts index cdcf0740714dcf..fab3b03ff3c452 100644 --- a/arch/arm64/boot/dts/apple/t6021-j414c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j414c.dts @@ -24,3 +24,24 @@ &bluetooth0 { brcm,board-type = "apple,tokara"; }; + +&panel { + compatible = "apple,panel-j414", "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; +}; + +&aop_audio { + apple,chassis-name = "J414"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j414-macaudio", "apple,j314-macaudio", "apple,macaudio"; + model = "MacBook Pro J414"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j414c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j416c.dts b/arch/arm64/boot/dts/apple/t6021-j416c.dts index 6d8146b9417036..b476e235639ffc 100644 --- a/arch/arm64/boot/dts/apple/t6021-j416c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j416c.dts @@ -17,6 +17,26 @@ model = "Apple MacBook Pro (16-inch, M2 Max, 2023)"; }; +/* This machine model (only) has two extra boost CPU P-states * + * Disabled: Only the highest CPU bin (38 GPU cores) has this. + * Keep this disabled until m1n1 learns how to remove these OPPs + * for unsupported machines, otherwise it breaks cpufreq. +&avalanche_opp { + opp18 { + opp-hz = /bits/ 64 <3528000000>; + opp-level = <18>; + clock-latency-ns = <67000>; + turbo-mode; + }; + opp19 { + opp-hz = /bits/ 64 <3696000000>; + opp-level = <19>; + clock-latency-ns = <67000>; + turbo-mode; + }; +}; +*/ + &wifi0 { brcm,board-type = "apple,amami"; }; @@ -24,3 +44,24 @@ &bluetooth0 { brcm,board-type = "apple,amami"; }; + +&panel { + compatible = "apple,panel-j416", "apple,panel-mini-led", "apple,panel"; + width-mm = <346>; + height-mm = <223>; + adj-height-mm = <216>; +}; + +&aop_audio { + apple,chassis-name = "J416"; + apple,machine-kind = "MacBook Pro"; +}; + +&sound { + compatible = "apple,j416-macaudio", "apple,j316-macaudio", "apple,macaudio"; + model = "MacBook Pro J416"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j416c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6021-j475c.dts b/arch/arm64/boot/dts/apple/t6021-j475c.dts index 533e3577487469..e4321cfc556838 100644 --- a/arch/arm64/boot/dts/apple/t6021-j475c.dts +++ b/arch/arm64/boot/dts/apple/t6021-j475c.dts @@ -29,9 +29,99 @@ /* enable PCIe port01 with SDHCI */ &port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; status = "okay"; }; &pcie0_dart_1 { status = "okay"; }; + +&pinctrl_ap { + usb_hub_oe-hog { + gpio-hog; + gpios = <231 0>; + input; + line-name = "usb-hub-oe"; + }; + + usb_hub_rst-hog { + gpio-hog; + gpios = <232 GPIO_ACTIVE_LOW>; + output-low; + line-name = "usb-hub-rst"; + }; +}; + +&sound { + compatible = "apple,j475-macaudio", "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J475"; +}; + +&lpdptxphy { + status = "okay"; +}; + + +#define USE_DCPEXT0 1 + +#if USE_DCPEXT0 +/ { + aliases { + dcpext0 = &dcpext0; + /delete-property/ dcp; + }; +}; + +&framebuffer0 { + power-domains = <&ps_dispext0_cpu0>, <&ps_dptx_phy_ps>; +}; + +&dcp { + status = "disabled"; +}; +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext0_dart { + status = "okay"; +}; +&dcpext0_mbox { + status = "okay"; +}; +&dpaudio1 { + status = "okay"; +}; +&dcpext0 { +#else +&dpaudio0 { + status = "okay"; +}; +&dcp { +#endif + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; +}; + +&gpu { + apple,idleoff-standby-timer = <3000>; + apple,perf-base-pstate = <5>; + apple,perf-boost-ce-step = <100>; + apple,perf-boost-min-util = <75>; + apple,perf-tgt-utilization = <70>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 62907ad6a54683..bb0e66851f1b59 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -16,6 +16,13 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif + #include "t602x-common.dtsi" / { @@ -28,6 +35,8 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; // filled via templated includes at the end of the file }; @@ -65,5 +74,12 @@ }; &gpu { - compatible = "apple,agx-g14c", "apple,agx-g14s"; + compatible = "apple,agx-t6021", "apple,agx-g14x", "apple,agx-g14c", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <1.5125>; + apple,avg-power-kp = <0.38>; + apple,fast-die0-integral-gain = <700.0>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; }; diff --git a/arch/arm64/boot/dts/apple/t6022-j180d.dts b/arch/arm64/boot/dts/apple/t6022-j180d.dts index f76b887429dd27..4a528c28031805 100644 --- a/arch/arm64/boot/dts/apple/t6022-j180d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j180d.dts @@ -11,6 +11,7 @@ #include "t6022.dtsi" #include "t6022-jxxxd.dtsi" +#include "t6022-pcie-ge.dtsi" / { compatible = "apple,j180d", "apple,t6022", "apple,arm-platform"; @@ -26,8 +27,12 @@ atcphy5 = &atcphy1_die1; atcphy6 = &atcphy2_die1; atcphy7 = &atcphy3_die1; - nvram = &nvram; + bluetooth0 = &bluetooth0; + dcpext0 = &dcpext0; + ethernet0 = ðernet0; + ethernet1 = ðernet1; serial0 = &serial0; + wifi0 = &wifi0; }; chosen { @@ -46,6 +51,13 @@ }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@10000000000 { device_type = "memory"; reg = <0x100 0 0x2 0>; /* To be filled by loader */ @@ -56,6 +68,46 @@ status = "okay"; }; +&lpdptxphy { + status = "okay"; +}; + +&display { + iommus = <&dispext0_dart_die1 0>, <&dispext0_dart 0>; +}; + +&dispext0_dart { + status = "okay"; +}; + +&dcpext0_dart { + status = "okay"; +}; + +&dcpext0_mbox { + status = "okay"; +}; + +&dcpext0 { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; + + // shared between dp2hdmi-gpio0 / dp2hdmi-gpio1 + // hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + apple,dptx-die = <0>; +}; + +&dpaudio1 { + status = "okay"; +}; + /* USB Type C Rear */ &i2c0 { hpm2: usb-pd@3b { @@ -521,6 +573,429 @@ }; }; +/* Audio */ +&i2c1 { + status = "okay"; + + speaker_tweeter: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Tweeter"; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + }; + + speaker_woofer: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Woofer"; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&nco_clkref { + clock-frequency = <1068000000>; +}; + +/ { + sound: sound { + compatible = "apple,j180-macaudio", "apple,macaudio"; + model = "Mac Pro J180"; + + dai-link@0 { + link-name = "Speakers"; + /* + * DANGER ZONE: You can blow your speakers! + * + * The drivers are not ready, and unless you are careful + * to attenuate the audio stream, you run the risk of + * blowing your speakers. + */ + status = "disabled"; + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_woofer>, <&speaker_tweeter>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +/* PCIe devices */ +&port_ge00 { + bus-range = <0x01 0x09>; + + pci@0,0 { + // compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x10000 0x00 0x00 0x00 0x00>; + bus-range = <0x02 0x07>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges = <0x82010000 0x00 0x80000000 0x82010000 0x00 0x80000000 0x00 0x500000>, + <0xc3010000 0x18 0x00000000 0xc3010000 0x18 0x00000000 0x00 0x500000>; + + #interrupt-cells = <0x01>; + interrupt-map-mask = <0xffff00 0x00 0x00 0x07>; + interrupt-map = <0x20000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x00>, + <0x20000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x01>, + <0x20000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x02>, + <0x20000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x01>, + <0x20800 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x02>, + <0x20800 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x02>, + <0x21000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x03>, + <0x21000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x03>, + <0x21800 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x00>, + <0x21800 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x01 &port_ge00 0x00 0x00 0x00 0x00>, + <0x22000 0x00 0x00 0x02 &port_ge00 0x00 0x00 0x00 0x01>, + <0x22000 0x00 0x00 0x03 &port_ge00 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x04 &port_ge00 0x00 0x00 0x00 0x03>; + + /* pci-slot1-dsp, PCIe slot-1 */ + pci@0,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20000 0x00 0x00 0x00 0x00>; + bus-range = <0x03 0x03>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot2-dsp, PCIe slot-2 */ + pci@1,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20800 0x00 0x00 0x00 0x00>; + bus-range = <0x04 0x04>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot3-dsp, PCIe slot-3 */ + pci@2,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21000 0x00 0x00 0x00 0x00>; + bus-range = <0x05 0x05>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot4-dsp, PCIe slot-4 */ + pci@3,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21800 0x00 0x00 0x00 0x00>; + bus-range = <0x06 0x06>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + /* pci-slot5-dsp, PCIe slot-5 */ + pci@4,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22000 0x00 0x00 0x00 0x00>; + bus-range = <0x07 0x07>; + + #address-cells = <0x03>; + #size-cells = <0x02>; + ranges; + }; + + }; +}; + +&port_ge00_die1 { + bus-range = <0x01 0x09>; + + /* + * Add mulptiple "reset-gpios" since there is no mechanismen to access + * PERST# for devices behind the PCIe switch. + * The "pwren" GPIO is from the wifi/bt chip which faces the same + * problem without pci-pwrctrl integration. + */ + reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>, + <&pinctrl_ap 6 GPIO_ACTIVE_LOW>, + <&pinctrl_ap 7 GPIO_ACTIVE_LOW>, + <&pinctrl_ap_die1 9 GPIO_ACTIVE_LOW>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + pci@0,0 { + device_type = "pci"; + reg = <0x10000 0x00 0x00 0x00 0x00>; + bus-range = <0x02 0x09>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0xffff00 0x00 0x00 0x07>; + interrupt-map = <0x20000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x20000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x20000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x20000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x20800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x20800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x20800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x21000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x21000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x21800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x21800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x21800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x22000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x22000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x22800 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x22800 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x22800 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x22800 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x23000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x23000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x03>, + <0x23000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x23000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>; + + /* pci-usba-dsp, internal USB-A port */ + pci@0,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20000 0x00 0x00 0x00 0x00>; + bus-range = <0x03 0x03>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x30000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x30000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x30000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 6 GPIO_ACTIVE_LOW>; + }; + + /* pci-sata-dsp, internal AHCI controller */ + pci@1,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x20800 0x00 0x00 0x00 0x00>; + bus-range = <0x04 0x04>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x40000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x40000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x40000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>; + }; + + /* pci-bio-dsp, I/O board USB-A ports */ + pci@2,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21000 0x00 0x00 0x00 0x00>; + bus-range = <0x05 0x05>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x50000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x50000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x50000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x00>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 7 GPIO_ACTIVE_LOW>; + }; + + /* pci-lan-dsp, Qtion AQC113 10G etherner controller (0) */ + pci@3,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x21800 0x00 0x00 0x00 0x00>; + bus-range = <0x06 0x06>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x60000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x60000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x60000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x01>; + + ethernet0: ethernet@0,0 { + reg = <0x60000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; + }; + + /* pci-lan-b-dsp, Qtion AQC113 10G etherner controller (1) */ + pci@4,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22000 0x00 0x00 0x00 0x00>; + bus-range = <0x07 0x07>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x70000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x70000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x70000 0x00 0x00 0x04 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + ethernet1: ethernet@0,0 { + reg = <0x70000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; + }; + + /* pci-wifibt-dsp, Broadcom BCM4388 Wlan/BT */ + pci@5,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x22800 0x00 0x00 0x00 0x00>; + bus-range = <0x08 0x08>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0x80000 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x80000 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x80000 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>, + <0x80100 0x00 0x00 0x01 &port_ge00_die1 0x00 0x00 0x00 0x00>, + <0x80100 0x00 0x00 0x02 &port_ge00_die1 0x00 0x00 0x00 0x01>, + <0x80100 0x00 0x00 0x03 &port_ge00_die1 0x00 0x00 0x00 0x02>; + + /* temporarily handled in the root port */ + // reset-gpios = <&pinctrl_ap 4 GPIO_ACTIVE_LOW>; + // pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + wifi0: wifi@0,0 { + reg = <0x80000 0x0 0x0 0x0 0x0>; + compatible = "pci14e4,4433"; + brcm,board-type = "apple,sumatra"; + apple,antenna-sku = "XX"; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + }; + + bluetooth0: network@0,1 { + compatible = "pci14e4,5f71"; + brcm,board-type = "apple,sumatra"; + // reg = <0x80100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; + }; + + /* pci-slot6-dsp, PCIe slot-6 */ + pci@6,0 { + compatible = "pci11f8,4000", "pciclass,060400", "pciclass,0604"; + device_type = "pci"; + reg = <0x23000 0x00 0x00 0x00 0x00>; + bus-range = <0x09 0x09>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; +}; + +&pcie_ge { + status = "ok"; +}; + +&pcie_ge_dart { + status = "ok"; +}; + +&pcie_ge_die1 { + status = "ok"; +}; + +&pcie_ge_dart_die1 { + status = "ok"; +}; + /* * Delete unused PCIe nodes, the Mac Pro uses slightly different PCIe * controllers with a single port connected to a PM40100 PCIe switch diff --git a/arch/arm64/boot/dts/apple/t6022-j475d.dts b/arch/arm64/boot/dts/apple/t6022-j475d.dts index 31f24bbda9689b..cdfc78a1703c7a 100644 --- a/arch/arm64/boot/dts/apple/t6022-j475d.dts +++ b/arch/arm64/boot/dts/apple/t6022-j475d.dts @@ -9,6 +9,8 @@ /dts-v1/; +#define NO_DCP + #include "t6022.dtsi" #include "t602x-j474-j475.dtsi" #include "t6022-jxxxd.dtsi" @@ -20,15 +22,27 @@ aliases { atcphy4 = &atcphy0_die1; atcphy5 = &atcphy1_die1; + /delete-property/ dcp; + /delete-property/ sio; }; }; +&sio { + status = "disabled"; +}; + &framebuffer0 { power-domains = <&ps_dispext0_cpu0_die1>, <&ps_dptx_phy_ps_die1>; }; +&dcpext0_die1 { + // J180 misses "function-dp2hdmi_pwr_en" + dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; +}; + /* enable PCIe port01 with SDHCI */ &port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -49,11 +63,13 @@ /delete-node/ &dwc3_2_dart_1_die1; /delete-node/ &dwc3_2_die1; /delete-node/ &atcphy2_die1; +/delete-node/ &atcphy2_xbar_die1; /delete-node/ &dwc3_3_dart_0_die1; /delete-node/ &dwc3_3_dart_1_die1; /delete-node/ &dwc3_3_die1; /delete-node/ &atcphy3_die1; +/delete-node/ &atcphy3_xbar_die1; /* delete unused always-on power-domains on die 1 */ /delete-node/ &ps_atc2_usb_aon_die1; @@ -71,3 +87,10 @@ compatible = "pci14e4,5f72"; brcm,board-type = "apple,canary"; }; + +&sound { + compatible = "apple,j475-macaudio", "apple,j375-macaudio", "apple,macaudio"; + model = "Mac Studio J475"; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi b/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi index dc877bd604f827..fa0183441d791b 100644 --- a/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi +++ b/arch/arm64/boot/dts/apple/t6022-jxxxd.dtsi @@ -9,9 +9,66 @@ * Copyright The Asahi Linux Contributors */ +/ { + aliases { + dcpext4 = &dcpext0_die1; + disp0 = &display; + sio1 = &sio_die1; + }; +}; + +&lpdptxphy_die1 { + status = "okay"; +}; + +&display { + iommus = <&dispext0_dart_die1 0>; +}; + +&dispext0_dart_die1 { + status = "okay"; +}; + +&dcpext0_dart_die1 { + status = "okay"; +}; + +&dcpext0_mbox_die1 { + status = "okay"; +}; + +&dcpext0_die1 { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 41 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + // J180 misses "function-dp2hdmi_pwr_en" + // dp2hdmi-pwren-gpios = <&smc_gpio 25 GPIO_ACTIVE_HIGH>; + + phys = <&lpdptxphy_die1>; + phy-names = "dp-phy"; + apple,dptx-phy = <4>; + apple,dptx-die = <1>; +}; + +&dpaudio1_die1 { + status = "okay"; +}; + +/* delete missing dcp0/disp0 */ +/delete-node/ &disp0_dart; +/delete-node/ &dcp_dart; +/delete-node/ &dcp_mbox; +/delete-node/ &dcp; +/delete-node/ &dpaudio0; + /* delete power-domains for missing disp0 / disp0_die1 */ /delete-node/ &ps_disp0_cpu0; /delete-node/ &ps_disp0_fe; +/delete-node/ &pmp_report_disp0; /delete-node/ &ps_disp0_cpu0_die1; /delete-node/ &ps_disp0_fe_die1; diff --git a/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi b/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi new file mode 100644 index 00000000000000..f78c483c29133f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6022-pcie-ge.dtsi @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Include PCIe-GE nodes presen on both dies of T6022 (M2 Ultra) in the + * Mac Pro (2023). + * + * Copyright The Asahi Linux Contributors + */ + +#define DIE +#define DIE_NO 0 + +&die0 { + #include "t602x-pcie-ge.dtsi" +}; + +#undef DIE +#undef DIE_NO + +#define DIE _die1 +#define DIE_NO 1 + +&die1 { + #include "t602x-pcie-ge.dtsi" +}; + +#undef DIE +#undef DIE_NO diff --git a/arch/arm64/boot/dts/apple/t6022.dtsi b/arch/arm64/boot/dts/apple/t6022.dtsi index e73bf2f7510ae2..fe065d48726a3d 100644 --- a/arch/arm64/boot/dts/apple/t6022.dtsi +++ b/arch/arm64/boot/dts/apple/t6022.dtsi @@ -16,6 +16,13 @@ #include "multi-die-cpp.h" +#ifndef GPU_REPEAT +# define GPU_REPEAT(x) +#endif +#ifndef GPU_DIE_REPEAT +# define GPU_DIE_REPEAT(x) +#endif + #include "t602x-common.dtsi" / { @@ -339,11 +346,58 @@ }; }; +&dcpext0_die1 { + apple,bw-scratch = <&pmgr_dcp 0 4 0x1240>; +}; + +&dcpext1_die1 { + apple,bw-scratch = <&pmgr_dcp 0 4 0x1248>; +}; + &ps_gfx { // On t6022, the die0 GPU power domain needs both AFR power domains power-domains = <&ps_afr>, <&ps_afr_die1>; }; +&pmp_report { + pmp_report_dispext0_die1: report@1f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1f>; + label = "pmp-dispext0_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0_die1>; + }; + + pmp_report_dispext1_die1: report@20 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext1_die1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0_die1>; + }; +}; + &gpu { - compatible = "apple,agx-g14d", "apple,agx-g14s"; + compatible = "apple,agx-t6022", "apple,agx-g14x", "apple,agx-g14d", "apple,agx-g14s"; + + apple,avg-power-filter-tc-ms = <302>; + apple,avg-power-ki-only = <1.0125>; + apple,avg-power-kp = <0.15>; + apple,fast-die0-integral-gain = <9.6>; + apple,fast-die0-proportional-gain = <24.0>; + apple,idleoff-standby-timer = <3000>; + apple,perf-base-pstate = <5>; + apple,perf-boost-ce-step = <100>; + apple,perf-boost-min-util = <75>; + apple,perf-tgt-utilization = <70>; + apple,ppm-ki = <11.0>; + apple,ppm-kp = <0.15>; +}; + +&pinctrl_ap_die1 { + pcie_ge_pins_die1: pcie-ge1-pins { + pinmux = ; + }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-common.dtsi b/arch/arm64/boot/dts/apple/t602x-common.dtsi index 9c800a391e7e87..2fa6b93a14165f 100644 --- a/arch/arm64/boot/dts/apple/t602x-common.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-common.dtsi @@ -13,6 +13,9 @@ aliases { gpu = &gpu; + #ifdef APPLE_USE_PMP + pmp = &pmp; + #endif }; cpus { @@ -387,6 +390,134 @@ }; }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = GPU_REPEAT(400000); + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_REPEAT(637000); + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_REPEAT(656000); + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_REPEAT(687000); + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = GPU_REPEAT(725000); + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = GPU_REPEAT(790000); + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_REPEAT(843000); + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = GPU_REPEAT(887000); + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = GPU_REPEAT(918000); + opp-microwatt = <22800000>; + }; + }; + + gpu_cs_opp: opp-table-gpu-cs { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <1024000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1140000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + + gpu_afr_opp: opp-table-gpu-afr { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <24>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp01 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = GPU_DIE_REPEAT(668000); + }; + opp02 { + opp-hz = /bits/ 64 <552000000>; + opp-microvolt = GPU_DIE_REPEAT(678000); + }; + opp03 { + opp-hz = /bits/ 64 <760000000>; + opp-microvolt = GPU_DIE_REPEAT(737000); + }; + opp04 { + opp-hz = /bits/ 64 <980000000>; + opp-microvolt = GPU_DIE_REPEAT(815000); + }; + opp05 { + opp-hz = /bits/ 64 <1098000000>; + opp-microvolt = GPU_DIE_REPEAT(862000); + }; + opp06 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = GPU_DIE_REPEAT(893000); + }; + }; + pmu-e { compatible = "apple,blizzard-pmu"; interrupt-parent = <&aic>; @@ -423,6 +554,41 @@ clock-output-names = "clk_200m"; }; + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <257142848>; /* TODO: check */ + clock-output-names = "clk_disp0"; + }; + + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + + clk_dispext0_die1: clock-dispext0_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0_die1"; + }; + + clk_dispext1: clock-dispext1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1"; + }; + + clk_dispext1_die1: clock-dispext1_die1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext1_die1"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -451,15 +617,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-die0.dtsi b/arch/arm64/boot/dts/apple/t602x-die0.dtsi index 8622ddea7b4453..c0c0626249a5a6 100644 --- a/arch/arm64/boot/dts/apple/t602x-die0.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-die0.dtsi @@ -7,7 +7,7 @@ */ nco: clock-controller@28e03c000 { - compatible = "apple,t6020-nco", "apple,t8103-nco"; + compatible = "apple,t6020-nco", "apple,t8103-nco", "apple,nco"; reg = <0x2 0x8e03c000 0x0 0x14000>; clocks = <&nco_clkref>; #clock-cells = <1>; @@ -23,8 +23,234 @@ power-domains = <&ps_aic>; }; + pmgr_misc: power-management@28e20c000 { + compatible = "apple,t6020-pmgr-misc", "apple,t6000-pmgr-misc"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x2 0x8e20c000 0 0x400>, + <0x2 0x8e20c400 0 0x400>; + reg-names = "fabric-ps", "dcs-ps"; + }; + + pmp_dart: iommu@28e300000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x8e300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@28e3c0000 { + compatible = "apple,t6020-pmp-v2-report"; + reg = <0x2 0x8e3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_ane_sys: report@b { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@c { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + pmp_report_disp0: report@d { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext0: report@e { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xe>; + label = "pmp-dispext0"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext1: report@f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xf>; + label = "pmp-dispext1"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_cpu0>; + apple,always-on; + }; + + pmp_report_venc_sys: report@10 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x10>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@11 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr0: report@12 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + status = "disabled"; + }; + + pmp_report_jpg: report@13 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x13>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@14 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x14>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_afnc4_ioa: report@1d { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1d>; + label = "pmp-afnc4-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_afnc5_ioa: report@1e { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1e>; + label = "pmp-afnc5-ioa"; + #power-domain-cells = <0>; + apple,always-on; + }; + + pmp_report_dispext2: report@1f { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x1f>; + label = "pmp-dispext2"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_cpu0>; + status = "disabled"; + }; + + pmp_report_dispext3: report@20 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x20>; + label = "pmp-dispext3"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_cpu0>; + status = "disabled"; + }; + + pmp_report_venc1: report@21 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x21>; + label = "pmp-venc1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + status = "disabled"; + }; + + pmp_report_msr1: report@22 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x22>; + label = "pmp-msr1"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + status = "disabled"; + }; + + pmp_report_prores: report@23 { + compatible = "apple,t6020-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x23>; + label = "pmp-prores"; + #power-domain-cells = <0>; + power-domains = <&ps_prores>; + status = "disabled"; + }; + }; + + pmgr_dcp: power-management@28e3d0000 { + reg = <0x2 0x8e3d0000 0x0 0x4000>; + reg-names = "dcp-fw-pmgr"; + #apple,bw-scratch-cells = <3>; + }; + + pmp: pmp@28e700000 { + compatible = "apple,t6020-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x2 0x8e700000 0x0 0x100000>, + <0x2 0x8ec00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + + pmp_mbox: mbox@28ec08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8ec08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + nub_spmi0: spmi@29e114000 { - compatible = "apple,t6020-spmi", "apple,t8103-spmi"; + compatible = "apple,t6020-spmi", "apple,t8103-spmi", "apple,spmi"; reg = <0x2 0x9e114000 0x0 0x100>; #address-cells = <2>; #size-cells = <0>; @@ -81,7 +307,7 @@ }; wdt: watchdog@29e2c4000 { - compatible = "apple,t6020-wdt", "apple,t8103-wdt"; + compatible = "apple,t6020-wdt", "apple,t8103-wdt", "apple,wdt"; reg = <0x2 0x9e2c4000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -102,7 +328,7 @@ }; smc: smc@2a2400000 { - compatible = "apple,t6020-smc", "apple,t8103-smc"; + compatible = "apple,t6020-smc", "apple,t8103-smc", "apple,smc"; reg = <0x2 0xa2400000 0x0 0x4000>, <0x2 0xa3e00000 0x0 0x100000>; reg-names = "smc", "sram"; @@ -114,12 +340,16 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; }; rtc { @@ -130,7 +360,7 @@ }; pinctrl_smc: pinctrl@2a2820000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0xa2820000 0x0 0x4000>; gpio-controller; @@ -150,13 +380,317 @@ ; }; - sio_dart: iommu@39b008000 { + aop_mbox: mbox@2a6408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa6408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@2a6808000 { compatible = "apple,t6020-dart", "apple,t8110-dart"; - reg = <0x3 0x9b008000 0x0 0x8000>; + reg = <0x2 0xa6808000 0x0 0x4000>; + #iommu-cells = <1>; interrupt-parent = <&aic>; - interrupts = ; + interrupts = ; + status = "disabled"; + // apple,dma-range = <0x100 0x0 0x300 0x0>; + }; + + aop_admac: dma-controller@2a6980000 { + compatible = "apple,t6020-admac", "apple,admac"; + reg = <0x2 0xa6980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 0 631 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 10>; + status = "disabled"; + }; + + aop: aop@2a6c00000 { + compatible = "apple,t6020-aop"; + reg = <0x2 0xa6c00000 0x0 0x250000>, + <0x2 0xa6400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t6020-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t6020-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t6020-aop-las", "apple,aop-las"; + }; + }; + + mtp: mtp@2a9400000 { + compatible = "apple,t6020-mtp", "apple,t6020-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xa9400000 0x0 0x4000>, + <0x2 0xa9c00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2a9408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa9408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@2a9808000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0xa9808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2a9b14000 { + compatible = "apple,t6020-dockchannel", "apple,dockchannel"; + reg = <0x2 0xa9b14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xa9b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + + isp_dart0: iommu@3860e8000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_isp_sys>; + #else + power-domains = <&ps_isp_sys>; + #endif + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp_dart1: iommu@3860f4000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp_dart2: iommu@3860fc000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x860fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + apple,dma-range = <0x100 0x0 0x1 0x0>; + status = "disabled"; + }; + + isp: isp@384000000 { + compatible = "apple,t6020-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x3 0x84000000 0x0 0x2000000>, + <0x3 0x86104000 0x0 0x100>, + <0x3 0x86104170 0x0 0x100>, + <0x3 0x861043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_cpu>, <&ps_isp_fe>, + <&ps_dprx>, <&ps_isp_vis>, <&ps_isp_be>, + <&ps_isp_clr>, <&ps_isp_raw>; + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + + disp0_dart: iommu@389304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_dart: iommu@38930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x8930c000 0x0 0x4000>; #iommu-cells = <1>; - power-domains = <&ps_sio_cpu>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp_mbox: mbox@389c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp: dcp@389c00000 { + compatible = "apple,t6020-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x89c00000 0x0 0x4000>, // check? + <0x3 0x88000000 0x0 0x61c000>, + <0x3 0x89320000 0x0 0x4000>, + <0x3 0x89344000 0x0 0x4000>, + <0x3 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1208>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_disp0>; + #else + power-domains = <&ps_disp0_cpu0>; + #endif + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + sep_dart: iommu@394ac0000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x94ac0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + sep: sep@396400000 { + compatible = "apple,sep"; + reg = <0x3 0x96400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + power-domains = <&ps_sep>; + status = "disabled"; + }; + + sep_mbox: mbox@396408000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x96408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; }; fpwm0: pwm@39b030000 { @@ -169,7 +703,7 @@ }; i2c0: i2c@39b040000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b040000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -182,7 +716,7 @@ }; i2c1: i2c@39b044000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b044000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -196,7 +730,7 @@ }; i2c2: i2c@39b048000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b048000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -210,7 +744,7 @@ }; i2c3: i2c@39b04c000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b04c000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -224,7 +758,7 @@ }; i2c4: i2c@39b050000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b050000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -238,7 +772,7 @@ }; i2c5: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -252,7 +786,7 @@ }; i2c6: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -266,7 +800,7 @@ }; i2c7: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -280,7 +814,7 @@ }; i2c8: i2c@39b054000 { - compatible = "apple,t6020-i2c", "apple,t8103-i2c"; + compatible = "apple,t6020-i2c", "apple,t8103-i2c", "apple,i2c"; reg = <0x3 0x9b054000 0x0 0x4000>; clocks = <&clkref>; interrupt-parent = <&aic>; @@ -294,7 +828,7 @@ }; spi1: spi@39b104000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b104000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -308,7 +842,7 @@ }; spi2: spi@39b108000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b108000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -322,7 +856,7 @@ }; spi4: spi@39b110000 { - compatible = "apple,t6020-spi", "apple,t8103-spi"; + compatible = "apple,t6020-spi", "apple,t8103-spi", "apple,spi"; reg = <0x3 0x9b110000 0x0 0x4000>; interrupt-parent = <&aic>; interrupts = ; @@ -352,7 +886,7 @@ }; admac: dma-controller@39b400000 { - compatible = "apple,t6020-admac", "apple,t8103-admac"; + compatible = "apple,t6020-admac", "apple,t8103-admac", "apple,admac"; reg = <0x3 0x9b400000 0x0 0x34000>; #dma-cells = <1>; dma-channels = <16>; @@ -365,8 +899,29 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@39b500000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + mca: mca@39b600000 { - compatible = "apple,t6020-mca", "apple,t8103-mca"; + compatible = "apple,t6020-mca", "apple,t8103-mca", "apple,mca"; reg = <0x3 0x9b600000 0x0 0x10000>, <0x3 0x9b500000 0x0 0x20000>; clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>; @@ -394,6 +949,14 @@ reg = <0x4 0x6400000 0 0x40000>, <0x4 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; mboxes = <&agx_mbox>; power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, @@ -402,6 +965,55 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + apple,firmware-version = <0 0 0>; + apple,firmware-compat = <0 0 0>; + + operating-points-v2 = <&gpu_opp>; + apple,cs-opp = <&gpu_cs_opp>; + apple,afr-opp = <&gpu_afr_opp>; + + apple,min-sram-microvolt = <790000>; + apple,csafr-min-sram-microvolt = <812000>; + apple,perf-base-pstate = <1>; + + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-proportional-gain = <34.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <1.62>; + apple,perf-integral-gain2 = <1.62>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <5.4>; + apple,perf-proportional-gain = <5.4>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <18.0>; + apple,ppm-kp = <0.1>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + apple,pwr-sample-period-aic-clks = <200000>; + apple,se-engagement-criteria = <700>; + apple,se-filter-time-constant = <9>; + apple,se-filter-time-constant-1 = <3>; + apple,se-inactive-threshold = <2500>; + apple,se-ki = <-50.0>; + apple,se-ki-1 = <-100.0>; + apple,se-kp = <-5.0>; + apple,se-kp-1 = <-10.0>; + apple,se-reset-criteria = <50>; + + apple,core-leak-coef = GPU_REPEAT(1200.0); + apple,sram-leak-coef = GPU_REPEAT(20.0); + apple,cs-leak-coef = GPU_DIE_REPEAT(400.0); + apple,afr-leak-coef = GPU_DIE_REPEAT(200.0); }; agx_mbox: mbox@406408000 { @@ -461,6 +1073,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi index ae3d535c5acb37..ae0038a4c28710 100644 --- a/arch/arm64/boot/dts/apple/t602x-dieX.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-dieX.dtsi @@ -23,6 +23,87 @@ #performance-domain-cells = <0>; }; + DIE_NODE(dispext0_dart): iommu@289304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x89304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_dart): iommu@28930c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x2 0x8930c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext0_mbox): mbox@289c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext0): dcp@289c00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext0_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext0_dart) 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x89c00000 0x0 0x4000>, + <0x2 0x88000000 0x0 0x4000000>, + <0x2 0x89320000 0x0 0x4000>, + <0x2 0x89344000 0x0 0x4000>, + <0x2 0x89800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1210>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext0)>; + #else + power-domains = <&DIE_NODE(ps_dispext0_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext0_cpu0)>; + clocks = <&DIE_NODE(clk_dispext0)>; + phandle = <&DIE_NODE(dcpext0)>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext0_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext0_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio1_dcp)>; + }; + }; + }; + }; + DIE_NODE(pmgr): power-management@28e080000 { compatible = "apple,t6020-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -45,7 +126,7 @@ }; DIE_NODE(pinctrl_nub): pinctrl@29e1f0000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x9e1f0000 0x0 0x4000>; power-domains = <&DIE_NODE(ps_nub_gpio)>; @@ -74,7 +155,7 @@ }; DIE_NODE(pinctrl_aop): pinctrl@2a6820000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0xa6820000 0x0 0x4000>; gpio-controller; @@ -94,8 +175,99 @@ ; }; + DIE_NODE(dispext1_dart): iommu@315304000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x15304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_dart): iommu@31530c000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x1530c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + DIE_NODE(dcpext1_mbox): mbox@315c08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x15c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + status = "disabled"; + }; + + DIE_NODE(dcpext1): dcp@315c00000 { + compatible = "apple,t6020-dcpext", "apple,dcpext"; + mboxes = <&DIE_NODE(dcpext1_mbox)>; + mbox-names = "mbox"; + iommus = <&DIE_NODE(dcpext1_dart) 5>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x3 0x15c00000 0x0 0x4000>, + <0x3 0x14000000 0x0 0x4000000>, + <0x3 0x15320000 0x0 0x4000>, + <0x3 0x15344000 0x0 0x4000>, + <0x3 0x15800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x1218>; + #ifdef APPLE_USE_PMP + power-domains = <&DIE_NODE(pmp_report_dispext1)>; + #else + power-domains = <&DIE_NODE(ps_dispext1_cpu0)>; + #endif + resets = <&DIE_NODE(ps_dispext1_cpu0)>; + clocks = <&DIE_NODE(clk_dispext1)>; + phandle = <&DIE_NODE(dcpext1)>; + apple,dcp-index = <2>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&DIE_NODE(dispext1_dart) 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dcpext1_audio): endpoint { + remote-endpoint = <&DIE_NODE(dpaudio2_dcp)>; + }; + }; + }; + }; + + DIE_NODE(sio_dart): iommu@39b008000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x3 0x9b008000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + power-domains = <&DIE_NODE(ps_sio)>; + //apple,dma-range = <0x100 0x0001c000 0x2ff 0xfffe4000>; + }; + DIE_NODE(pinctrl_ap): pinctrl@39b028000 { - compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl"; + compatible = "apple,t6020-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x3 0x9b028000 0x0 0x4000>; interrupt-parent = <&aic>; @@ -119,6 +291,131 @@ #interrupt-cells = <2>; }; + DIE_NODE(sio_mbox): mbox@39bc08000 { + compatible = "apple,t6020-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x9bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + }; + + DIE_NODE(sio): sio@39bc00000 { + compatible = "apple,t6020-sio", "apple,sio"; + reg = <0x3 0x9bc00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&DIE_NODE(sio_mbox)>; + iommus = <&DIE_NODE(sio_dart) 0>; + power-domains = <&DIE_NODE(ps_sio_cpu)>; + resets = <&DIE_NODE(ps_sio_cpu)>; + status = "disabled"; + }; + + DIE_NODE(dpaudio1): audio-controller@39b504000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b540000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x66>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa1)>; + reset-domains = <&DIE_NODE(ps_dpa1)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio1_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext0_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio2): audio-controller@39b508000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b580000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x68>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa2)>; + reset-domains = <&DIE_NODE(ps_dpa2)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio2_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext1_audio)>; + }; + }; + }; + }; + + /* + * omit dpaudio3 / 4 as long as the linked dcpext nodes don't exist + * + DIE_NODE(dpaudio3): audio-controller@39b50c000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b5c0000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6a>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa3)>; + reset-domains = <&DIE_NODE(ps_dpa3)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio3_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext2_audio)>; + }; + }; + }; + }; + + DIE_NODE(dpaudio4): audio-controller@39b510000 { + compatible = "apple,t6020-dpaudio", "apple,dpaudio"; + reg = <0x3 0x9b500000 0x0 0x4000>; + dmas = <&DIE_NODE(sio) 0x6c>; + dma-names = "tx"; + power-domains = <&DIE_NODE(ps_dpa4)>; + reset-domains = <&DIE_NODE(ps_dpa4)>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + DIE_NODE(dpaudio4_dcp): endpoint { + remote-endpoint = <&DIE_NODE(dcpext3_audio)>; + }; + }; + }; + }; + */ + + DIE_NODE(lpdptxphy): phy@39c000000 { + compatible = "apple,t6020-dptx-phy", "apple,dptx-phy"; + reg = <0x3 0x9c000000 0x0 0x4000>, + <0x3 0x9c040000 0x0 0xc000>; + reg-names = "core", "dptx"; + power-domains = <&DIE_NODE(ps_dptx_phy_ps)>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; /* only exposed on desktop devices */ + }; + DIE_NODE(pmgr_gfx): power-management@404e80000 { compatible = "apple,t6020-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -180,6 +477,14 @@ power-domains = <&DIE_NODE(ps_atc0_usb)>; }; + DIE_NODE(atcphy0_xbar): mux@70304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x7 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc0_usb)>; + status = "disabled"; + }; + DIE_NODE(dwc3_1): usb@b02280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0xb 0x02280000 0x0 0xcd00>, <0xb 0x0228cd00 0x0 0x3200>; @@ -233,6 +538,14 @@ power-domains = <&DIE_NODE(ps_atc1_usb)>; }; + DIE_NODE(atcphy1_xbar): mux@b0304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xb 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc1_usb)>; + status = "disabled"; + }; + DIE_NODE(dwc3_2): usb@f02280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0xf 0x02280000 0x0 0xcd00>, <0xf 0x0228cd00 0x0 0x3200>; @@ -286,6 +599,14 @@ power-domains = <&DIE_NODE(ps_atc2_usb)>; }; + DIE_NODE(atcphy2_xbar): mux@f0304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xf 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc2_usb)>; + status = "disabled"; + }; + DIE_NODE(dwc3_3): usb@1302280000 { compatible = "apple,t6020-dwc3", "apple,t8103-dwc3"; reg = <0x13 0x02280000 0x0 0xcd00>, <0x13 0x0228cd00 0x0 0x3200>; @@ -338,3 +659,11 @@ mode-switch; power-domains = <&DIE_NODE(ps_atc3_usb)>; }; + + DIE_NODE(atcphy3_xbar): mux@130304c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0x13 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&DIE_NODE(ps_atc3_usb)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi index e41b6475f79218..c5de99bd2e5cf3 100644 --- a/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-gpio-pins.dtsi @@ -78,4 +78,8 @@ , ; }; + + pcie_ge_pins: pcie-ge-pins { + pinmux = ; + }; }; diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 0e806d8ddf81b1..0057e6a9465f9d 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -14,12 +14,31 @@ * the GPIO indices. */ +#define NO_SPI_TRACKPAD #include "t600x-j314-j316.dtsi" +/ { + aliases { + keyboard = &keyboard; + }; +}; + &framebuffer0 { power-domains = <&ps_disp0_cpu0>, <&ps_dptx_phy_ps>; }; +/* HACK: keep dptx_phy_ps power-domain always-on + * it is unclear how to sequence with dcp for the integrated display + */ +&ps_dptx_phy_ps { + apple,always-on; +}; + +&dcpext0 { + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 25 GPIO_ACTIVE_HIGH>; +}; + &hpm0 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; @@ -36,6 +55,40 @@ interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; +/* Redefine GPIO for SDZ */ +&speaker_sdz { + gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; +}; + +&speaker_left_tweet { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof1 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_left_woof2 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_tweet { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof1 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&speaker_right_woof2 { + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; + &wifi0 { compatible = "pci14e4,4434"; }; @@ -43,3 +96,56 @@ &bluetooth0 { compatible = "pci14e4,5f72"; }; + +&port01 { + pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; +}; + +&ps_mtp_fabric { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; + + mtp_mt: multi-touch { + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +&isp { + apple,platform-id = <7>; + /delete-node/ sensor-presets; /* Override j31[46] below */ +}; + +#include "isp-imx558-cfg0.dtsi" diff --git a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi index ee12fea5b12cb3..287348628eb177 100644 --- a/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j474-j475.dtsi @@ -36,3 +36,23 @@ &hpm3 { interrupts = <44 IRQ_TYPE_LEVEL_LOW>; }; + +/* PCIe devices */ +&port00 { + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; +}; + +&port03 { + /* USB xHCI */ + pwren-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; +}; + +&speaker { + shutdown-gpios = <&pinctrl_ap 57 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 58 IRQ_TYPE_LEVEL_LOW>; +}; + +&jack_codec { + reset-gpios = <&pinctrl_nub 8 GPIO_ACTIVE_HIGH>; + interrupts-extended = <&pinctrl_ap 59 IRQ_TYPE_LEVEL_LOW>; +}; diff --git a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi index 590cec8ac804c0..eb8c4e359079e5 100644 --- a/arch/arm64/boot/dts/apple/t602x-nvme.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-nvme.dtsi @@ -26,7 +26,7 @@ }; DIE_NODE(nvme): nvme@34bcc0000 { - compatible = "apple,t6020-nvme-ans2", "apple,t8103-nvme-ans2"; + compatible = "apple,t6020-nvme-ans2", "apple,t8103-nvme-ans2", "apple,nvme-ans2"; reg = <0x3 0x4bcc0000 0x0 0x40000>, <0x3 0x47400000 0x0 0x4000>; reg-names = "nvme", "ans"; interrupt-parent = <&aic>; diff --git a/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi b/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi new file mode 100644 index 00000000000000..4a509cae0e5766 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t602x-pcie-ge.dtsi @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PCIe-GE Nodes present on both dies of a T6022 (M2 Ultra) and M2 Pro/Max but + * only used on T6022 in the Mac Pro (2023). + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(pcie_ge): pcie@1680000000 { + compatible = "apple,t6020-pcie-ge", "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x16 0x80000000 0x0 0x1000000>, /* config */ + <0x16 0x91000000 0x0 0x4000>, /* rc */ + <0x16 0x94008000 0x0 0x4000>, /* port0 */ + <0x16 0x9e01c000 0x0 0x4000>, /* phy0 */ + <0x16 0x9401c000 0x0 0x1000>; /* ltssm0 */ + reg-names = "config", "rc", "port0", "phy0", "ltssm0"; + + interrupt-parent = <&aic>; + interrupts = ; + + msi-controller; + msi-parent = <&DIE_NODE(pcie_ge)>; + msi-ranges = <&aic AIC_IRQ DIE_NO 1672 IRQ_TYPE_EDGE_RISING 128>; + + iommu-map = <0x000 &DIE_NODE(pcie_ge_dart) 0 0>, + <0x100 &DIE_NODE(pcie_ge_dart) 1 1>, + <0x200 &DIE_NODE(pcie_ge_dart) 2 2>, + <0x300 &DIE_NODE(pcie_ge_dart) 3 3>, + <0x400 &DIE_NODE(pcie_ge_dart) 4 4>, + <0x500 &DIE_NODE(pcie_ge_dart) 5 5>, + <0x600 &DIE_NODE(pcie_ge_dart) 6 6>, + <0x700 &DIE_NODE(pcie_ge_dart) 7 7>, + <0x800 &DIE_NODE(pcie_ge_dart) 8 8>, + <0x900 &DIE_NODE(pcie_ge_dart) 9 9>, + <0xa00 &DIE_NODE(pcie_ge_dart) 10 10>, + <0xb00 &DIE_NODE(pcie_ge_dart) 11 11>, + <0xc00 &DIE_NODE(pcie_ge_dart) 12 12>, + <0xd00 &DIE_NODE(pcie_ge_dart) 13 13>, + <0xe00 &DIE_NODE(pcie_ge_dart) 14 14>, + <0xf00 &DIE_NODE(pcie_ge_dart) 15 15>; + iommu-map-mask = <0xff00>; + + bus-range = <0 15>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x18 0x00000000 0x18 0x00000000 0x4 0x00000000>, + <0x02000000 0x00 0x80000000 0x17 0x80000000 0x0 0x80000000>; + + power-domains = <&DIE_NODE(ps_apcie_ge_sys)>; + pinctrl-0 = <&DIE_NODE(pcie_ge_pins)>; + pinctrl-names = "default"; + + dma-coherent; + + status = "disabled"; + + DIE_NODE(port_ge00): pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&DIE_NODE(pinctrl_ap) 9 GPIO_ACTIVE_LOW>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &DIE_NODE(port_ge00) 0 0 0 0>, + <0 0 0 2 &DIE_NODE(port_ge00) 0 0 0 1>, + <0 0 0 3 &DIE_NODE(port_ge00) 0 0 0 2>, + <0 0 0 4 &DIE_NODE(port_ge00) 0 0 0 3>; + }; + }; + + DIE_NODE(pcie_ge_dart): iommu@1694000000 { + compatible = "apple,t6020-dart", "apple,t8110-dart"; + reg = <0x16 0x94000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&DIE_NODE(ps_apcie_ge_sys)>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi index f5382a2faf0b25..4ead781fea6893 100644 --- a/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-pmgr.dtsi @@ -7,7 +7,7 @@ &DIE_NODE(pmgr) { DIE_NODE(ps_afi): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -16,7 +16,7 @@ }; DIE_NODE(ps_aic): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -25,7 +25,7 @@ }; DIE_NODE(ps_dwi): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -33,7 +33,7 @@ }; DIE_NODE(ps_pms): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -42,7 +42,7 @@ }; DIE_NODE(ps_gpio): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -51,7 +51,7 @@ }; DIE_NODE(ps_soc_dpe): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -60,7 +60,7 @@ }; DIE_NODE(ps_pms_c1ppt): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -69,7 +69,7 @@ }; DIE_NODE(ps_pmgr_soc_ocla): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -78,7 +78,7 @@ }; DIE_NODE(ps_amcc0): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -87,7 +87,7 @@ }; DIE_NODE(ps_amcc2): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -96,7 +96,7 @@ }; DIE_NODE(ps_dcs_00): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -105,7 +105,7 @@ }; DIE_NODE(ps_dcs_01): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -114,7 +114,7 @@ }; DIE_NODE(ps_dcs_02): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -123,7 +123,7 @@ }; DIE_NODE(ps_dcs_03): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -132,7 +132,7 @@ }; DIE_NODE(ps_dcs_08): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -141,7 +141,7 @@ }; DIE_NODE(ps_dcs_09): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -150,7 +150,7 @@ }; DIE_NODE(ps_dcs_10): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -159,7 +159,7 @@ }; DIE_NODE(ps_dcs_11): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -168,7 +168,7 @@ }; DIE_NODE(ps_afnc1_ioa): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -178,7 +178,7 @@ }; DIE_NODE(ps_afc): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -187,7 +187,7 @@ }; DIE_NODE(ps_afnc0_ioa): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -197,7 +197,7 @@ }; DIE_NODE(ps_afnc1_ls): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -207,7 +207,7 @@ }; DIE_NODE(ps_afnc0_ls): power-controller@1f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -217,7 +217,7 @@ }; DIE_NODE(ps_afnc1_lw0): power-controller@200 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x200 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -227,7 +227,7 @@ }; DIE_NODE(ps_afnc1_lw1): power-controller@208 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x208 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -237,7 +237,7 @@ }; DIE_NODE(ps_afnc1_lw2): power-controller@210 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x210 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -247,7 +247,7 @@ }; DIE_NODE(ps_afnc0_lw0): power-controller@218 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x218 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -257,7 +257,7 @@ }; DIE_NODE(ps_scodec): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -266,7 +266,7 @@ }; DIE_NODE(ps_atc0_common): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -275,7 +275,7 @@ }; DIE_NODE(ps_atc1_common): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -284,7 +284,7 @@ }; DIE_NODE(ps_atc2_common): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -293,7 +293,7 @@ }; DIE_NODE(ps_atc3_common): power-controller@240 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x240 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -302,7 +302,7 @@ }; DIE_NODE(ps_dispext1_sys): power-controller@248 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x248 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -311,7 +311,7 @@ }; DIE_NODE(ps_pms_bridge): power-controller@250 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x250 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -321,7 +321,7 @@ }; DIE_NODE(ps_dispext0_sys): power-controller@258 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x258 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -330,7 +330,7 @@ }; DIE_NODE(ps_ane_sys): power-controller@260 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x260 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -339,7 +339,7 @@ }; DIE_NODE(ps_avd_sys): power-controller@268 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -348,7 +348,7 @@ }; DIE_NODE(ps_atc0_cio): power-controller@270 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x270 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -357,7 +357,7 @@ }; DIE_NODE(ps_atc0_pcie): power-controller@278 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x278 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -366,7 +366,7 @@ }; DIE_NODE(ps_atc1_cio): power-controller@280 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x280 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -375,7 +375,7 @@ }; DIE_NODE(ps_atc1_pcie): power-controller@288 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -384,7 +384,7 @@ }; DIE_NODE(ps_atc2_cio): power-controller@290 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x290 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -393,7 +393,7 @@ }; DIE_NODE(ps_atc2_pcie): power-controller@298 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x298 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -402,7 +402,7 @@ }; DIE_NODE(ps_atc3_cio): power-controller@2a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -411,7 +411,7 @@ }; DIE_NODE(ps_atc3_pcie): power-controller@2a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -420,7 +420,7 @@ }; DIE_NODE(ps_dispext1_fe): power-controller@2b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -429,7 +429,7 @@ }; DIE_NODE(ps_dispext1_cpu0): power-controller@2b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -439,7 +439,7 @@ }; DIE_NODE(ps_dispext0_fe): power-controller@2c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -447,24 +447,27 @@ power-domains = <&DIE_NODE(ps_dispext0_sys)>; }; + /* PMP is only present on die 0 of the M2 Ultra */ DIE_NODE(ps_pmp): power-controller@2c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pmp); + apple,always-on; }; DIE_NODE(ps_pms_sram): power-controller@2d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(pms_sram); + apple,always-on; }; DIE_NODE(ps_dispext0_cpu0): power-controller@2d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -474,7 +477,7 @@ }; DIE_NODE(ps_ane_cpu): power-controller@2e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -483,7 +486,7 @@ }; DIE_NODE(ps_atc0_cio_pcie): power-controller@2e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -492,7 +495,7 @@ }; DIE_NODE(ps_atc0_cio_usb): power-controller@2f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -501,7 +504,7 @@ }; DIE_NODE(ps_atc1_cio_pcie): power-controller@2f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -510,7 +513,7 @@ }; DIE_NODE(ps_atc1_cio_usb): power-controller@300 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -519,7 +522,7 @@ }; DIE_NODE(ps_atc2_cio_pcie): power-controller@308 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x308 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -528,7 +531,7 @@ }; DIE_NODE(ps_atc2_cio_usb): power-controller@310 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x310 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -537,7 +540,7 @@ }; DIE_NODE(ps_atc3_cio_pcie): power-controller@318 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x318 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -546,7 +549,7 @@ }; DIE_NODE(ps_atc3_cio_usb): power-controller@320 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x320 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -555,7 +558,7 @@ }; DIE_NODE(ps_trace_fab): power-controller@390 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x390 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -563,7 +566,7 @@ }; DIE_NODE(ps_ane_sys_mpm): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -572,7 +575,7 @@ }; DIE_NODE(ps_ane_td): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -581,7 +584,7 @@ }; DIE_NODE(ps_ane_base): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -590,7 +593,7 @@ }; DIE_NODE(ps_ane_set1): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -599,7 +602,7 @@ }; DIE_NODE(ps_ane_set2): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -608,7 +611,7 @@ }; DIE_NODE(ps_ane_set3): power-controller@4028 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -617,7 +620,7 @@ }; DIE_NODE(ps_ane_set4): power-controller@4030 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -628,7 +631,7 @@ &DIE_NODE(pmgr_south) { DIE_NODE(ps_amcc4): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -637,7 +640,7 @@ }; DIE_NODE(ps_amcc5): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -646,7 +649,7 @@ }; DIE_NODE(ps_amcc6): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -655,7 +658,7 @@ }; DIE_NODE(ps_amcc7): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -664,7 +667,7 @@ }; DIE_NODE(ps_dcs_16): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -673,7 +676,7 @@ }; DIE_NODE(ps_dcs_17): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -682,7 +685,7 @@ }; DIE_NODE(ps_dcs_18): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -691,7 +694,7 @@ }; DIE_NODE(ps_dcs_19): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -700,7 +703,7 @@ }; DIE_NODE(ps_dcs_20): power-controller@140 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x140 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -709,7 +712,7 @@ }; DIE_NODE(ps_dcs_21): power-controller@148 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x148 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -718,7 +721,7 @@ }; DIE_NODE(ps_dcs_22): power-controller@150 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x150 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -727,7 +730,7 @@ }; DIE_NODE(ps_dcs_23): power-controller@158 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x158 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -736,7 +739,7 @@ }; DIE_NODE(ps_dcs_24): power-controller@160 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x160 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -745,7 +748,7 @@ }; DIE_NODE(ps_dcs_25): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -754,7 +757,7 @@ }; DIE_NODE(ps_dcs_26): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -763,7 +766,7 @@ }; DIE_NODE(ps_dcs_27): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -772,7 +775,7 @@ }; DIE_NODE(ps_dcs_28): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -781,7 +784,7 @@ }; DIE_NODE(ps_dcs_29): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -790,7 +793,7 @@ }; DIE_NODE(ps_dcs_30): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -799,7 +802,7 @@ }; DIE_NODE(ps_dcs_31): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -808,7 +811,7 @@ }; DIE_NODE(ps_afnc4_ioa): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -818,7 +821,7 @@ }; DIE_NODE(ps_afnc4_ls): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -828,7 +831,7 @@ }; DIE_NODE(ps_afnc4_lw0): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -838,7 +841,7 @@ }; DIE_NODE(ps_afnc5_ioa): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -848,7 +851,7 @@ }; DIE_NODE(ps_afnc5_ls): power-controller@1c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -858,7 +861,7 @@ }; DIE_NODE(ps_afnc5_lw0): power-controller@1c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -868,7 +871,7 @@ }; DIE_NODE(ps_dispext2_sys): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -876,7 +879,7 @@ }; DIE_NODE(ps_msr1): power-controller@1d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -884,7 +887,7 @@ }; DIE_NODE(ps_dispext2_fe): power-controller@1e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -893,7 +896,7 @@ }; DIE_NODE(ps_dispext2_cpu0): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -903,7 +906,7 @@ }; DIE_NODE(ps_msr1_ase_core): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -912,7 +915,7 @@ }; DIE_NODE(ps_dispext3_sys): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -920,7 +923,7 @@ }; DIE_NODE(ps_venc1_sys): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -928,7 +931,7 @@ }; DIE_NODE(ps_dispext3_fe): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -937,7 +940,7 @@ }; DIE_NODE(ps_dispext3_cpu0): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -947,7 +950,7 @@ }; DIE_NODE(ps_venc1_dma): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -956,7 +959,7 @@ }; DIE_NODE(ps_venc1_pipe4): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -965,7 +968,7 @@ }; DIE_NODE(ps_venc1_pipe5): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -974,7 +977,7 @@ }; DIE_NODE(ps_venc1_me0): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -983,7 +986,7 @@ }; DIE_NODE(ps_venc1_me1): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -994,7 +997,7 @@ &DIE_NODE(pmgr_east) { DIE_NODE(ps_clvr_spmi0): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1003,7 +1006,7 @@ }; DIE_NODE(ps_clvr_spmi1): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1012,7 +1015,7 @@ }; DIE_NODE(ps_clvr_spmi2): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1021,7 +1024,7 @@ }; DIE_NODE(ps_clvr_spmi3): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1030,7 +1033,7 @@ }; DIE_NODE(ps_clvr_spmi4): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1039,7 +1042,7 @@ }; DIE_NODE(ps_ispsens0): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1047,7 +1050,7 @@ }; DIE_NODE(ps_ispsens1): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1055,7 +1058,7 @@ }; DIE_NODE(ps_ispsens2): power-controller@138 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x138 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1063,7 +1066,7 @@ }; DIE_NODE(ps_ispsens3): power-controller@140 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x140 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1071,7 +1074,7 @@ }; DIE_NODE(ps_afnc6_ioa): power-controller@148 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x148 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1081,7 +1084,7 @@ }; DIE_NODE(ps_afnc6_ls): power-controller@150 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x150 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1091,7 +1094,7 @@ }; DIE_NODE(ps_afnc6_lw0): power-controller@158 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x158 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1101,7 +1104,7 @@ }; DIE_NODE(ps_afnc2_ioa): power-controller@160 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x160 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1111,7 +1114,7 @@ }; DIE_NODE(ps_afnc2_ls): power-controller@168 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x168 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1121,7 +1124,7 @@ }; DIE_NODE(ps_afnc2_lw0): power-controller@170 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x170 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1131,7 +1134,7 @@ }; DIE_NODE(ps_afnc2_lw1): power-controller@178 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x178 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1141,7 +1144,7 @@ }; DIE_NODE(ps_afnc3_ioa): power-controller@180 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x180 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1151,7 +1154,7 @@ }; DIE_NODE(ps_afnc3_ls): power-controller@188 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x188 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1161,7 +1164,7 @@ }; DIE_NODE(ps_afnc3_lw0): power-controller@190 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x190 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1171,7 +1174,7 @@ }; DIE_NODE(ps_apcie_gp): power-controller@198 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x198 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1180,7 +1183,7 @@ }; DIE_NODE(ps_apcie_st): power-controller@1a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1189,7 +1192,7 @@ }; DIE_NODE(ps_ans2): power-controller@1a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1198,7 +1201,7 @@ }; DIE_NODE(ps_disp0_sys): power-controller@1b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1207,7 +1210,7 @@ }; DIE_NODE(ps_jpg): power-controller@1b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1216,7 +1219,7 @@ }; DIE_NODE(ps_sio): power-controller@1c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1225,7 +1228,7 @@ }; DIE_NODE(ps_isp_sys): power-controller@1c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1235,7 +1238,7 @@ }; DIE_NODE(ps_disp0_fe): power-controller@1d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1244,7 +1247,7 @@ }; DIE_NODE(ps_disp0_cpu0): power-controller@1d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1254,16 +1257,16 @@ }; DIE_NODE(ps_sio_cpu): power-controller@1e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(sio_cpu); - power-domains = <&DIE_NODE(ps_sio)>; + power-domains = <&DIE_NODE(ps_sio) &DIE_NODE(ps_uart_p) &DIE_NODE(ps_spi_p) &DIE_NODE(ps_audio_p)>; }; DIE_NODE(ps_fpwm0): power-controller@1e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1272,7 +1275,7 @@ }; DIE_NODE(ps_fpwm1): power-controller@1f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1281,7 +1284,7 @@ }; DIE_NODE(ps_fpwm2): power-controller@1f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x1f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1290,7 +1293,7 @@ }; DIE_NODE(ps_i2c0): power-controller@200 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x200 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1299,7 +1302,7 @@ }; DIE_NODE(ps_i2c1): power-controller@208 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x208 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1308,7 +1311,7 @@ }; DIE_NODE(ps_i2c2): power-controller@210 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x210 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1317,7 +1320,7 @@ }; DIE_NODE(ps_i2c3): power-controller@218 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x218 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1326,7 +1329,7 @@ }; DIE_NODE(ps_i2c4): power-controller@220 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x220 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1335,7 +1338,7 @@ }; DIE_NODE(ps_i2c5): power-controller@228 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x228 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1344,7 +1347,7 @@ }; DIE_NODE(ps_i2c6): power-controller@230 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x230 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1353,7 +1356,7 @@ }; DIE_NODE(ps_i2c7): power-controller@238 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x238 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1362,7 +1365,7 @@ }; DIE_NODE(ps_i2c8): power-controller@240 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x240 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1371,7 +1374,7 @@ }; DIE_NODE(ps_spi_p): power-controller@248 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x248 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1380,7 +1383,7 @@ }; DIE_NODE(ps_sio_spmi0): power-controller@250 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x250 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1389,7 +1392,7 @@ }; DIE_NODE(ps_sio_spmi1): power-controller@258 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x258 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1398,7 +1401,7 @@ }; DIE_NODE(ps_sio_spmi2): power-controller@260 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x260 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1407,7 +1410,7 @@ }; DIE_NODE(ps_uart_p): power-controller@268 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x268 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1416,7 +1419,7 @@ }; DIE_NODE(ps_audio_p): power-controller@270 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x270 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1425,7 +1428,7 @@ }; DIE_NODE(ps_sio_adma): power-controller@278 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x278 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1434,7 +1437,7 @@ }; DIE_NODE(ps_aes): power-controller@280 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x280 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1444,7 +1447,7 @@ }; DIE_NODE(ps_dptx_phy_ps): power-controller@288 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x288 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1453,7 +1456,7 @@ }; DIE_NODE(ps_spi0): power-controller@2d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1462,7 +1465,7 @@ }; DIE_NODE(ps_spi1): power-controller@2e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1471,7 +1474,7 @@ }; DIE_NODE(ps_spi2): power-controller@2e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1480,7 +1483,7 @@ }; DIE_NODE(ps_spi3): power-controller@2f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1489,7 +1492,7 @@ }; DIE_NODE(ps_spi4): power-controller@2f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x2f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1498,7 +1501,7 @@ }; DIE_NODE(ps_spi5): power-controller@300 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x300 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1507,7 +1510,7 @@ }; DIE_NODE(ps_uart_n): power-controller@308 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x308 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1516,7 +1519,7 @@ }; DIE_NODE(ps_uart0): power-controller@310 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x310 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1525,7 +1528,7 @@ }; DIE_NODE(ps_amcc1): power-controller@318 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x318 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1534,7 +1537,7 @@ }; DIE_NODE(ps_amcc3): power-controller@320 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x320 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1543,7 +1546,7 @@ }; DIE_NODE(ps_dcs_04): power-controller@328 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x328 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1552,7 +1555,7 @@ }; DIE_NODE(ps_dcs_05): power-controller@330 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x330 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1561,7 +1564,7 @@ }; DIE_NODE(ps_dcs_06): power-controller@338 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x338 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1570,7 +1573,7 @@ }; DIE_NODE(ps_dcs_07): power-controller@340 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x340 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1579,7 +1582,7 @@ }; DIE_NODE(ps_dcs_12): power-controller@348 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x348 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1588,7 +1591,7 @@ }; DIE_NODE(ps_dcs_13): power-controller@350 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x350 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1597,7 +1600,7 @@ }; DIE_NODE(ps_dcs_14): power-controller@358 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x358 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1606,7 +1609,7 @@ }; DIE_NODE(ps_dcs_15): power-controller@360 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x360 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1615,7 +1618,7 @@ }; DIE_NODE(ps_uart1): power-controller@368 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x368 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1624,7 +1627,7 @@ }; DIE_NODE(ps_uart2): power-controller@370 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x370 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1633,7 +1636,7 @@ }; DIE_NODE(ps_uart3): power-controller@378 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x378 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1642,7 +1645,7 @@ }; DIE_NODE(ps_uart4): power-controller@380 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x380 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1651,7 +1654,7 @@ }; DIE_NODE(ps_uart5): power-controller@388 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x388 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1660,7 +1663,7 @@ }; DIE_NODE(ps_uart6): power-controller@390 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x390 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1669,43 +1672,47 @@ }; DIE_NODE(ps_mca0): power-controller@398 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x398 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca0); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca1): power-controller@3a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3a0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca1); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca2): power-controller@3a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3a8 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca2); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_mca3): power-controller@3b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3b0 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(mca3); power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + apple,externally-clocked; }; DIE_NODE(ps_dpa0): power-controller@3b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3b8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1714,7 +1721,7 @@ }; DIE_NODE(ps_dpa1): power-controller@3c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3c0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1723,7 +1730,7 @@ }; DIE_NODE(ps_dpa2): power-controller@3c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3c8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1732,7 +1739,7 @@ }; DIE_NODE(ps_dpa3): power-controller@3d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3d0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1741,7 +1748,7 @@ }; DIE_NODE(ps_msr0): power-controller@3d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3d8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1749,7 +1756,7 @@ }; DIE_NODE(ps_venc_sys): power-controller@3e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3e0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1757,7 +1764,7 @@ }; DIE_NODE(ps_dpa4): power-controller@3e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3e8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1766,7 +1773,7 @@ }; DIE_NODE(ps_msr0_ase_core): power-controller@3f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3f0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1775,7 +1782,7 @@ }; DIE_NODE(ps_apcie_gpshr_sys): power-controller@3f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x3f8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1784,7 +1791,7 @@ }; DIE_NODE(ps_apcie_st_sys): power-controller@408 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x408 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1793,7 +1800,7 @@ }; DIE_NODE(ps_apcie_st1_sys): power-controller@410 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x410 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1802,7 +1809,7 @@ }; DIE_NODE(ps_apcie_gp_sys): power-controller@418 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x418 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1812,7 +1819,7 @@ }; DIE_NODE(ps_apcie_ge_sys): power-controller@420 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x420 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1821,7 +1828,7 @@ }; DIE_NODE(ps_apcie_phy_sw): power-controller@428 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x428 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1830,7 +1837,7 @@ }; DIE_NODE(ps_sep): power-controller@c00 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc00 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1847,16 +1854,17 @@ * have to enable/disable everything in the per-model DTs. */ DIE_NODE(ps_isp_cpu): power-controller@4000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4000 4>; #power-domain-cells = <0>; #reset-cells = <0>; label = DIE_LABEL(isp_cpu); /* power-domains = <&DIE_NODE(ps_isp_sys)>; */ + apple,force-disable; }; DIE_NODE(ps_isp_fe): power-controller@4008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1865,7 +1873,7 @@ }; DIE_NODE(ps_dprx): power-controller@4010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1874,7 +1882,7 @@ }; DIE_NODE(ps_isp_vis): power-controller@4018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1883,7 +1891,7 @@ }; DIE_NODE(ps_isp_be): power-controller@4020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1892,7 +1900,7 @@ }; DIE_NODE(ps_isp_raw): power-controller@4028 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4028 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1901,7 +1909,7 @@ }; DIE_NODE(ps_isp_clr): power-controller@4030 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x4030 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1910,7 +1918,7 @@ }; DIE_NODE(ps_venc_dma): power-controller@8000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1919,7 +1927,7 @@ }; DIE_NODE(ps_venc_pipe4): power-controller@8008 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8008 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1928,7 +1936,7 @@ }; DIE_NODE(ps_venc_pipe5): power-controller@8010 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8010 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1937,7 +1945,7 @@ }; DIE_NODE(ps_venc_me0): power-controller@8018 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8018 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1946,7 +1954,7 @@ }; DIE_NODE(ps_venc_me1): power-controller@8020 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8020 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1955,7 +1963,7 @@ }; DIE_NODE(ps_prores): power-controller@c000 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc000 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1966,7 +1974,7 @@ &DIE_NODE(pmgr_mini) { DIE_NODE(ps_debug): power-controller@58 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x58 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1975,7 +1983,7 @@ }; DIE_NODE(ps_nub_spmi0): power-controller@60 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x60 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1984,7 +1992,7 @@ }; DIE_NODE(ps_nub_spmi1): power-controller@68 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x68 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -1993,7 +2001,7 @@ }; DIE_NODE(ps_nub_aon): power-controller@70 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x70 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2002,7 +2010,7 @@ }; DIE_NODE(ps_msg): power-controller@78 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x78 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2011,7 +2019,7 @@ }; DIE_NODE(ps_nub_gpio): power-controller@80 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x80 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2020,7 +2028,7 @@ }; DIE_NODE(ps_nub_fabric): power-controller@88 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x88 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2029,7 +2037,7 @@ }; DIE_NODE(ps_atc0_usb_aon): power-controller@90 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x90 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2038,7 +2046,7 @@ }; DIE_NODE(ps_atc1_usb_aon): power-controller@98 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x98 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2047,7 +2055,7 @@ }; DIE_NODE(ps_atc2_usb_aon): power-controller@a0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xa0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2056,7 +2064,7 @@ }; DIE_NODE(ps_atc3_usb_aon): power-controller@a8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xa8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2065,7 +2073,7 @@ }; DIE_NODE(ps_mtp_fabric): power-controller@b0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xb0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2076,7 +2084,7 @@ }; DIE_NODE(ps_nub_sram): power-controller@b8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xb8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2085,7 +2093,7 @@ }; DIE_NODE(ps_debug_switch): power-controller@c0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2094,7 +2102,7 @@ }; DIE_NODE(ps_atc0_usb): power-controller@c8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xc8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2103,7 +2111,7 @@ }; DIE_NODE(ps_atc1_usb): power-controller@d0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xd0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2112,7 +2120,7 @@ }; DIE_NODE(ps_atc2_usb): power-controller@d8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xd8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2121,7 +2129,7 @@ }; DIE_NODE(ps_atc3_usb): power-controller@e0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xe0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2132,7 +2140,7 @@ #if 0 /* MTP stuff is self-managed */ DIE_NODE(ps_mtp_gpio): power-controller@e8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xe8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2142,7 +2150,7 @@ }; DIE_NODE(ps_mtp_base): power-controller@f0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xf0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2152,7 +2160,7 @@ }; DIE_NODE(ps_mtp_periph): power-controller@f8 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0xf8 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2162,7 +2170,7 @@ }; DIE_NODE(ps_mtp_spi0): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2172,7 +2180,7 @@ }; DIE_NODE(ps_mtp_i2cm0): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2182,7 +2190,7 @@ }; DIE_NODE(ps_mtp_uart0): power-controller@110 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x110 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2192,7 +2200,7 @@ }; DIE_NODE(ps_mtp_cpu): power-controller@118 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x118 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2202,7 +2210,7 @@ }; DIE_NODE(ps_mtp_scm_fabric): power-controller@120 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x120 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2212,7 +2220,7 @@ }; DIE_NODE(ps_mtp_sram): power-controller@128 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x128 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2222,7 +2230,7 @@ }; DIE_NODE(ps_mtp_dma): power-controller@130 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x130 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2235,7 +2243,7 @@ &DIE_NODE(pmgr_gfx) { DIE_NODE(ps_gpx): power-controller@0 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x0 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2245,7 +2253,7 @@ }; DIE_NODE(ps_afr): power-controller@100 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x100 4>; #power-domain-cells = <0>; #reset-cells = <0>; @@ -2255,7 +2263,7 @@ }; DIE_NODE(ps_gfx): power-controller@108 { - compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + compatible = "apple,t6020-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x108 4>; #power-domain-cells = <0>; #reset-cells = <0>; diff --git a/arch/arm64/boot/dts/apple/t6030-j514s.dts b/arch/arm64/boot/dts/apple/t6030-j514s.dts new file mode 100644 index 00000000000000..0077e01cd2e05b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6030-j514s.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14-inch, M3 Pro, Nov 2023) + * + * target-type: J514s + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6030.dtsi" +#include "t603x-j514-j516.dtsi" + +/ { + compatible = "apple,j514s", "apple,t6030", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M3 Pro, Nov 2023)"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j514s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6030-j516s.dts b/arch/arm64/boot/dts/apple/t6030-j516s.dts new file mode 100644 index 00000000000000..f1c071568d086c --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6030-j516s.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (16-inch, M3 Pro, Nov 2023) + * + * target-type: J516s + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6030.dtsi" +#include "t603x-j514-j516.dtsi" + +/ { + compatible = "apple,j516s", "apple,t6030", "apple,arm-platform"; + model = "Apple MacBook Pro (16-inch, M3 Pro, Nov 2023)"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j516s.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi new file mode 100644 index 00000000000000..a227538e03e87f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi @@ -0,0 +1,1436 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PMGR Power domains for Apple T6030 "M3 Pro" SoC + * + * Copyright The Asahi Linux Contributors + */ + +&pmgr_gfx { + ps_gpx: power-controller@0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpx"; + apple,always-on; + }; + + ps_afr: power-controller@100 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "afr"; + apple,always-on; + }; + + ps_gfx: power-controller@110 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gfx"; + power-domains = <&ps_gpx>, <&ps_afr>; + }; +}; + +&pmgr { + ps_msg: power-controller@108 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msg"; + apple,always-on; + }; + + ps_aic: power-controller@110 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aic"; + }; + + ps_dwi: power-controller@118 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dwi"; + }; + + ps_gpio: power-controller@120 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "gpio"; + }; + + ps_pms_fpwm0: power-controller@138 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm0"; + }; + + ps_pms_fpwm1: power-controller@140 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm1"; + }; + + ps_pms_fpwm2: power-controller@148 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm2"; + }; + + ps_pms_fpwm3: power-controller@150 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm3"; + }; + + ps_pms_fpwm4: power-controller@158 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_fpwm4"; + }; + + ps_pms_c1ppt: power-controller@160 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_c1ppt"; + }; + + ps_soc_rc: power-controller@168 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_rc"; + }; + + ps_soc_dpe: power-controller@170 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "soc_dpe"; + apple,always-on; + }; + + ps_pmgr_soc_ocla: power-controller@178 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmgr_soc_ocla"; + }; + + ps_ispsens0: power-controller@180 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ispsens0"; + }; + + ps_aft0: power-controller@190 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aft0"; + }; + + ps_ap_tmm: power-controller@1a8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ap_tmm"; + }; + + ps_sio: power-controller@1b8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio"; + }; + + ps_disp_sys: power-controller@1c0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_sys"; + }; + + ps_jpg: power-controller@1c8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "jpg"; + }; + + ps_sio_cpu: power-controller@1d0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_cpu"; + power-domains = <&ps_sio>; + }; + + ps_fpwm0: power-controller@1d8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + power-domains = <&ps_sio>; + }; + + ps_fpwm1: power-controller@1e0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm1"; + power-domains = <&ps_sio>; + }; + + ps_fpwm2: power-controller@1e8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm2"; + power-domains = <&ps_sio>; + }; + + ps_i2c0: power-controller@1f0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c0"; + power-domains = <&ps_sio>; + }; + + ps_i2c1: power-controller@1f8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c1"; + power-domains = <&ps_sio>; + }; + + ps_i2c2: power-controller@200 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c2"; + power-domains = <&ps_sio>; + }; + + ps_i2c3: power-controller@208 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c3"; + power-domains = <&ps_sio>; + }; + + ps_i2c4: power-controller@210 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c4"; + power-domains = <&ps_sio>; + }; + + ps_i2c5: power-controller@218 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c5"; + power-domains = <&ps_sio>; + }; + + ps_i2c6: power-controller@220 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c6"; + power-domains = <&ps_sio>; + }; + + ps_i2c7: power-controller@228 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c7"; + power-domains = <&ps_sio>; + }; + + ps_i2c8: power-controller@230 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "i2c8"; + power-domains = <&ps_sio>; + }; + + ps_spi_p: power-controller@238 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi_p"; + power-domains = <&ps_sio>; + }; + + ps_uart_p: power-controller@240 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_p"; + power-domains = <&ps_sio>; + }; + + ps_audio_p: power-controller@248 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "audio_p"; + power-domains = <&ps_sio>; + }; + + ps_aes: power-controller@250 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "aes"; + power-domains = <&ps_sio>; + }; + + ps_disp_fe: power-controller@258 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_fe"; + power-domains = <&ps_disp_sys>; + }; + + ps_spi0: power-controller@260 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi0"; + power-domains = <&ps_spi_p>; + }; + + ps_spi1: power-controller@268 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi1"; + power-domains = <&ps_spi_p>; + }; + + ps_spi2: power-controller@270 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi2"; + power-domains = <&ps_spi_p>; + }; + + ps_spi3: power-controller@278 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi3"; + power-domains = <&ps_spi_p>; + }; + + ps_spi4: power-controller@280 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi4"; + power-domains = <&ps_spi_p>; + }; + + ps_spi5: power-controller@288 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "spi5"; + power-domains = <&ps_spi_p>; + }; + + ps_qspi: power-controller@290 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "qspi"; + power-domains = <&ps_spi_p>; + }; + + ps_uart_n: power-controller@298 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart_n"; + power-domains = <&ps_uart_p>; + }; + + ps_uart0: power-controller@2a0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart0"; + power-domains = <&ps_uart_p>; + }; + + ps_uart1: power-controller@2a8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart1"; + power-domains = <&ps_uart_p>; + }; + + ps_uart2: power-controller@2b0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart2"; + power-domains = <&ps_uart_p>; + }; + + ps_uart3: power-controller@2b8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart3"; + power-domains = <&ps_uart_p>; + }; + + ps_uart4: power-controller@2c0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart4"; + power-domains = <&ps_uart_p>; + }; + + ps_uart5: power-controller@2c8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart5"; + power-domains = <&ps_uart_p>; + }; + + ps_uart6: power-controller@2d0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "uart6"; + power-domains = <&ps_uart_p>; + }; + + ps_sio_adma: power-controller@2d8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sio_adma"; + power-domains = <&ps_audio_p>; + }; + + ps_dpa0: power-controller@2e0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa0"; + apple,always-on; + power-domains = <&ps_audio_p>; + }; + + ps_dpa1: power-controller@2e8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa1"; + apple,always-on; + power-domains = <&ps_audio_p>; + }; + + ps_dpa2: power-controller@2f0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa2"; + apple,always-on; + power-domains = <&ps_audio_p>; + }; + + ps_dpa3: power-controller@2f8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa3"; + apple,always-on; + power-domains = <&ps_audio_p>; + }; + + ps_dpa4: power-controller@300 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dpa4"; + apple,always-on; + power-domains = <&ps_audio_p>; + }; + + ps_mca0: power-controller@308 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca0"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_dcs0: power-controller@320 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs0"; + apple,always-on; + }; + + ps_dcs2: power-controller@328 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs2"; + apple,always-on; + }; + + ps_dcs1: power-controller@330 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs1"; + apple,always-on; + }; + + ps_dcs3: power-controller@338 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs3"; + apple,always-on; + }; + + ps_dcs4: power-controller@340 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs4"; + apple,always-on; + }; + + ps_dcs5: power-controller@348 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs5"; + apple,always-on; + }; + + ps_dcs6: power-controller@358 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs6"; + apple,always-on; + }; + + ps_dcs7: power-controller@360 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs7"; + apple,always-on; + }; + + ps_dcs8: power-controller@368 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs8"; + apple,always-on; + }; + + ps_dcs9: power-controller@370 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs9"; + apple,always-on; + }; + + ps_dcs10: power-controller@378 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs10"; + apple,always-on; + }; + + ps_dcs11: power-controller@380 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dcs11"; + apple,always-on; + }; + + ps_mca1: power-controller@388 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca1"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_mca2: power-controller@390 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca2"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_mca3: power-controller@398 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "mca3"; + power-domains = <&ps_sio_adma>, <&ps_audio_p>; + }; + + ps_ioa1: power-controller@3a0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ioa1"; + apple,always-on; + }; + + ps_ls1: power-controller@3a8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ls1"; + apple,always-on; + power-domains = <&ps_ioa1>; + }; + + ps_lw10: power-controller@3b0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "lw10"; + apple,always-on; + power-domains = <&ps_ls1>; + }; + + ps_dispext0_sys: power-controller@3b8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_sys"; + power-domains = <&ps_lw10>; + }; + + ps_dispext1_sys: power-controller@3c0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext1_sys"; + power-domains = <&ps_lw10>; + }; + + ps_isp_sys: power-controller@3c8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_sys"; + power-domains = <&ps_lw10>; + }; + + ps_venc_sys: power-controller@3d0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_sys"; + power-domains = <&ps_lw10>; + }; + + ps_dispext0_fe: power-controller@3d8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_fe"; + power-domains = <&ps_dispext0_sys>; + }; + + ps_dispext0_cpu: power-controller@3e0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext0_cpu"; + power-domains = <&ps_dispext0_fe>; + }; + + ps_trace_fab: power-controller@3e8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "trace_fab"; + }; + + ps_dispext1_fe: power-controller@3f0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext1_fe"; + power-domains = <&ps_dispext1_sys>; + }; + + ps_dispext1_cpu: power-controller@3f8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dispext1_cpu"; + power-domains = <&ps_dispext1_fe>; + }; + + ps_dptx_phy: power-controller@408 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dptx_phy"; + apple,always-on; + }; + + ps_avd_sys: power-controller@490 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x490 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "avd_sys"; + }; + + ps_ane_sys: power-controller@498 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x498 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ane_sys"; + }; + + ps_scodec: power-controller@4a0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec"; + }; + + ps_scodec_stream: power-controller@4a8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "scodec_stream"; + power-domains = <&ps_scodec>; + }; + + ps_ioa3: power-controller@4b0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ioa3"; + apple,always-on; + }; + + ps_ls3: power-controller@4b8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ls3"; + apple,always-on; + power-domains = <&ps_ioa3>; + }; + + ps_lw30: power-controller@4c0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "lw30"; + apple,always-on; + power-domains = <&ps_ls3>; + }; + + ps_pmp: power-controller@4c8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pmp"; + }; + + ps_pms_sram: power-controller@4d0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "pms_sram"; + }; + + ps_lw31: power-controller@4d8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "lw31"; + apple,always-on; + power-domains = <&ps_ls3>; + }; + + ps_atc0_common: power-controller@4e0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_common"; + power-domains = <&ps_lw30>; + }; + + ps_atc1_common: power-controller@4e8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_common"; + power-domains = <&ps_lw30>; + }; + + ps_atc2_common: power-controller@4f0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_common"; + power-domains = <&ps_lw30>; + }; + + ps_atc3_common: power-controller@4f8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_common"; + power-domains = <&ps_lw30>; + }; + + ps_apcie_gp: power-controller@500 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x500 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_gp"; + power-domains = <&ps_lw31>; + }; + + ps_msr: power-controller@508 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x508 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr"; + power-domains = <&ps_lw31>; + }; + + ps_ans: power-controller@510 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x510 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "ans"; + power-domains = <&ps_lw31>; + }; + + ps_atc0_pcie: power-controller@518 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x518 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_pcie"; + power-domains = <&ps_atc0_common>; + }; + + ps_atc0_cio: power-controller@520 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio"; + power-domains = <&ps_atc0_common>; + }; + + ps_atc1_pcie: power-controller@528 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x528 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_pcie"; + power-domains = <&ps_atc1_common>; + }; + + ps_atc1_cio: power-controller@530 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x530 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio"; + power-domains = <&ps_atc1_common>; + }; + + ps_atc2_pcie: power-controller@538 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x538 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_pcie"; + power-domains = <&ps_atc2_common>; + }; + + ps_atc2_cio: power-controller@540 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x540 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_cio"; + power-domains = <&ps_atc2_common>; + }; + + ps_atc3_pcie: power-controller@548 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x548 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_pcie"; + power-domains = <&ps_atc3_common>; + }; + + ps_atc3_cio: power-controller@550 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x550 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_cio"; + power-domains = <&ps_atc3_common>; + }; + + ps_apcie_sys_gp: power-controller@558 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x558 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_gp"; + power-domains = <&ps_apcie_gp>; + }; + + ps_msr_ase_core: power-controller@560 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "msr_ase_core"; + power-domains = <&ps_msr>; + }; + + ps_apcie_st: power-controller@568 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x568 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_st"; + power-domains = <&ps_ans>; + }; + + ps_atc0_cio_pcie: power-controller@570 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x570 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_pcie"; + power-domains = <&ps_atc0_cio>; + }; + + ps_atc0_cio_usb: power-controller@578 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x578 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_cio_usb"; + power-domains = <&ps_atc0_cio>; + }; + + ps_atc1_cio_pcie: power-controller@580 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x580 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_pcie"; + power-domains = <&ps_atc1_cio>; + }; + + ps_atc1_cio_usb: power-controller@588 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x588 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_cio_usb"; + power-domains = <&ps_atc1_cio>; + }; + + ps_atc2_cio_pcie: power-controller@590 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x590 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_cio_pcie"; + power-domains = <&ps_atc2_cio>; + }; + + ps_atc2_cio_usb: power-controller@598 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_cio_usb"; + power-domains = <&ps_atc2_cio>; + }; + + ps_atc3_cio_pcie: power-controller@5a0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_cio_pcie"; + power-domains = <&ps_atc3_cio>; + }; + + ps_atc3_cio_usb: power-controller@5a8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_cio_usb"; + power-domains = <&ps_atc3_cio>; + }; + + ps_apcie_sys_st: power-controller@5b0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_sys_st"; + power-domains = <&ps_apcie_st>, <&ps_ans>; + }; + + ps_apcie_phy_sw: power-controller@5b8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "apcie_phy_sw"; + apple,always-on; /* macOS does not turn this off */ + }; + + ps_sep: power-controller@c00 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "sep"; + apple,always-on; + }; + + ps_isp_cpu: power-controller@4000 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_cpu"; + power-domains = <&ps_isp_sys>; + }; + + ps_isp_fe: power-controller@4008 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + power-domains = <&ps_isp_sys>; + }; + + ps_dprx: power-controller@4010 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "dprx"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_secure: power-controller@4018 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_secure"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_be: power-controller@4020 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_be"; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_clr: power-controller@4028 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_clr"; + power-domains = <&ps_isp_be>; + }; + + ps_venc_dma: power-controller@8000 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_dma"; + power-domains = <&ps_venc_sys>; + }; + + ps_venc_pipe4: power-controller@8008 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_pipe4"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_pipe5: power-controller@8010 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_pipe5"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_me0: power-controller@8018 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_me0"; + power-domains = <&ps_venc_dma>; + }; + + ps_venc_me1: power-controller@8020 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "venc_me1"; + power-domains = <&ps_venc_me0>; + }; + + ps_disp_cpu: power-controller@10000 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x10000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "disp_cpu"; + power-domains = <&ps_disp_fe>; + }; +}; + +&pmgr_mini { + ps_debug_gated: power-controller@0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_gated"; + apple,always-on; + }; + + ps_nub_spmi0: power-controller@58 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi0"; + apple,always-on; + }; + + ps_nub_spmi1: power-controller@60 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi1"; + apple,always-on; + }; + + ps_nub_spmi2: power-controller@68 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi2"; + apple,always-on; + }; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a0"; + apple,always-on; + }; + + ps_nub_spmi_a1: power-controller@78 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spmi_a1"; + apple,always-on; + }; + + ps_nub_spi0: power-controller@88 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_spi0"; + apple,always-on; + }; + + ps_nub_ocla: power-controller@90 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x90 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_ocla"; + apple,always-on; + }; + + ps_nub_gpio: power-controller@98 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x98 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_gpio"; + apple,always-on; + }; + + ps_nub_sram: power-controller@a8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub_sram"; + apple,always-on; + }; + + ps_debug_switch: power-controller@b0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "debug_switch"; + apple,always-on; + }; + + ps_atc0_usb_aon: power-controller@b8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb_aon"; + }; + + ps_atc1_usb_aon: power-controller@c0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_usb_aon"; + }; + + ps_atc2_usb_aon: power-controller@c8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_usb_aon"; + }; + + ps_atc3_usb_aon: power-controller@d0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_usb_aon"; + }; + + ps_atc0_usb: power-controller@d8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0_usb"; + power-domains = <&ps_atc0_usb_aon>, <&ps_atc0_common>; + }; + + ps_atc1_usb: power-controller@e0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1_usb"; + power-domains = <&ps_atc1_usb_aon>, <&ps_atc1_common>; + }; + + ps_atc2_usb: power-controller@e8 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc2_usb"; + power-domains = <&ps_atc2_usb_aon>, <&ps_atc2_common>; + }; + + ps_atc3_usb: power-controller@f0 { + compatible = "apple,t6030-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc3_usb"; + power-domains = <&ps_atc3_usb_aon>, <&ps_atc3_common>; + }; +}; + diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi new file mode 100644 index 00000000000000..58406393a30964 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6030.dtsi @@ -0,0 +1,740 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6030 "M3 Pro" SoC + * + * Other names: H15J, "Lobos" + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e00>; + }; + core1 { + cpu = <&cpu_e01>; + }; + core2 { + cpu = <&cpu_e02>; + }; + core3 { + cpu = <&cpu_e03>; + }; + core4 { + cpu = <&cpu_e04>; + }; + core5 { + cpu = <&cpu_e05>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_p00>; + }; + core1 { + cpu = <&cpu_p01>; + }; + core2 { + cpu = <&cpu_p02>; + }; + core3 { + cpu = <&cpu_p03>; + }; + core4 { + cpu = <&cpu_p04>; + }; + core5 { + cpu = <&cpu_p05>; + }; + }; + }; + + cpu_e00: cpu@0 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e01: cpu@1 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e02: cpu@2 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e03: cpu@3 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e04: cpu@4 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x4>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e05: cpu@5 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x5>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_p00: cpu@10100 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p01: cpu@10101 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p02: cpu@10102 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p03: cpu@10103 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p04: cpu@10104 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10104>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p05: cpu@10105 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10105>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + }; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + i2c1: i2c@289014000 { + compatible = "apple,t6030-i2c", "apple,t8103-i2c"; + reg = <0x2 0x89014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c1>; + status = "disabled"; + }; + + i2c2: i2c@289018000 { + compatible = "apple,t6030-i2c", "apple,t8103-i2c"; + reg = <0x2 0x89018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c2>; + status = "disabled"; + }; + + i2c3: i2c@28901c000 { + compatible = "apple,t6030-i2c", "apple,t8103-i2c"; + reg = <0x2 0x8901c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c3>; + status = "disabled"; + }; + + i2c4: i2c@289020000 { + compatible = "apple,t6030-i2c", "apple,t8103-i2c"; + reg = <0x2 0x89020000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c4_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c4>; + status = "disabled"; + }; + + i2c6: i2c@289028000 { + compatible = "apple,t6030-i2c", "apple,t8103-i2c"; + reg = <0x2 0x89028000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c6_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c6>; + status = "disabled"; + }; + + i2c8: i2c@289030000 { + compatible = "apple,t6030-i2c", "apple,t8103-i2c"; + reg = <0x2 0x89030000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c8_pins>; + pinctrl-names = "default"; + #address-cells = <0x1>; + #size-cells = <0x0>; + power-domains = <&ps_i2c8>; + status = "disabled"; + }; + + fpwm0: pwm@289040000 { + compatible = "apple,t6030-fpwm", "apple,s5l-fpwm"; + reg = <0x2 0x89040000 0x0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + status = "disabled"; + }; + + pmgr_gfx: power-management@290e80000 { + compatible = "apple,t6030-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x2 0x90e80000 0 0x4000>; + /* child nodes are added in t6030-pmgr.dtsi */ + }; + + serial0: serial@2892000000 { + compatible = "apple,s5l-uart"; + reg = <0x2 0x89200000 0x0 0x1000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + /* + * TODO: figure out the clocking properly, there may + * be a third selectable clock. + */ + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; + + pinctrl_ap: pinctrl@347100000 { + compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x47100000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + power-domains = <&ps_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 224>; + apple,npins = <224>; + + interrupt-controller; + #interrupt-cells = <2>; + + i2c1_pins: i2c1-pins { + pinmux = , + ; + }; + + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + + i2c3_pins: i2c3-pins { + pinmux = , + ; + }; + + i2c4_pins: i2c4-pins { + pinmux = , + ; + }; + + i2c6_pins: i2c6-pins { + pinmux = , + ; + }; + + i2c8_pins: i2c8-pins { + pinmux = , + ; + }; + }; + + pmgr: power-management@350700000 { + compatible = "apple,t6030-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x3 0x50700000 0 0x14000>; + /* child nodes are added in t6030-pmgr.dtsi */ + }; + + aic: interrupt-controller@351000000 { + compatible = "apple,t6030-aic3", "apple,t8122-aic3"; + #interrupt-cells = <3>; + interrupt-controller; + + reg = <0x3 0x51000000 0x0 0x184000>, + <0x3 0x51040000 0x0 0x4000>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + }; + + pinctrl_nub: pinctrl@3641f0000 { + compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x641f0000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 37>; + apple,npins = <37>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + pmgr_mini: power-management@364280000 { + compatible = "apple,t6030-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x3 0x64280000 0 0x4000>; + /* child nodes are added in t6030-pmgr.dtsi */ + }; + + wdt: watchdog@3642b0000 { + compatible = "apple,t6030-wdt", "apple,t8103-wdt"; + reg = <0x3 0x642b0000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + nub_spmi: spmi@364714000 { + compatible = "apple,t6030-spmi", "apple,t8103-spmi"; + reg = <0x3 0x64714000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + + pmic1: pmic@b { + compatible = "apple,chapel-pmic", "apple,spmi-nvmem"; + reg = <0xb SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@2001 { + reg = <0x2001 0x1>; + }; + + rtc_offset: rtc-offset@2100 { + reg = <0x2100 0x6>; + }; + + fault_shadow: fault-shadow@867b { + reg = <0x867b 0x10>; + }; + + socd: socd@8b00 { + reg = <0x8b00 0x400>; + }; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@f803 { + reg = <0xf803 0x1>; + }; + + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; + + smc_mbox: mbox@36c408000 { + compatible = "apple,t6030-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x6c408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@36c400000 { + compatible = "apple,t6030-smc", "apple,t8103-smc"; + reg = <0x3 0x6c400000 0x0 0x4000>, + <0x3 0x6de00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + smc_reboot: reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + }; + + pinctrl_smc: pinctrl@36c820000 { + compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x6c820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 18>; + apple,npins = <18>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + pinctrl_aop: pinctrl@374824000 { + compatible = "apple,t6030-pinctrl", "apple,t8103-pinctrl"; + reg = <0x3 0x74824000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + clocks = <&clkref>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 54>; + apple,npins = <54>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + mtp: mtp@37a400000 { + compatible = "apple,t6030-mtp", "apple,t8122-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x3 0x7a400000 0x0 0x4000>, + <0x3 0x7ac00000 0x0 0x100000>; + reg-names = "asc", "sram"; + + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@37a408000 { + compatible = "apple,t6030-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x7a408000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + mtp_dart: iommu@37a808000 { + compatible = "apple,t6030-dart", "apple,t8110-dart"; + reg = <0x3 0x7a808000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = ; + + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@37ab30000 { + compatible = "apple,t6030-dockchannel", "apple,dockchannel"; + reg = <0x3 0x7ab14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x3 0x7ab28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@0 { + compatible = "apple,dockchannel-hid"; + reg = <0x0000 0x4000>, + <0x4000 0x4000>, + <0x8000 0x4000>, + <0xc000 0x4000>; + reg-names = "rmt-config", "rmt-data", "config", "data"; + + iommus = <&mtp_dart 1>; + + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; + + ans_mbox: mbox@389408000 { + compatible = "apple,t6030-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x89408000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + + #mbox-cells = <0>; + power-domains = <&ps_ans>; + }; + + sart: sart@38dc50000 { + compatible = "apple,t6030-sart", "apple,t6000-sart"; + reg = <0x3 0x8dc50000 0x0 0xc000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@38dcc0000 { + compatible = "apple,t6030-nvme-ans3", "apple,t8103-nvme-ans2"; + reg = <0x3 0x8dcc0000 0x0 0x60000>, + <0x3 0x89400000 0x0 0x4000>; + reg-names = "nvme", "ans"; + + interrupt-parent = <&aic>; + interrupts = ; + + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + + power-domains = <&ps_ans>, <&ps_apcie_sys_st>; + power-domain-names = "ans", "apcie0"; + resets = <&ps_ans>; + }; + }; +}; + +#include "t6030-pmgr.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6031-base.dtsi b/arch/arm64/boot/dts/apple/t6031-base.dtsi new file mode 100644 index 00000000000000..0bb7373b3f0648 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-base.dtsi @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Nodes common for T6031, T6032 and T6034 family SoCs (M3 Max/Ultra) + * + * Other names: H15J, H15S, "Palma" + * + * Copyright The Asahi Linux Contributors + */ + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_e00>; + }; + core1 { + cpu = <&cpu_e01>; + }; + core2 { + cpu = <&cpu_e02>; + }; + core3 { + cpu = <&cpu_e03>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_p00>; + }; + core1 { + cpu = <&cpu_p01>; + }; + core2 { + cpu = <&cpu_p02>; + }; + core3 { + cpu = <&cpu_p03>; + }; + core4 { + cpu = <&cpu_p04>; + }; + core5 { + cpu = <&cpu_p05>; + }; + }; + + cluster2 { + core0 { + cpu = <&cpu_p10>; + }; + core1 { + cpu = <&cpu_p11>; + }; + core2 { + cpu = <&cpu_p12>; + }; + core3 { + cpu = <&cpu_p13>; + }; + core4 { + cpu = <&cpu_p14>; + }; + core5 { + cpu = <&cpu_p15>; + }; + }; + }; + + cpu_e00: cpu@0 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e01: cpu@1 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e02: cpu@2 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e03: cpu@3 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_p00: cpu@10100 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p01: cpu@10101 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p02: cpu@10102 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p03: cpu@10103 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p04: cpu@10104 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10104>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p05: cpu@10105 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10105>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_1>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p10: cpu@10200 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10200>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p11: cpu@10201 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10201>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p12: cpu@10202 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10202>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p13: cpu@10203 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10203>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p14: cpu@10204 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10204>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p15: cpu@10205 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10205>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_2>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + + l2_cache_2: l2-cache-2 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-die0.dtsi b/arch/arm64/boot/dts/apple/t6031-die0.dtsi new file mode 100644 index 00000000000000..8997674c00e8cc --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-die0.dtsi @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Devices used on die 0 on the Apple T6032 "M3 Ultra" SoC and present on + * Apple T6030 ("M3 Pro") and T6031 / T6034 ("M3 Max"). + * + * Copyright The Asahi Linux Contributors + */ + + wdt: watchdog@2a02d4000 { + compatible = "apple,t6031-wdt", "apple,t8103-wdt"; + reg = <0x2 0xa02d4000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + aic: interrupt-controller@292400000 { + compatible = "apple,t6031-aic3", "apple,t8122-aic3"; + #interrupt-cells = <4>; + interrupt-controller; + + reg = <0x00000002 0x92400000 0x00000000 0x1cc000>, + <0x00000002 0x92440000 0x00000000 0x4000>; + reg-names = "core", "event"; + power-domains = <&ps_aic>; + }; + + nub_spmi: spmi@2a1014000 { + compatible = "apple,t6031-spmi", "apple,t8103-spmi"; + reg = <0x2 0xa1014000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + + pmic1: pmic@b { + compatible = "apple,chapel-pmic", "apple,spmi-nvmem"; + reg = <0xb SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@2001 { + reg = <0x2001 0x1>; + }; + + rtc_offset: rtc-offset@2100 { + reg = <0x2100 0x6>; + }; + + fault_shadow: fault-shadow@867b { + reg = <0x867b 0x10>; + }; + + socd: socd@8b00 { + reg = <0x8b00 0x400>; + }; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@f803 { + reg = <0xf803 0x1>; + }; + + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; + + smc_mbox: mbox@2a4408000 { + compatible = "apple,t6031-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xa4408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@2a4400000 { + compatible = "apple,t6031-smc", "apple,t8103-smc"; + reg = <0x2 0xa4400000 0x0 0x4000>, + <0x2 0xa5e00000 0x0 0x100000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + smc_reboot: reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + }; + + pinctrl_smc: pinctrl@2a4820000 { + compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl"; + reg = <0x2 0xa4820000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_smc 0 0 30>; + apple,npins = <30>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + mtp: mtp@2ac400000 { + compatible = "apple,t6031-mtp", "apple,t8122-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xac400000 0x0 0x4000>, + <0x2 0xacc00000 0x0 0x100000>; + reg-names = "asc", "sram"; + + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2ac408000 { + compatible = "apple,t6031-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xac408000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + mtp_dart: iommu@2ac808000 { + compatible = "apple,t6031-dart", "apple,t8110-dart"; + reg = <0x2 0xac808000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = ; + + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2acb14000 { + compatible = "apple,t6031-dockchannel", "apple,dockchannel"; + reg = <0x2 0xacb14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xacb28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@0 { + compatible = "apple,dockchannel-hid"; + reg = <0x0000 0x4000>, + <0x4000 0x4000>, + <0x8000 0x4000>, + <0xc000 0x4000>; + reg-names = "rmt-config", "rmt-data", "config", "data"; + + iommus = <&mtp_dart 1>; + + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; + + i2c0: i2c@391010000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91010000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c0_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c0>; + #address-cells = <0x1>; + #size-cells = <0x0>; + }; + + i2c1: i2c@391014000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c1_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c1>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c2: i2c@391018000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c2_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c2>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c3: i2c@39101c000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x9101c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c3_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c3>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c4: i2c@391020000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91020000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c4_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c4>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c5: i2c@391024000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91024000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c5_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c5>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c6: i2c@39102c800 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91028000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c6_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c6>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c7: i2c@39102c000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x9102c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c7_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c7>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + i2c8: i2c@391030000 { + compatible = "apple,t6031-i2c", "apple,t8103-i2c"; + reg = <0x3 0x91030000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + pinctrl-0 = <&i2c8_pins>; + pinctrl-names = "default"; + power-domains = <&ps_i2c8>; + #address-cells = <0x1>; + #size-cells = <0x0>; + status = "disabled"; + }; + + fpwm0: pwm@391040000 { + compatible = "apple,t6031-fpwm", "apple,s5l-fpwm"; + reg = <0x3 0x91040000 0x0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + status = "disabled"; + }; + + serial0: serial@391200000 { + compatible = "apple,s5l-uart"; + reg = <0x3 0x91200000 0x0 0x4000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + /* + * TODO: figure out the clocking properly, there may + * be a third selectable clock. + */ + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + power-domains = <&ps_uart0>; + status = "disabled"; + }; diff --git a/arch/arm64/boot/dts/apple/t6031-dieX.dtsi b/arch/arm64/boot/dts/apple/t6031-dieX.dtsi new file mode 100644 index 00000000000000..66f55f82c37412 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-dieX.dtsi @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Nodes present on both dies of T6032 (M3 Ultra) and present on T6031 / T6034 + * (M3 Max). + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(pmgr): power-management@292280000 { + compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x2 0x92280000 0 0xc000>; + }; + + DIE_NODE(pmgr1): power-management@292800000 { + compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x2 0x92800000 0 0x4000>; + }; + + DIE_NODE(pinctrl_nub): pinctrl@2a01f0000 { + compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0xa01f0000 0x0 0x4000>; + power-domains = <&DIE_NODE(ps_nub_gpio)>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&DIE_NODE(pinctrl_nub) 0 0 59>; + apple,npins = <59>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + DIE_NODE(pmgr_mini): power-management@2a0280000 { + compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x2 0xa0280000 0 0x4000>; + }; + + DIE_NODE(pinctrl_aop): pinctrl@2a8824000 { + compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0xa8824000 0x0 0x4000>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&DIE_NODE(pinctrl_aop) 0 0 102>; + apple,npins = <102>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + }; + + DIE_NODE(pinctrl_ap): pinctrl@2b3000000 { + compatible = "apple,t6031-pinctrl", "apple,t8103-pinctrl", "apple,pinctrl"; + reg = <0x2 0xb3000000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + + clocks = <&clkref>; + power-domains = <&DIE_NODE(ps_gpio)>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&DIE_NODE(pinctrl_ap) 0 0 200>; + apple,npins = <200>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + DIE_NODE(pmgr_gfx): power-management@408e80000 { + compatible = "apple,t6031-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + #address-cells = <1>; + #size-cells = <1>; + + reg = <0x4 0x8e80000 0 0x4000>; + }; diff --git a/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi b/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi new file mode 100644 index 00000000000000..5e4727e357d218 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-gpio-pins.dtsi @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * GPIO pin mappings for Apple T6031/T6032/T6034 SoCs. + * + * Copyright The Asahi Linux Contributors + */ + +&pinctrl_ap { + i2c0_pins: i2c0-pins { + pinmux = , + ; + }; + + i2c1_pins: i2c1-pins { + pinmux = , + ; + }; + + i2c2_pins: i2c2-pins { + pinmux = , + ; + }; + + i2c3_pins: i2c3-pins { + pinmux = , + ; + }; + + i2c4_pins: i2c4-pins { + pinmux = , + ; + }; + + i2c5_pins: i2c5-pins { + pinmux = , + ; + }; + + i2c6_pins: i2c6-pins { + pinmux = , + ; + }; + + i2c7_pins: i2c7-pins { + pinmux = , + ; + }; + + i2c8_pins: i2c8-pins { + pinmux = , + ; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j514c.dts b/arch/arm64/boot/dts/apple/t6031-j514c.dts new file mode 100644 index 00000000000000..0dc2f32e808fad --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-j514c.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14-inch, M3 Max, 16 CPU cores, Nov 2023) + * + * target-type: J514c + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6031.dtsi" +#include "t603x-j514-j516.dtsi" + +/ { + compatible = "apple,j514c", "apple,t6031", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M3 Max, 16 CPU cores, Nov 2023)"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j514c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-j516c.dts b/arch/arm64/boot/dts/apple/t6031-j516c.dts new file mode 100644 index 00000000000000..5b6d96e79bf4f8 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-j516c.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (16-inch, M3 Max, 16 CPU cores, Nov 2023) + * + * target-type: J516c + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6031.dtsi" +#include "t603x-j514-j516.dtsi" + +/ { + compatible = "apple,j516c", "apple,t6031", "apple,arm-platform"; + model = "Apple MacBook Pro (16-inch, M3 Max, 16 CPU cores, Nov 2023)"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j516c.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6031-nvme.dtsi b/arch/arm64/boot/dts/apple/t6031-nvme.dtsi new file mode 100644 index 00000000000000..e69cf0e73a2a73 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-nvme.dtsi @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * NVMe related devices for Apple T603x SoCs. + * + * Copyright The Asahi Linux Contributors + */ + + DIE_NODE(ans_mbox): mbox@349408000 { + compatible = "apple,t6031-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x3 0x49408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&DIE_NODE(ps_ans)>; + #mbox-cells = <0>; + }; + + DIE_NODE(sart): sart@34dc50000 { + compatible = "apple,t6031-sart", "apple,t6000-sart"; + reg = <0x3 0x4dc50000 0x0 0x10000>; + power-domains = <&DIE_NODE(ps_ans)>; + }; + + DIE_NODE(nvme): nvme@34dcc0000 { + compatible = "apple,t6031-nvme-ans2", "apple,t8103-nvme-ans2"; + reg = <0x3 0x4dcc0000 0x0 0x40000>, + <0x3 0x49400000 0x0 0x4000>; + reg-names = "nvme", "ans"; + interrupt-parent = <&aic>; + /* The NVME interrupt is always routed to die 0 */ + interrupts = ; + mboxes = <&DIE_NODE(ans_mbox)>; + apple,sart = <&DIE_NODE(sart)>; + power-domains = <&DIE_NODE(ps_ans)>, + <&DIE_NODE(ps_apcie_sys_st)>, + <&DIE_NODE(ps_apcie_sys_st1)>; + power-domain-names = "ans", "apcie0", "apcie1"; + resets = <&DIE_NODE(ps_ans)>; + }; diff --git a/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi new file mode 100644 index 00000000000000..1917737078abb8 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031-pmgr.dtsi @@ -0,0 +1,2399 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PMGR Power domains for Apple T6031 "M3 Max/Ultra" SoC + * + * Copyright The Asahi Linux Contributors + */ + +&DIE_NODE(pmgr) { + DIE_NODE(ps_ispsens0): power-controller@100 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ispsens0); + }; + + DIE_NODE(ps_apcie_gp): power-controller@108 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_gp); + }; + + DIE_NODE(ps_apcie_ge): power-controller@110 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_ge); + }; + + DIE_NODE(ps_apcie_st): power-controller@118 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_st); + }; + + DIE_NODE(ps_afnc3_ioa): power-controller@120 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_ioa); + apple,always-on; + }; + + DIE_NODE(ps_afnc3_ls): power-controller@128 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc3_ioa)>; + }; + + DIE_NODE(ps_afnc3_lw0): power-controller@138 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc3_ls)>; + }; + + DIE_NODE(ps_afnc3_lw1): power-controller@148 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc3_lw1); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc3_ls)>; + }; + + DIE_NODE(ps_apcie_sys_gp): power-controller@158 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_sys_gp); + power-domains = <&DIE_NODE(ps_apcie_gp)>, <&DIE_NODE(ps_afnc3_lw0)>; + }; + + DIE_NODE(ps_afnc4_ioa): power-controller@168 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc4_ioa); + apple,always-on; + }; + + DIE_NODE(ps_afnc4_ls): power-controller@178 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc4_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc4_ioa)>; + }; + + DIE_NODE(ps_afnc4_lw0): power-controller@188 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc4_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc4_ls)>; + }; + + DIE_NODE(ps_afnc5_ioa): power-controller@198 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc5_ioa); + apple,always-on; + }; + + DIE_NODE(ps_afnc5_ls): power-controller@1a8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc5_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc5_ioa)>; + }; + + DIE_NODE(ps_afnc5_lw0): power-controller@1b8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc5_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc5_ls)>; + }; + + DIE_NODE(ps_afnc6_ioa): power-controller@1c8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc6_ioa); + apple,always-on; + }; + + DIE_NODE(ps_afnc6_ls): power-controller@1d8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc6_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc6_ioa)>; + }; + + DIE_NODE(ps_afnc6_lw0): power-controller@1e8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc6_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc6_ls)>; + }; + + DIE_NODE(ps_sio): power-controller@1f8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio); + power-domains = <&DIE_NODE(ps_afnc4_lw0)>; + }; + + DIE_NODE(ps_disp_sys): power-controller@200 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(disp_sys); + power-domains = <&DIE_NODE(ps_afnc4_lw0)>; + }; + + DIE_NODE(ps_isp_sys): power-controller@208 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_sys); + power-domains = <&DIE_NODE(ps_afnc4_lw0)>; + }; + + DIE_NODE(ps_sio_cpu): power-controller@210 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_cpu); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_fpwm0): power-controller@218 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x218 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(fpwm0); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_fpwm1): power-controller@220 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(fpwm1); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_fpwm2): power-controller@228 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(fpwm2); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c0): power-controller@230 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c0); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c1): power-controller@238 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c1); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c2): power-controller@240 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c2); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c3): power-controller@248 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c3); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c4): power-controller@250 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c4); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c5): power-controller@258 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c5); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c6): power-controller@260 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c6); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c7): power-controller@268 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c7); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_i2c8): power-controller@270 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(i2c8); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_spi_p): power-controller@278 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi_p); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_uart_p): power-controller@280 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart_p); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_audio_p): power-controller@288 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(audio_p); + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_aes): power-controller@290 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(aes); + apple,always-on; + power-domains = <&DIE_NODE(ps_sio)>; + }; + + DIE_NODE(ps_disp_fe): power-controller@298 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(disp_fe); + power-domains = <&DIE_NODE(ps_disp_cpu)>; + }; + + DIE_NODE(ps_disp_cpu): power-controller@2a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(disp_cpu); + power-domains = <&DIE_NODE(ps_disp_sys)>; + }; + + DIE_NODE(ps_spi0): power-controller@2a8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi0); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi1): power-controller@2b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi1); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi2): power-controller@2b8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi2); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi3): power-controller@2c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi3); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi4): power-controller@2c8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi4); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_spi5): power-controller@2d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(spi5); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_qspi): power-controller@2d8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(qspi); + power-domains = <&DIE_NODE(ps_spi_p)>; + }; + + DIE_NODE(ps_uart_n): power-controller@2e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart_n); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart0): power-controller@2e8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart0); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart1): power-controller@2f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart1); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart2): power-controller@2f8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart2); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart3): power-controller@300 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart3); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart4): power-controller@308 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart4); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart5): power-controller@310 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart5); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_uart6): power-controller@318 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(uart6); + power-domains = <&DIE_NODE(ps_uart_p)>; + }; + + DIE_NODE(ps_sio_adma): power-controller@320 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sio_adma); + power-domains = <&DIE_NODE(ps_sio)>, <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa0): power-controller@328 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa0); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa1): power-controller@330 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa1); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa2): power-controller@338 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa2); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa3): power-controller@340 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa3); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_dpa4): power-controller@348 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dpa4); + power-domains = <&DIE_NODE(ps_audio_p)>; + }; + + DIE_NODE(ps_amcc1): power-controller@350 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc1); + apple,always-on; + }; + + DIE_NODE(ps_amcc3): power-controller@360 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc3); + apple,always-on; + }; + + DIE_NODE(ps_amcc5): power-controller@370 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc5); + apple,always-on; + }; + + DIE_NODE(ps_amcc7): power-controller@380 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc7); + apple,always-on; + }; + + DIE_NODE(ps_dcs_04): power-controller@390 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_04); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc1)>; + }; + + DIE_NODE(ps_dcs_05): power-controller@3a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_05); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc1)>; + }; + + DIE_NODE(ps_dcs_06): power-controller@3b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_06); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc1)>; + }; + + DIE_NODE(ps_dcs_07): power-controller@3c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_07); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc1)>; + }; + + DIE_NODE(ps_dcs_12): power-controller@3d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_12); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc3)>; + }; + + DIE_NODE(ps_dcs_13): power-controller@3e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_13); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc3)>; + }; + + DIE_NODE(ps_dcs_14): power-controller@3f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_14); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc3)>; + }; + + DIE_NODE(ps_dcs_15): power-controller@400 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_15); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc3)>; + }; + + DIE_NODE(ps_dcs_20): power-controller@410 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_20); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc5)>; + }; + + DIE_NODE(ps_dcs_21): power-controller@420 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_21); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc5)>; + }; + + DIE_NODE(ps_dcs_22): power-controller@430 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x430 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_22); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc5)>; + }; + + DIE_NODE(ps_dcs_23): power-controller@440 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x440 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_23); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc5)>; + }; + + DIE_NODE(ps_dcs_28): power-controller@450 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x450 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_28); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc7)>; + }; + + DIE_NODE(ps_dcs_29): power-controller@460 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_29); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc7)>; + }; + + DIE_NODE(ps_dcs_30): power-controller@470 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x470 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_30); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc7)>; + }; + + DIE_NODE(ps_dcs_31): power-controller@480 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x480 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_31); + apple,always-on; /* LPDDR5 interface */ + power-domains = <&DIE_NODE(ps_amcc7)>; + }; + + DIE_NODE(ps_mca0): power-controller@4e8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca0); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_mca1): power-controller@4f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca1); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_mca2): power-controller@4f8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca2); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_mca3): power-controller@500 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x500 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mca3); + power-domains = <&DIE_NODE(ps_audio_p)>, <&DIE_NODE(ps_sio_adma)>; + }; + + DIE_NODE(ps_msr1): power-controller@508 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x508 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr1); + power-domains = <&DIE_NODE(ps_afnc5_lw0)>; + }; + + DIE_NODE(ps_venc1_sys): power-controller@510 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x510 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_sys); + power-domains = <&DIE_NODE(ps_afnc5_lw0)>; + }; + + DIE_NODE(ps_msr0): power-controller@518 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x518 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr0); + power-domains = <&DIE_NODE(ps_afnc6_lw0)>; + }; + + DIE_NODE(ps_ane_sys): power-controller@520 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_sys); + power-domains = <&DIE_NODE(ps_afnc5_lw0)>, <&DIE_NODE(ps_afnc6_lw0)>; + }; + + DIE_NODE(ps_msr1_ase_core): power-controller@538 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x538 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr1_ase_core); + power-domains = <&DIE_NODE(ps_msr1)>; + }; + + DIE_NODE(ps_apcie_sys_ge): power-controller@540 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x540 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_sys_ge); + power-domains = <&DIE_NODE(ps_apcie_ge)>, <&DIE_NODE(ps_afnc3_lw0)>; + }; + + DIE_NODE(ps_ans): power-controller@550 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x550 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ans); + power-domains = <&DIE_NODE(ps_afnc3_lw1)>; + }; + + DIE_NODE(ps_apcie_sys_st): power-controller@560 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_sys_st); + power-domains = <&DIE_NODE(ps_apcie_st)>, <&DIE_NODE(ps_ans)>; + }; + + DIE_NODE(ps_apcie_sys_st1): power-controller@570 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x570 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_sys_st1); + power-domains = <&DIE_NODE(ps_apcie_st)>, <&DIE_NODE(ps_ans)>; + }; + + DIE_NODE(ps_apcie_phy_sw): power-controller@580 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x580 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(apcie_phy_sw); + apple,always-on; /* macOS does not turn this off */ + }; + + DIE_NODE(ps_msr0_ase_core): power-controller@590 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x590 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msr0_ase_core); + power-domains = <&DIE_NODE(ps_msr0)>; + }; + + DIE_NODE(ps_ane_mpm): power-controller@5a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_mpm); + power-domains = <&DIE_NODE(ps_ane_sys)>; + }; + + DIE_NODE(ps_ane_cpu): power-controller@5a8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_cpu); + power-domains = <&DIE_NODE(ps_ane_sys)>; + }; + + DIE_NODE(ps_ane_td): power-controller@5b8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_td); + power-domains = <&DIE_NODE(ps_ane_sys)>; + }; + + DIE_NODE(ps_ane_base): power-controller@5c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ane_base); + power-domains = <&DIE_NODE(ps_ane_td)>; + }; + + DIE_NODE(ps_sep): power-controller@c00 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc00 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sep); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc4_lw0)>; + }; + + DIE_NODE(ps_isp_cpu): power-controller@4008 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_cpu); + power-domains = <&DIE_NODE(ps_isp_sys)>; + }; + + DIE_NODE(ps_isp_fe): power-controller@4030 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_fe); + power-domains = <&DIE_NODE(ps_isp_sys)>; + }; + + DIE_NODE(ps_dprx): power-controller@4038 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dprx); + power-domains = <&DIE_NODE(ps_isp_fe)>; + }; + + DIE_NODE(ps_isp_secure): power-controller@4040 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_secure); + power-domains = <&DIE_NODE(ps_isp_fe)>; + }; + + DIE_NODE(ps_isp_be): power-controller@4048 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_be); + power-domains = <&DIE_NODE(ps_isp_fe)>; + }; + + DIE_NODE(ps_isp_clr): power-controller@4050 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(isp_clr); + power-domains = <&DIE_NODE(ps_isp_be)>; + }; + + DIE_NODE(ps_venc1_dma): power-controller@8000 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_dma); + power-domains = <&DIE_NODE(ps_venc1_sys)>; + }; + + DIE_NODE(ps_venc1_pipe4): power-controller@8008 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_pipe4); + power-domains = <&DIE_NODE(ps_venc1_dma)>; + }; + + DIE_NODE(ps_venc1_pipe5): power-controller@8010 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_pipe5); + power-domains = <&DIE_NODE(ps_venc1_dma)>; + }; + + DIE_NODE(ps_venc1_me0): power-controller@8018 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_me0); + power-domains = <&DIE_NODE(ps_venc1_dma)>; + }; + + DIE_NODE(ps_venc1_me1): power-controller@8020 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc1_me1); + power-domains = <&DIE_NODE(ps_venc1_me0)>; + }; +}; + +&DIE_NODE(pmgr1) { + DIE_NODE(ps_aic): power-controller@100 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(aic); + }; + + DIE_NODE(ps_dwi): power-controller@108 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dwi); + }; + + DIE_NODE(ps_sbr): power-controller@110 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(sbr); + apple,always-on; + }; + + DIE_NODE(ps_pms): power-controller@118 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms); + apple,always-on; + }; + + DIE_NODE(ps_soc_dpe): power-controller@120 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(soc_dpe); + apple,always-on; + }; + + DIE_NODE(ps_pms_c1ppt): power-controller@128 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_c1ppt); + apple,always-on; + power-domains = <&DIE_NODE(ps_soc_dpe)>; + }; + + DIE_NODE(ps_pmgr_soc_ocla): power-controller@130 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pmgr_soc_ocla); + }; + + DIE_NODE(ps_pms_fpwm0): power-controller@138 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_fpwm0); + }; + + DIE_NODE(ps_pms_fpwm1): power-controller@140 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_fpwm1); + }; + + DIE_NODE(ps_pms_fpwm2): power-controller@148 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_fpwm2); + }; + + DIE_NODE(ps_pms_fpwm3): power-controller@150 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_fpwm3); + }; + + DIE_NODE(ps_pms_fpwm4): power-controller@158 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_fpwm4); + }; + + DIE_NODE(ps_gpio): power-controller@160 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(gpio); + power-domains = <&DIE_NODE(ps_sbr)>; + }; + + DIE_NODE(ps_msg): power-controller@168 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(msg); + }; + + DIE_NODE(ps_afc): power-controller@170 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afc); + apple,always-on; + }; + + DIE_NODE(ps_amcc0): power-controller@180 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afc)>; + }; + + DIE_NODE(ps_amcc2): power-controller@190 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc2); + apple,always-on; + power-domains = <&DIE_NODE(ps_afc)>; + }; + + DIE_NODE(ps_amcc4): power-controller@1a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc4); + apple,always-on; + power-domains = <&DIE_NODE(ps_afc)>; + }; + + DIE_NODE(ps_amcc6): power-controller@1b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(amcc6); + apple,always-on; + power-domains = <&DIE_NODE(ps_afc)>; + }; + + DIE_NODE(ps_dcs_00): power-controller@1c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_00); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc0)>; + }; + + DIE_NODE(ps_dcs_01): power-controller@1d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_01); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc0)>; + }; + + DIE_NODE(ps_dcs_02): power-controller@1e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_02); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc0)>; + }; + + DIE_NODE(ps_dcs_03): power-controller@1f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_03); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc0)>; + }; + + DIE_NODE(ps_dcs_08): power-controller@200 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_08); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc2)>; + }; + + DIE_NODE(ps_dcs_09): power-controller@210 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_09); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc2)>; + }; + + DIE_NODE(ps_dcs_10): power-controller@220 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_10); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc2)>; + }; + + DIE_NODE(ps_dcs_11): power-controller@230 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_11); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc2)>; + }; + + DIE_NODE(ps_dcs_16): power-controller@240 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_16); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc4)>; + }; + + DIE_NODE(ps_dcs_17): power-controller@250 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_17); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc4)>; + }; + + DIE_NODE(ps_dcs_18): power-controller@260 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_18); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc4)>; + }; + + DIE_NODE(ps_dcs_19): power-controller@270 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_19); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc4)>; + }; + + DIE_NODE(ps_dcs_24): power-controller@280 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_24); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc6)>; + }; + + DIE_NODE(ps_dcs_25): power-controller@290 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_25); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc6)>; + }; + + DIE_NODE(ps_dcs_26): power-controller@2a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_26); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc6)>; + }; + + DIE_NODE(ps_afi): power-controller@2b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afi); + apple,always-on; + }; + + DIE_NODE(ps_dcs_27): power-controller@2c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dcs_27); + apple,always-on; + power-domains = <&DIE_NODE(ps_amcc6)>; + }; + + DIE_NODE(ps_afi_d2d_0): power-controller@2d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afi_d2d_0); + apple,always-on; + status = "disabled"; + }; + + DIE_NODE(ps_afi_d2d_1): power-controller@2e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afi_d2d_1); + apple,always-on; + status = "disabled"; + }; + + DIE_NODE(ps_afc_d2d_0): power-controller@2f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afc_d2d_0); + apple,always-on; + status = "disabled"; + }; + + DIE_NODE(ps_afc_d2d_1): power-controller@300 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afc_d2d_1); + apple,always-on; + status = "disabled"; + }; + + DIE_NODE(ps_afr_d2d_0): power-controller@310 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afr_d2d_0); + apple,always-on; + status = "disabled"; + }; + + DIE_NODE(ps_afr_d2d_1): power-controller@320 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afr_d2d_1); + apple,always-on; + status = "disabled"; + }; + + DIE_NODE(ps_afnc1_ioa): power-controller@330 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc0_ioa): power-controller@340 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc0_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc2_ioa): power-controller@350 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc7_ioa): power-controller@360 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc7_ioa); + apple,always-on; + power-domains = <&DIE_NODE(ps_afi)>; + }; + + DIE_NODE(ps_afnc1_ls): power-controller@370 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc1_ioa)>; + }; + + DIE_NODE(ps_afnc0_ls): power-controller@380 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc0_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc0_ioa)>; + }; + + DIE_NODE(ps_afnc2_ls): power-controller@390 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc2_ioa)>; + }; + + DIE_NODE(ps_afnc7_ls): power-controller@3a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc7_ls); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc7_ioa)>; + }; + + DIE_NODE(ps_afnc1_lw0): power-controller@3b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc1_ls)>; + }; + + DIE_NODE(ps_afnc1_lw1): power-controller@3c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc1_lw1); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc1_ls)>; + }; + + DIE_NODE(ps_afnc0_lw0): power-controller@3d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc0_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc0_ls)>; + }; + + DIE_NODE(ps_afnc2_lw0): power-controller@3e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_lw0); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc2_ls)>; + }; + + DIE_NODE(ps_afnc2_lw1): power-controller@3f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc2_lw1); + apple,always-on; + power-domains = <&DIE_NODE(ps_afnc2_ls)>; + }; + + DIE_NODE(ps_afnc7_lw0): power-controller@400 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afnc7_lw0); + apple,always-on; + }; + + DIE_NODE(ps_avd_sys): power-controller@410 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(avd_sys); + power-domains = <&DIE_NODE(ps_afnc1_lw0)>; + }; + + DIE_NODE(ps_jpg): power-controller@418 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x418 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(jpg); + power-domains = <&DIE_NODE(ps_afnc1_lw0)>; + }; + + DIE_NODE(ps_dispext3_sys): power-controller@420 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext3_sys); + power-domains = <&DIE_NODE(ps_afnc1_lw1)>; + }; + + DIE_NODE(ps_scodec): power-controller@428 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x428 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(scodec); + power-domains = <&DIE_NODE(ps_afnc1_lw1)>; + }; + + DIE_NODE(ps_venc0_sys): power-controller@430 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x430 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc0_sys); + power-domains = <&DIE_NODE(ps_afnc0_lw0)>; + }; + + DIE_NODE(ps_prores): power-controller@438 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x438 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(prores); + power-domains = <&DIE_NODE(ps_afnc0_lw0)>; + }; + + DIE_NODE(ps_dispext0_sys): power-controller@440 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x440 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext0_sys); + power-domains = <&DIE_NODE(ps_afnc2_lw0)>; + }; + + DIE_NODE(ps_atc0_common): power-controller@448 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x448 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_common); + power-domains = <&DIE_NODE(ps_afnc2_lw0)>; + }; + + DIE_NODE(ps_atc1_common): power-controller@450 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x450 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_common); + power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + }; + + DIE_NODE(ps_atc2_common): power-controller@458 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x458 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_common); + power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + }; + + DIE_NODE(ps_atc3_common): power-controller@460 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_common); + power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + }; + + DIE_NODE(ps_trace_fab): power-controller@468 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x468 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(trace_fab); + power-domains = <&DIE_NODE(ps_afnc2_lw1)>; + }; + + DIE_NODE(ps_dispext1_sys): power-controller@470 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x470 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext1_sys); + power-domains = <&DIE_NODE(ps_afnc7_lw0)>; + }; + + DIE_NODE(ps_dispext2_sys): power-controller@478 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x478 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext2_sys); + power-domains = <&DIE_NODE(ps_afnc7_lw0)>; + }; + + DIE_NODE(ps_dispext3_fe): power-controller@480 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x480 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext3_fe); + power-domains = <&DIE_NODE(ps_dispext3_cpu)>; + }; + + DIE_NODE(ps_dispext3_cpu): power-controller@488 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x488 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext3_cpu); + power-domains = <&DIE_NODE(ps_dispext3_sys)>; + }; + + DIE_NODE(ps_scodec_streaming): power-controller@490 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x490 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(scodec_streaming); + power-domains = <&DIE_NODE(ps_scodec)>; + }; + + DIE_NODE(ps_venc0_dma): power-controller@498 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x498 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc0_dma); + power-domains = <&DIE_NODE(ps_venc0_sys)>; + }; + + DIE_NODE(ps_dispext0_fe): power-controller@4a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext0_fe); + power-domains = <&DIE_NODE(ps_dispext0_cpu)>; + }; + + DIE_NODE(ps_dispext0_cpu): power-controller@4a8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext0_cpu); + power-domains = <&DIE_NODE(ps_dispext0_sys)>; + }; + + DIE_NODE(ps_atc0_cio): power-controller@4b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_cio); + power-domains = <&DIE_NODE(ps_atc0_common)>; + }; + + DIE_NODE(ps_atc0_pcie): power-controller@4b8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_pcie); + power-domains = <&DIE_NODE(ps_atc0_common)>; + }; + + DIE_NODE(ps_atc1_cio): power-controller@4c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_cio); + power-domains = <&DIE_NODE(ps_atc1_common)>; + }; + + DIE_NODE(ps_atc1_pcie): power-controller@4c8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_pcie); + power-domains = <&DIE_NODE(ps_atc1_common)>; + }; + + DIE_NODE(ps_atc2_cio): power-controller@4d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_cio); + power-domains = <&DIE_NODE(ps_atc2_common)>; + }; + + DIE_NODE(ps_atc2_pcie): power-controller@4d8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4d8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_pcie); + power-domains = <&DIE_NODE(ps_atc2_common)>; + }; + + DIE_NODE(ps_atc3_cio): power-controller@4e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4e0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_cio); + power-domains = <&DIE_NODE(ps_atc3_common)>; + }; + + DIE_NODE(ps_atc3_pcie): power-controller@4e8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4e8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_pcie); + power-domains = <&DIE_NODE(ps_atc3_common)>; + }; + + DIE_NODE(ps_dispext1_fe): power-controller@4f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4f0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext1_fe); + power-domains = <&DIE_NODE(ps_dispext1_cpu)>; + }; + + DIE_NODE(ps_dispext1_cpu): power-controller@4f8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4f8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext1_cpu); + power-domains = <&DIE_NODE(ps_dispext1_sys)>; + }; + + DIE_NODE(ps_dispext2_fe): power-controller@500 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x500 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext2_fe); + power-domains = <&DIE_NODE(ps_dispext2_cpu)>; + }; + + DIE_NODE(ps_dispext2_cpu): power-controller@508 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x508 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(dispext2_cpu); + power-domains = <&DIE_NODE(ps_dispext2_sys)>; + }; + + DIE_NODE(ps_venc0_pipe4): power-controller@538 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x538 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc0_pipe4); + power-domains = <&DIE_NODE(ps_venc0_dma)>; + }; + + DIE_NODE(ps_venc0_pipe5): power-controller@540 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x540 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc0_pipe5); + power-domains = <&DIE_NODE(ps_venc0_dma)>; + }; + + DIE_NODE(ps_venc0_me0): power-controller@548 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x548 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc0_me0); + power-domains = <&DIE_NODE(ps_venc0_dma)>; + }; + + DIE_NODE(ps_pmp): power-controller@550 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x550 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pmp); + }; + + DIE_NODE(ps_pms_sram): power-controller@560 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x560 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(pms_sram); + }; + + DIE_NODE(ps_atc0_cio_pcie): power-controller@598 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_cio_pcie); + power-domains = <&DIE_NODE(ps_atc0_cio)>; + }; + + DIE_NODE(ps_atc0_cio_usb): power-controller@5a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_cio_usb); + power-domains = <&DIE_NODE(ps_atc0_cio)>; + }; + + DIE_NODE(ps_atc1_cio_pcie): power-controller@5a8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_cio_pcie); + power-domains = <&DIE_NODE(ps_atc1_cio)>; + }; + + DIE_NODE(ps_atc1_cio_usb): power-controller@5b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_cio_usb); + power-domains = <&DIE_NODE(ps_atc1_cio)>; + }; + + DIE_NODE(ps_atc2_cio_pcie): power-controller@5b8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_cio_pcie); + power-domains = <&DIE_NODE(ps_atc2_cio)>; + }; + + DIE_NODE(ps_atc2_cio_usb): power-controller@5c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5c0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_cio_usb); + power-domains = <&DIE_NODE(ps_atc2_cio)>; + }; + + DIE_NODE(ps_atc3_cio_pcie): power-controller@5c8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_cio_pcie); + power-domains = <&DIE_NODE(ps_atc3_cio)>; + }; + + DIE_NODE(ps_atc3_cio_usb): power-controller@5d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_cio_usb); + power-domains = <&DIE_NODE(ps_atc3_cio)>; + }; + + DIE_NODE(ps_venc0_me1): power-controller@638 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x638 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(venc0_me1); + power-domains = <&DIE_NODE(ps_venc0_me0)>; + }; + + DIE_NODE(ps_ap_tmm): power-controller@670 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x670 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(ap_tmm); + }; +}; + +&DIE_NODE(pmgr_mini) { + DIE_NODE(ps_debug_gated): power-controller@58 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(debug_gated); + apple,always-on; + }; + + DIE_NODE(ps_nub_spmi_ahb_fab): power-controller@60 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi_ahb_fab); + apple,always-on; + }; + + DIE_NODE(ps_nub_spmi0): power-controller@68 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi0); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_spmi_ahb_fab)>; + }; + + DIE_NODE(ps_nub_spmi_a0): power-controller@70 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi_a0); + apple,always-on; + }; + + DIE_NODE(ps_nub_spmi_a1): power-controller@78 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x78 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi_a1); + apple,always-on; + }; + + DIE_NODE(ps_nub_spmi_a2): power-controller@80 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi_a2); + apple,always-on; + }; + + DIE_NODE(ps_nub_aon): power-controller@90 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x90 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_aon); + apple,always-on; + }; + + DIE_NODE(ps_nub_gpio): power-controller@98 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x98 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_gpio); + apple,always-on; + }; + + DIE_NODE(ps_nub_ocla): power-controller@a0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_ocla); + apple,always-on; + }; + + DIE_NODE(ps_atc0_common_dp): power-controller@a8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_common_dp); + }; + + DIE_NODE(ps_atc1_common_dp): power-controller@b0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_common_dp); + }; + + DIE_NODE(ps_atc2_common_dp): power-controller@b8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_common_dp); + }; + + DIE_NODE(ps_atc3_common_dp): power-controller@c0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_common_dp); + }; + + DIE_NODE(ps_nub_spmi1): power-controller@c8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi1); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_spmi_ahb_fab)>; + }; + + DIE_NODE(ps_atc0_usb_aon): power-controller@d0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_usb_aon); + }; + + DIE_NODE(ps_atc1_usb_aon): power-controller@d8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_usb_aon); + }; + + DIE_NODE(ps_atc2_usb_aon): power-controller@e0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_usb_aon); + }; + + DIE_NODE(ps_atc3_usb_aon): power-controller@e8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_usb_aon); + }; + + DIE_NODE(ps_nub_spmi2): power-controller@f0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi2); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_spmi_ahb_fab)>; + }; + + DIE_NODE(ps_nub_spmi3): power-controller@f8 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi3); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_spmi_ahb_fab)>; + }; + + DIE_NODE(ps_nub_spmi4): power-controller@100 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi4); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_spmi_ahb_fab)>; + }; + + DIE_NODE(ps_nub_spmi5): power-controller@108 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_spmi5); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_spmi_ahb_fab)>; + }; + + DIE_NODE(ps_nub_fabric): power-controller@110 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_fabric); + apple,always-on; + }; + + DIE_NODE(ps_nub_sram): power-controller@120 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(nub_sram); + apple,always-on; + }; + + DIE_NODE(ps_debug_switch): power-controller@130 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(debug_switch); + apple,always-on; + }; + + DIE_NODE(ps_atc0_usb): power-controller@140 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc0_usb); + power-domains = <&DIE_NODE(ps_atc0_usb_aon)>, <&DIE_NODE(ps_atc0_common)>; + }; + + DIE_NODE(ps_atc1_usb): power-controller@148 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc1_usb); + power-domains = <&DIE_NODE(ps_atc1_usb_aon)>, <&DIE_NODE(ps_atc1_common)>; + }; + + DIE_NODE(ps_atc2_usb): power-controller@150 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc2_usb); + power-domains = <&DIE_NODE(ps_atc2_usb_aon)>, <&DIE_NODE(ps_atc2_common)>; + }; + + DIE_NODE(ps_atc3_usb): power-controller@158 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(atc3_usb); + power-domains = <&DIE_NODE(ps_atc3_usb_aon)>, <&DIE_NODE(ps_atc3_common)>; + }; + +#if 0 + /* MTP stuff is self-managed */ + DIE_NODE(ps_mtp_fabric): power-controller@2000 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_fabric); + apple,always-on; + power-domains = <&DIE_NODE(ps_nub_fabric)>; + }; + + DIE_NODE(ps_mtp_gpio): power-controller@2008 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_gpio); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_base): power-controller@2010 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_base); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_periph): power-controller@2018 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_periph); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_uart0): power-controller@2020 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_uart0); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_cpu): power-controller@2028 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_cpu); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>; + }; + + DIE_NODE(ps_mtp_scm_fabric): power-controller@2030 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_scm_fabric); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_periph)>; + }; + + DIE_NODE(ps_mtp_spi0): power-controller@2038 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_spi0); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>, <&DIE_NODE(ps_mtp_periph)>; + }; + + DIE_NODE(ps_mtp_i2cm0): power-controller@2040 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_i2cm0); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>, <&DIE_NODE(ps_mtp_periph)>; + }; + + DIE_NODE(ps_mtp_sram): power-controller@2048 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_sram); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_cpu)>, <&DIE_NODE(ps_mtp_scm_fabric)>; + }; + + DIE_NODE(ps_mtp_dma): power-controller@2050 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(mtp_dma); + apple,always-on; + power-domains = <&DIE_NODE(ps_mtp_fabric)>, <&DIE_NODE(ps_mtp_sram)>; + }; +#endif +}; + +&DIE_NODE(pmgr_gfx) { + DIE_NODE(ps_gpx): power-controller@0 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(gpx); + apple,always-on; + }; + + DIE_NODE(ps_afr): power-controller@100 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(afr); + apple,always-on; + }; + + DIE_NODE(ps_gfx): power-controller@110 { + compatible = "apple,t6031-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = DIE_LABEL(gfx); + power-domains = <&DIE_NODE(ps_afr)>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6031.dtsi b/arch/arm64/boot/dts/apple/t6031.dtsi new file mode 100644 index 00000000000000..f9f687bdf619a1 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6031.dtsi @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6031 "M3 Max" SoC + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include + +#include "multi-die-cpp.h" + +#include "t6031-base.dtsi" + +/ { + compatible = "apple,t6031", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + // filled via templated includes at the end of the file + }; +}; + +#define DIE +#define DIE_NO 0 + +&soc { + #include "t6031-die0.dtsi" + #include "t6031-dieX.dtsi" + #include "t6031-nvme.dtsi" +}; + +#include "t6031-gpio-pins.dtsi" +#include "t6031-pmgr.dtsi" + +#undef DIE +#undef DIE_NO diff --git a/arch/arm64/boot/dts/apple/t6032-j575d.dts b/arch/arm64/boot/dts/apple/t6032-j575d.dts new file mode 100644 index 00000000000000..56edfb1139fbf7 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6032-j575d.dts @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple Mac Studio (M3 Ultra, 2025) + * + * target-type: J575d + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6032.dtsi" + +/ { + compatible = "apple,j575d", "apple,t6032", "apple,arm-platform"; + model = "Apple Mac Studio (M3 Ultra, 2025)"; + + aliases { + serial0 = &serial0; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* To be filled by loader */ + /* Format properties will be added by loader */ + status = "disabled"; + power-domains = <&ps_dispext0_cpu>; + }; + }; + + memory@10000000000 { + device_type = "memory"; + reg = <0x100 0 0x2 0>; /* To be filled by loader */ + }; +}; + +&serial0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t6032.dtsi b/arch/arm64/boot/dts/apple/t6032.dtsi new file mode 100644 index 00000000000000..dbbce59da2de41 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6032.dtsi @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6032 "M3 Ultra" SoC + * + * Other names: H15J, "Palma 2C" + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "multi-die-cpp.h" + +#include "t6031-base.dtsi" + +/ { + compatible = "apple,t6032", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster3 { + core0 { + cpu = <&cpu_e10>; + }; + core1 { + cpu = <&cpu_e11>; + }; + core2 { + cpu = <&cpu_e12>; + }; + core3 { + cpu = <&cpu_e13>; + }; + }; + + cluster4 { + core0 { + cpu = <&cpu_p20>; + }; + core1 { + cpu = <&cpu_p21>; + }; + core2 { + cpu = <&cpu_p22>; + }; + core3 { + cpu = <&cpu_p23>; + }; + core4 { + cpu = <&cpu_p24>; + }; + core5 { + cpu = <&cpu_p25>; + }; + }; + + cluster5 { + core0 { + cpu = <&cpu_p30>; + }; + core1 { + cpu = <&cpu_p31>; + }; + core2 { + cpu = <&cpu_p32>; + }; + core3 { + cpu = <&cpu_p33>; + }; + core4 { + cpu = <&cpu_p34>; + }; + core5 { + cpu = <&cpu_p35>; + }; + }; + }; + + cpu_e10: cpu@800 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x800>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_3>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e11: cpu@801 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x801>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_3>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e12: cpu@802 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x802>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_3>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_e13: cpu@803 { + compatible = "apple,sawtooth"; + device_type = "cpu"; + reg = <0x0 0x803>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_3>; + i-cache-size = <0x20000>; + d-cache-size = <0x10000>; + }; + + cpu_p20: cpu@10900 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10900>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_4>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p21: cpu@10901 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10901>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_4>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p22: cpu@10902 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10902>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_4>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p23: cpu@10903 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10903>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_4>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p24: cpu@10904 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10904>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_4>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p25: cpu@10905 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10105>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_4>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p30: cpu@10a00 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10a00>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_5>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p31: cpu@10a01 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10a01>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_5>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p32: cpu@10a02 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10a02>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_5>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p33: cpu@10a03 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10a03>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_5>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p34: cpu@10a04 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10a04>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_5>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + cpu_p35: cpu@10a05 { + compatible = "apple,everest"; + device_type = "cpu"; + reg = <0x0 0x10a05>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* To be filled by loader */ + next-level-cache = <&l2_cache_5>; + i-cache-size = <0x30000>; + d-cache-size = <0x20000>; + }; + + l2_cache_3: l2-cache-3 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x400000>; + }; + + l2_cache_4: l2-cache-4 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + + l2_cache_5: l2-cache-5 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + cache-size = <0x1000000>; + }; + }; + + die0: soc@200000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x02 0x00000000 0x02 0x00000000 0x4 0x00000000>, + <0x05 0x80000000 0x05 0x80000000 0x1 0x80000000>, + <0x07 0x00000000 0x07 0x00000000 0xf 0x80000000>, + <0x16 0x80000000 0x16 0x80000000 0x5 0x80000000>; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + // filled via templated includes at the end of the file + + aic: interrupt-controller@292400000 { + compatible = "apple,t8122-aic3"; + #interrupt-cells = <4>; + interrupt-controller; + + /* + * reg[0]: Main MMIO range (approx 1.8 MB) + * reg[1]: CPU Event/IACK register page (Base + 0x40000) + */ + reg = <0x00000002 0x92400000 0x00000000 0x1cc000>, + <0x00000002 0x92440000 0x00000000 0x4000>; + }; + }; + + die1: soc@2200000000 { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges = <0x02 0x00000000 0x22 0x00000000 0x4 0x00000000>, + <0x07 0x00000000 0x27 0x00000000 0xf 0x80000000>, + <0x16 0x80000000 0x36 0x80000000 0x5 0x80000000>; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + + // filled via templated includes at the end of the file + }; +}; + + +#define DIE +#define DIE_NO 0 + +&die0 { + #include "t6031-die0.dtsi" + #include "t6031-dieX.dtsi" +}; + +#include "t6031-pmgr.dtsi" +#include "t6031-gpio-pins.dtsi" + +#undef DIE +#undef DIE_NO + +#define DIE _die1 +#define DIE_NO 1 + +&die1 { + #include "t6031-dieX.dtsi" + #include "t6031-nvme.dtsi" +}; + +#include "t6031-pmgr.dtsi" + +/delete-node/ &ps_pmp_die1; + +#undef DIE +#undef DIE_NO + +/* delete non-present DISP power-states */ +/delete-node/ &ps_disp_cpu; +/delete-node/ &ps_disp_cpu_die1; +/delete-node/ &ps_disp_fe; +/delete-node/ &ps_disp_fe_die1; +/delete-node/ &ps_disp_sys; +/delete-node/ &ps_disp_sys_die1; + +/* delete non-present ISP power-states */ +/delete-node/ &ps_dprx; +/delete-node/ &ps_dprx_die1; +/delete-node/ &ps_isp_be; +/delete-node/ &ps_isp_be_die1; +/delete-node/ &ps_isp_clr; +/delete-node/ &ps_isp_clr_die1; +/delete-node/ &ps_isp_cpu; +/delete-node/ &ps_isp_cpu_die1; +/delete-node/ &ps_isp_fe; +/delete-node/ &ps_isp_fe_die1; +/delete-node/ &ps_isp_secure; +/delete-node/ &ps_isp_secure_die1; +/delete-node/ &ps_isp_sys; +/delete-node/ &ps_isp_sys_die1; + +&ps_afi_d2d_0 { + status = "okay"; +}; +&ps_afi_d2d_0_die1 { + status = "okay"; +}; +&ps_afi_d2d_1 { + status = "okay"; +}; +&ps_afi_d2d_1_die1 { + status = "okay"; +}; +&ps_afc_d2d_0 { + status = "okay"; +}; +&ps_afc_d2d_0_die1{ + status = "okay"; +}; +&ps_afc_d2d_1 { + status = "okay"; +}; +&ps_afc_d2d_1_die1 { + status = "okay"; +}; +&ps_afr_d2d_0 { + status = "okay"; +}; +&ps_afr_d2d_0_die1 { + status = "okay"; +}; +&ps_afr_d2d_1 { + status = "okay"; +}; +&ps_afr_d2d_1_die1 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t6034-j514m.dts b/arch/arm64/boot/dts/apple/t6034-j514m.dts new file mode 100644 index 00000000000000..88b11819f6e901 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6034-j514m.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14-inch, M3 Max, 14 CPU cores, Nov 2023) + * + * target-type: J514m + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6034.dtsi" +#include "t603x-j514-j516.dtsi" + +/ { + compatible = "apple,j514m", "apple,t6031", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M3 Max, 14 CPU cores, Nov 2023)"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j514m.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6034-j516m.dts b/arch/arm64/boot/dts/apple/t6034-j516m.dts new file mode 100644 index 00000000000000..3f89ff77c8682c --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6034-j516m.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (16-inch, M3 Max, 14 CPU cores, Nov 2023) + * + * target-type: J516m + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include "t6034.dtsi" +#include "t603x-j514-j516.dtsi" + +/ { + compatible = "apple,j514m", "apple,t6031", "apple,arm-platform"; + model = "Apple MacBook Pro (16-inch, M3 Max, 14 CPU cores, Nov 2023)"; +}; + +&mtp_mt { + firmware-name = "apple/tpmtfw-j516m.bin"; +}; diff --git a/arch/arm64/boot/dts/apple/t6034.dtsi b/arch/arm64/boot/dts/apple/t6034.dtsi new file mode 100644 index 00000000000000..aa73af9c512dbc --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6034.dtsi @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6034 "M3 Max" SoC + * + * Copyright The Asahi Linux Contributors + */ + +#include "t6031.dtsi" + +/ { + compatible = "apple,t6034", "apple,arm-platform"; +}; diff --git a/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi b/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi new file mode 100644 index 00000000000000..cc22f42ea9dd43 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t603x-j514-j516.dtsi @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * MacBook Pro (14/16-inch, 2023) + * + * This file contains the parts common to J514 and J516 devices with t6030, + * t6031 and t6034. + * + * target-type: J514s / J514m / J514c / J516s / J516m / J516c + * + * Copyright The Asahi Linux Contributors + */ + +/* + * These models are essentially identical to the previous generations, other + * than the GPIO indices and using SPMI based USB Type-C port controllers. + */ + +#include + +/ { + chassis-type = "laptop"; + + aliases { + serial0 = &serial0; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* To be filled by loader */ + /* Format properties will be added by loader */ + status = "disabled"; + power-domains = <&ps_disp_fe>; + }; + }; + + memory@10000000000 { + device_type = "memory"; + reg = <0x100 0 0x2 0>; /* To be filled by loader */ + }; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&serial0 { + status = "okay"; +}; + +&fpwm0 { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 25 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 26 GPIO_ACTIVE_LOW>; + + mtp_mt: multi-touch { + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts index 52965258200da3..f55683c48784b8 100644 --- a/arch/arm64/boot/dts/apple/t8103-j274.dts +++ b/arch/arm64/boot/dts/apple/t8103-j274.dts @@ -19,9 +19,23 @@ aliases { ethernet0 = ðernet0; + sio = &sio; }; }; +&dcp { + apple,connector-type = "HDMI-A"; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio0 { + status = "okay"; +}; + &bluetooth0 { brcm,board-type = "apple,atlantisb"; }; @@ -71,6 +85,65 @@ status = "okay"; }; +&i2c1 { + speaker_amp: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + shutdown-gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-zero-fill; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; }; + +/ { + sound { + compatible = "apple,j274-macaudio", "apple,macaudio"; + model = "Mac mini J274"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts index 52f63ae7a58dd8..6fa04626b1d08b 100644 --- a/arch/arm64/boot/dts/apple/t8103-j293.dts +++ b/arch/arm64/boot/dts/apple/t8103-j293.dts @@ -24,6 +24,7 @@ */ aliases { touchbar0 = &touchbar0; + sep = &sep; }; led-controller { @@ -39,6 +40,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j293", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,honshu"; }; @@ -59,8 +74,105 @@ label = "USB-C Left-front"; }; +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-tas5770-sdz { + compatible = "regulator-fixed"; + regulator-name = "tas5770-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_rear: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + ti,pdm-slot-no = <12>; + }; + + speaker_left_front: codec@32 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x32>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,pdm-slot-no = <4>; + ti,sdout-pull-down; + }; +}; + &i2c2 { status = "okay"; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + speaker_right_rear: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + ti,pdm-slot-no = <16>; + }; + + speaker_right_front: codec@35 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x35>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,pdm-slot-no = <8>; + ti,sdout-pull-down; + }; }; &i2c4 { @@ -132,3 +244,67 @@ &displaydfr_dart { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J293"; + apple,machine-kind = "MacBook Pro"; +}; + +/ { + sound { + compatible = "apple,j293-macaudio", "apple,macaudio"; + model = "MacBook Pro J293"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <1>; +}; + +#include "hwmon-fan.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts index 9eb2825d25dcba..883ba4a1f0100a 100644 --- a/arch/arm64/boot/dts/apple/t8103-j313.dts +++ b/arch/arm64/boot/dts/apple/t8103-j313.dts @@ -18,6 +18,10 @@ model = "Apple MacBook Air (M1, 2020)"; chassis-type = "laptop"; + aliases { + sep = &sep; + }; + led-controller { compatible = "pwm-leds"; led-0 { @@ -31,6 +35,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j313", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <420>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,shikoku"; }; @@ -54,3 +72,136 @@ &typec1 { label = "USB-C Left-front"; }; + +&spi3 { + status = "okay"; + + hid-transport@0 { + compatible = "apple,spi-hid-transport"; + reg = <0>; + spi-max-frequency = <8000000>; + /* + * Apple's ADT specifies 20us CS change delays, and the + * SPI HID interface metadata specifies 45us. Using either + * seems not to be reliable, but adding both works, so + * best guess is they are cumulative. + */ + spi-cs-setup-delay-ns = <65000>; + spi-cs-hold-delay-ns = <65000>; + spi-cs-inactive-delay-ns = <250000>; + spien-gpios = <&pinctrl_ap 195 0>; + interrupts-extended = <&pinctrl_nub 13 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-tas5770-sdz { + compatible = "regulator-fixed"; + regulator-name = "tas5770-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 181 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left: codec@31 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x31>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-zero-fill; + }; +}; + +&i2c3 { + speaker_right: codec@34 { + compatible = "ti,tas5770l", "ti,tas2770"; + reg = <0x34>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right"; + interrupts-extended = <&pinctrl_ap 182 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-zero-fill; + }; + + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J313"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j313-macaudio", "apple,macaudio"; + model = "MacBook Air J313"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left>, <&speaker_right>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <1>; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts index 090c97bb781b32..c7da4815fb94c0 100644 --- a/arch/arm64/boot/dts/apple/t8103-j456.dts +++ b/arch/arm64/boot/dts/apple/t8103-j456.dts @@ -22,6 +22,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j456", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &bluetooth0 { brcm,board-type = "apple,capri"; }; @@ -88,3 +102,72 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J456"; + apple,machine-kind = "iMac"; + apple,no-beamforming; +}; + +/ { + sound { + compatible = "apple,j456-macaudio", "apple,macaudio"; + model = "iMac J456"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "isp-imx364.dtsi" + +&isp { + apple,platform-id = <2>; +}; + +#include "hwmon-fan-dual.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts index ebddde75455c69..fc0f28fb1c4367 100644 --- a/arch/arm64/boot/dts/apple/t8103-j457.dts +++ b/arch/arm64/boot/dts/apple/t8103-j457.dts @@ -22,6 +22,20 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j457", "apple,panel"; + width-mm = <522>; + height-mm = <294>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Adjust pcie0's iommu-map to account for the disabled port01. */ @@ -69,3 +83,72 @@ &pcie0_dart_2 { status = "okay"; }; + +&i2c1 { + jack_codec: codec@48 { + compatible = "cirrus,cs42l83"; + reg = <0x48>; + reset-gpios = <&pinctrl_nub 11 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <183 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&sep { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J457"; + apple,machine-kind = "iMac"; + apple,no-beamforming; +}; + +/ { + sound { + compatible = "apple,j457-macaudio", "apple,macaudio"; + model = "iMac J457"; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "isp-imx364.dtsi" + +&isp { + apple,platform-id = <2>; +}; + +#include "hwmon-fan.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi index 686fb1dd215d2d..67a57fc507df92 100644 --- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi @@ -12,6 +12,9 @@ / { aliases { bluetooth0 = &bluetooth0; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; wifi0 = &wifi0; @@ -29,11 +32,19 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* To be filled by loader */ + }; + memory@800000000 { device_type = "memory"; reg = <0x8 0 0x2 0>; /* To be filled by loader */ @@ -205,6 +216,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -225,4 +237,6 @@ clock-frequency = <900000000>; }; +#include "hwmon-common.dtsi" + #include "spi1-nvram.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi index fef8a4058f1415..5d3846d44e3578 100644 --- a/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8103-pmgr.dtsi @@ -234,7 +234,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_dpa0>; }; ps_fpwm0: power-controller@1d8 { @@ -493,6 +493,7 @@ #reset-cells = <0>; label = "mca0"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca1: power-controller@2c0 { @@ -502,6 +503,7 @@ #reset-cells = <0>; label = "mca1"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca2: power-controller@2c8 { @@ -511,6 +513,7 @@ #reset-cells = <0>; label = "mca2"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca3: power-controller@2d0 { @@ -520,6 +523,7 @@ #reset-cells = <0>; label = "mca3"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca4: power-controller@2d8 { @@ -529,6 +533,7 @@ #reset-cells = <0>; label = "mca4"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_mca5: power-controller@2e0 { @@ -538,6 +543,7 @@ #reset-cells = <0>; label = "mca5"; power-domains = <&ps_audio_p>, <&ps_sio_adma>; + apple,externally-clocked; }; ps_dpa0: power-controller@2e8 { @@ -645,8 +651,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_rmx>; - apple,always-on; /* TODO: figure out if we can enable PM here */ + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_fe: power-controller@368 { @@ -655,7 +660,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_rmx>; + power-domains = <&ps_rmx>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@378 { @@ -717,6 +722,7 @@ #reset-cells = <0>; label = "apcie_gp"; power-domains = <&ps_apcie>; + apple,always-on; /* Breaks things if shut down */ }; ps_ans2: power-controller@3f0 { @@ -806,6 +812,7 @@ #reset-cells = <0>; label = "isp_sys"; power-domains = <&ps_rmx>; + status = "disabled"; }; ps_venc_sys: power-controller@408 { @@ -1001,9 +1008,125 @@ #reset-cells = <0>; label = "disp0_cpu0"; power-domains = <&ps_disp0_fe>; - apple,always-on; /* TODO: figure out if we can enable PM here */ apple,min-state = <4>; }; + + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + ps_isp_set0: power-controller@4000 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set0"; + apple,force-disable; + }; + + ps_isp_set1: power-controller@4008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set1"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_set2: power-controller@4010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set2"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + }; + + ps_isp_set4: power-controller@4020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set4"; + }; + + ps_isp_set5: power-controller@4028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set5"; + }; + + ps_isp_set6: power-controller@4030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set6"; + }; + + ps_isp_set7: power-controller@4038 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set7"; + }; + + ps_isp_set8: power-controller@4040 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set8"; + }; + + ps_isp_set9: power-controller@4048 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set9"; + }; + + ps_isp_set10: power-controller@4050 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set10"; + }; + + ps_isp_set11: power-controller@4058 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set11"; + }; + + ps_isp_set12: power-controller@4060 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set12"; + }; }; &pmgr_mini { @@ -1096,6 +1219,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_atc0_usb_aon: power-controller@88 { diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index da774096b6674b..d3fc50b8f901b5 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -194,26 +194,31 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <47296>; }; opp02 { opp-hz = /bits/ 64 <972000000>; opp-level = <2>; clock-latency-ns = <22000>; + opp-microwatt = <99715>; }; opp03 { opp-hz = /bits/ 64 <1332000000>; opp-level = <3>; clock-latency-ns = <27000>; + opp-microwatt = <188860>; }; opp04 { opp-hz = /bits/ 64 <1704000000>; opp-level = <4>; clock-latency-ns = <33000>; + opp-microwatt = <288891>; }; opp05 { opp-hz = /bits/ 64 <2064000000>; opp-level = <5>; clock-latency-ns = <50000>; + opp-microwatt = <412979>; }; }; @@ -224,83 +229,140 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <8000>; + opp-microwatt = <290230>; }; opp02 { opp-hz = /bits/ 64 <828000000>; opp-level = <2>; clock-latency-ns = <19000>; + opp-microwatt = <449013>; }; opp03 { opp-hz = /bits/ 64 <1056000000>; opp-level = <3>; clock-latency-ns = <21000>; + opp-microwatt = <647097>; }; opp04 { opp-hz = /bits/ 64 <1284000000>; opp-level = <4>; clock-latency-ns = <23000>; + opp-microwatt = <865620>; }; opp05 { opp-hz = /bits/ 64 <1500000000>; opp-level = <5>; clock-latency-ns = <24000>; + opp-microwatt = <1112838>; }; opp06 { opp-hz = /bits/ 64 <1728000000>; opp-level = <6>; clock-latency-ns = <29000>; + opp-microwatt = <1453271>; }; opp07 { opp-hz = /bits/ 64 <1956000000>; opp-level = <7>; clock-latency-ns = <31000>; + opp-microwatt = <1776667>; }; opp08 { opp-hz = /bits/ 64 <2184000000>; opp-level = <8>; clock-latency-ns = <34000>; + opp-microwatt = <2366690>; }; opp09 { opp-hz = /bits/ 64 <2388000000>; opp-level = <9>; clock-latency-ns = <36000>; + opp-microwatt = <2892193>; }; opp10 { opp-hz = /bits/ 64 <2592000000>; opp-level = <10>; clock-latency-ns = <51000>; + opp-microwatt = <3475417>; }; opp11 { opp-hz = /bits/ 64 <2772000000>; opp-level = <11>; clock-latency-ns = <54000>; + opp-microwatt = <3959410>; }; opp12 { opp-hz = /bits/ 64 <2988000000>; opp-level = <12>; clock-latency-ns = <55000>; + opp-microwatt = <4540620>; }; -#if 0 /* Not available until CPU deep sleep is implemented */ opp13 { opp-hz = /bits/ 64 <3096000000>; opp-level = <13>; clock-latency-ns = <55000>; + opp-microwatt = <4745031>; turbo-mode; }; opp14 { opp-hz = /bits/ 64 <3144000000>; opp-level = <14>; clock-latency-ns = <56000>; + opp-microwatt = <4822390>; turbo-mode; }; opp15 { opp-hz = /bits/ 64 <3204000000>; opp-level = <15>; clock-latency-ns = <56000>; + opp-microwatt = <4951324>; turbo-mode; }; -#endif + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <396000000>; + opp-microvolt = <603000>; + opp-microwatt = <3714690>; + }; + opp02 { + opp-hz = /bits/ 64 <528000000>; + opp-microvolt = <640000>; + opp-microwatt = <5083260>; + }; + opp03 { + opp-hz = /bits/ 64 <720000000>; + opp-microvolt = <690000>; + opp-microwatt = <7429380>; + }; + opp04 { + opp-hz = /bits/ 64 <924000000>; + opp-microvolt = <784000>; + opp-microwatt = <11730600>; + }; + opp05 { + opp-hz = /bits/ 64 <1128000000>; + opp-microvolt = <862000>; + opp-microwatt = <17009370>; + }; + opp06 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <931000>; + opp-microwatt = <19551000>; + }; }; timer { @@ -346,6 +408,22 @@ clock-output-names = "clk_200m"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + /* * This is a fabulated representation of the input clock * to NCO since we don't know the true clock tree. @@ -374,15 +452,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; @@ -393,9 +471,11 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { - compatible = "apple,agx-g13g"; + compatible = "apple,agx-t8103", "apple,agx-g13g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; @@ -407,6 +487,40 @@ "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + + apple,firmware-version = <12 3 0>; + apple,firmware-compat = <12 3 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <850000>; + apple,avg-power-filter-tc-ms = <1000>; + apple,avg-power-ki-only = <7.5>; + apple,avg-power-kp = <4.0>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <125>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <50>; + apple,perf-integral-gain2 = <0.197392>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain2 = <6.853981>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,power-zones = <30000 100 6875>; + apple,ppm-filter-time-constant-ms = <100>; + apple,ppm-ki = <91.5>; + apple,ppm-kp = <6.9>; + apple,pwr-filter-time-constant = <313>; + apple,pwr-integral-gain = <0.0202129>; + apple,pwr-integral-min-clamp = <0>; + apple,pwr-min-duty-cycle = <40>; + apple,pwr-proportional-gain = <5.2831855>; + + apple,core-leak-coef = <1000.0>; + apple,sram-leak-coef = <45.0>; }; agx_mbox: mbox@206408000 { @@ -493,6 +607,146 @@ }; }; + disp0_dart: iommu@231304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + apple,dma-range = <0xf 0x00000000 0x0 0xfc000000>; + power-domains = <&ps_disp0_cpu0>; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x14>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + + isp_dart0: iommu@22c0e8000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0e8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp_dart1: iommu@22c0f4000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0f4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp_dart2: iommu@22c0fc000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x2c0fc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + + status = "disabled"; + }; + + isp: isp@22a000000 { + compatible = "apple,t8103-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x2 0x2a000000 0x0 0x2000000>, + <0x2 0x2c104000 0x0 0x100>, + <0x2 0x2c104170 0x0 0x100>, + <0x2 0x2c1043f0 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + <&ps_isp_set1>, <&ps_isp_set2>, <&ps_isp_fe>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, + <&ps_isp_set10>, <&ps_isp_set11>, + <&ps_isp_set12>; + + apple,dart-vm-size = <0x0 0xa0000000>; + + status = "disabled"; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8103-dart"; reg = <0x2 0x35004000 0x0 0x4000>; @@ -648,6 +902,32 @@ status = "disabled"; }; + sio_mbox: mbox@236408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x36408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio>; + }; + + sio: sio@236400000 { + compatible = "apple,t8103-sio", "apple,sio"; + reg = <0x2 0x36400000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + resets = <&ps_sio>; /* TODO: verify reset does something */ + status = "disabled"; + }; + admac: dma-controller@238200000 { compatible = "apple,t8103-admac", "apple,admac"; reg = <0x2 0x38200000 0x0 0x34000>; @@ -662,6 +942,48 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@238330000 { + compatible = "apple,t8103-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38330000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller@238334000 { + compatible = "apple,t8103-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38334000 0x0 0x4000>; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + reset-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; + mca: i2s@238400000 { compatible = "apple,t8103-mca", "apple,mca"; reg = <0x2 0x38400000 0x0 0x18000>, @@ -730,6 +1052,14 @@ reg = <0x2 0x3b700000 0 0x14000>; }; + pmgr_dcp: power-management@23b738000 { + reg = <0x2 0x3b738000 0x0 0x1000>, + <0x2 0x3bc3c000 0x0 0x1000>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + pinctrl_ap: pinctrl@23c100000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x3c100000 0x0 0x100000>; @@ -910,12 +1240,16 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; }; rtc { @@ -959,6 +1293,36 @@ ; }; + sep_dart: iommu@2412c0000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x412c0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@242400000 { + compatible = "apple,sep"; + reg = <0x2 0x42400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + status = "disabled"; + }; + + sep_mbox: mbox@242408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x42408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + pinctrl_aop: pinctrl@24a820000 { compatible = "apple,t8103-pinctrl", "apple,pinctrl"; reg = <0x2 0x4a820000 0x0 0x4000>; @@ -980,6 +1344,147 @@ ; }; + aop_mbox: mbox@24a408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@24a808000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x4a808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@24a980000 { + compatible = "apple,t8103-admac", "apple,admac"; + reg = <0x2 0x4a980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 321 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 7>; + status = "disabled"; + }; + + aop: aop@24ac00000 { + compatible = "apple,t8103-aop"; + reg = <0x2 0x4ac00000 0x0 0x1e0000>, + <0x2 0x4a400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t8103-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t8103-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t8103-aop-las", "apple,aop-las"; + }; + }; + + dispext0_dart: iommu@271304000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x71304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + apple,dma-range = <0x0 0x0 0x0 0xfc000000>; + status = "disabled"; + }; + + dcpext_dart: iommu@27130c000 { + compatible = "apple,t8103-dart"; + reg = <0x2 0x7130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + apple,dma-range = <0xf 0x00000000 0x0 0xfc000000>; + status = "disabled"; + }; + + dcpext_mbox: mbox@271c08000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x71c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext: dcp@271c00000 { + compatible = "apple,t8103-dcpext", "apple,dcpext"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext_dart 0>; + phandle = <&dcpext>; + + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + reg = <0x2 0x71c00000 0x0 0x4000>, + <0x2 0x70000000 0x0 0x118000>, + <0x2 0x71320000 0x0 0x4000>, + <0x2 0x71344000 0x0 0x4000>, + <0x2 0x71800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x18>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + clocks = <&clk_dispext0>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext0_dart 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; @@ -1066,6 +1571,13 @@ power-domains = <&ps_atc0_usb>; }; + atcphy0_xbar: mux@38304c000 { + compatible = "apple,t8103-display-crossbar"; + reg = <0x3 0x8304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + dwc3_1: usb@502280000 { compatible = "apple,t8103-dwc3"; reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; @@ -1118,6 +1630,13 @@ power-domains = <&ps_atc1_usb>; }; + atcphy1_xbar: mux@50304c000 { + compatible = "apple,t8103-display-crossbar"; + reg = <0x5 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart_0: iommu@681008000 { compatible = "apple,t8103-dart"; reg = <0x6 0x81008000 0x0 0x4000>; @@ -1183,6 +1702,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8112-j413.dts b/arch/arm64/boot/dts/apple/t8112-j413.dts index 1a08a41f369bfe..f36d40cb7fe611 100644 --- a/arch/arm64/boot/dts/apple/t8112-j413.dts +++ b/arch/arm64/boot/dts/apple/t8112-j413.dts @@ -20,6 +20,7 @@ aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; wifi0 = &wifi0; }; @@ -36,6 +37,21 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j413", "apple,panel"; + width-mm = <290>; + height-mm = <189>; + adj-height-mm = <181>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -43,6 +59,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -84,6 +101,76 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_woof: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0>; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; +}; + +&i2c3 { + speaker_right_woof: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f>; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -91,3 +178,102 @@ &fpwm1 { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J413"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j413-macaudio", "apple,macaudio"; + model = "MacBook Air J413"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof>, <&speaker_left_tweet>, + <&speaker_right_woof>, <&speaker_right_tweet>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j413.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx558-cfg0.dtsi" + +&isp { + apple,platform-id = <14>; + apple,temporal-filter = <1>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j415.dts b/arch/arm64/boot/dts/apple/t8112-j415.dts index e37c56d9fb4d66..7f63969ede2ab9 100644 --- a/arch/arm64/boot/dts/apple/t8112-j415.dts +++ b/arch/arm64/boot/dts/apple/t8112-j415.dts @@ -20,6 +20,7 @@ aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; wifi0 = &wifi0; }; @@ -36,6 +37,21 @@ }; }; +&dcp { + panel: panel { + compatible = "apple,panel-j415", "apple,panel"; + width-mm = <327>; + height-mm = <211>; + adj-height-mm = <204>; + apple,max-brightness = <500>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + /* * Force the bus number assignments so that we can declare some of the * on-board devices and properties that are populated by the bootloader @@ -43,6 +59,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4433"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -84,6 +101,98 @@ }; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0f0>; + }; + + speaker_left_tweet: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_woof2: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c3 { + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f0f>; + }; + + speaker_right_tweet: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_woof2: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -91,3 +200,106 @@ &fpwm1 { status = "okay"; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J415"; + apple,machine-kind = "MacBook Air"; +}; + +/ { + sound { + compatible = "apple,j415-macaudio", "apple,macaudio"; + model = "MacBook Air J415"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_left_tweet>, + <&speaker_left_woof2>, + <&speaker_right_woof1>, + <&speaker_right_tweet>, + <&speaker_right_woof2>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j415.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx558-cfg0.dtsi" + +&isp { + apple,platform-id = <15>; + apple,temporal-filter = <1>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j473.dts b/arch/arm64/boot/dts/apple/t8112-j473.dts index 438f972546b813..4fc96779806ea3 100644 --- a/arch/arm64/boot/dts/apple/t8112-j473.dts +++ b/arch/arm64/boot/dts/apple/t8112-j473.dts @@ -18,10 +18,63 @@ chassis-type = "desktop"; aliases { + bluetooth0 = &bluetooth0; + /delete-property/ dcp; + dcpext = &dcpext; ethernet0 = ðernet0; + sio = &sio; + wifi0 = &wifi0; }; }; +&framebuffer0 { + power-domains = <&ps_dispext_cpu0>, <&ps_dptx_ext_phy>; +}; + +&dptxphy { + status = "okay"; +}; + +&dcp { + status = "disabled"; +}; + +&display { + iommus = <&dispext0_dart 0>; +}; +&dispext0_dart { + status = "okay"; +}; +&dcpext_dart { + status = "okay"; +}; +&dcpext_mbox { + status = "okay"; +}; +&dcpext { + status = "okay"; + apple,connector-type = "HDMI-A"; + + /* HDMI HPD gpio, used as interrupt*/ + hdmi-hpd-gpios = <&pinctrl_aop 49 GPIO_ACTIVE_HIGH>; + + hdmi-pwren-gpios = <&smc_gpio 21 GPIO_ACTIVE_HIGH>; + dp2hdmi-pwren-gpios = <&smc_gpio 22 GPIO_ACTIVE_HIGH>; + + phys = <&dptxphy>; + phy-names = "dp-phy"; + apple,dptx-phy = <5>; +}; + +/* remove once m1n1 enables sio nodes after setup */ +&sio { + status = "okay"; +}; + +&dpaudio1 { + status = "okay"; +}; + /* * Keep the power-domains used for the HDMI port on. */ @@ -48,10 +101,28 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 10]; + apple,antenna-sku = "XX"; + brcm,board-type = "apple,miyake"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + brcm,board-type = "apple,miyake"; + }; }; &port01 { bus-range = <2 2>; + pwren-gpios = <&smc_gpio 24 GPIO_ACTIVE_HIGH>; status = "okay"; }; @@ -84,3 +155,63 @@ &typec1 { label = "USB-C Back-right"; }; + +&i2c1 { + speaker_amp: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&pinctrl_ap>; + interrupts = <149 IRQ_TYPE_LEVEL_LOW>; + #sound-dai-cells = <0>; + cirrus,ts-inv = <1>; + sound-name-prefix = "Jack"; + }; +}; + +/ { + sound { + compatible = "apple,j473-macaudio", "apple,macaudio"; + model = "Mac mini J473"; + + dai-link@0 { + link-name = "Speaker"; + + cpu { + sound-dai = <&mca 0>; + }; + codec { + sound-dai = <&speaker_amp>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + }; +}; + +&gpu { + apple,perf-base-pstate = <3>; +}; + +#include "hwmon-mini.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-j493.dts b/arch/arm64/boot/dts/apple/t8112-j493.dts index ec116da3e4dd9e..61bbb7a3c6f486 100644 --- a/arch/arm64/boot/dts/apple/t8112-j493.dts +++ b/arch/arm64/boot/dts/apple/t8112-j493.dts @@ -24,6 +24,7 @@ */ aliases { bluetooth0 = &bluetooth0; + keyboard = &keyboard; touchbar0 = &touchbar0; wifi0 = &wifi0; }; @@ -51,6 +52,20 @@ apple,always-on; }; +&dcp { + panel: panel { + compatible = "apple,panel-j493", "apple,panel"; + width-mm = <286>; + height-mm = <179>; + apple,max-brightness = <525>; + }; +}; + +&framebuffer0 { + panel = <&panel>; + post-init-providers = <&panel>; +}; + &display_dfr { status = "okay"; }; @@ -91,6 +106,7 @@ */ &port00 { bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; wifi0: wifi@0,0 { compatible = "pci14e4,4425"; reg = <0x10000 0x0 0x0 0x0 0x0>; @@ -121,6 +137,76 @@ label = "USB-C Left-front"; }; +/* Virtual regulator representing the shared shutdown GPIO */ +/ { + speaker_sdz: fixed-regulator-sn012776-sdz { + compatible = "regulator-fixed"; + regulator-name = "sn012776-sdz"; + startup-delay-us = <5000>; + gpios = <&pinctrl_ap 88 GPIO_ACTIVE_HIGH>; + enable-active-high; + }; +}; + +&i2c1 { + speaker_left_rear: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Rear"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_front: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Front"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + ti,sdout-force-zero-mask = <0xf0f0>; + }; +}; + +&i2c3 { + speaker_right_rear: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Rear"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_front: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + SDZ-supply = <&speaker_sdz>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Front"; + interrupts-extended = <&pinctrl_ap 11 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + ti,sdout-force-zero-mask = <0x0f0f>; + }; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_nub 12 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 149 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + &i2c4 { status = "okay"; }; @@ -146,3 +232,102 @@ touchscreen-inverted-y; }; }; + +&aop_mbox { + status = "okay"; +}; + +&aop_dart { + status = "okay"; +}; + +&aop_admac { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J493"; + apple,machine-kind = "MacBook Pro"; +}; + +/ { + sound { + compatible = "apple,j493-macaudio", "apple,macaudio"; + model = "MacBook Pro J493"; + + dai-link@0 { + link-name = "Speakers"; + + cpu { + sound-dai = <&mca 0>, <&mca 1>; + }; + codec { + sound-dai = <&speaker_left_front>, <&speaker_left_rear>, + <&speaker_right_front>, <&speaker_right_rear>; + }; + }; + + dai-link@1 { + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&mca 2>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + }; +}; + +&mtp { + status = "okay"; +}; +&mtp_mbox { + status = "okay"; +}; +&mtp_dart { + status = "okay"; +}; +&mtp_dockchannel { + status = "okay"; +}; +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j493.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "isp-imx248.dtsi" + +&isp { + apple,platform-id = <6>; +}; + +&pmp_report_isp_sys { + status = "okay"; +}; + +#include "hwmon-fan.dtsi" +#include "hwmon-laptop.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi index 562e7a25a1e884..fb957f785d82c5 100644 --- a/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-jxxx.dtsi @@ -13,6 +13,9 @@ aliases { atcphy0 = &atcphy0; atcphy1 = &atcphy1; + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; serial0 = &serial0; serial2 = &serial2; }; @@ -27,6 +30,7 @@ framebuffer0: framebuffer@0 { compatible = "apple,simple-framebuffer", "simple-framebuffer"; reg = <0 0 0 0>; /* To be filled by loader */ + power-domains = <&ps_disp0_cpu0>; /* Format properties will be added by loader */ status = "disabled"; }; @@ -214,4 +218,6 @@ clock-frequency = <900000000>; }; +#include "hwmon-common.dtsi" + #include "spi1-nvram.dtsi" diff --git a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi index 7c050c6f2707a1..ab8ec9bd4e4401 100644 --- a/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8112-pmgr.dtsi @@ -176,7 +176,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_dpa0>; }; ps_fpwm0: power-controller@1c8 { @@ -465,6 +465,7 @@ #reset-cells = <0>; label = "mca0"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca1: power-controller@2c8 { @@ -474,6 +475,7 @@ #reset-cells = <0>; label = "mca1"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca2: power-controller@2d0 { @@ -483,6 +485,7 @@ #reset-cells = <0>; label = "mca2"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca3: power-controller@2d8 { @@ -492,6 +495,7 @@ #reset-cells = <0>; label = "mca3"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca4: power-controller@2e0 { @@ -501,6 +505,7 @@ #reset-cells = <0>; label = "mca4"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mca5: power-controller@2e8 { @@ -510,6 +515,7 @@ #reset-cells = <0>; label = "mca5"; power-domains = <&ps_sio_adma>, <&ps_audio_p>; + apple,externally-clocked; }; ps_mcc: power-controller@2f0 { @@ -663,7 +669,6 @@ #reset-cells = <0>; label = "disp0_sys"; power-domains = <&ps_rmx1>; - apple,always-on; /* TODO: figure out if we can enable PM here */ }; ps_disp0_fe: power-controller@378 { @@ -672,8 +677,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "disp0_fe"; - power-domains = <&ps_disp0_sys>; - apple,always-on; /* TODO: figure out if we can enable PM here */ + power-domains = <&ps_disp0_sys>, <&ps_pmp>; }; ps_dispext_sys: power-controller@380 { @@ -691,7 +695,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "dispext_fe"; - power-domains = <&ps_dispext_sys>; + power-domains = <&ps_dispext_sys>, <&ps_pmp>; }; ps_dispext_cpu0: power-controller@3c8 { @@ -773,7 +777,6 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "pmp"; - apple,always-on; }; ps_pms_sram: power-controller@418 { @@ -818,6 +821,7 @@ #reset-cells = <0>; label = "isp_sys"; power-domains = <&ps_rmx1>; + status = "disabled"; }; ps_venc_sys: power-controller@440 { @@ -964,6 +968,123 @@ apple,always-on; }; + /* There is a dependency tree involved with these PDs, + * but we do not express it here since the ISP driver + * is supposed to sequence them in the right order anyway + * (and we do not know the exact tree structure). + * + * This also works around spurious parent PD activation + * on machines with ISP disabled (desktops). + */ + ps_isp_set0: power-controller@4000 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4000 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set0"; + apple,force-disable; + }; + + ps_isp_set1: power-controller@4008 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4008 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set1"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_set2: power-controller@4010 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4010 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set2"; + apple,force-disable; + apple,force-reset; + }; + + ps_isp_fe: power-controller@4018 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4018 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_fe"; + }; + + ps_isp_set4: power-controller@4020 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4020 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set4"; + }; + + ps_isp_set5: power-controller@4028 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4028 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set5"; + }; + + ps_isp_set6: power-controller@4030 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4030 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set6"; + }; + + ps_isp_set7: power-controller@4038 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4038 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set7"; + }; + + ps_isp_set8: power-controller@4040 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4040 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set8"; + }; + + ps_isp_set9: power-controller@4048 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4048 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set9"; + }; + + ps_isp_set12: power-controller@4050 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4050 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set10"; + }; + + ps_isp_set10: power-controller@4058 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4058 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set11"; + }; + + ps_isp_set11: power-controller@4060 { + compatible = "apple,t8103-pmgr-pwrstate", "apple,pmgr-pwrstate"; + reg = <0x4060 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "isp_set12"; + }; + ps_venc_dma: power-controller@8000 { compatible = "apple,t8112-pmgr-pwrstate", "apple,pmgr-pwrstate"; reg = <0x8000 4>; @@ -1064,6 +1185,7 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "msg"; + apple,always-on; /* Core AON device? */ }; ps_nub_gpio: power-controller@80 { diff --git a/arch/arm64/boot/dts/apple/t8112.dtsi b/arch/arm64/boot/dts/apple/t8112.dtsi index 85c47422d4e8e3..b667944f6dc5c3 100644 --- a/arch/arm64/boot/dts/apple/t8112.dtsi +++ b/arch/arm64/boot/dts/apple/t8112.dtsi @@ -22,6 +22,9 @@ aliases { gpu = &gpu; + #ifdef APPLE_USE_PMP + pmp = &pmp; + #endif }; cpus { @@ -195,36 +198,43 @@ opp-hz = /bits/ 64 <600000000>; opp-level = <1>; clock-latency-ns = <7500>; + opp-microwatt = <26000>; }; opp02 { opp-hz = /bits/ 64 <912000000>; opp-level = <2>; clock-latency-ns = <20000>; + opp-microwatt = <56000>; }; opp03 { opp-hz = /bits/ 64 <1284000000>; opp-level = <3>; clock-latency-ns = <22000>; + opp-microwatt = <88000>; }; opp04 { opp-hz = /bits/ 64 <1752000000>; opp-level = <4>; clock-latency-ns = <30000>; + opp-microwatt = <155000>; }; opp05 { opp-hz = /bits/ 64 <2004000000>; opp-level = <5>; clock-latency-ns = <35000>; + opp-microwatt = <231000>; }; opp06 { opp-hz = /bits/ 64 <2256000000>; opp-level = <6>; clock-latency-ns = <39000>; + opp-microwatt = <254000>; }; opp07 { opp-hz = /bits/ 64 <2424000000>; opp-level = <7>; clock-latency-ns = <53000>; + opp-microwatt = <351000>; }; }; @@ -236,93 +246,161 @@ opp-hz = /bits/ 64 <660000000>; opp-level = <1>; clock-latency-ns = <9000>; + opp-microwatt = <133000>; }; opp02 { opp-hz = /bits/ 64 <924000000>; opp-level = <2>; clock-latency-ns = <19000>; + opp-microwatt = <212000>; }; opp03 { opp-hz = /bits/ 64 <1188000000>; opp-level = <3>; clock-latency-ns = <22000>; + opp-microwatt = <261000>; }; opp04 { opp-hz = /bits/ 64 <1452000000>; opp-level = <4>; clock-latency-ns = <24000>; + opp-microwatt = <345000>; }; opp05 { opp-hz = /bits/ 64 <1704000000>; opp-level = <5>; clock-latency-ns = <26000>; + opp-microwatt = <441000>; }; opp06 { opp-hz = /bits/ 64 <1968000000>; opp-level = <6>; clock-latency-ns = <28000>; + opp-microwatt = <619000>; }; opp07 { opp-hz = /bits/ 64 <2208000000>; opp-level = <7>; clock-latency-ns = <30000>; + opp-microwatt = <740000>; }; opp08 { opp-hz = /bits/ 64 <2400000000>; opp-level = <8>; clock-latency-ns = <33000>; + opp-microwatt = <855000>; }; opp09 { opp-hz = /bits/ 64 <2568000000>; opp-level = <9>; clock-latency-ns = <34000>; + opp-microwatt = <1006000>; }; opp10 { opp-hz = /bits/ 64 <2724000000>; opp-level = <10>; clock-latency-ns = <36000>; + opp-microwatt = <1217000>; }; opp11 { opp-hz = /bits/ 64 <2868000000>; opp-level = <11>; clock-latency-ns = <41000>; + opp-microwatt = <1534000>; }; opp12 { opp-hz = /bits/ 64 <2988000000>; opp-level = <12>; clock-latency-ns = <42000>; + opp-microwatt = <1714000>; }; opp13 { opp-hz = /bits/ 64 <3096000000>; opp-level = <13>; clock-latency-ns = <44000>; + opp-microwatt = <1877000>; }; opp14 { opp-hz = /bits/ 64 <3204000000>; opp-level = <14>; clock-latency-ns = <46000>; + opp-microwatt = <2159000>; }; - /* Not available until CPU deep sleep is implemented */ -#if 0 opp15 { opp-hz = /bits/ 64 <3324000000>; opp-level = <15>; clock-latency-ns = <62000>; + opp-microwatt = <2393000>; turbo-mode; }; opp16 { opp-hz = /bits/ 64 <3408000000>; opp-level = <16>; clock-latency-ns = <62000>; + opp-microwatt = <2497000>; turbo-mode; }; opp17 { opp-hz = /bits/ 64 <3504000000>; opp-level = <17>; clock-latency-ns = <62000>; + opp-microwatt = <2648000>; turbo-mode; }; -#endif + }; + + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The voltage and power values are device-specific and + * must be filled in by the bootloader. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <400000>; + opp-microwatt = <0>; + }; + opp01 { + opp-hz = /bits/ 64 <444000000>; + opp-microvolt = <603000>; + opp-microwatt = <4295000>; + }; + opp02 { + opp-hz = /bits/ 64 <612000000>; + opp-microvolt = <675000>; + opp-microwatt = <6251000>; + }; + opp03 { + opp-hz = /bits/ 64 <808000000>; + opp-microvolt = <710000>; + opp-microwatt = <8625000>; + }; + opp04 { + opp-hz = /bits/ 64 <968000000>; + opp-microvolt = <775000>; + opp-microwatt = <11948000>; + }; + opp05 { + opp-hz = /bits/ 64 <1110000000>; + opp-microvolt = <820000>; + opp-microwatt = <15071000>; + }; + opp06 { + opp-hz = /bits/ 64 <1236000000>; + opp-microvolt = <875000>; + opp-microwatt = <18891000>; + }; + opp07 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <915000>; + opp-microwatt = <21960000>; + }; + opp08 { + opp-hz = /bits/ 64 <1398000000>; + opp-microvolt = <950000>; + opp-microwatt = <22800000>; + }; }; timer { @@ -371,6 +449,22 @@ clock-output-names = "nco_ref"; }; + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533333328>; + clock-output-names = "clk_disp0"; + }; + + /* Pixel clock? frequency in Hz (compare: 4K@60 VGA clock 533.250 MHz) */ + clk_dispext0: clock-dispext0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <0>; + clock-output-names = "clk_dispext0"; + }; + reserved-memory { #address-cells = <2>; #size-cells = <2>; @@ -389,15 +483,15 @@ }; uat_handoff: uat-handoff { - status = "disabled"; + reg = <0 0 0 0>; }; uat_pagetables: uat-pagetables { - status = "disabled"; + reg = <0 0 0 0>; }; uat_ttbs: uat-ttbs { - status = "disabled"; + reg = <0 0 0 0>; }; }; @@ -408,20 +502,57 @@ ranges; nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; gpu: gpu@206400000 { - compatible = "apple,agx-g14g"; + compatible = "apple,agx-t8112", "apple,agx-g14g"; reg = <0x2 0x6400000 0 0x40000>, <0x2 0x4000000 0 0x1000000>; reg-names = "asc", "sgx"; mboxes = <&agx_mbox>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_gfx>; + #else power-domains = <&ps_gfx>; + #endif memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>, <&gpu_hw_cal_a>, <&gpu_hw_cal_b>, <&gpu_globals>; memory-region-names = "ttbs", "pagetables", "handoff", "hw-cal-a", "hw-cal-b", "globals"; apple,firmware-abi = <0 0 0>; + apple,firmware-version = <12 4 0>; + apple,firmware-compat = <12 4 0>; + + operating-points-v2 = <&gpu_opp>; + apple,perf-base-pstate = <1>; + apple,min-sram-microvolt = <780000>; + apple,avg-power-filter-tc-ms = <300>; + apple,avg-power-ki-only = <9.375>; + apple,avg-power-kp = <3.22>; + apple,avg-power-min-duty-cycle = <40>; + apple,avg-power-target-filter-tc = <1>; + apple,fast-die0-integral-gain = <200.0>; + apple,fast-die0-proportional-gain = <5.0>; + apple,perf-boost-ce-step = <50>; + apple,perf-boost-min-util = <90>; + apple,perf-filter-drop-threshold = <0>; + apple,perf-filter-time-constant = <5>; + apple,perf-filter-time-constant2 = <200>; + apple,perf-integral-gain = <5.94>; + apple,perf-integral-gain2 = <5.94>; + apple,perf-integral-min-clamp = <0>; + apple,perf-proportional-gain = <14.85>; + apple,perf-proportional-gain2 = <14.85>; + apple,perf-tgt-utilization = <85>; + apple,power-sample-period = <8>; + apple,ppm-filter-time-constant-ms = <34>; + apple,ppm-ki = <205.0>; + apple,ppm-kp = <0.75>; + apple,pwr-min-duty-cycle = <40>; + apple,core-leak-coef = <1920.0>; + apple,sram-leak-coef = <74.0>; }; agx_mbox: mbox@206408000 { @@ -454,7 +585,11 @@ reg = <0x2 0x28200000 0x0 0xc000>, <0x2 0x28400000 0x0 0x4000>; reg-names = "be", "fe"; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_dispdfr>, <&ps_dispdfr_be>; + #else power-domains = <&ps_dispdfr_fe>, <&ps_dispdfr_be>; + #endif interrupt-parent = <&aic>; interrupts = , ; @@ -508,6 +643,148 @@ }; }; + isp_dart0: iommu@22c4a8000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4a8000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart1: iommu@22c4b4000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4b4000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp_dart2: iommu@22c4bc000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x2c4bc000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + isp: isp@22a000000 { + compatible = "apple,t8112-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2"; + reg = <0x2 0x2a000000 0x0 0x2000000>, + <0x2 0x2c4c4000 0x0 0x100>, + <0x2 0x2c4c41b0 0x0 0x100>, + <0x2 0x2c4c4430 0x0 0x100>; + interrupt-parent = <&aic>; + interrupts = ; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_isp_sys>, <&ps_isp_set0>, + #else + power-domains = <&ps_isp_sys>, <&ps_isp_set0>, + #endif + <&ps_isp_set1>, <&ps_isp_set2>, <&ps_isp_fe>, + <&ps_isp_set4>, <&ps_isp_set5>, <&ps_isp_set6>, + <&ps_isp_set7>, <&ps_isp_set8>, <&ps_isp_set9>, + <&ps_isp_set10>, <&ps_isp_set11>, + <&ps_isp_set12>; + + apple,dart-vm-size = <0x0 0xa0000000>; + status = "disabled"; + }; + + disp0_dart: iommu@231304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x31304000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x0 0x0 0xf 0xffff0000>; + status = "disabled"; + }; + + dcp_dart: iommu@23130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x3130c000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_disp0_cpu0>; + apple,dma-range = <0x8 0x00000000 0x7 0xffff0000>; + }; + + dcp_mbox: mbox@231c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x31c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + resets = <&ps_disp0_cpu0>; + }; + + dcp: dcp@231c00000 { + compatible = "apple,t8112-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 5>; + + /* the ADT has 2 additional regs which seems to be unused */ + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x61c000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x5d8>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_disp0>; + #else + power-domains = <&ps_disp0_cpu0>; + #endif + resets = <&ps_disp0_cpu0>; + clocks = <&clk_disp0>; + phandle = <&dcp>; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 4>; + phandle = <&disp0_piodma>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcp_audio: endpoint { + remote-endpoint = <&dpaudio0_dcp>; + }; + }; + }; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp_dart0 must be 1st since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; + sio_dart: iommu@235004000 { compatible = "apple,t8110-dart"; reg = <0x2 0x35004000 0x0 0x4000>; @@ -652,6 +929,32 @@ status = "disabled"; }; + sio_mbox: mbox@236408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x36408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio_cpu>; + }; + + sio: sio@236400000 { + compatible = "apple,t8112-sio", "apple,sio"; + reg = <0x2 0x36400000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + resets = <&ps_sio>; /* TODO: verify reset does something */ + status = "disabled"; + }; + admac: dma-controller@238200000 { compatible = "apple,t8112-admac", "apple,admac"; reg = <0x2 0x38200000 0x0 0x34000>; @@ -666,6 +969,48 @@ resets = <&ps_audio_p>; }; + dpaudio0: audio-controller@238330000 { + compatible = "apple,t8112-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38330000 0x0 0x4000>; + dmas = <&sio 0x64>; + dma-names = "tx"; + power-domains = <&ps_dpa0>; + reset-domains = <&ps_dpa0>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio0_dcp: endpoint { + remote-endpoint = <&dcp_audio>; + }; + }; + }; + }; + + dpaudio1: audio-controller@238334000 { + compatible = "apple,t8112-dpaudio", "apple,dpaudio"; + reg = <0x2 0x38334000 0x0 0x4000>; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + reset-domains = <&ps_dpa1>; + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dpaudio1_dcp: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; + mca: i2s@238400000 { compatible = "apple,t8112-mca", "apple,mca"; reg = <0x2 0x38400000 0x0 0x18000>, @@ -729,6 +1074,148 @@ }; }; + pmp_dart: iommu@23b300000 { + compatible = "apple,t8110-dart"; + reg = <0x2 0x3b300000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_pmp>; + }; + + pmp_report: pmp_report@23b3c0000 { + compatible = "apple,t8112-pmp-v2-report"; + reg = <0x2 0x3b3c0000 0x0 0x20000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + + pmp_report_gfx: report@4 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x4>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + + pmp_report_ane_sys: report@5 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x5>; + label = "pmp-ane-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + + pmp_report_isp_sys: report@6 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x6>; + label = "pmp-isp-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_isp_sys>; + status = "disabled"; + }; + + pmp_report_disp0: report@7 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x7>; + label = "pmp-disp0"; + #power-domain-cells = <0>; + power-domains = <&ps_disp0_cpu0>; + apple,always-on; + }; + + pmp_report_dispext: report@8 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x8>; + label = "pmp-dispext"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + }; + + pmp_report_venc_sys: report@9 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-venc-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_venc_sys>; + status = "disabled"; + }; + + pmp_report_avd_sys: report@a { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xa>; + label = "pmp-avd-sys"; + #power-domain-cells = <0>; + power-domains = <&ps_avd_sys>; + status = "disabled"; + }; + + pmp_report_msr: report@b { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xb>; + label = "pmp-msr"; + #power-domain-cells = <0>; + power-domains = <&ps_msr>; + status = "disabled"; + }; + + pmp_report_jpg: report@c { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xc>; + label = "pmp-jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_jpg>; + status = "disabled"; + }; + + pmp_report_scodec: report@d { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xd>; + label = "pmp-scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + status = "disabled"; + }; + + pmp_report_dispdfr: report@11 { + compatible = "apple,t8112-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x11>; + label = "pmp-dispdfr"; + #power-domain-cells = <0>; + power-domains = <&ps_dispdfr_fe>; + }; + }; + + pmgr_dcp: power-management@23b3d0000 { + reg = <0x2 0x3b3d0000 0x0 0x4000>; + reg-names = "dcp-bw-scratch"; + #apple,bw-scratch-cells = <3>; + }; + + pmp: pmp@23b500000 { + compatible = "apple,t8112-pmp-v2", "apple,t6000-pmp-v2"; + reg = <0x2 0x3b500000 0x0 0x80000>, + <0x2 0x3bc00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + power-domains = <&ps_pmp>; + status = "disabled"; + }; + pmgr: power-management@23b700000 { compatible = "apple,t8112-pmgr", "apple,pmgr", "syscon", "simple-mfd"; #address-cells = <1>; @@ -737,6 +1224,20 @@ /* child nodes are added in t8103-pmgr.dtsi */ }; + pmp_mbox: mbox@23bc08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x3bc08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_pmp>, <&ps_pms_sram>; + }; + pinctrl_ap: pinctrl@23c100000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3c100000 0x0 0x100000>; @@ -805,6 +1306,17 @@ }; }; + dptxphy: phy@23c500000 { + compatible = "apple,t8112-dptx-phy", "apple,dptx-phy"; + reg = <0x2 0x3c500000 0x0 0x4000>, + <0x2 0x3c540000 0x0 0xc000>; + reg-names = "core", "dptx"; + power-domains = <&ps_dptx_ext_phy>; + #phy-cells = <0>; + #reset-cells = <0>; + status = "disabled"; /* only used on j473 */ + }; + pinctrl_nub: pinctrl@23d1f0000 { compatible = "apple,t8112-pinctrl", "apple,pinctrl"; reg = <0x2 0x3d1f0000 0x0 0x4000>; @@ -913,12 +1425,16 @@ #gpio-cells = <2>; }; + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + smc_reboot: reboot { compatible = "apple,smc-reboot"; nvmem-cells = <&shutdown_flag>, <&boot_stage>, - <&boot_error_count>, <&panic_count>; + <&boot_error_count>, <&panic_count>, <&pm_setting>; nvmem-cell-names = "shutdown_flag", "boot_stage", - "boot_error_count", "panic_count"; + "boot_error_count", "panic_count", "pm_setting"; }; rtc { @@ -983,6 +1499,254 @@ ; }; + aop_mbox: mbox@24a408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4a408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@24a808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4a808000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop_admac: dma-controller@24a980000 { + compatible = "apple,t8112-admac", "apple,admac"; + reg = <0x2 0x4a980000 0x0 0x34000>; + #dma-cells = <1>; + dma-channels = <16>; + interrupts-extended = <0>, + <0>, + <&aic AIC_IRQ 359 IRQ_TYPE_LEVEL_HIGH>, + <0>; + iommus = <&aop_dart 10>; + status = "disabled"; + }; + + aop: aop@24ac00000 { + compatible = "apple,t8112-aop"; + reg = <0x2 0x4ac00000 0x0 0x1e0000>, + <0x2 0x4a400000 0x0 0x6c000>; + mboxes = <&aop_mbox>; + mbox-names = "mbox"; + iommus = <&aop_dart 0>; + + status = "disabled"; + + aop_audio: audio { + compatible = "apple,t8112-aop-audio", "apple,aop-audio"; + dmas = <&aop_admac 1>; + dma-names = "dma"; + }; + + aop_als: als { + compatible = "apple,t8112-aop-als", "apple,aop-als"; + // intentionally empty + }; + + las { + compatible = "apple,t8112-aop-las", "apple,aop-las"; + }; + }; + + mtp: mtp@24e400000 { + compatible = "apple,t8112-mtp", "apple,t8112-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0x4e400000 0x0 0x4000>, + <0x2 0x4ec00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@24e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x4e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + + status = "disabled"; + }; + + mtp_dart: iommu@24e808000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x4e808000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = ; + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@24eb14000 { + compatible = "apple,t8112-dockchannel", "apple,dockchannel"; + reg = <0x2 0x4eb14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0x4eb28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x4000>, + <0xc000 0x4000>, + <0x0000 0x4000>, + <0x4000 0x4000>; + reg-names = "config", "data", + "rmt-config", "rmt-data"; + iommus = <&mtp_dart 1>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + + }; + + sep_dart: iommu@25d2c0000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x5d2c0000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + sep: sep@25e400000 { + compatible = "apple,sep"; + reg = <0x2 0x5e400000 0x0 0x6C000>; + mboxes = <&sep_mbox>; + mbox-names = "mbox"; + iommus = <&sep_dart 0>; + status = "disabled"; + }; + + sep_mbox: mbox@25e408000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x5e408000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + dispext0_dart: iommu@271304000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x71304000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x0 0x0 0xf 0xffff0000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext_dart: iommu@27130c000 { + compatible = "apple,t8112-dart", "apple,t8110-dart"; + reg = <0x2 0x7130c000 0x0 0x4000>; + #iommu-cells = <1>; + apple,dma-range = <0x8 0x0 0x7 0xffff0000>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext_mbox: mbox@271c08000 { + compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x71c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_dispext_cpu0>; + resets = <&ps_dispext_cpu0>; + status = "disabled"; + }; + + dcpext: dcp@271c00000 { + compatible = "apple,t8112-dcpext", "apple,dcpext"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + iommus = <&dcpext_dart 5>; + phandle = <&dcpext>; + + /* the ADT has 2 additional regs which seems to be unused */ + reg-names = "coproc", "disp-0", "disp-1", "disp-2", "disp-3"; + reg = <0x2 0x71c00000 0x0 0x4000>, + <0x2 0x70000000 0x0 0x61C000>, + <0x2 0x71320000 0x0 0x4000>, + <0x2 0x71344000 0x0 0x4000>, + <0x2 0x71800000 0x0 0x800000>; + apple,bw-scratch = <&pmgr_dcp 0 4 0x5e0>; + #ifdef APPLE_USE_PMP + power-domains = <&pmp_report_dispext>; + #else + power-domains = <&ps_dispext_cpu0>; + #endif + resets = <&ps_dispext_cpu0>; + clocks = <&clk_dispext0>; + apple,dcp-index = <1>; + status = "disabled"; + // required bus properties for 'piodma' subdevice + #address-cells = <2>; + #size-cells = <2>; + + piodma { + iommus = <&dispext0_dart 4>; + }; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcp>; + }; + }; + }; + }; + ans_mbox: mbox@277408000 { compatible = "apple,t8112-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0x77408000 0x0 0x4000>; @@ -1069,6 +1833,13 @@ power-domains = <&ps_atc0_usb>; }; + atcphy0_xbar: mux@38304c000 { + compatible = "apple,t8112-display-crossbar", "apple,t8103-display-crossbar"; + reg = <0x3 0x8304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc0_usb>; + }; + dwc3_1: usb@502280000 { compatible = "apple,t8112-dwc3", "apple,t8103-dwc3"; reg = <0x5 0x02280000 0x0 0xcd00>, <0x5 0x0228cd00 0x0 0x3200>; @@ -1121,6 +1892,13 @@ power-domains = <&ps_atc1_usb>; }; + atcphy1_xbar: mux@50304c000 { + compatible = "apple,t8112-display-crossbar", "apple,t8103-display-crossbar"; + reg = <0x5 0x0304c000 0x0 0x4000>; + #mux-control-cells = <1>; + power-domains = <&ps_atc1_usb>; + }; + pcie0_dart: iommu@681008000 { compatible = "apple,t8110-dart"; reg = <0x6 0x81008000 0x0 0x4000>; @@ -1198,6 +1976,8 @@ pinctrl-0 = <&pcie_pins>; pinctrl-names = "default"; + dma-coherent; + port00: pci@0,0 { device_type = "pci"; reg = <0x0 0x0 0x0 0x0 0x0>; diff --git a/arch/arm64/boot/dts/apple/t8122-j433.dts b/arch/arm64/boot/dts/apple/t8122-j433.dts index 34205d173a9bd1..937f159741b736 100644 --- a/arch/arm64/boot/dts/apple/t8122-j433.dts +++ b/arch/arm64/boot/dts/apple/t8122-j433.dts @@ -16,4 +16,30 @@ / { compatible = "apple,j433", "apple,t8122", "apple,arm-platform"; model = "Apple iMac (24-inch, 2x USB-C, M3, 2023)"; + + aliases { + ethernet0 = ðernet0; + }; +}; + +&wifi0 { + brcm,board-type = "apple,azores"; +}; + +&bluetooth0 { + brcm,board-type = "apple,azores"; +}; + +&port01 { + bus-range = <2 2>; + status = "okay"; + ethernet0: ethernet@0,0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&pcie1_dart { + status = "okay"; }; diff --git a/arch/arm64/boot/dts/apple/t8122-j434.dts b/arch/arm64/boot/dts/apple/t8122-j434.dts index ead5afd77efbc6..fd79ec61091391 100644 --- a/arch/arm64/boot/dts/apple/t8122-j434.dts +++ b/arch/arm64/boot/dts/apple/t8122-j434.dts @@ -17,3 +17,35 @@ compatible = "apple,j434", "apple,t8122", "apple,arm-platform"; model = "Apple iMac (24-inch, 4x USB-C, M3, 2023)"; }; + +&wifi0 { + brcm,board-type = "apple,iona"; +}; + +&bluetooth0 { + brcm,board-type = "apple,iona"; +}; + +&port01 { + bus-range = <2 2>; + status = "okay"; + ethernet0: ethernet@0,0 { + reg = <0x20000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 10 18 00 00 00]; + }; +}; + +&port02 { + bus-range = <3 3>; + pwren-gpios = <&smc_gpio 14 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; + +&pcie1_dart { + status = "okay"; +}; + +&pcie2_dart { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-j504.dts b/arch/arm64/boot/dts/apple/t8122-j504.dts index 464491b55b019c..7cc0ffc4a8a925 100644 --- a/arch/arm64/boot/dts/apple/t8122-j504.dts +++ b/arch/arm64/boot/dts/apple/t8122-j504.dts @@ -31,7 +31,73 @@ }; }; +&wifi0 { + brcm,board-type = "apple,tresco"; +}; + +&bluetooth0 { + brcm,board-type = "apple,tresco"; +}; + +&port01 { + /* SD card reader */ + bus-range = <2 2>; + pwren-gpios = <&smc_gpio 23 GPIO_ACTIVE_HIGH>; + status = "okay"; + + sdhci0: mmc@0,0 { + compatible = "pci17a0,9755"; + reg = <0x20000 0x0 0x0 0x0 0x0>; + cd-inverted; + wp-inverted; + }; +}; + +&pcie1_dart { + status = "okay"; +}; + + &fpwm1 { status = "okay"; }; +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j504.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + diff --git a/arch/arm64/boot/dts/apple/t8122-j613.dts b/arch/arm64/boot/dts/apple/t8122-j613.dts index 51894ea705e765..0e0ff85f7e793a 100644 --- a/arch/arm64/boot/dts/apple/t8122-j613.dts +++ b/arch/arm64/boot/dts/apple/t8122-j613.dts @@ -30,6 +30,53 @@ }; }; +&wifi0 { + brcm,board-type = "apple,dnieper"; +}; + +&bluetooth0 { + brcm,board-type = "apple,dnieper"; +}; + &fpwm1 { status = "okay"; }; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j613.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-j615.dts b/arch/arm64/boot/dts/apple/t8122-j615.dts index 2a1970c1bc90c0..77b249dda6fcfc 100644 --- a/arch/arm64/boot/dts/apple/t8122-j615.dts +++ b/arch/arm64/boot/dts/apple/t8122-j615.dts @@ -30,6 +30,53 @@ }; }; +&wifi0 { + brcm,board-type = "apple,tuzla"; +}; + +&bluetooth0 { + brcm,board-type = "apple,tuzla"; +}; + &fpwm1 { status = "okay"; }; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + apple,afe-reset-gpios = <&smc_gpio 8 GPIO_ACTIVE_LOW>; + apple,stm-reset-gpios = <&smc_gpio 24 GPIO_ACTIVE_LOW>; + + multi-touch { + firmware-name = "apple/tpmtfw-j615.bin"; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi index dd85f0c9fb1e11..3eac7384882040 100644 --- a/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi +++ b/arch/arm64/boot/dts/apple/t8122-jxxx.dtsi @@ -11,7 +11,9 @@ / { aliases { + bluetooth0 = &bluetooth0; serial0 = &serial0; + wifi0 = &wifi0; }; chosen { @@ -46,3 +48,28 @@ &serial0 { status = "okay"; }; + +/* + * Force the bus number assignments so that we can declare some of the + * on-board devices and properties that are populated by the bootloader + * (such as MAC addresses). + */ +&port00 { + bus-range = <1 1>; + pwren-gpios = <&smc_gpio 13 GPIO_ACTIVE_HIGH>; + + wifi0: wifi@0,0 { + compatible = "pci14e4,4434"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-mac-address = [00 00 00 00 00 00]; + apple,antenna-sku = "XX"; + }; + + bluetooth0: bluetooth@0,1 { + compatible = "pci14e4,5f72"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + /* To be filled by the loader */ + local-bd-address = [00 00 00 00 00 00]; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8122.dtsi b/arch/arm64/boot/dts/apple/t8122.dtsi index 6627a45bac6db7..1287e968af7759 100644 --- a/arch/arm64/boot/dts/apple/t8122.dtsi +++ b/arch/arm64/boot/dts/apple/t8122.dtsi @@ -357,6 +357,13 @@ ; }; + pcie_pins: pcie-pins { + // clkreq pins + pinmux = , + , + , + ; + }; }; pinctrl_nub: pinctrl@2e41f0000 { @@ -397,6 +404,63 @@ interrupts = ; }; + nub_spmi: spmi@2e4714000 { + compatible = "apple,t8122-spmi", "apple,t8103-spmi"; + reg = <0x2 0xe4714000 0x0 0x100>; + #address-cells = <2>; + #size-cells = <0>; + + pmic1: pmic@e { + compatible = "apple,stowe-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + fault_shadow: fault-shadow@867b { + reg = <0x867b 0x10>; + }; + + socd: socd@8b00 { + reg = <0x8b00 0x400>; + }; + + boot_stage: boot-stage@f701 { + reg = <0xf701 0x1>; + }; + + boot_error_count: boot-error-count@f702,0 { + reg = <0xf702 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f702,4 { + reg = <0xf702 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@f703 { + reg = <0xf703 0x1>; + }; + + shutdown_flag: shutdown-flag@f70f,3 { + reg = <0xf70f 0x1>; + bits = <3 1>; + }; + + pm_setting: pm-setting@f801 { + reg = <0xf801 0x1>; + }; + + rtc_offset: rtc-offset@f900 { + reg = <0xf900 0x6>; + }; + }; + }; + }; + smc_mbox: mbox@2ec408000 { compatible = "apple,t8122-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x2 0xec408000 0x0 0x4000>; @@ -422,6 +486,20 @@ gpio-controller; #gpio-cells = <2>; }; + + smc_reboot: reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + + rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; }; pinctrl_smc: pinctrl@2ec820000 { @@ -466,6 +544,83 @@ ; }; + mtp: mtp@2fa400000 { + compatible = "apple,t8122-mtp", "apple,t8122-rtk-helper-asc4", "apple,mtp", "apple,rtk-helper-asc4"; + reg = <0x2 0xfa400000 0x0 0x4000>, + <0x2 0xfac00000 0x0 0x100000>; + reg-names = "asc", "sram"; + + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 1>; + #helper-cells = <0>; + + status = "disabled"; + }; + + mtp_mbox: mbox@2fa408000 { + compatible = "apple,t8122-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0xfa408000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + mtp_dart: iommu@2fa808000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x2 0xfa808000 0x0 0x4000>; + + interrupt-parent = <&aic>; + interrupts = ; + + #iommu-cells = <1>; + + status = "disabled"; + }; + + mtp_dockchannel: fifo@2fab30000 { + compatible = "apple,t8122-dockchannel", "apple,dockchannel"; + reg = <0x2 0xfab14000 0x0 0x4000>; + reg-names = "irq"; + interrupt-parent = <&aic>; + interrupts = ; + + ranges = <0 0x2 0xfab28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + + interrupt-controller; + #interrupt-cells = <2>; + + status = "disabled"; + + mtp_hid: input@0 { + compatible = "apple,dockchannel-hid"; + reg = <0x0000 0x1000>, + <0x4000 0x1000>, + <0x8000 0x1000>, + <0xc000 0x1000>; + reg-names = "rmt-config", "rmt-data", "config", "data"; + + iommus = <&mtp_dart 1>; + + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, + <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; + ans_mbox: mbox@309408000 { compatible = "apple,t8122-asc-mailbox", "apple,asc-mailbox-v4"; reg = <0x3 0x09408000 0x0 0x4000>; @@ -504,6 +659,167 @@ power-domain-names = "ans", "apcie0"; resets = <&ps_ans>; }; + + pcie0_dart: iommu@594000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x5 0x94000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + }; + + pcie1_dart: iommu@595000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x5 0x95000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie2_dart: iommu@596000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x6 0x96000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie3_dart: iommu@597000000 { + compatible = "apple,t8122-dart", "apple,t8110-dart"; + reg = <0x6 0x97000000 0x0 0x4000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_apcie_gp>; + status = "disabled"; + }; + + pcie0: pcie@580000000 { + compatible = "apple,t8122-pcie", "apple,t6020-pcie"; + device_type = "pci"; + + reg = <0x5 0x80000000 0x0 0x1000000>, /* config */ + <0x5 0x91000000 0x0 0x4000>, /* rc */ + <0x5 0x94008000 0x0 0x4000>, /* port0 */ + <0x5 0x95008000 0x0 0x4000>, /* port1 */ + <0x5 0x96008000 0x0 0x4000>, /* port2 */ + <0x5 0x97008000 0x0 0x4000>, /* port3 */ + <0x5 0x9e00c000 0x0 0x4000>, /* phy0 */ + <0x5 0x9e010000 0x0 0x4000>, /* phy1 */ + <0x5 0x9e014000 0x0 0x4000>, /* phy2 */ + <0x5 0x9e018000 0x0 0x4000>; /* phy3 */ + reg-names = "config", "rc", "port0", "port1", "port2", "port3", "phy0", "phy1", "phy2", "phy3"; + + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + + msi-controller; + msi-parent = <&pcie0>; + msi-ranges = <&aic AIC_IRQ 1075 IRQ_TYPE_EDGE_RISING 32>; + + iommu-map = <0x100 &pcie0_dart 0 1>, + <0x200 &pcie1_dart 1 1>, + <0x300 &pcie2_dart 2 1>, + <0x300 &pcie3_dart 3 1>; + iommu-map-mask = <0xff00>; + + bus-range = <0 4>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x43000000 0x5 0xa0000000 0x5 0xa0000000 0x0 0x20000000>, + <0x02000000 0x0 0xc0000000 0x5 0xc0000000 0x0 0x40000000>; + + power-domains = <&ps_apcie_gp>; + pinctrl-0 = <&pcie_pins>; + pinctrl-names = "default"; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 187 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + }; + + port01: pci@1,0 { + device_type = "pci"; + reg = <0x800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 188 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port01 0 0 0 0>, + <0 0 0 2 &port01 0 0 0 1>, + <0 0 0 3 &port01 0 0 0 2>, + <0 0 0 4 &port01 0 0 0 3>; + status = "disabled"; + }; + + port02: pci@2,0 { + device_type = "pci"; + reg = <0x1000 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 189 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port02 0 0 0 0>, + <0 0 0 2 &port02 0 0 0 1>, + <0 0 0 3 &port02 0 0 0 2>, + <0 0 0 4 &port02 0 0 0 3>; + status = "disabled"; + }; + + port03: pci@3,0 { + device_type = "pci"; + reg = <0x1800 0x0 0x0 0x0 0x0>; + reset-gpios = <&pinctrl_ap 190 GPIO_ACTIVE_LOW>; // perst + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port03 0 0 0 0>, + <0 0 0 2 &port03 0 0 0 1>, + <0 0 0 3 &port03 0 0 0 2>, + <0 0 0 4 &port03 0 0 0 3>; + status = "disabled"; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts b/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts index 7de24d60bcd1a7..127be0fc27c22e 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts +++ b/arch/arm64/boot/dts/broadcom/bcm2712-d-rpi-5-b.dts @@ -35,3 +35,17 @@ "PMIC_SCL", // AON_SGPIO_04 "PMIC_SDA"; // AON_SGPIO_05 }; + +&pinctrl { + compatible = "brcm,bcm2712d0-pinctrl"; + reg = <0x7d504100 0x20>; +}; + +&pinctrl_aon { + compatible = "brcm,bcm2712d0-aon-pinctrl"; + reg = <0x7d510700 0x1c>; +}; + +&uart10 { + interrupts = ; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-cex7.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a-cex7.dtsi index eec2cd6c6d32a7..7f6e39e27ce5ce 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a-cex7.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-cex7.dtsi @@ -162,6 +162,8 @@ }; &fspi { + pinctrl-names = "default"; + pinctrl-0 = <&fspi_data74_pins>, <&fspi_data30_pins>, <&fspi_dqs_sck_cs10_pins>; status = "okay"; flash@0 { @@ -177,6 +179,11 @@ }; }; +&pinmux_i2crv { + pinctrl-names = "default"; + pinctrl-0 = <&gpio0_14_12_pins>; +}; + &usb0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi index af6258b2fe8265..580ee9b3026e30 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a-clearfog-itx.dtsi @@ -89,6 +89,8 @@ }; &esdhc0 { + pinctrl-names = "default"; + pinctrl-0 = <&esdhc0_cd_wp_pins>, <&esdhc0_cmd_data30_clk_vsel_pins>; sd-uhs-sdr104; sd-uhs-sdr50; sd-uhs-sdr25; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi index 853b01452813a7..479982948ee539 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi @@ -750,9 +750,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c0_scl>; - pinctrl-1 = <&i2c0_scl_gpio>; + pinctrl-0 = <&i2c0_pins>; + pinctrl-1 = <&gpio0_3_2_pins>; scl-gpios = <&gpio0 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -766,9 +767,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c1_scl>; - pinctrl-1 = <&i2c1_scl_gpio>; + pinctrl-0 = <&i2c1_pins>; + pinctrl-1 = <&gpio0_31_30_pins>; scl-gpios = <&gpio0 31 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 30 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -782,9 +784,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c2_scl>; - pinctrl-1 = <&i2c2_scl_gpio>; + pinctrl-0 = <&i2c2_pins>; + pinctrl-1 = <&gpio0_29_28_pins>; scl-gpios = <&gpio0 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 28 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -798,9 +801,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c3_scl>; - pinctrl-1 = <&i2c3_scl_gpio>; + pinctrl-0 = <&i2c3_pins>; + pinctrl-1 = <&gpio0_27_26_pins>; scl-gpios = <&gpio0 27 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 26 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -814,9 +818,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c4_scl>; - pinctrl-1 = <&i2c4_scl_gpio>; + pinctrl-0 = <&i2c4_pins>; + pinctrl-1 = <&gpio0_25_24_pins>; scl-gpios = <&gpio0 25 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 24 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -830,9 +835,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c5_scl>; - pinctrl-1 = <&i2c5_scl_gpio>; + pinctrl-0 = <&i2c5_pins>; + pinctrl-1 = <&gpio0_23_22_pins>; scl-gpios = <&gpio0 23 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio0 22 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -846,9 +852,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c6_scl>; - pinctrl-1 = <&i2c6_scl_gpio>; + pinctrl-0 = <&i2c6_i2c7_pins>; + pinctrl-1 = <&gpio1_18_15_pins>; scl-gpios = <&gpio1 16 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio1 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -862,9 +869,10 @@ clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(16)>; pinctrl-names = "default", "gpio"; - pinctrl-0 = <&i2c7_scl>; - pinctrl-1 = <&i2c7_scl_gpio>; + pinctrl-0 = <&i2c6_i2c7_pins>; + pinctrl-1 = <&gpio1_18_15_pins>; scl-gpios = <&gpio1 18 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; + sda-gpios = <&gpio1 17 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>; status = "disabled"; }; @@ -1713,68 +1721,159 @@ pinctrl-single,register-width = <32>; pinctrl-single,function-mask = <0x7>; - i2c1_scl: i2c1-scl-pins { - pinctrl-single,bits = <0x0 0 0x7>; + /* RCWSR12 */ + i2c1_pins: iic2-i2c-pins { + pinctrl-single,bits = <0x0 0x0 0x7>; }; - i2c1_scl_gpio: i2c1-scl-gpio-pins { + gpio0_31_30_pins: iic2-gpio-pins { pinctrl-single,bits = <0x0 0x1 0x7>; }; - i2c2_scl: i2c2-scl-pins { - pinctrl-single,bits = <0x0 0 (0x7 << 3)>; + ftm0_ch10_pins: iic2-ftm-pins { + pinctrl-single,bits = <0x0 0x2 0x7>; }; - i2c2_scl_gpio: i2c2-scl-gpio-pins { + esdhc0_cd_wp_pins: iic2-sdhc-pins { + pinctrl-single,bits = <0x0 0x6 0x7>; + }; + + i2c2_pins: iic3-i2c-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 3)>; + }; + + gpio0_29_28_pins: iic3-gpio-pins { pinctrl-single,bits = <0x0 (0x1 << 3) (0x7 << 3)>; }; - i2c3_scl: i2c3-scl-pins { - pinctrl-single,bits = <0x0 0 (0x7 << 6)>; + can0_pins: iic3-can-pins { + pinctrl-single,bits = <0x0 (0x2 << 3) (0x7 << 3)>; + }; + + event65_pins: iic3-event-pins { + pinctrl-single,bits = <0x0 (0x6 << 3) (0x7 << 3)>; }; - i2c3_scl_gpio: i2c3-scl-gpio-pins { + i2c3_pins: iic4-i2c-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 6)>; + }; + + gpio0_27_26_pins: iic4-gpio-pins { pinctrl-single,bits = <0x0 (0x1 << 6) (0x7 << 6)>; }; - i2c4_scl: i2c4-scl-pins { - pinctrl-single,bits = <0x0 0 (0x7 << 9)>; + can1_pins: iic4-can-pins { + pinctrl-single,bits = <0x0 (0x2 << 6) (0x7 << 6)>; + }; + + event87_pins: iic4-event-pins { + pinctrl-single,bits = <0x0 (0x6 << 6) (0x7 << 6)>; + }; + + i2c4_pins: iic5-i2c-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 9)>; }; - i2c4_scl_gpio: i2c4-scl-gpio-pins { + gpio0_25_24_pins: iic5-gpio-pins { pinctrl-single,bits = <0x0 (0x1 << 9) (0x7 << 9)>; }; - i2c5_scl: i2c5-scl-pins { - pinctrl-single,bits = <0x0 0 (0x7 << 12)>; + esdhc0_clksync_pins: iic5-sdhc-clk-pins { + pinctrl-single,bits = <0x0 (0x2 << 9) (0x7 << 9)>; }; - i2c5_scl_gpio: i2c5-scl-gpio-pins { + dspi2_miso_mosi_pins: iic5-spi3-pins { + pinctrl-single,bits = <0x3 (0x2 << 9) (0x7 << 9)>; + }; + + i2c5_pins: iic6-i2c-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 12)>; + }; + + gpio0_23_22_pins: iic6-gpio-pins { pinctrl-single,bits = <0x0 (0x1 << 12) (0x7 << 12)>; }; - i2c6_scl: i2c6-scl-pins { - pinctrl-single,bits = <0x4 0x2 0x7>; + esdhc1_clksync_pins: iic6-sdhc-clk-pins { + pinctrl-single,bits = <0x0 (0x2 << 12) (0x7 << 12)>; }; - i2c6_scl_gpio: i2c6-scl-gpio-pins { - pinctrl-single,bits = <0x4 0x1 0x7>; + fspi_data74_pins: xspi1-data74-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 15)>; }; - i2c7_scl: i2c7-scl-pins { - pinctrl-single,bits = <0x4 0x2 0x7>; + gpio1_31_28_pins: xspi1-data74-gpio-pins { + pinctrl-single,bits = <0x0 0x1 (0x7 << 15)>; + }; + + fspi_data30_pins: xspi1-data30-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 18)>; + }; + + gpio1_27_24_pins: xspi1-data30-gpio-pins { + pinctrl-single,bits = <0x0 0x1 (0x7 << 18)>; + }; + + fspi_dqs_sck_cs10_pins: xspi1-base-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 21)>; + }; + + gpio1_23_20_pins: xspi1-base-gpio-pins { + pinctrl-single,bits = <0x0 0x1 (0x7 << 21)>; + }; + + esdhc0_cmd_data30_clk_vsel_pins: sdhc1-base-sdhc-vsel-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 24)>; + }; + + gpio0_21_15_pins: sdhc1-base-gpio-pins { + pinctrl-single,bits = <0x0 (0x1 << 24) (0x7 << 24)>; + }; + + dspi0_pins: sdhc1-base-spi1-pins { + pinctrl-single,bits = <0x0 (0x2 << 24) (0x7 << 24)>; + }; + + esdhc0_cmd_data30_clk_dspi2_cs0_pins: sdhc1-base-sdhc-spi3-pins { + pinctrl-single,bits = <0x0 (0x3 << 24) (0x7 << 24)>; + }; + + esdhc0_cmd_data30_clk_data4_pins: sdhc1-base-sdhc-data4-pins { + pinctrl-single,bits = <0x0 (0x4 << 24) (0x7 << 24)>; + }; + + esdhc0_dir_pins: sdhc1-dir-pins { + pinctrl-single,bits = <0x0 0x0 (0x7 << 27)>; + }; + + gpio0_14_12_pins: sdhc1-dir-gpio-pins { + pinctrl-single,bits = <0x0 (0x1 << 27) (0x7 << 27)>; + }; + + dspi2_cs31_pins: sdhc1-dir-spi3-pins { + pinctrl-single,bits = <0x0 (0x3 << 27) (0x7 << 27)>; + }; + + esdhc0_data75_pins: sdhc1-dir-sdhc-pins { + pinctrl-single,bits = <0x0 (0x4 << 27) (0x7 << 27)>; }; - i2c7_scl_gpio: i2c7-scl-gpio-pins { + /* RCWSR13 */ + gpio1_18_15_pins: iic8-iic7-gpio-pins { pinctrl-single,bits = <0x4 0x1 0x7>; }; - i2c0_scl: i2c0-scl-pins { - pinctrl-single,bits = <0x8 0 (0x7 << 10)>; + i2c6_i2c7_pins: iic8-iic7-i2c-pins { + pinctrl-single,bits = <0x4 0x2 0x7>; + }; + + /* RCWSR14 */ + i2c0_pins: iic1-i2c-pins { + pinctrl-single,bits = <0x8 0x0 (0x1 << 10)>; }; - i2c0_scl_gpio: i2c0-scl-gpio-pins { - pinctrl-single,bits = <0x8 (0x1 << 10) (0x7 << 10)>; + gpio0_3_2_pins: iic1-gpio-pins { + pinctrl-single,bits = <0x8 (0x1 << 10) (0x1 << 10)>; }; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts b/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts index eafef8718a0fe6..8920326a067351 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts +++ b/arch/arm64/boot/dts/freescale/fsl-lx2162a-clearfog.dts @@ -223,6 +223,8 @@ }; &esdhc0 { + pinctrl-names = "default"; + pinctrl-0 = <&esdhc0_cd_wp_pins>, <&esdhc0_cmd_data30_clk_vsel_pins>; sd-uhs-sdr104; sd-uhs-sdr50; sd-uhs-sdr25; diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2162a-sr-som.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2162a-sr-som.dtsi index e914291e63a1ae..e1344942eaaeea 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2162a-sr-som.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2162a-sr-som.dtsi @@ -30,6 +30,8 @@ }; &fspi { + pinctrl-names = "default"; + pinctrl-0 = <&fspi_data74_pins>, <&fspi_data30_pins>, <&fspi_dqs_sck_cs10_pins>; status = "okay"; flash@0 { @@ -80,3 +82,8 @@ reg = <0x6f>; }; }; + +&pinmux_i2crv { + pinctrl-names = "default"; + pinctrl-0 = <&gpio0_14_12_pins>; +}; diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi index 7022de46b8bff4..abb131d247c30e 100644 --- a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.1.dtsi @@ -21,6 +21,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <1>; gpios = <&lsio_gpio5 27 GPIO_ACTIVE_HIGH>; }; @@ -29,6 +30,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <1>; gpios = <&lsio_gpio5 29 GPIO_ACTIVE_HIGH>; }; @@ -37,6 +39,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <2>; gpios = <&lsio_gpio5 20 GPIO_ACTIVE_HIGH>; }; @@ -45,6 +48,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <2>; gpios = <&lsio_gpio5 21 GPIO_ACTIVE_HIGH>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi index 12732ed7f81197..a0b452b92b3e6e 100644 --- a/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-apalis-ixora-v1.2.dtsi @@ -21,6 +21,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <1>; gpios = <&lsio_gpio5 27 GPIO_ACTIVE_HIGH>; }; @@ -29,6 +30,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <1>; gpios = <&lsio_gpio5 29 GPIO_ACTIVE_HIGH>; }; @@ -37,6 +39,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <2>; gpios = <&lsio_gpio5 20 GPIO_ACTIVE_HIGH>; }; @@ -45,6 +48,7 @@ color = ; default-state = "off"; function = LED_FUNCTION_STATUS; + function-enumerator = <2>; gpios = <&lsio_gpio5 21 GPIO_ACTIVE_HIGH>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts index 5c68d33e19f22e..bc62ae5ca812dd 100644 --- a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts @@ -259,33 +259,37 @@ }; sound-wm8960-2 { - compatible = "fsl,imx-audio-wm8960"; - model = "wm8960-audio-2"; - audio-cpu = <&sai2>; - audio-codec = <&wm8960_2>; - audio-routing = "Headphone Jack", "HP_L", - "Headphone Jack", "HP_R", - "Ext Spk", "SPK_LP", - "Ext Spk", "SPK_LN", - "Ext Spk", "SPK_RP", - "Ext Spk", "SPK_RN", - "LINPUT1", "Mic Jack", - "Mic Jack", "MICB"; + compatible = "audio-graph-card2"; + label = "wm8960-audio-2"; + links = <&sai2_port2>; + routing = "Headphones", "HP_L", + "Headphones", "HP_R", + "Ext Spk", "SPK_LP", + "Ext Spk", "SPK_LN", + "Ext Spk", "SPK_RP", + "Ext Spk", "SPK_RN", + "LINPUT1", "Mic Jack", + "Mic Jack", "MICB"; + widgets = "Headphone", "Headphones", + "Speaker", "Ext Spk", + "Microphone", "Mic Jack"; }; sound-wm8960-3 { - compatible = "fsl,imx-audio-wm8960"; - model = "wm8960-audio-3"; - audio-cpu = <&sai3>; - audio-codec = <&wm8960_3>; - audio-routing = "Headphone Jack", "HP_L", - "Headphone Jack", "HP_R", - "Ext Spk", "SPK_LP", - "Ext Spk", "SPK_LN", - "Ext Spk", "SPK_RP", - "Ext Spk", "SPK_RN", - "LINPUT1", "Mic Jack", - "Mic Jack", "MICB"; + compatible = "audio-graph-card2"; + label = "wm8960-audio-3"; + links = <&sai3_port2>; + routing = "Headphones", "HP_L", + "Headphones", "HP_R", + "Ext Spk", "SPK_LP", + "Ext Spk", "SPK_LN", + "Ext Spk", "SPK_RP", + "Ext Spk", "SPK_RN", + "LINPUT1", "Mic Jack", + "Mic Jack", "MICB"; + widgets = "Headphone", "Headphones", + "Speaker", "Ext Spk", + "Microphone", "Mic Jack"; }; }; @@ -481,6 +485,16 @@ DCVDD-supply = <®_audio_1v8>; SPKVDD1-supply = <®_audio_5v>; SPKVDD2-supply = <®_audio_5v>; + + port { + capture-only; + + wm8960_2_ep: endpoint { + bitclock-master; + frame-master; + remote-endpoint = <&sai2_endpoint2>; + }; + }; }; }; @@ -510,6 +524,16 @@ DCVDD-supply = <®_audio_1v8>; SPKVDD1-supply = <®_audio_5v>; SPKVDD2-supply = <®_audio_5v>; + + port { + capture-only; + + wm8960_3_ep: endpoint { + bitclock-master; + frame-master; + remote-endpoint = <&sai3_endpoint2>; + }; + }; }; }; @@ -700,6 +724,27 @@ pinctrl-0 = <&pinctrl_sai2>; fsl,sai-asynchronous; status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + sai2_port1: port@1 { + reg = <1>; + endpoint { /* not used */ }; + }; + + sai2_port2: port@2 { + reg = <2>; + capture-only; + + sai2_endpoint2: endpoint { + dai-format = "i2s"; + remote-endpoint = <&wm8960_2_ep>; + system-clock-direction-out; + }; + }; + }; }; &sai3 { @@ -712,6 +757,27 @@ pinctrl-0 = <&pinctrl_sai3>; fsl,sai-asynchronous; status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + sai3_port1: port@1 { + reg = <1>; + endpoint { /* not used */ }; + }; + + sai3_port2: port@2 { + reg = <2>; + capture-only; + + sai3_endpoint2: endpoint { + dai-format = "i2s"; + remote-endpoint = <&wm8960_3_ep>; + system-clock-direction-out; + }; + }; + }; }; &thermal_zones { diff --git a/arch/arm64/boot/dts/freescale/imx8mm-emtop-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-emtop-som.dtsi index 67d22d3768aa8a..507d1824d99d99 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-emtop-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-emtop-som.dtsi @@ -60,7 +60,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_pmic>; interrupt-parent = <&gpio1>; - interrupts = <3 IRQ_TYPE_EDGE_RISING>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; regulators { buck1: BUCK1 { @@ -194,7 +194,7 @@ pinctrl_pmic: emtop-pmic-grp { fsl,pins = < - MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3 0x41 + MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3 0x141 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi index 29b298af0d7391..1b5ba3c47164f2 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml.dtsi @@ -292,7 +292,7 @@ }; pinctrl_pmic: pmicgrp { - fsl,pins = ; + fsl,pins = ; }; pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp { diff --git a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi index 31a3ca137e6364..48a687926aa1b8 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn-tqma8mqnl.dtsi @@ -283,7 +283,7 @@ }; pinctrl_pmic: pmicgrp { - fsl,pins = ; + fsl,pins = ; }; pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-aristainetos3a-som-v1.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-aristainetos3a-som-v1.dtsi index f654d866e58c06..e7666e54310be0 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-aristainetos3a-som-v1.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-aristainetos3a-som-v1.dtsi @@ -903,7 +903,7 @@ pinctrl_pmic: aristainetos3-pmic-grp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts b/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts index 8290f187b79fd2..7bc213499f094a 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-cubox-m.dts @@ -68,7 +68,7 @@ regulator-name = "vmmc"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; - gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>; + gpio = <&gpio2 19 GPIO_ACTIVE_LOW>; startup-delay-us = <250>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts index 7e46537a22a01d..cb28cf1cdd23fa 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-data-modul-edm-sbc.dts @@ -1001,7 +1001,7 @@ pinctrl_pmic: pmic-grp { fsl,pins = < /* PMIC_nINT */ - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x40000090 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-debix-model-a.dts b/arch/arm64/boot/dts/freescale/imx8mp-debix-model-a.dts index 9422beee30b29c..201cf7f5eb0ea0 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-debix-model-a.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-debix-model-a.dts @@ -440,7 +440,7 @@ pinctrl_pmic: pmicirqgrp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a-bmb-08.dts b/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a-bmb-08.dts index 04619a7229065b..1471ff361b54cb 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a-bmb-08.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a-bmb-08.dts @@ -499,7 +499,7 @@ pinctrl_pmic: pmicgrp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a.dtsi index 91094c2277443c..b31e8fe95ca745 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-debix-som-a.dtsi @@ -241,7 +241,7 @@ pinctrl_pmic: pmicgrp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi index f8303b7e2bd22b..0a6a60670f762f 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-som.dtsi @@ -989,7 +989,7 @@ pinctrl_pmic: dhcom-pmic-grp { fsl,pins = < /* PMIC_nINT */ - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x40000090 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-edm-g.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-edm-g.dtsi index 3f1e0837f349fa..91b87a7248dd1f 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-edm-g.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-edm-g.dtsi @@ -563,7 +563,7 @@ pinctrl_pmic: pmicirqgrp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts index b256be710ea128..f981504f019d16 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts @@ -595,7 +595,8 @@ hdmi@3d { compatible = "adi,adv7535"; - reg = <0x3d>; + reg = <0x3d>, <0x3f>, <0x3b>, <0x38>; + reg-names = "main", "edid", "cec", "packet"; interrupt-parent = <&gpio1>; interrupts = <9 IRQ_TYPE_EDGE_FALLING>; adi,dsi-lanes = <4>; @@ -1064,7 +1065,7 @@ pinctrl_pcie0_reg: pcie0reggrp { fsl,pins = < - MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06 0x40 + MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06 0x140 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi index fa7cb9759d01c7..0b4e5f300eb165 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-common.dtsi @@ -73,7 +73,7 @@ regulator-name = "vmmc"; regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; - gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>; + gpio = <&gpio2 19 GPIO_ACTIVE_LOW>; startup-delay-us = <250>; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi index 46916ddc053355..0e5f4607c7c1b3 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-hummingboard-pulse-mini-hdmi.dtsi @@ -41,7 +41,7 @@ reg = <0>; adv7535_from_dsim: endpoint { - remote-endpoint = <&dsim_to_adv7535>; + remote-endpoint = <&mipi_dsi_out>; }; }; @@ -71,11 +71,8 @@ &mipi_dsi { samsung,esc-clock-frequency = <10000000>; status = "okay"; +}; - port@1 { - dsim_to_adv7535: endpoint { - remote-endpoint = <&adv7535_from_dsim>; - attach-bridge; - }; - }; +&mipi_dsi_out { + remote-endpoint = <&adv7535_from_dsim>; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi index a6319824ea2eb1..69558ffefa9a67 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi @@ -132,7 +132,7 @@ pinctrl_pmic: pmicgrp { fsl,pins = < - MX8MP_IOMUXC_NAND_CE0_B__GPIO3_IO01 0x41 + MX8MP_IOMUXC_NAND_CE0_B__GPIO3_IO01 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso b/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso index a3cba41d2b5312..7131e9a499ae18 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso +++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-dl.dtso @@ -77,6 +77,8 @@ touchscreen@5d { compatible = "goodix,gt928"; reg = <0x5d>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_touch>; interrupt-parent = <&gpio1>; interrupts = <6 8>; irq-gpios = <&gpio1 6 0>; @@ -98,6 +100,16 @@ status = "okay"; }; +/* redefine to remove touch controller GPIOs */ +&pinctrl_gpio1 { + fsl,pins = < + MX8MP_IOMUXC_GPIO1_IO00__GPIO1_IO00 0x19 /* GPIO_A_0 */ + MX8MP_IOMUXC_GPIO1_IO01__GPIO1_IO01 0x19 /* GPIO_A_1 */ + MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05 0x19 /* GPIO_A_2 */ + MX8MP_IOMUXC_GPIO1_IO08__GPIO1_IO08 0x19 /* GPIO_A_5 */ + >; +}; + &pwm1 { status = "okay"; }; @@ -108,4 +120,11 @@ MX8MP_IOMUXC_SAI3_RXFS__GPIO4_IO28 0x19 >; }; + + pinctrl_touch: touchgrp { + fsl,pins = < + MX8MP_IOMUXC_GPIO1_IO06__GPIO1_IO06 0x19 + MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07 0x150 + >; + }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi index b97bfeb1c30f89..bc1a261bb000ed 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi @@ -330,6 +330,12 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_rtc>; interrupts-extended = <&gpio3 24 IRQ_TYPE_LEVEL_LOW>; + /* + * While specifying the vdd-supply is normally not strictly necessary, + * here it also makes sure that the PMIC driver enables the level- + * shifter for the RTC before the RTC is probed. + */ + vdd-supply = <®_vdd_3v3>; }; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts b/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts index 2173a36ff69178..74d620dd06b7bc 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-smarc-eval-carrier.dts @@ -249,6 +249,5 @@ }; &usdhc2 { - vmmc-supply = <®_vdd_3v3>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-navqp.dts b/arch/arm64/boot/dts/freescale/imx8mp-navqp.dts index 4a4f7c1adc23fe..9dedb9f11145ea 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-navqp.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-navqp.dts @@ -356,7 +356,7 @@ pinctrl_pmic: pmicgrp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-som.dtsi index f658309612efff..8465b36d440ae3 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-nitrogen-som.dtsi @@ -296,7 +296,7 @@ pinctrl_pmic: pmicirqgrp { fsl,pins = < - MX8MP_IOMUXC_NAND_ALE__GPIO3_IO00 0x41 + MX8MP_IOMUXC_NAND_ALE__GPIO3_IO00 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-sr-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-sr-som.dtsi index 3cdb0bc0ab7217..c3f7daa773eaf3 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-sr-som.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-sr-som.dtsi @@ -174,7 +174,7 @@ pinctrl-0 = <&pmic_pins>; pinctrl-names = "default"; interrupt-parent = <&gpio1>; - interrupts = <3 GPIO_ACTIVE_LOW>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; nxp,i2c-lt-enable; regulators { @@ -417,7 +417,7 @@ pmic_pins: pinctrl-pmic-grp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x41 + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts index b7f69c92b7748f..1665a5030b993c 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mp-ras314.dts @@ -848,8 +848,8 @@ pinctrl_uart1: uart1grp { fsl,pins = , , - , - ; + , + ; }; pinctrl_uart1_gpio: uart1gpiogrp { diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ultra-mach-sbc.dts b/arch/arm64/boot/dts/freescale/imx8mp-ultra-mach-sbc.dts index 9ecec1a418781a..3e6f9c88cc200b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-ultra-mach-sbc.dts +++ b/arch/arm64/boot/dts/freescale/imx8mp-ultra-mach-sbc.dts @@ -275,7 +275,7 @@ reg = <0x25>; pinctrl-0 = <&pinctrl_pmic>; interrupt-parent = <&gpio1>; - interrupts = <3 GPIO_ACTIVE_LOW>; + interrupts = <3 IRQ_TYPE_LEVEL_LOW>; /* * i.MX 8M Plus Data Sheet for Consumer Products @@ -739,7 +739,7 @@ pinctrl_pmic: pmic-grp { fsl,pins = < - MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x40 /* #PMIC_INT */ + MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03 0x1c0 /* #PMIC_INT */ >; }; diff --git a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts index dadc136aec6e62..011a89d85961d9 100644 --- a/arch/arm64/boot/dts/freescale/imx8qm-mek.dts +++ b/arch/arm64/boot/dts/freescale/imx8qm-mek.dts @@ -611,9 +611,17 @@ usb_con1: connector { compatible = "usb-c-connector"; label = "USB-C"; - power-role = "source"; + power-role = "dual"; data-role = "dual"; + try-power-role = "sink"; source-pdos = ; + /* + * Set operational current to 0mA as we don't want EN_SNK + * enable 12V VBUS switch when it work as a sink. + */ + sink-pdos = ; + op-sink-microwatt = <0>; + self-powered; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts index 40a0bc9f4e8485..623169f7ddb5fa 100644 --- a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts +++ b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts @@ -566,9 +566,17 @@ usb_con1: connector { compatible = "usb-c-connector"; label = "USB-C"; - power-role = "source"; + power-role = "dual"; data-role = "dual"; + try-power-role = "sink"; source-pdos = ; + /* + * Set operational current to 0mA as we don't want EN_SNK + * enable 12V VBUS switch when it work as a sink. + */ + sink-pdos = ; + op-sink-microwatt = <0>; + self-powered; ports { #address-cells = <1>; diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts index 03f460d62f7a58..6a066a0d86bc29 100644 --- a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts +++ b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts @@ -514,6 +514,7 @@ pinctrl-1 = <&pinctrl_usdhc1_100mhz>; pinctrl-2 = <&pinctrl_usdhc1_200mhz>; pinctrl-names = "default", "state_100mhz", "state_200mhz"; + fsl,tuning-step = <1>; status = "okay"; }; @@ -528,6 +529,7 @@ pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>; pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep"; vmmc-supply = <®_usdhc2_vmmc>; + fsl,tuning-step = <1>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/freescale/imx91.dtsi b/arch/arm64/boot/dts/freescale/imx91.dtsi index f075592bfc01f1..d63569b39bbc5d 100644 --- a/arch/arm64/boot/dts/freescale/imx91.dtsi +++ b/arch/arm64/boot/dts/freescale/imx91.dtsi @@ -11,7 +11,7 @@ cpu-thermal { polling-delay-passive = <250>; polling-delay = <2000>; - thermal-sensors = <&tmu 0>; + thermal-sensors = <&tmu>; trips { cpu_alert: cpu-alert { diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi index 5932ba238a8ac5..f64c05dc50f802 100644 --- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi +++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi @@ -262,7 +262,6 @@ "", "", "", - "", "PMIC_SD2_VSEL"; status = "okay"; }; diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi index 24282084570787..12deacb741ccbe 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi @@ -15,6 +15,11 @@ #include "armada-372x.dtsi" / { + aliases { + ethernet0 = ð0; + ethernet1 = ð1; + }; + chosen { stdout-path = "serial0:115200n8"; }; @@ -156,7 +161,7 @@ &usb3 { status = "okay"; phys = <&usb2_utmi_otg_phy>; - phy-names = "usb2-utmi-otg-phy"; + phy-names = "usb2-phy"; }; &uart0 { diff --git a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi index 87f9367aec1227..cbc411bfa38100 100644 --- a/arch/arm64/boot/dts/marvell/armada-37xx.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-37xx.dtsi @@ -373,7 +373,7 @@ interrupts = ; clocks = <&sb_periph_clk 12>; phys = <&comphy0 0>, <&usb2_utmi_otg_phy>; - phy-names = "usb3-phy", "usb2-utmi-otg-phy"; + phy-names = "usb3-phy", "usb2-phy"; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt6795.dtsi b/arch/arm64/boot/dts/mediatek/mt6795.dtsi index ae2aaa51c9ad29..134cfa77e3b1fa 100644 --- a/arch/arm64/boot/dts/mediatek/mt6795.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt6795.dtsi @@ -371,7 +371,7 @@ ; gpio-controller; #gpio-cells = <2>; - gpio-ranges = <&pio 0 0 196>; + gpio-ranges = <&pio 0 0 197>; interrupt-controller; #interrupt-cells = <2>; }; diff --git a/arch/arm64/boot/dts/mediatek/mt7981b.dtsi b/arch/arm64/boot/dts/mediatek/mt7981b.dtsi index 4084f4dfa3e5e3..1bbe219380f990 100644 --- a/arch/arm64/boot/dts/mediatek/mt7981b.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt7981b.dtsi @@ -332,7 +332,7 @@ interrupt-controller; interrupts = ; interrupt-parent = <&gic>; - gpio-ranges = <&pio 0 0 56>; + gpio-ranges = <&pio 0 0 57>; gpio-controller; #gpio-cells = <2>; #interrupt-cells = <2>; diff --git a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi index 9693f62fd01362..9ebc196107e5a0 100644 --- a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi @@ -187,7 +187,7 @@ "iocfg_lb", "iocfg_tr", "iocfg_tl", "eint"; gpio-controller; #gpio-cells = <2>; - gpio-ranges = <&pio 0 0 100>; + gpio-ranges = <&pio 0 0 101>; interrupt-controller; interrupts = ; interrupt-parent = <&gic>; diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts index c7ea6e88c4f485..621d01e3cd8961 100644 --- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts +++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-4e.dts @@ -9,7 +9,7 @@ #include "mt7988a-bananapi-bpi-r4-pro.dtsi" / { - model = "Bananapi BPI-R4"; + model = "Bananapi BPI-R4 Pro 4E"; compatible = "bananapi,bpi-r4-pro-4e", "bananapi,bpi-r4-pro", "mediatek,mt7988a"; diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts index c9a0e69e9dd51b..bb15bfa5e6ae53 100644 --- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts +++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-8x.dts @@ -9,7 +9,7 @@ #include "mt7988a-bananapi-bpi-r4-pro.dtsi" / { - model = "Bananapi BPI-R4"; + model = "Bananapi BPI-R4 Pro 8X"; compatible = "bananapi,bpi-r4-pro-8x", "bananapi,bpi-r4-pro", "mediatek,mt7988a"; diff --git a/arch/arm64/boot/dts/mediatek/mt8365.dtsi b/arch/arm64/boot/dts/mediatek/mt8365.dtsi index a5ca3cda6ef30d..2e782558fb7761 100644 --- a/arch/arm64/boot/dts/mediatek/mt8365.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8365.dtsi @@ -536,10 +536,9 @@ #iommu-cells = <1>; }; - infracfg_nao: infracfg@1020e000 { - compatible = "mediatek,mt8365-infracfg", "syscon"; + infracfg_nao: syscon@1020e000 { + compatible = "mediatek,mt8365-infracfg-nao", "syscon"; reg = <0 0x1020e000 0 0x1000>; - #clock-cells = <1>; }; rng: rng@1020f000 { diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3701.dtsi b/arch/arm64/boot/dts/nvidia/tegra234-p3701.dtsi index 58bf55c0e414ca..c10d041c183bef 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234-p3701.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234-p3701.dtsi @@ -9,6 +9,7 @@ mmc0 = "/bus@0/mmc@3460000"; mmc1 = "/bus@0/mmc@3400000"; rtc0 = "/bpmp/i2c/pmic@3c"; + rtc1 = "/bus@0/rtc@c2a0000"; }; bus@0 { diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3767.dtsi b/arch/arm64/boot/dts/nvidia/tegra234-p3767.dtsi index ab391a71c3d338..9e9e80d57623c4 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234-p3767.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234-p3767.dtsi @@ -8,6 +8,7 @@ aliases { mmc0 = "/bus@0/mmc@3400000"; rtc0 = "/bpmp/i2c/pmic@3c"; + rtc1 = "/bus@0/rtc@c2a0000"; }; bus@0 { diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile index f80b5d9cf1e80e..b05e8adc02f655 100644 --- a/arch/arm64/boot/dts/qcom/Makefile +++ b/arch/arm64/boot/dts/qcom/Makefile @@ -374,6 +374,8 @@ x1e80100-lenovo-yoga-slim7x-el2-dtbs := x1e80100-lenovo-yoga-slim7x.dtb x1-el2.d dtb-$(CONFIG_ARCH_QCOM) += x1e80100-lenovo-yoga-slim7x.dtb x1e80100-lenovo-yoga-slim7x-el2.dtb x1e80100-medion-sprchrgd-14-s1-el2-dtbs := x1e80100-medion-sprchrgd-14-s1.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1e80100-medion-sprchrgd-14-s1.dtb x1e80100-medion-sprchrgd-14-s1-el2.dtb +x1e80100-microsoft-denali-oled-el2-dtbs := x1e80100-microsoft-denali-oled.dtb x1-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += x1e80100-microsoft-denali-oled.dtb x1e80100-microsoft-denali-oled-el2.dtb x1e80100-microsoft-romulus13-el2-dtbs := x1e80100-microsoft-romulus13.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1e80100-microsoft-romulus13.dtb x1e80100-microsoft-romulus13-el2.dtb x1e80100-microsoft-romulus15-el2-dtbs := x1e80100-microsoft-romulus15.dtb x1-el2.dtbo diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi index 4b0784af4bd395..cb95549275ca89 100644 --- a/arch/arm64/boot/dts/qcom/hamoa.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -4714,7 +4714,7 @@ clocks = <&gcc GCC_SDCC2_AHB_CLK>, <&gcc GCC_SDCC2_APPS_CLK>, - <&rpmhcc RPMH_CXO_CLK>; + <&bi_tcxo_div2>; clock-names = "iface", "core", "xo"; iommus = <&apps_smmu 0x520 0>; qcom,dll-config = <0x0007642c>; @@ -4767,7 +4767,7 @@ clocks = <&gcc GCC_SDCC4_AHB_CLK>, <&gcc GCC_SDCC4_APPS_CLK>, - <&rpmhcc RPMH_CXO_CLK>; + <&bi_tcxo_div2>; clock-names = "iface", "core", "xo"; iommus = <&apps_smmu 0x160 0>; qcom,dll-config = <0x0007642c>; @@ -5432,19 +5432,19 @@ opp-366000000 { opp-hz = /bits/ 64 <366000000>; - required-opps = <&rpmhpd_opp_svs_l1>, + required-opps = <&rpmhpd_opp_svs>, <&rpmhpd_opp_svs_l1>; }; opp-444000000 { opp-hz = /bits/ 64 <444000000>; - required-opps = <&rpmhpd_opp_nom>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_nom>; }; opp-481000000 { opp-hz = /bits/ 64 <481000000>; - required-opps = <&rpmhpd_opp_turbo>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_turbo>; }; }; diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi index 9ef57ad0ca71d5..9be86479ceef12 100644 --- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi +++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi @@ -1239,7 +1239,7 @@ gic_its: msi-controller@17040000 { compatible = "arm,gic-v3-its"; - reg = <0x0 0x17040000 0x0 0x20000>; + reg = <0x0 0x17040000 0x0 0x40000>; msi-controller; #msi-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi index 6079e67ea829b5..ba0f7e5c89a0de 100644 --- a/arch/arm64/boot/dts/qcom/kodiak.dtsi +++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi @@ -2445,7 +2445,7 @@ reg = <0 0x01c0e000 0 0x1000>; clocks = <&gcc GCC_PCIE_1_AUX_CLK>, <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_CLKREF_EN>, + <&rpmhcc RPMH_CXO_CLK>, <&gcc GCC_PCIE1_PHY_RCHNG_CLK>, <&gcc GCC_PCIE_1_PIPE_CLK>; clock-names = "aux", diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi index 808827b83553dd..be8c8c59ef5ab8 100644 --- a/arch/arm64/boot/dts/qcom/lemans.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans.dtsi @@ -1512,7 +1512,7 @@ reg = <0x0 0x898000 0x0 0x4000>; #address-cells = <1>; #size-cells = <0>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; clock-names = "se"; pinctrl-0 = <&qup_i2c20_default>; @@ -1539,7 +1539,7 @@ reg = <0x0 0x898000 0x0 0x4000>; #address-cells = <1>; #size-cells = <0>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; clock-names = "se"; pinctrl-0 = <&qup_spi20_default>; @@ -1564,7 +1564,7 @@ uart20: serial@898000 { compatible = "qcom,geni-uart"; reg = <0x0 0x00898000 0x0 0x4000>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP2_S6_CLK>; clock-names = "se"; pinctrl-0 = <&qup_uart20_default>; @@ -2510,7 +2510,7 @@ reg = <0x0 0xa98000 0x0 0x4000>; #address-cells = <1>; #size-cells = <0>; - interrupts = ; + interrupts = ; clocks = <&gcc GCC_QUPV3_WRAP1_S6_CLK>; clock-names = "se"; pinctrl-0 = <&qup_i2c13_default>; @@ -4625,19 +4625,19 @@ opp-444000000 { opp-hz = /bits/ 64 <444000000>; - required-opps = <&rpmhpd_opp_nom>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_nom>; }; opp-533000000 { opp-hz = /bits/ 64 <533000000>; - required-opps = <&rpmhpd_opp_turbo>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_turbo>; }; opp-560000000 { opp-hz = /bits/ 64 <560000000>; - required-opps = <&rpmhpd_opp_turbo_l1>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_turbo_l1>; }; }; diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi index e1a51d43943ff4..098f9ceaa4f381 100644 --- a/arch/arm64/boot/dts/qcom/milos.dtsi +++ b/arch/arm64/boot/dts/qcom/milos.dtsi @@ -802,6 +802,8 @@ <0>, /* ufs_phy_tx_symbol_0_clk */ <0>; /* usb3_phy_wrapper_gcc_usb30_pipe_clk */ + power-domains = <&rpmhpd RPMHPD_CX>; + #clock-cells = <1>; #reset-cells = <1>; #power-domain-cells = <1>; @@ -1911,7 +1913,7 @@ gic_its: msi-controller@17140000 { compatible = "arm,gic-v3-its"; - reg = <0x0 0x17140000 0x0 0x20000>; + reg = <0x0 0x17140000 0x0 0x40000>; msi-controller; #msi-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi index 0cb9fd154b684e..37d0515e889360 100644 --- a/arch/arm64/boot/dts/qcom/monaco.dtsi +++ b/arch/arm64/boot/dts/qcom/monaco.dtsi @@ -5293,19 +5293,19 @@ opp-444000000 { opp-hz = /bits/ 64 <444000000>; - required-opps = <&rpmhpd_opp_nom>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_nom>; }; opp-533000000 { opp-hz = /bits/ 64 <533000000>; - required-opps = <&rpmhpd_opp_turbo>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_turbo>; }; opp-560000000 { opp-hz = /bits/ 64 <560000000>; - required-opps = <&rpmhpd_opp_turbo_l1>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_turbo_l1>; }; }; diff --git a/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts b/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts index 9db503e2188866..1bfb16f90ddd55 100644 --- a/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts +++ b/arch/arm64/boot/dts/qcom/msm8917-xiaomi-riva.dts @@ -18,7 +18,7 @@ chassis-type = "handset"; qcom,msm-id = ; - qcom,board-id = <0x1000b 2>, <0x2000b 2>; + qcom,board-id = <0x1000b 1>, <0x1000b 2>; pwm_backlight: backlight { compatible = "pwm-backlight"; diff --git a/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts b/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts index 91837ff940f1b6..4f301e7c651710 100644 --- a/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts +++ b/arch/arm64/boot/dts/qcom/msm8937-xiaomi-land.dts @@ -178,7 +178,7 @@ qcom,num-strings = <2>; qcom,external-pfet; qcom,current-limit-microamp = <20000>; - qcom,ovp-millivolt = <29600>; + qcom,ovp-millivolt = <29500>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts index ddd7af61679429..59f873a06e4ddd 100644 --- a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts +++ b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-daisy.dts @@ -157,7 +157,7 @@ &pmi8950_wled { qcom,current-limit-microamp = <20000>; - qcom,num-strings = <2>; + qcom,num-strings = <3>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts index d46325e7991769..c2a290bf493c17 100644 --- a/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts +++ b/arch/arm64/boot/dts/qcom/msm8953-xiaomi-vince.dts @@ -169,7 +169,7 @@ &pmi8950_wled { qcom,current-limit-microamp = <20000>; - qcom,ovp-millivolt = <29600>; + qcom,ovp-millivolt = <29500>; qcom,num-strings = <2>; qcom,external-pfet; qcom,cabc; diff --git a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts index 0b64a0b912021d..f47efca42d48d8 100644 --- a/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts +++ b/arch/arm64/boot/dts/qcom/qcs6490-thundercomm-rubikpi3.dts @@ -755,10 +755,10 @@ #address-cells = <1>; #size-cells = <0>; - port@0 { - reg = <0>; + port@1 { + reg = <1>; - lt9611_a: endpoint { + lt9611_b: endpoint { remote-endpoint = <&mdss_dsi0_out>; }; }; @@ -801,7 +801,7 @@ }; &mdss_dsi0_out { - remote-endpoint = <<9611_a>; + remote-endpoint = <<9611_b>; data-lanes = <0 1 2 3>; }; diff --git a/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts b/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts index 197ab6eb1666fa..5ab605cc56c801 100644 --- a/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts +++ b/arch/arm64/boot/dts/qcom/qrb2210-arduino-imola.dts @@ -325,21 +325,13 @@ &spi5 { status = "okay"; - spidev@0 { - reg = <0>; + mcu@0 { compatible = "arduino,unoq-mcu"; - pinctrl-0 = <&spidev_cs>; - pinctrl-names = "default"; + reg = <0>; }; }; &tlmm { - spidev_cs: spidev-cs-state { - pins = "gpio17"; - function = "gpio"; - drive-strength = <16>; - }; - jmisc_gpio18: jmisc-gpio18-state { pins = "gpio18"; function = "gpio"; diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi index 01b570d0880d6d..1298485c42142a 100644 --- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi @@ -148,6 +148,7 @@ regulator-min-microvolt = <880000>; regulator-max-microvolt = <880000>; regulator-initial-mode = ; + regulator-boot-on; }; vreg_l5a_0p8: ldo5 { diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts index 6b68e391cf3ea1..c3edeee3af3ef7 100644 --- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts +++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-ginkgo.dts @@ -13,14 +13,18 @@ #include "sm6125.dtsi" #include "pm6125.dtsi" +/delete-node/ &adsp_pil_mem; +/delete-node/ &cont_splash_mem; +/delete-node/ &gpu_mem; +/delete-node/ &ipa_fw_mem; +/delete-node/ &ipa_gsi_mem; + / { model = "Xiaomi Redmi Note 8"; compatible = "xiaomi,ginkgo", "qcom,sm6125"; chassis-type = "handset"; - /* required for bootloader to select correct board */ qcom,msm-id = ; - qcom,board-id = <22 0>; chosen { #address-cells = <2>; @@ -38,33 +42,39 @@ }; reserved-memory { - debug_mem: debug@ffb00000 { - reg = <0x0 0xffb00000 0x0 0xc0000>; + adsp_pil_mem: adsp_pil_mem@55300000 { + reg = <0x0 0x55300000 0x0 0x2200000>; no-map; }; - last_log_mem: lastlog@ffbc0000 { - reg = <0x0 0xffbc0000 0x0 0x80000>; + ipa_fw_mem: ipa_fw_mem@57500000 { + reg = <0x0 0x57500000 0x0 0x10000>; no-map; }; - pstore_mem: ramoops@ffc00000 { - compatible = "ramoops"; - reg = <0x0 0xffc40000 0x0 0xc0000>; - record-size = <0x1000>; - console-size = <0x40000>; - pmsg-size = <0x20000>; + ipa_gsi_mem: ipa_gsi_mem@57510000 { + reg = <0x0 0x57510000 0x0 0x5000>; + no-map; }; - cmdline_mem: memory@ffd00000 { - reg = <0x0 0xffd40000 0x0 0x1000>; + gpu_mem: gpu_mem@57515000 { + reg = <0x0 0x57515000 0x0 0x2000>; no-map; }; - }; - extcon_usb: extcon-usb { - compatible = "linux,extcon-usb-gpio"; - id-gpios = <&tlmm 102 GPIO_ACTIVE_HIGH>; + framebuffer@5c000000 { + reg = <0x0 0x5c000000 0x0 (2340 * 1080 * 4)>; + no-map; + }; + + /* Matching with recovery values to be able to get the results. */ + ramoops@61600000 { + compatible = "ramoops"; + reg = <0x0 0x61600000 0x0 0x400000>; + record-size = <0x80000>; + pmsg-size = <0x200000>; + console-size = <0x100000>; + }; }; gpio-keys { @@ -283,13 +293,9 @@ }; &tlmm { - gpio-reserved-ranges = <22 2>, <28 6>; + gpio-reserved-ranges = <0 4>, <30 4>; }; &usb3 { status = "okay"; }; - -&usb3_dwc3 { - extcon = <&extcon_usb>; -}; diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts index a3c2b26736f471..3964aae47fd4e9 100644 --- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts +++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts @@ -1019,12 +1019,14 @@ * the Bluetooth module drives the pin in either * direction or leaves the pin fully unpowered. */ + /delete-property/ bias-disable; bias-bus-hold; }; &qup_uart1_rts { /* We'll drive RTS, so no pull */ drive-strength = <2>; + /delete-property/ bias-pull-down; bias-disable; }; @@ -1035,12 +1037,14 @@ * in tri-state (module powered off or not driving the * signal yet). */ + /delete-property/ bias-disable; bias-pull-up; }; &qup_uart1_tx { /* We'll drive TX, so no pull */ drive-strength = <2>; + /delete-property/ bias-pull-up; bias-disable; }; diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index c7dffa44007407..37c41cc1abdd01 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -665,6 +665,11 @@ opp-hz = /bits/ 64 <2841600000>; opp-peak-kBps = <8368000 51609600>; }; + + cpu7_opp21: opp-3091200000 { + opp-hz = /bits/ 64 <3091200000>; + opp-peak-kBps = <8368000 51609600>; + }; }; firmware { diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 920a2d1c04d0c5..dd60843e022efb 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -5104,7 +5104,7 @@ gic_its: msi-controller@17140000 { compatible = "arm,gic-v3-its"; - reg = <0x0 0x17140000 0x0 0x20000>; + reg = <0x0 0x17140000 0x0 0x40000>; msi-controller; #msi-cells = <1>; }; @@ -5429,9 +5429,6 @@ bus-width = <4>; dma-coherent; - /* Forbid SDR104/SDR50 - broken hw! */ - sdhci-caps-mask = <0x3 0x0>; - status = "disabled"; sdhc2_opp_table: opp-table { diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi index e3f93f4f412ded..42f9deaadd86a3 100644 --- a/arch/arm64/boot/dts/qcom/sm8550.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi @@ -3210,7 +3210,7 @@ clocks = <&gcc GCC_SDCC2_AHB_CLK>, <&gcc GCC_SDCC2_APPS_CLK>, - <&rpmhcc RPMH_CXO_CLK>; + <&bi_tcxo_div2>; clock-names = "iface", "core", "xo"; iommus = <&apps_smmu 0x540 0>; qcom,dll-config = <0x0007642c>; @@ -3227,9 +3227,6 @@ max-sd-hs-hz = <37500000>; dma-coherent; - /* Forbid SDR104/SDR50 - broken hw! */ - sdhci-caps-mask = <0x3 0>; - status = "disabled"; sdhc2_opp_table: opp-table { @@ -3320,19 +3317,19 @@ opp-366000000 { opp-hz = /bits/ 64 <366000000>; - required-opps = <&rpmhpd_opp_svs_l1>, + required-opps = <&rpmhpd_opp_svs>, <&rpmhpd_opp_svs_l1>; }; opp-444000000 { opp-hz = /bits/ 64 <444000000>; - required-opps = <&rpmhpd_opp_nom>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_nom>; }; opp-533333334 { opp-hz = /bits/ 64 <533333334>; - required-opps = <&rpmhpd_opp_turbo>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_turbo>; }; }; @@ -5274,7 +5271,7 @@ gic_its: msi-controller@17140000 { compatible = "arm,gic-v3-its"; - reg = <0 0x17140000 0 0x20000>; + reg = <0 0x17140000 0 0x40000>; msi-controller; #msi-cells = <1>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8650.dtsi b/arch/arm64/boot/dts/qcom/sm8650.dtsi index 357e43b907405f..64a7480291d841 100644 --- a/arch/arm64/boot/dts/qcom/sm8650.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8650.dtsi @@ -4957,7 +4957,7 @@ clocks = <&gcc GCC_SDCC2_AHB_CLK>, <&gcc GCC_SDCC2_APPS_CLK>, - <&rpmhcc RPMH_CXO_CLK>; + <&bi_tcxo_div2>; clock-names = "iface", "core", "xo"; @@ -4976,9 +4976,6 @@ bus-width = <4>; - /* Forbid SDR104/SDR50 - broken hw! */ - sdhci-caps-mask = <0x3 0>; - qcom,dll-config = <0x0007642c>; qcom,ddr-config = <0x80040868>; @@ -5236,13 +5233,13 @@ opp-300000000 { opp-hz = /bits/ 64 <300000000>; - required-opps = <&rpmhpd_opp_low_svs>, + required-opps = <&rpmhpd_opp_svs>, <&rpmhpd_opp_low_svs>; }; opp-380000000 { opp-hz = /bits/ 64 <380000000>; - required-opps = <&rpmhpd_opp_svs>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_svs>; }; @@ -5254,13 +5251,13 @@ opp-480000000 { opp-hz = /bits/ 64 <480000000>; - required-opps = <&rpmhpd_opp_nom>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_nom>; }; opp-533333334 { opp-hz = /bits/ 64 <533333334>; - required-opps = <&rpmhpd_opp_turbo>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_turbo>; }; }; @@ -7219,7 +7216,7 @@ gic_its: msi-controller@17140000 { compatible = "arm,gic-v3-its"; - reg = <0 0x17140000 0 0x20000>; + reg = <0 0x17140000 0 0x40000>; msi-controller; #msi-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index f56b1f889b857a..4efdead3583f9d 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -2945,19 +2945,19 @@ opp-240000000 { opp-hz = /bits/ 64 <240000000>; - required-opps = <&rpmhpd_opp_low_svs_d1>, + required-opps = <&rpmhpd_opp_svs>, <&rpmhpd_opp_low_svs_d1>; }; opp-338000000 { opp-hz = /bits/ 64 <338000000>; - required-opps = <&rpmhpd_opp_low_svs>, + required-opps = <&rpmhpd_opp_svs>, <&rpmhpd_opp_low_svs>; }; opp-420000000 { opp-hz = /bits/ 64 <420000000>; - required-opps = <&rpmhpd_opp_svs>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_svs>; }; @@ -2969,19 +2969,19 @@ opp-533333334 { opp-hz = /bits/ 64 <533333334>; - required-opps = <&rpmhpd_opp_nom>, + required-opps = <&rpmhpd_opp_svs_l1>, <&rpmhpd_opp_nom>; }; opp-570000000 { opp-hz = /bits/ 64 <570000000>; - required-opps = <&rpmhpd_opp_nom_l1>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_nom_l1>; }; opp-630000000 { opp-hz = /bits/ 64 <630000000>; - required-opps = <&rpmhpd_opp_turbo>, + required-opps = <&rpmhpd_opp_nom>, <&rpmhpd_opp_turbo>; }; }; @@ -4658,7 +4658,7 @@ gic_its: msi-controller@16040000 { compatible = "arm,gic-v3-its"; - reg = <0x0 0x16040000 0x0 0x20000>; + reg = <0x0 0x16040000 0x0 0x40000>; msi-controller; #msi-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi index 75716b4a58d6d3..6dab0d5dcbb8a0 100644 --- a/arch/arm64/boot/dts/qcom/talos.dtsi +++ b/arch/arm64/boot/dts/qcom/talos.dtsi @@ -666,6 +666,9 @@ clocks = <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>, <&sleep_clk>; + clock-names = "bi_tcxo", + "bi_tcxo_ao", + "sleep_clk"; #clock-cells = <1>; #reset-cells = <1>; diff --git a/arch/arm64/boot/dts/rockchip/rk3328-a1.dts b/arch/arm64/boot/dts/rockchip/rk3328-a1.dts index 30bdb38f0727a5..e810ed146451c1 100644 --- a/arch/arm64/boot/dts/rockchip/rk3328-a1.dts +++ b/arch/arm64/boot/dts/rockchip/rk3328-a1.dts @@ -58,24 +58,6 @@ gpios = <&gpio2 RK_PA2 GPIO_ACTIVE_LOW>; linux,rc-map-name = "rc-beelink-gs1"; }; - - spdif_dit: spdif-dit { - compatible = "linux,spdif-dit"; - #sound-dai-cells = <0>; - }; - - spdif_sound: spdif-sound { - compatible = "simple-audio-card"; - simple-audio-card,name = "SPDIF"; - - simple-audio-card,cpu { - sound-dai = <&spdif>; - }; - - simple-audio-card,codec { - sound-dai = <&spdif_dit>; - }; - }; }; &analog_sound { @@ -343,11 +325,6 @@ status = "okay"; }; -&spdif { - pinctrl-0 = <&spdifm0_tx>; - status = "okay"; -}; - &tsadc { rockchip,hw-tshut-mode = <0>; rockchip,hw-tshut-polarity = <0>; diff --git a/arch/arm64/boot/dts/rockchip/rk3562-evb2-v10.dts b/arch/arm64/boot/dts/rockchip/rk3562-evb2-v10.dts index 6a84db154a7d53..387062eea52085 100644 --- a/arch/arm64/boot/dts/rockchip/rk3562-evb2-v10.dts +++ b/arch/arm64/boot/dts/rockchip/rk3562-evb2-v10.dts @@ -13,7 +13,7 @@ #include "rk3562.dtsi" / { - model = "Rockchip RK3562 EVB V20 Board"; + model = "Rockchip RK3562 EVB2 V10 Board"; compatible = "rockchip,rk3562-evb2-v10", "rockchip,rk3562"; chosen: chosen { diff --git a/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts b/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts index ed65d312044460..18a560a6e2a4af 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts +++ b/arch/arm64/boot/dts/rockchip/rk3566-lckfb-tspi.dts @@ -635,10 +635,10 @@ status = "okay"; bluetooth: bluetooth { - compatible = "brcm,bcm43438-bt"; + compatible = "brcm,bcm43430a1-bt"; clocks = <&rk809 1>; clock-names = "lpo"; - max-speed = <3000000>; + max-speed = <1500000>; pinctrl-names = "default"; pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>; shutdown-gpios = <&gpio2 RK_PB7 GPIO_ACTIVE_HIGH>; diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi index 49ccdf12ef7eb4..8149e2bbde79ae 100644 --- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi @@ -1868,8 +1868,9 @@ pinctrl-0 = <&ufs_refclk &ufs_rstgpio>; pinctrl-names = "default"; resets = <&cru SRST_A_UFS_BIU>, <&cru SRST_A_UFS_SYS>, - <&cru SRST_A_UFS>, <&cru SRST_P_UFS_GRF>; - reset-names = "biu", "sys", "ufs", "grf"; + <&cru SRST_A_UFS>, <&cru SRST_P_UFS_GRF>, + <&cru SRST_MPHY_INIT>; + reset-names = "biu", "sys", "ufs", "grf", "mphy"; reset-gpios = <&gpio4 RK_PD0 GPIO_ACTIVE_LOW>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts index 952affaf455cf2..500a0bad1ea30f 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts @@ -588,7 +588,7 @@ pcie30x4 { pcie30x4_clkreqn_m0: pcie30x4-clkreqn-m0 { - rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>; + rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_up>; }; pcie30x4_perstn_m0: pcie30x4-perstn-m0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts index e8ad525ba3f9bc..89618394c0bfb4 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts @@ -60,8 +60,8 @@ reg = <0>; abs-flat = <40>; abs-fuzz = <30>; - abs-range = <0 4095>; - linux,code = ; + abs-range = <4095 0>; + linux,code = ; }; axis@1 { @@ -69,7 +69,7 @@ abs-flat = <40>; abs-fuzz = <30>; abs-range = <0 4095>; - linux,code = ; + linux,code = ; }; axis@2 { @@ -77,7 +77,7 @@ abs-flat = <40>; abs-fuzz = <30>; abs-range = <0 4095>; - linux,code = ; + linux,code = ; }; axis@3 { @@ -85,7 +85,7 @@ abs-flat = <40>; abs-fuzz = <30>; abs-range = <0 4095>; - linux,code = ; + linux,code = ; }; }; @@ -318,7 +318,7 @@ compatible = "pwm-fan"; #cooling-cells = <2>; cooling-levels = <0 120 150 180 210 240 255>; - fan-supply = <&vcc5v0_sys>; + fan-supply = <&vcc5v0_spk>; interrupt-parent = <&gpio4>; interrupts = ; pulses-per-revolution = <4>; diff --git a/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts b/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts index 3e2d8f66953513..8a556fbbe08b7b 100644 --- a/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts @@ -88,13 +88,13 @@ AM62X_IOPAD(0x220, PIN_INPUT, 0) /* (V3) MMC0_CMD */ AM62X_IOPAD(0x218, PIN_INPUT, 0) /* (Y1) MMC0_CLK */ AM62X_IOPAD(0x214, PIN_INPUT, 0) /* (V2) MMC0_DAT0 */ - AM62X_IOPAD(0x210, PIN_INPUT, 0) /* (V1) MMC0_DAT1 */ - AM62X_IOPAD(0x20c, PIN_INPUT, 0) /* (W2) MMC0_DAT2 */ - AM62X_IOPAD(0x208, PIN_INPUT, 0) /* (W1) MMC0_DAT3 */ - AM62X_IOPAD(0x204, PIN_INPUT, 0) /* (Y2) MMC0_DAT4 */ - AM62X_IOPAD(0x200, PIN_INPUT, 0) /* (W3) MMC0_DAT5 */ - AM62X_IOPAD(0x1fc, PIN_INPUT, 0) /* (W4) MMC0_DAT6 */ - AM62X_IOPAD(0x1f8, PIN_INPUT, 0) /* (V4) MMC0_DAT7 */ + AM62X_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (V1) MMC0_DAT1 */ + AM62X_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (W2) MMC0_DAT2 */ + AM62X_IOPAD(0x208, PIN_INPUT_PULLUP, 0) /* (W1) MMC0_DAT3 */ + AM62X_IOPAD(0x204, PIN_INPUT_PULLUP, 0) /* (Y2) MMC0_DAT4 */ + AM62X_IOPAD(0x200, PIN_INPUT_PULLUP, 0) /* (W3) MMC0_DAT5 */ + AM62X_IOPAD(0x1fc, PIN_INPUT_PULLUP, 0) /* (W4) MMC0_DAT6 */ + AM62X_IOPAD(0x1f8, PIN_INPUT_PULLUP, 0) /* (V4) MMC0_DAT7 */ >; }; diff --git a/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi b/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi index 2a7242a2fef878..20dbfa3001eacb 100644 --- a/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62-verdin.dtsi @@ -278,7 +278,7 @@ }; /* Verdin SPI_1 CS as GPIO */ - pinctrl_qspi1_io4_gpio: main-gpio0-7-default-pins { + pinctrl_spi1_cs_gpio: main-gpio0-7-default-pins { pinctrl-single,pins = < AM62X_IOPAD(0x001c, PIN_INPUT, 7) /* (J23) OSPI0_D4.GPIO0_7 */ /* SODIMM 202 */ >; @@ -572,16 +572,16 @@ /* On-module eMMC */ pinctrl_sdhci0: main-mmc0-default-pins { pinctrl-single,pins = < - AM62X_IOPAD(0x220, PIN_INPUT, 0) /* (Y3) MMC0_CMD */ - AM62X_IOPAD(0x218, PIN_INPUT, 0) /* (AB1) MMC0_CLK */ - AM62X_IOPAD(0x214, PIN_INPUT, 0) /* (AA2) MMC0_DAT0 */ - AM62X_IOPAD(0x210, PIN_INPUT, 0) /* (AA1) MMC0_DAT1 */ - AM62X_IOPAD(0x20c, PIN_INPUT, 0) /* (AA3) MMC0_DAT2 */ - AM62X_IOPAD(0x208, PIN_INPUT, 0) /* (Y4) MMC0_DAT3 */ - AM62X_IOPAD(0x204, PIN_INPUT, 0) /* (AB2) MMC0_DAT4 */ - AM62X_IOPAD(0x200, PIN_INPUT, 0) /* (AC1) MMC0_DAT5 */ - AM62X_IOPAD(0x1fc, PIN_INPUT, 0) /* (AD2) MMC0_DAT6 */ - AM62X_IOPAD(0x1f8, PIN_INPUT, 0) /* (AC2) MMC0_DAT7 */ + AM62X_IOPAD(0x220, PIN_INPUT, 0) /* (Y3) MMC0_CMD */ + AM62X_IOPAD(0x218, PIN_INPUT, 0) /* (AB1) MMC0_CLK */ + AM62X_IOPAD(0x214, PIN_INPUT, 0) /* (AA2) MMC0_DAT0 */ + AM62X_IOPAD(0x210, PIN_INPUT_PULLUP, 0) /* (AA1) MMC0_DAT1 */ + AM62X_IOPAD(0x20c, PIN_INPUT_PULLUP, 0) /* (AA3) MMC0_DAT2 */ + AM62X_IOPAD(0x208, PIN_INPUT_PULLUP, 0) /* (Y4) MMC0_DAT3 */ + AM62X_IOPAD(0x204, PIN_INPUT_PULLUP, 0) /* (AB2) MMC0_DAT4 */ + AM62X_IOPAD(0x200, PIN_INPUT_PULLUP, 0) /* (AC1) MMC0_DAT5 */ + AM62X_IOPAD(0x1fc, PIN_INPUT_PULLUP, 0) /* (AD2) MMC0_DAT6 */ + AM62X_IOPAD(0x1f8, PIN_INPUT_PULLUP, 0) /* (AC2) MMC0_DAT7 */ >; }; diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts index e99bdbc2e0cbdf..b1a6f10adf26b8 100644 --- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts @@ -398,7 +398,7 @@ vddshv_sdio_pins_default: vddshv-sdio-default-pins { pinctrl-single,pins = < - AM62AX_IOPAD(0x07c, PIN_OUTPUT, 7) /* (M19) GPMC0_CLK.GPIO0_31 */ + AM62AX_IOPAD(0x07c, PIN_OUTPUT, 7) /* (N22) GPMC0_CLK.GPIO0_31 */ >; }; diff --git a/arch/arm64/boot/dts/ti/k3-am62l.dtsi b/arch/arm64/boot/dts/ti/k3-am62l.dtsi index 23acdbb301fe38..e01e342c26daaa 100644 --- a/arch/arm64/boot/dts/ti/k3-am62l.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am62l.dtsi @@ -92,7 +92,7 @@ <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00001400>, /* VTM */ <0x00 0x04080000 0x00 0x04080000 0x00 0x00008000>, /* PDCFG */ <0x00 0x04201000 0x00 0x04201000 0x00 0x00000100>, /* GPIO */ - <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00100100>, /* Wakeup Peripheral Window */ + <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00200200>, /* Wakeup Peripheral Window */ <0x00 0x40800000 0x00 0x40800000 0x00 0x00014000>, /* DMA */ <0x00 0x43000000 0x00 0x43000000 0x00 0x00080000>; /* CTRL MMRs */ #address-cells = <2>; @@ -104,7 +104,7 @@ <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00001400>, /* VTM */ <0x00 0x04080000 0x00 0x04080000 0x00 0x00008000>, /* PDCFG */ <0x00 0x04201000 0x00 0x04201000 0x00 0x00000100>, /* GPIO */ - <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00100100>, /* Wakeup Peripheral Window */ + <0x00 0x2b100000 0x00 0x2b100000 0x00 0x00200200>, /* Wakeup Peripheral Window */ <0x00 0x40800000 0x00 0x40800000 0x00 0x00014000>, /* DMA */ <0x00 0x43000000 0x00 0x43000000 0x00 0x00080000>; /* CTRL MMRs */ #address-cells = <2>; diff --git a/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts b/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts index cae04cce337366..bd876c68aa347e 100644 --- a/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts +++ b/arch/arm64/boot/dts/ti/k3-am62l3-evm.dts @@ -272,9 +272,9 @@ AM62LX_IOPAD(0x0230, PIN_INPUT, 0) /* (Y3) MMC1_CMD */ AM62LX_IOPAD(0x0228, PIN_OUTPUT, 0) /* (Y2) MMC1_CLK */ AM62LX_IOPAD(0x0224, PIN_INPUT, 0) /* (AA1) MMC1_DAT0 */ - AM62LX_IOPAD(0x0220, PIN_INPUT_PULLUP, 0) /* (Y4) MMC1_DAT1 */ - AM62LX_IOPAD(0x021c, PIN_INPUT_PULLUP, 0) /* (AA2) MMC1_DAT2 */ - AM62LX_IOPAD(0x0218, PIN_INPUT_PULLUP, 0) /* (AB2) MMC1_DAT3 */ + AM62LX_IOPAD(0x0220, PIN_INPUT, 0) /* (Y4) MMC1_DAT1 */ + AM62LX_IOPAD(0x021c, PIN_INPUT, 0) /* (AA2) MMC1_DAT2 */ + AM62LX_IOPAD(0x0218, PIN_INPUT, 0) /* (AB2) MMC1_DAT3 */ AM62LX_IOPAD(0x0234, PIN_INPUT, 0) /* (B6) MMC1_SDCD */ >; bootph-all; diff --git a/arch/arm64/boot/dts/ti/k3-am62p5-sk.dts b/arch/arm64/boot/dts/ti/k3-am62p5-sk.dts index 4f7f6f95b02ef9..35baa777b9121a 100644 --- a/arch/arm64/boot/dts/ti/k3-am62p5-sk.dts +++ b/arch/arm64/boot/dts/ti/k3-am62p5-sk.dts @@ -271,9 +271,9 @@ AM62PX_IOPAD(0x023c, PIN_INPUT, 0) /* (H20) MMC1_CMD */ AM62PX_IOPAD(0x0234, PIN_OUTPUT, 0) /* (J24) MMC1_CLK */ AM62PX_IOPAD(0x0230, PIN_INPUT, 0) /* (H21) MMC1_DAT0 */ - AM62PX_IOPAD(0x022c, PIN_INPUT_PULLUP, 0) /* (H23) MMC1_DAT1 */ - AM62PX_IOPAD(0x0228, PIN_INPUT_PULLUP, 0) /* (H22) MMC1_DAT2 */ - AM62PX_IOPAD(0x0224, PIN_INPUT_PULLUP, 0) /* (H25) MMC1_DAT3 */ + AM62PX_IOPAD(0x022c, PIN_INPUT, 0) /* (H23) MMC1_DAT1 */ + AM62PX_IOPAD(0x0228, PIN_INPUT, 0) /* (H22) MMC1_DAT2 */ + AM62PX_IOPAD(0x0224, PIN_INPUT, 0) /* (H25) MMC1_DAT3 */ AM62PX_IOPAD(0x0240, PIN_INPUT, 0) /* (D23) MMC1_SDCD */ >; bootph-all; diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts index ec8ff458771574..dc0d3cf2f985d1 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-clover.dts @@ -26,7 +26,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_21_dp>; /* Aquila GPIO_21_DP (AQUILA B57) */ - gpio = <&main_gpio0 37 GPIO_ACTIVE_HIGH>; + gpio = <&main_gpio0 21 GPIO_ACTIVE_HIGH>; enable-active-high; regulator-max-microvolt = <3300000>; regulator-min-microvolt = <3300000>; diff --git a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts index f48601ae38b7c7..d3677c2c25473d 100644 --- a/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts +++ b/arch/arm64/boot/dts/ti/k3-am69-aquila-dev.dts @@ -33,7 +33,7 @@ pinctrl-names = "default"; pinctrl-0 = <&pinctrl_gpio_21_dp>; /* Aquila GPIO_21_DP (AQUILA B57) */ - gpio = <&main_gpio0 37 GPIO_ACTIVE_HIGH>; + gpio = <&main_gpio0 21 GPIO_ACTIVE_HIGH>; enable-active-high; regulator-max-microvolt = <3300000>; regulator-min-microvolt = <3300000>; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config new file mode 100644 index 00000000000000..c76f2c727fb0dd --- /dev/null +++ b/arch/arm64/configs/asahi.config @@ -0,0 +1,77 @@ +CONFIG_RUST=y +CONFIG_ARM64_ACTLR_STATE=y +CONFIG_ARCH_APPLE=y +# CONFIG_ARM64_4K_PAGES is not set +CONFIG_ARM64_16K_PAGES=y +# CONFIG_ARM64_64K_PAGES is not set +CONFIG_ARM64_MEMORY_MODEL_CONTROL=y +CONFIG_ARM_APPLE_CPUIDLE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=m +CONFIG_BT_BRCMEXT=y +CONFIG_BT_HCIBCM4377=m +CONFIG_PCIE_APPLE=m +CONFIG_NVME_APPLE=m +CONFIG_BRCMFMAC=m +CONFIG_BRCMFMAC_PCIE=y +CONFIG_TOUCHSCREEN_APPLE_Z2=m +CONFIG_INPUT_MACSMC_INPUT=m +CONFIG_I2C_APPLE=m +CONFIG_SPI_APPLE=m +CONFIG_SPMI_APPLE=m +CONFIG_PINCTRL_APPLE_GPIO=m +CONFIG_GPIO_MACSMC=m +CONFIG_POWER_RESET_MACSMC=m +CONFIG_CHARGER_MACSMC=m +CONFIG_SENSORS_MACSMC_HWMON=m +CONFIG_APPLE_WATCHDOG=m +CONFIG_VIDEO_APPLE_ISP=m +CONFIG_DRM=y +CONFIG_DRM_ASAHI=m +CONFIG_DRM_ADP=m +CONFIG_DRM_APPLE=m +CONFIG_DRM_APPLE_AUDIO=y +CONFIG_SND_SOC_APPLE_AOP_AUDIO=m +CONFIG_SND_SOC_APPLE_MCA=m +CONFIG_SND_SOC_APPLE_MACAUDIO=m +CONFIG_SND_SOC_CS42L83=m +CONFIG_SND_SOC_CS42L84=m +CONFIG_SND_SOC_TAS2764=m +CONFIG_SND_SOC_TAS2770=m +CONFIG_HID_APPLE=m +CONFIG_HID_MAGICMOUSE=m +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_HID_DOCKCHANNEL=m +CONFIG_SPI_HID_APPLE_OF=m +CONFIG_SPI_HID_APPLE_CORE=m +CONFIG_USB_DWC3_APPLE=m +CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_RTC_DRV_MACSMC=m +CONFIG_APPLE_ADMAC=m +CONFIG_APPLE_SIO=m +CONFIG_MFD_MACSMC=m +CONFIG_COMMON_CLK_APPLE_NCO=m +CONFIG_APPLE_DART=m +CONFIG_APPLE_DOCKCHANNEL=m +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_PMGR_MISC=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=m +CONFIG_APPLE_SART=m +CONFIG_RUST_APPLE_RTKIT=y +CONFIG_APPLE_AOP=m +CONFIG_APPLE_SEP=m +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_APPLE_PMP=m +CONFIG_APPLE_PMP_REPORT=y +CONFIG_IIO_AOP_SENSOR_LAS=m +CONFIG_IIO_AOP_SENSOR_ALS=m +CONFIG_RUST_FW_LOADER_ABSTRACTIONS=y +CONFIG_PWM_APPLE=m +CONFIG_APPLE_AIC=y +CONFIG_PHY_APPLE_ATC=m +CONFIG_PHY_APPLE_DPTX=m +CONFIG_APPLE_M1_CPU_PMU=y +CONFIG_NVMEM_APPLE_EFUSES=m +CONFIG_NVMEM_APPLE_SPMI=m +CONFIG_MUX_APPLE_DPXBAR=m diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 0e834a2c062cf2..e793478f37c1eb 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -838,7 +838,7 @@ AES_FUNC_START(aes_mac_update) encrypt_block v0, w2, x1, x7, w8 eor v0.16b, v0.16b, v4.16b cmp w3, wzr - csinv x5, x6, xzr, eq + csinv w5, w6, wzr, eq cbz w5, .Lmacout encrypt_block v0, w2, x1, x7, w8 st1 {v0.16b}, [x4] /* return dg */ @@ -852,7 +852,7 @@ AES_FUNC_START(aes_mac_update) eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ subs w3, w3, #1 - csinv x5, x6, xzr, eq + csinv w5, w6, wzr, eq cbz w5, .Lmacout .Lmacenc: diff --git a/arch/arm64/include/asm/apple_cpufeature.h b/arch/arm64/include/asm/apple_cpufeature.h new file mode 100644 index 00000000000000..4370d91ffa3ec9 --- /dev/null +++ b/arch/arm64/include/asm/apple_cpufeature.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +#ifndef __ASM_APPLE_CPUFEATURES_H +#define __ASM_APPLE_CPUFEATURES_H + +#include +#include + +#define AIDR_APPLE_TSO_SHIFT 9 +#define AIDR_APPLE_TSO BIT(9) + +#define ACTLR_APPLE_TSO_SHIFT 1 +#define ACTLR_APPLE_TSO BIT(1) + +#endif diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 4de51f8d92cbac..a69eae348990bf 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -955,6 +955,12 @@ static inline unsigned int get_vmid_bits(u64 mmfr1) return 8; } +static __always_inline bool system_has_actlr_state(void) +{ + return IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && + alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE); +} + s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new, s64 cur); struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id); @@ -1078,6 +1084,10 @@ static inline bool cpu_has_lpa2(void) #endif } +void __init init_cpucap_indirect_list_impdef(void); +void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps); +bool cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry); + #endif /* __ASSEMBLER__ */ #endif diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h index cab8cd78f69385..20f0a7c7bde159 100644 --- a/arch/arm64/include/asm/entry-common.h +++ b/arch/arm64/include/asm/entry-common.h @@ -29,14 +29,19 @@ static __always_inline void arch_exit_to_user_mode_work(struct pt_regs *regs, static inline bool arch_irqentry_exit_need_resched(void) { - /* - * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC - * priority masking is used the GIC irqchip driver will clear DAIF.IF - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in - * DAIF we must have handled an NMI, so skip preemption. - */ - if (system_uses_irq_prio_masking() && read_sysreg(daif)) - return false; + if (system_uses_irq_prio_masking()) { + /* + * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC + * priority masking is used the GIC irqchip driver will clear DAIF.IF + * using gic_arch_enable_irqs() for normal IRQs. If anything is set in + * DAIF we must have handled an NMI, so skip preemption. + */ + if (read_sysreg(daif)) + return false; + } else { + if (read_sysreg(daif) & (PSR_D_BIT | PSR_A_BIT)) + return false; + } /* * Preempting a task from an IRQ means we leave copies of PSTATE diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 74a4f738c5f52e..229ee7976f6934 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -68,7 +68,12 @@ #define KERNEL_SEGMENT_COUNT 5 #if SWAPPER_BLOCK_SIZE > SEGMENT_ALIGN -#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 1) +/* + * KERNEL_SEGMENT_COUNT counts the permanent kernel VMAs. The early mapping + * has one additional split, [_text, _stext). Reserve one more page for the + * SWAPPER_BLOCK_SIZE-unaligned boundaries. + */ +#define EARLY_SEGMENT_EXTRA_PAGES (KERNEL_SEGMENT_COUNT + 2) /* * The initial ID map consists of the kernel image, mapped as two separate * segments, and may appear misaligned wrt the swapper block size. This means diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 5bf3d7e1d92c77..0df8cc5eaa7c5a 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -105,6 +105,11 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) { if (!vcpu_has_run_once(vcpu)) vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE) && ( + alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT) || + alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE) + )) + vcpu->arch.hcr_el2 &= ~HCR_TACR; /* * For non-FWB CPUs, we trap VM ops (HCR_EL2.TVM) until M+C diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 70cb9cfd760a36..9d82f9a644cdde 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1506,7 +1506,7 @@ static inline bool __vcpu_has_feature(const struct kvm_arch *ka, int feature) #define kvm_vcpu_has_feature(k, f) __vcpu_has_feature(&(k)->arch, (f)) #define vcpu_has_feature(v, f) __vcpu_has_feature(&(v)->kvm->arch, (f)) -#define kvm_vcpu_initialized(v) vcpu_get_flag(vcpu, VCPU_INITIALIZED) +#define kvm_vcpu_initialized(v) vcpu_get_flag(v, VCPU_INITIALIZED) int kvm_trng_call(struct kvm_vcpu *vcpu); #ifdef CONFIG_KVM diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index a2b7a33966ff1d..ff4a0425db1001 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -112,7 +112,7 @@ #define DIRECT_MAP_PHYSMEM_END __pa(PAGE_END - 1) -#define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT) +#define MIN_THREAD_SHIFT (15 + KASAN_THREAD_SHIFT) /* * VMAP'd stacks are allocated at page granularity, so we must ensure that such diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 137a173df1ff88..472610433aaea9 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -112,5 +112,7 @@ void kpti_install_ng_mappings(void); static inline void kpti_install_ng_mappings(void) {} #endif +extern bool page_alloc_available; + #endif /* !__ASSEMBLER__ */ #endif diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index f560e64202674d..212ce1b02e15ed 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -25,6 +25,8 @@ */ #define PTE_PRESENT_INVALID (PTE_NG) /* only when !PTE_VALID */ +#define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG) + #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */ #define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */ diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index b3e58735c49bdd..dd062179b9b66a 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -322,9 +322,11 @@ static inline pte_t pte_mknoncont(pte_t pte) return clear_pte_bit(pte, __pgprot(PTE_CONT)); } -static inline pte_t pte_mkvalid(pte_t pte) +static inline pte_t pte_mkvalid_k(pte_t pte) { - return set_pte_bit(pte, __pgprot(PTE_VALID)); + pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID)); + pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL)); + return pte; } static inline pte_t pte_mkinvalid(pte_t pte) @@ -594,6 +596,7 @@ static inline int pmd_protnone(pmd_t pmd) #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd))) #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd))) #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd))) +#define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd))) #define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd))) #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP #define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd)) @@ -635,6 +638,8 @@ static inline pmd_t pmd_mkspecial(pmd_t pmd) #define pud_young(pud) pte_young(pud_pte(pud)) #define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud))) +#define pud_mkwrite_novma(pud) pte_pud(pte_mkwrite_novma(pud_pte(pud))) +#define pud_mkvalid_k(pud) pte_pud(pte_mkvalid_k(pud_pte(pud))) #define pud_write(pud) pte_write(pud_pte(pud)) static inline pud_t pud_mkhuge(pud_t pud) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e30c4c8e3a7a7d..499e0a85c08828 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -199,6 +199,9 @@ struct thread_struct { u64 gcs_base; u64 gcs_size; #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + u64 actlr; +#endif }; static inline unsigned int thread_get_vl(struct thread_struct *thread, diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 9810106a3f664a..eafc83d255d8e4 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -478,7 +478,7 @@ extern __must_check long strnlen_user(const char __user *str, long n); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE extern unsigned long __must_check __copy_user_flushcache(void *to, const void __user *from, unsigned long n); -static inline int __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) +static inline size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size) { kasan_check_write(dst, size); return __copy_user_flushcache(dst, __uaccess_mask_ptr(src), size); diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h index c38e3d017a79ec..bb7428d4ebc65e 100644 --- a/arch/arm64/include/asm/xor.h +++ b/arch/arm64/include/asm/xor.h @@ -13,7 +13,7 @@ #ifdef CONFIG_KERNEL_MODE_NEON -extern struct xor_block_template const xor_block_inner_neon; +extern struct xor_block_template xor_block_inner_neon __ro_after_init; static void xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index fe627100d19903..3391c586aa24b8 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,6 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idle.o patching.o pi/ \ + cpufeature_impdef.o \ rsi.o jump_label.o obj-$(CONFIG_COMPAT) += sys32.o signal32.o \ diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 32c2dbcc0c6416..2fe69a0e81bc45 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -565,7 +565,7 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { * We can instantiate multiple PMU instances with different levels * of support. */ - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6), ARM64_FTR_END, }; @@ -709,7 +709,7 @@ static const struct arm64_ftr_bits ftr_id_pfr2[] = { static const struct arm64_ftr_bits ftr_id_dfr0[] = { /* [31:28] TraceFilt */ - S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0), @@ -1081,7 +1081,7 @@ static void init_cpu_ftr_reg(u32 sys_reg, u64 new) extern const struct arm64_cpu_capabilities arm64_errata[]; static const struct arm64_cpu_capabilities arm64_features[]; -static void __init +void __init init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) { @@ -1593,8 +1593,8 @@ has_always(const struct arm64_cpu_capabilities *entry, int scope) return true; } -static bool -feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) +bool +cpufeature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { int val, min, max; u64 tmp; @@ -1647,14 +1647,14 @@ has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) if (!mask) return false; - return feature_matches(val, entry); + return cpufeature_matches(val, entry); } static bool has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) { u64 val = read_scoped_sysreg(entry, scope); - return feature_matches(val, entry); + return cpufeature_matches(val, entry); } const struct cpumask *system_32bit_el0_cpumask(void) @@ -3919,6 +3919,7 @@ void __init setup_boot_cpu_features(void) * handle the boot CPU. */ init_cpucap_indirect_list(); + init_cpucap_indirect_list_impdef(); /* * Detect broken pseudo-NMI. Must be called _before_ the call to diff --git a/arch/arm64/kernel/cpufeature_impdef.c b/arch/arm64/kernel/cpufeature_impdef.c new file mode 100644 index 00000000000000..aee7571fbadb84 --- /dev/null +++ b/arch/arm64/kernel/cpufeature_impdef.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Contains implementation-defined CPU feature definitions. + */ + +#define pr_fmt(fmt) "CPU features: " fmt + +#include +#include +#include +#include +#include + +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +static bool has_apple_feature(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible()); + + if (read_cpuid_implementor() != ARM_CPU_IMP_APPLE) + return false; + + val = read_sysreg(aidr_el1); + return cpufeature_matches(val, entry); +} + +static bool has_apple_tso(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 val; + + if (!has_apple_feature(entry, scope)) + return false; + + /* + * KVM and old versions of the macOS hypervisor will advertise TSO in + * AIDR_EL1, but then ignore writes to ACTLR_EL1. Test that the bit is + * actually writable before enabling TSO. + */ + + val = read_sysreg(actlr_el1); + write_sysreg(val ^ ACTLR_APPLE_TSO, actlr_el1); + if (!((val ^ read_sysreg(actlr_el1)) & ACTLR_APPLE_TSO)) { + pr_info_once("CPU advertises Apple TSO but it is broken, ignoring\n"); + return false; + } + + write_sysreg(val, actlr_el1); + return true; +} + +static bool has_tso_fixed(const struct arm64_cpu_capabilities *entry, int scope) +{ + /* List of CPUs that always use the TSO memory model */ + static const struct midr_range fixed_tso_list[] = { + MIDR_ALL_VERSIONS(MIDR_NVIDIA_DENVER), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL), + MIDR_ALL_VERSIONS(MIDR_FUJITSU_A64FX), + { /* sentinel */ } + }; + + return is_midr_in_range_list(fixed_tso_list); +} +#endif + +static bool has_apple_actlr_virt_impdef(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + return midr >= MIDR_APPLE_M1_ICESTORM && midr <= MIDR_APPLE_M1_FIRESTORM_MAX; +} + +static bool has_apple_actlr_virt(const struct arm64_cpu_capabilities *entry, int scope) +{ + u64 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + return midr >= MIDR_APPLE_M2_BLIZZARD && midr <= MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, 0xfff); +} + +static const struct arm64_cpu_capabilities arm64_impdef_features[] = { +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + { + .desc = "TSO memory model (Apple)", + .capability = ARM64_HAS_TSO_APPLE, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_tso, + .field_pos = AIDR_APPLE_TSO_SHIFT, + .field_width = 1, + .sign = FTR_UNSIGNED, + .min_field_value = 1, + .max_field_value = 1, + }, + { + .desc = "TSO memory model (Fixed)", + .capability = ARM64_HAS_TSO_FIXED, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_tso_fixed, + }, +#endif + { + .desc = "ACTLR virtualization (IMPDEF, Apple)", + .capability = ARM64_HAS_ACTLR_VIRT_APPLE, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_actlr_virt_impdef, + }, + { + .desc = "ACTLR virtualization (architectural?)", + .capability = ARM64_HAS_ACTLR_VIRT, + .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, + .matches = has_apple_actlr_virt, + }, + {}, +}; + +void __init init_cpucap_indirect_list_impdef(void) +{ + init_cpucap_indirect_list_from_array(arm64_impdef_features); +} diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 239c16e3d02f29..c5693a32e49b06 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -129,9 +129,6 @@ int machine_kexec_post_load(struct kimage *kimage) } /* Create a copy of the linear map */ - trans_pgd = kexec_page_alloc(kimage); - if (!trans_pgd) - return -ENOMEM; rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END); if (rc) return rc; diff --git a/arch/arm64/kernel/pi/patch-scs.c b/arch/arm64/kernel/pi/patch-scs.c index dac568e4a54f23..3944ad899021cd 100644 --- a/arch/arm64/kernel/pi/patch-scs.c +++ b/arch/arm64/kernel/pi/patch-scs.c @@ -196,9 +196,9 @@ static int scs_handle_fde_frame(const struct eh_frame *frame, loc += *opcode++ * code_alignment_factor; loc += (*opcode++ << 8) * code_alignment_factor; loc += (*opcode++ << 16) * code_alignment_factor; - loc += (*opcode++ << 24) * code_alignment_factor; + loc += ((u64)*opcode++ << 24) * code_alignment_factor; size -= 4; - break; + break; case DW_CFA_def_cfa: case DW_CFA_offset_extended: diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 489554931231e6..59c416cfcdd2a4 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -41,8 +41,10 @@ #include #include #include +#include #include +#include #include #include #include @@ -442,6 +444,11 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) if (system_supports_poe()) p->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); +#ifdef CONFIG_ARM64_ACTLR_STATE + if (system_has_actlr_state()) + p->thread.actlr = read_sysreg(actlr_el1); +#endif + if (stack_start) { if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; @@ -699,6 +706,65 @@ void update_sctlr_el1(u64 sctlr) isb(); } +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL +int arch_prctl_mem_model_get(struct task_struct *t) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE) && + t->thread.actlr & ACTLR_APPLE_TSO) + return PR_SET_MEM_MODEL_TSO; + + return PR_SET_MEM_MODEL_DEFAULT; +} + +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_FIXED) && + val == PR_SET_MEM_MODEL_TSO) + return 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_TSO_APPLE)) { + WARN_ON(!system_has_actlr_state()); + + switch (val) { + case PR_SET_MEM_MODEL_TSO: + t->thread.actlr |= ACTLR_APPLE_TSO; + break; + case PR_SET_MEM_MODEL_DEFAULT: + t->thread.actlr &= ~ACTLR_APPLE_TSO; + break; + default: + return -EINVAL; + } + write_sysreg(t->thread.actlr, actlr_el1); + return 0; + } + + if (val == PR_SET_MEM_MODEL_DEFAULT) + return 0; + + return -EINVAL; +} +#endif + +#ifdef CONFIG_ARM64_ACTLR_STATE +/* + * IMPDEF control register ACTLR_EL1 handling. Some CPUs use this to + * expose features that can be controlled by userspace. + */ +static void actlr_thread_switch(struct task_struct *next) +{ + if (!system_has_actlr_state()) + return; + + current->thread.actlr = read_sysreg(actlr_el1); + write_sysreg(next->thread.actlr, actlr_el1); +} +#else +static inline void actlr_thread_switch(struct task_struct *next) +{ +} +#endif + /* * Thread switching. */ @@ -718,6 +784,7 @@ struct task_struct *__switch_to(struct task_struct *prev, ptrauth_thread_switch_user(next); permission_overlay_switch(next); gcs_thread_switch(next); + actlr_thread_switch(next); /* * Complete any pending TLB or cache maintenance on this CPU in case the @@ -840,6 +907,10 @@ void arch_setup_new_exec(void) arch_prctl_spec_ctrl_set(current, PR_SPEC_STORE_BYPASS, PR_SPEC_ENABLE); } + +#ifdef CONFIG_ARM64_MEMORY_MODEL_CONTROL + arch_prctl_mem_model_set(current, PR_SET_MEM_MODEL_DEFAULT); +#endif } #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index ba5eab23fd9008..4d08598e2891d3 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -983,8 +983,8 @@ static int sve_set_common(struct task_struct *target, } /* Always zero V regs, FPSR, and FPCR */ - memset(¤t->thread.uw.fpsimd_state, 0, - sizeof(current->thread.uw.fpsimd_state)); + memset(&target->thread.uw.fpsimd_state, 0, + sizeof(target->thread.uw.fpsimd_state)); /* Registers: FPSIMD-only case */ diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 23c05dc7a8f2ac..0fa2403c6fc0e6 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -368,6 +368,14 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) */ init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); #endif +#ifdef CONFIG_ARM64_ACTLR_STATE + /* Store the boot CPU ACTLR_EL1 value as the default. This will only + * be actually restored during context switching iff the platform is + * known to use ACTLR_EL1 for exposable features and its layout is + * known to be the same on all CPUs. + */ + init_task.thread.actlr = read_sysreg(actlr_el1); +#endif if (boot_args[1] || boot_args[2] || boot_args[3]) { pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 08ffc5a5aea4cc..38e6fa204c17b5 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -67,6 +67,9 @@ struct rt_sigframe_user_layout { unsigned long end_offset; }; +#define TERMINATOR_SIZE round_up(sizeof(struct _aarch64_ctx), 16) +#define EXTRA_CONTEXT_SIZE round_up(sizeof(struct extra_context), 16) + /* * Holds any EL0-controlled state that influences unprivileged memory accesses. * This includes both accesses done in userspace and uaccess done in the kernel. @@ -74,13 +77,35 @@ struct rt_sigframe_user_layout { * This state needs to be carefully managed to ensure that it doesn't cause * uaccess to fail when setting up the signal frame, and the signal handler * itself also expects a well-defined state when entered. + * + * The struct should be zero-initialised. Its members should only be accessed + * via the accessors below. __valid_fields tracks which of the fields are valid + * (have been set to some value). */ struct user_access_state { - u64 por_el0; + unsigned int __valid_fields; + u64 __por_el0; }; -#define TERMINATOR_SIZE round_up(sizeof(struct _aarch64_ctx), 16) -#define EXTRA_CONTEXT_SIZE round_up(sizeof(struct extra_context), 16) +#define UA_STATE_HAS_POR_EL0 BIT(0) + +static void set_ua_state_por_el0(struct user_access_state *ua_state, + u64 por_el0) +{ + ua_state->__por_el0 = por_el0; + ua_state->__valid_fields |= UA_STATE_HAS_POR_EL0; +} + +static int get_ua_state_por_el0(const struct user_access_state *ua_state, + u64 *por_el0) +{ + if (ua_state->__valid_fields & UA_STATE_HAS_POR_EL0) { + *por_el0 = ua_state->__por_el0; + return 0; + } + + return -ENOENT; +} /* * Save the user access state into ua_state and reset it to disable any @@ -94,7 +119,7 @@ static void save_reset_user_access_state(struct user_access_state *ua_state) for (int pkey = 0; pkey < arch_max_pkey(); pkey++) por_enable_all |= POR_ELx_PERM_PREP(pkey, POE_RWX); - ua_state->por_el0 = read_sysreg_s(SYS_POR_EL0); + set_ua_state_por_el0(ua_state, read_sysreg_s(SYS_POR_EL0)); write_sysreg_s(por_enable_all, SYS_POR_EL0); /* * No ISB required as we can tolerate spurious Overlay faults - @@ -122,8 +147,10 @@ static void set_handler_user_access_state(void) */ static void restore_user_access_state(const struct user_access_state *ua_state) { - if (system_supports_poe()) - write_sysreg_s(ua_state->por_el0, SYS_POR_EL0); + u64 por_el0; + + if (get_ua_state_por_el0(ua_state, &por_el0) == 0) + write_sysreg_s(por_el0, SYS_POR_EL0); } static void init_user_layout(struct rt_sigframe_user_layout *user) @@ -333,11 +360,16 @@ static int restore_fpmr_context(struct user_ctxs *user) static int preserve_poe_context(struct poe_context __user *ctx, const struct user_access_state *ua_state) { - int err = 0; + int err; + u64 por_el0; + + err = get_ua_state_por_el0(ua_state, &por_el0); + if (WARN_ON_ONCE(err)) + return err; __put_user_error(POE_MAGIC, &ctx->head.magic, err); __put_user_error(sizeof(*ctx), &ctx->head.size, err); - __put_user_error(ua_state->por_el0, &ctx->por_el0, err); + __put_user_error(por_el0, &ctx->por_el0, err); return err; } @@ -353,7 +385,7 @@ static int restore_poe_context(struct user_ctxs *user, __get_user_error(por_el0, &(user->poe->por_el0), err); if (!err) - ua_state->por_el0 = por_el0; + set_ua_state_por_el0(ua_state, por_el0); return err; } @@ -1095,7 +1127,7 @@ SYSCALL_DEFINE0(rt_sigreturn) { struct pt_regs *regs = current_pt_regs(); struct rt_sigframe __user *frame; - struct user_access_state ua_state; + struct user_access_state ua_state = {}; /* Always make any pending restarted system calls return -EINTR */ current->restart_block.fn = do_no_restart_syscall; @@ -1507,7 +1539,7 @@ static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set, { struct rt_sigframe_user_layout user; struct rt_sigframe __user *frame; - struct user_access_state ua_state; + struct user_access_state ua_state = {}; int err = 0; fpsimd_save_and_flush_current_state(); diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 96bcfb9074430b..12a948f3a50437 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -89,7 +89,7 @@ COMPAT_SYSCALL_DEFINE4(aarch32_truncate64, const char __user *, pathname, COMPAT_SYSCALL_DEFINE4(aarch32_ftruncate64, unsigned int, fd, u32, __pad, arg_u32p(length)) { - return ksys_ftruncate(fd, arg_u64(length)); + return ksys_ftruncate(fd, arg_u64(length), FTRUNCATE_LFS); } COMPAT_SYSCALL_DEFINE5(aarch32_readahead, int, fd, u32, __pad, diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 410ffd41fd73aa..f9c9e7fb0997fc 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -805,6 +805,10 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) { bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE); + irq_lines |= (!irqchip_in_kernel(v->kvm) && + (kvm_timer_should_notify_user(v) || + kvm_pmu_should_notify_user(v))); + return ((irq_lines || kvm_vgic_vcpu_pending_irq(v)) && !kvm_arm_vcpu_stopped(v) && !v->arch.pause); } diff --git a/arch/arm64/kvm/config.c b/arch/arm64/kvm/config.c index d9f553cbf9dfdf..f95783a7fa4c7f 100644 --- a/arch/arm64/kvm/config.c +++ b/arch/arm64/kvm/config.c @@ -131,7 +131,6 @@ struct reg_feat_map_desc { } #define FEAT_SPE ID_AA64DFR0_EL1, PMSVer, IMP -#define FEAT_SPE_FnE ID_AA64DFR0_EL1, PMSVer, V1P2 #define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP #define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP #define FEAT_PMUv3 ID_AA64DFR0_EL1, PMUVer, IMP @@ -192,7 +191,7 @@ struct reg_feat_map_desc { #define FEAT_SRMASK ID_AA64MMFR4_EL1, SRMASK, IMP #define FEAT_PoPS ID_AA64MMFR4_EL1, PoPS, IMP #define FEAT_PFAR ID_AA64PFR1_EL1, PFAR, IMP -#define FEAT_Debugv8p9 ID_AA64DFR0_EL1, PMUVer, V3P9 +#define FEAT_Debugv8p9 ID_AA64DFR0_EL1, DebugVer, V8P9 #define FEAT_PMUv3_SS ID_AA64DFR0_EL1, PMSS, IMP #define FEAT_SEBEP ID_AA64DFR0_EL1, SEBEP, IMP #define FEAT_EBEP ID_AA64DFR1_EL1, EBEP, IMP @@ -301,6 +300,16 @@ static bool feat_spe_fds(struct kvm *kvm) (read_sysreg_s(SYS_PMSIDR_EL1) & PMSIDR_EL1_FDS)); } +static bool feat_spe_fne(struct kvm *kvm) +{ + /* + * Revisit this if KVM ever supports SPE -- this really should + * look at the guest's view of PMSIDR_EL1. + */ + return (kvm_has_feat(kvm, FEAT_SPEv1p2) && + (read_sysreg_s(SYS_PMSIDR_EL1) & PMSIDR_EL1_FnE)); +} + static bool feat_trbe_mpam(struct kvm *kvm) { /* @@ -536,7 +545,7 @@ static const struct reg_bits_to_feat_map hdfgrtr_feat_map[] = { HDFGRTR_EL2_PMBPTR_EL1 | HDFGRTR_EL2_PMBLIMITR_EL1, FEAT_SPE), - NEEDS_FEAT(HDFGRTR_EL2_nPMSNEVFR_EL1, FEAT_SPE_FnE), + NEEDS_FEAT(HDFGRTR_EL2_nPMSNEVFR_EL1, feat_spe_fne), NEEDS_FEAT(HDFGRTR_EL2_nBRBDATA | HDFGRTR_EL2_nBRBCTL | HDFGRTR_EL2_nBRBIDR, @@ -604,7 +613,7 @@ static const struct reg_bits_to_feat_map hdfgwtr_feat_map[] = { HDFGWTR_EL2_PMBPTR_EL1 | HDFGWTR_EL2_PMBLIMITR_EL1, FEAT_SPE), - NEEDS_FEAT(HDFGWTR_EL2_nPMSNEVFR_EL1, FEAT_SPE_FnE), + NEEDS_FEAT(HDFGWTR_EL2_nPMSNEVFR_EL1, feat_spe_fne), NEEDS_FEAT(HDFGWTR_EL2_nBRBDATA | HDFGWTR_EL2_nBRBCTL, FEAT_BRBE), @@ -1585,8 +1594,8 @@ static __always_inline void __compute_fgt(struct kvm_vcpu *vcpu, enum vcpu_sysre clear |= ~nested & m->nmask; } - val |= set; - val &= ~clear; + val |= set | m->res1; + val &= ~(clear | m->res0); *vcpu_fgt(vcpu, reg) = val; } diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index a17cbe7582de90..7c8383c809ea36 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -16,6 +16,9 @@ #include #include +#define SYS_IMP_APL_ACTLR_EL12 sys_reg(3, 6, 15, 14, 6) +#define SYS_ACTLR_EL12 sys_reg(3, 5, 1, 0, 1) + static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt); static inline struct kvm_vcpu *ctxt_to_vcpu(struct kvm_cpu_context *ctxt) @@ -172,6 +175,13 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) if (ctxt_has_sctlr2(ctxt)) ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2); + + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE)) { + if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT)) + ctxt_sys_reg(ctxt, ACTLR_EL1) = read_sysreg_s(SYS_ACTLR_EL12); + else if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)) + ctxt_sys_reg(ctxt, ACTLR_EL1) = read_sysreg_s(SYS_IMP_APL_ACTLR_EL12); + } } static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt) @@ -256,6 +266,13 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt, write_sysreg(ctxt_sys_reg(ctxt, PAR_EL1), par_el1); write_sysreg(ctxt_sys_reg(ctxt, TPIDR_EL1), tpidr_el1); + if (IS_ENABLED(CONFIG_ARM64_ACTLR_STATE)) { + if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT)) + write_sysreg_s(ctxt_sys_reg(ctxt, ACTLR_EL1), SYS_ACTLR_EL12); + else if (alternative_has_cap_unlikely(ARM64_HAS_ACTLR_VIRT_APPLE)) + write_sysreg_s(ctxt_sys_reg(ctxt, ACTLR_EL1), SYS_IMP_APL_ACTLR_EL12); + } + if (ctxt_has_mte(ctxt)) { write_sysreg_el1(ctxt_sys_reg(ctxt, TFSR_EL1), SYS_TFSR); write_sysreg_s(ctxt_sys_reg(ctxt, TFSRE0_EL1), SYS_TFSRE0_EL1); diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 2f029bfe4755a5..13db8979fe9dbc 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -258,7 +258,8 @@ struct pkvm_hyp_vcpu *pkvm_load_hyp_vcpu(pkvm_handle_t handle, if (!hyp_vm || hyp_vm->kvm.created_vcpus <= vcpu_idx) goto unlock; - hyp_vcpu = hyp_vm->vcpus[vcpu_idx]; + /* Pairs with smp_store_release() in register_hyp_vcpu(). */ + hyp_vcpu = smp_load_acquire(&hyp_vm->vcpus[vcpu_idx]); if (!hyp_vcpu) goto unlock; @@ -803,12 +804,30 @@ int __pkvm_init_vm(struct kvm *host_kvm, unsigned long vm_hva, * the page-aligned size of 'struct pkvm_hyp_vcpu'. * Return 0 on success, negative error code on failure. */ +static int register_hyp_vcpu(struct pkvm_hyp_vm *hyp_vm, + struct pkvm_hyp_vcpu *hyp_vcpu) +{ + unsigned int idx = hyp_vcpu->vcpu.vcpu_idx; + + if (idx >= hyp_vm->kvm.created_vcpus) + return -EINVAL; + + if (hyp_vm->vcpus[idx]) + return -EINVAL; + + /* + * Ensure the hyp_vcpu is initialised before publishing it to + * the vCPU-load path via 'hyp_vm->vcpus[]'. + */ + smp_store_release(&hyp_vm->vcpus[idx], hyp_vcpu); + return 0; +} + int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, unsigned long vcpu_hva) { struct pkvm_hyp_vcpu *hyp_vcpu; struct pkvm_hyp_vm *hyp_vm; - unsigned int idx; int ret; hyp_vcpu = map_donated_memory(vcpu_hva, sizeof(*hyp_vcpu)); @@ -827,18 +846,11 @@ int __pkvm_init_vcpu(pkvm_handle_t handle, struct kvm_vcpu *host_vcpu, if (ret) goto unlock; - idx = hyp_vcpu->vcpu.vcpu_idx; - if (idx >= hyp_vm->kvm.created_vcpus) { - ret = -EINVAL; - goto unlock; - } - - if (hyp_vm->vcpus[idx]) { - ret = -EINVAL; - goto unlock; + ret = register_hyp_vcpu(hyp_vm, hyp_vcpu); + if (ret) { + unpin_host_vcpu(host_vcpu); + unpin_host_sve_state(hyp_vcpu); } - - hyp_vm->vcpus[idx] = hyp_vcpu; unlock: hyp_spin_unlock(&vm_table_lock); diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index 90bd014e952fbc..97643fc02d9257 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -312,15 +312,15 @@ void __noreturn __pkvm_init_finalise(void) }; pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops; - ret = fix_host_ownership(); + ret = fix_hyp_pgtable_refcnt(); if (ret) goto out; - ret = fix_hyp_pgtable_refcnt(); + ret = hyp_create_fixmap(); if (ret) goto out; - ret = hyp_create_fixmap(); + ret = fix_host_ownership(); if (ret) goto out; diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v2.c b/arch/arm64/kvm/vgic/vgic-mmio-v2.c index 406845b3117cfb..0643e333db35dc 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v2.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v2.c @@ -91,7 +91,7 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu, * migration from old kernels to new kernels with legacy * userspace. */ - reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg); + reg = FIELD_GET(GICD_IIDR_REVISION_MASK, val); switch (reg) { case KVM_VGIC_IMP_REV_2: case KVM_VGIC_IMP_REV_3: diff --git a/arch/arm64/kvm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c index 89edb84d1ac6d2..5913a20d830191 100644 --- a/arch/arm64/kvm/vgic/vgic-mmio-v3.c +++ b/arch/arm64/kvm/vgic/vgic-mmio-v3.c @@ -194,7 +194,7 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK) return -EINVAL; - reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg); + reg = FIELD_GET(GICD_IIDR_REVISION_MASK, val); switch (reg) { case KVM_VGIC_IMP_REV_2: case KVM_VGIC_IMP_REV_3: diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c index cc5b40917d0dd5..37ce75f7f1f089 100644 --- a/arch/arm64/lib/insn.c +++ b/arch/arm64/lib/insn.c @@ -338,6 +338,8 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, long offset; offset = label_imm_common(pc, addr, SZ_1M); + if (offset >= SZ_1M) + return AARCH64_BREAK_FAULT; insn = aarch64_insn_get_bcond_value(); diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 96711b8578fd06..b9b248d24fd101 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -350,7 +350,6 @@ void __init arch_mm_preinit(void) } swiotlb_init(swiotlb, flags); - swiotlb_update_mem_attributes(); /* * Check boundaries twice: Some fundamental inconsistencies can be @@ -377,6 +376,14 @@ void __init arch_mm_preinit(void) } } +bool page_alloc_available __ro_after_init; + +void __init mem_init(void) +{ + page_alloc_available = true; + swiotlb_update_mem_attributes(); +} + void free_initmem(void) { void *lm_init_begin = lm_alias(__init_begin); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index a6a00accf4f938..97270947787fc8 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -602,6 +602,8 @@ static int split_pmd(pmd_t *pmdp, pmd_t pmd, gfp_t gfp, bool to_cont) tableprot |= PMD_TABLE_PXN; prot = __pgprot((pgprot_val(prot) & ~PTE_TYPE_MASK) | PTE_TYPE_PAGE); + if (!pmd_valid(pmd)) + prot = pte_pgprot(pte_mkinvalid(pfn_pte(0, prot))); prot = __pgprot(pgprot_val(prot) & ~PTE_CONT); if (to_cont) prot = __pgprot(pgprot_val(prot) | PTE_CONT); @@ -647,6 +649,8 @@ static int split_pud(pud_t *pudp, pud_t pud, gfp_t gfp, bool to_cont) tableprot |= PUD_TABLE_PXN; prot = __pgprot((pgprot_val(prot) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT); + if (!pud_valid(pud)) + prot = pmd_pgprot(pmd_mkinvalid(pfn_pmd(0, prot))); prot = __pgprot(pgprot_val(prot) & ~PTE_CONT); if (to_cont) prot = __pgprot(pgprot_val(prot) | PTE_CONT); @@ -768,30 +772,51 @@ static inline bool force_pte_mapping(void) } static DEFINE_MUTEX(pgtable_split_lock); +static bool linear_map_requires_bbml2; int split_kernel_leaf_mapping(unsigned long start, unsigned long end) { int ret; - /* - * !BBML2_NOABORT systems should not be trying to change permissions on - * anything that is not pte-mapped in the first place. Just return early - * and let the permission change code raise a warning if not already - * pte-mapped. - */ - if (!system_supports_bbml2_noabort()) - return 0; - /* * If the region is within a pte-mapped area, there is no need to try to * split. Additionally, CONFIG_DEBUG_PAGEALLOC and CONFIG_KFENCE may * change permissions from atomic context so for those cases (which are * always pte-mapped), we must not go any further because taking the - * mutex below may sleep. + * mutex below may sleep. Do not call force_pte_mapping() here because + * it could return a confusing result if called from a secondary cpu + * prior to finalizing caps. Instead, linear_map_requires_bbml2 gives us + * what we need. */ - if (force_pte_mapping() || is_kfence_address((void *)start)) + if (!linear_map_requires_bbml2 || is_kfence_address((void *)start)) return 0; + if (!system_supports_bbml2_noabort()) { + /* + * !BBML2_NOABORT systems should not be trying to change + * permissions on anything that is not pte-mapped in the first + * place. Just return early and let the permission change code + * raise a warning if not already pte-mapped. + */ + if (system_capabilities_finalized()) + return 0; + + /* + * Boot-time: split_kernel_leaf_mapping_locked() allocates from + * page allocator. Can't split until it's available. + */ + if (WARN_ON(!page_alloc_available)) + return -EBUSY; + + /* + * Boot-time: Started secondary cpus but don't know if they + * support BBML2_NOABORT yet. Can't allow splitting in this + * window in case they don't. + */ + if (WARN_ON(num_online_cpus() > 1)) + return -EBUSY; + } + /* * Ensure start and end are at least page-aligned since this is the * finest granularity we can split to. @@ -891,8 +916,6 @@ static int range_split_to_ptes(unsigned long start, unsigned long end, gfp_t gfp return ret; } -static bool linear_map_requires_bbml2 __initdata; - u32 idmap_kpti_bbml2_flag; static void __init init_idmap_kpti_bbml2_flag(void) @@ -1458,10 +1481,14 @@ static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr, WARN_ON(!pte_present(pte)); __pte_clear(&init_mm, addr, ptep); - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); - if (free_mapped) + if (free_mapped) { + /* CONT blocks are not supported in the vmemmap */ + WARN_ON(pte_cont(pte)); + flush_tlb_kernel_range(addr, addr + PAGE_SIZE); free_hotplug_page_range(pte_page(pte), PAGE_SIZE, altmap); + } + /* unmap_hotplug_range() flushes TLB for !free_mapped */ } while (addr += PAGE_SIZE, addr < end); } @@ -1482,15 +1509,14 @@ static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr, WARN_ON(!pmd_present(pmd)); if (pmd_sect(pmd)) { pmd_clear(pmdp); - - /* - * One TLBI should be sufficient here as the PMD_SIZE - * range is mapped with a single block entry. - */ - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); - if (free_mapped) + if (free_mapped) { + /* CONT blocks are not supported in the vmemmap */ + WARN_ON(pmd_cont(pmd)); + flush_tlb_kernel_range(addr, addr + PMD_SIZE); free_hotplug_page_range(pmd_page(pmd), PMD_SIZE, altmap); + } + /* unmap_hotplug_range() flushes TLB for !free_mapped */ continue; } WARN_ON(!pmd_table(pmd)); @@ -1515,15 +1541,12 @@ static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr, WARN_ON(!pud_present(pud)); if (pud_sect(pud)) { pud_clear(pudp); - - /* - * One TLBI should be sufficient here as the PUD_SIZE - * range is mapped with a single block entry. - */ - flush_tlb_kernel_range(addr, addr + PAGE_SIZE); - if (free_mapped) + if (free_mapped) { + flush_tlb_kernel_range(addr, addr + PUD_SIZE); free_hotplug_page_range(pud_page(pud), PUD_SIZE, altmap); + } + /* unmap_hotplug_range() flushes TLB for !free_mapped */ continue; } WARN_ON(!pud_table(pud)); @@ -1553,6 +1576,7 @@ static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr, static void unmap_hotplug_range(unsigned long addr, unsigned long end, bool free_mapped, struct vmem_altmap *altmap) { + unsigned long start = addr; unsigned long next; pgd_t *pgdp, pgd; @@ -1574,6 +1598,9 @@ static void unmap_hotplug_range(unsigned long addr, unsigned long end, WARN_ON(!pgd_present(pgd)); unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped, altmap); } while (addr = next, addr < end); + + if (!free_mapped) + flush_tlb_kernel_range(start, end); } static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr, diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 358d1dc9a576f0..ce035e1b4eaf66 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -25,6 +25,11 @@ static ptdesc_t set_pageattr_masks(ptdesc_t val, struct mm_walk *walk) { struct page_change_data *masks = walk->private; + /* + * Some users clear and set bits which alias each other (e.g. PTE_NG and + * PTE_PRESENT_INVALID). It is therefore important that we always clear + * first then set. + */ val &= ~(pgprot_val(masks->clear_mask)); val |= (pgprot_val(masks->set_mask)); @@ -36,7 +41,7 @@ static int pageattr_pud_entry(pud_t *pud, unsigned long addr, { pud_t val = pudp_get(pud); - if (pud_sect(val)) { + if (pud_leaf(val)) { if (WARN_ON_ONCE((next - addr) != PUD_SIZE)) return -EINVAL; val = __pud(set_pageattr_masks(pud_val(val), walk)); @@ -52,7 +57,7 @@ static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr, { pmd_t val = pmdp_get(pmd); - if (pmd_sect(val)) { + if (pmd_leaf(val)) { if (WARN_ON_ONCE((next - addr) != PMD_SIZE)) return -EINVAL; val = __pmd(set_pageattr_masks(pmd_val(val), walk)); @@ -132,11 +137,12 @@ static int __change_memory_common(unsigned long start, unsigned long size, ret = update_range_prot(start, size, set_mask, clear_mask); /* - * If the memory is being made valid without changing any other bits - * then a TLBI isn't required as a non-valid entry cannot be cached in - * the TLB. + * If the memory is being switched from present-invalid to valid without + * changing any other bits then a TLBI isn't required as a non-valid + * entry cannot be cached in the TLB. */ - if (pgprot_val(set_mask) != PTE_VALID || pgprot_val(clear_mask)) + if (pgprot_val(set_mask) != PTE_PRESENT_VALID_KERNEL || + pgprot_val(clear_mask) != PTE_PRESENT_INVALID) flush_tlb_kernel_range(start, start + size); return ret; } @@ -237,18 +243,18 @@ int set_memory_valid(unsigned long addr, int numpages, int enable) { if (enable) return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(PTE_VALID), - __pgprot(0)); + __pgprot(PTE_PRESENT_VALID_KERNEL), + __pgprot(PTE_PRESENT_INVALID)); else return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(0), - __pgprot(PTE_VALID)); + __pgprot(PTE_PRESENT_INVALID), + __pgprot(PTE_PRESENT_VALID_KERNEL)); } int set_direct_map_invalid_noflush(struct page *page) { - pgprot_t clear_mask = __pgprot(PTE_VALID); - pgprot_t set_mask = __pgprot(0); + pgprot_t clear_mask = __pgprot(PTE_PRESENT_VALID_KERNEL); + pgprot_t set_mask = __pgprot(PTE_PRESENT_INVALID); if (!can_set_direct_map()) return 0; @@ -259,8 +265,8 @@ int set_direct_map_invalid_noflush(struct page *page) int set_direct_map_default_noflush(struct page *page) { - pgprot_t set_mask = __pgprot(PTE_VALID | PTE_WRITE); - pgprot_t clear_mask = __pgprot(PTE_RDONLY); + pgprot_t set_mask = __pgprot(PTE_PRESENT_VALID_KERNEL | PTE_WRITE); + pgprot_t clear_mask = __pgprot(PTE_PRESENT_INVALID | PTE_RDONLY); if (!can_set_direct_map()) return 0; @@ -296,8 +302,8 @@ static int __set_memory_enc_dec(unsigned long addr, * entries or Synchronous External Aborts caused by RIPAS_EMPTY */ ret = __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(set_prot), - __pgprot(clear_prot | PTE_VALID)); + __pgprot(set_prot | PTE_PRESENT_INVALID), + __pgprot(clear_prot | PTE_PRESENT_VALID_KERNEL)); if (ret) return ret; @@ -311,8 +317,8 @@ static int __set_memory_enc_dec(unsigned long addr, return ret; return __change_memory_common(addr, PAGE_SIZE * numpages, - __pgprot(PTE_VALID), - __pgprot(0)); + __pgprot(PTE_PRESENT_VALID_KERNEL), + __pgprot(PTE_PRESENT_INVALID)); } static int realm_set_memory_encrypted(unsigned long addr, int numpages) @@ -404,15 +410,15 @@ bool kernel_page_present(struct page *page) pud = READ_ONCE(*pudp); if (pud_none(pud)) return false; - if (pud_sect(pud)) - return true; + if (pud_leaf(pud)) + return pud_valid(pud); pmdp = pmd_offset(pudp, addr); pmd = READ_ONCE(*pmdp); if (pmd_none(pmd)) return false; - if (pmd_sect(pmd)) - return true; + if (pmd_leaf(pmd)) + return pmd_valid(pmd); ptep = pte_offset_kernel(pmdp, addr); return pte_valid(__ptep_get(ptep)); diff --git a/arch/arm64/mm/trans_pgd.c b/arch/arm64/mm/trans_pgd.c index 18543b603c77bd..cca9706a875c30 100644 --- a/arch/arm64/mm/trans_pgd.c +++ b/arch/arm64/mm/trans_pgd.c @@ -31,36 +31,6 @@ static void *trans_alloc(struct trans_pgd_info *info) return info->trans_alloc_page(info->trans_alloc_arg); } -static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) -{ - pte_t pte = __ptep_get(src_ptep); - - if (pte_valid(pte)) { - /* - * Resume will overwrite areas that may be marked - * read only (code, rodata). Clear the RDONLY bit from - * the temporary mappings we use during restore. - */ - __set_pte(dst_ptep, pte_mkwrite_novma(pte)); - } else if (!pte_none(pte)) { - /* - * debug_pagealloc will removed the PTE_VALID bit if - * the page isn't in use by the resume kernel. It may have - * been in use by the original kernel, in which case we need - * to put it back in our copy to do the restore. - * - * Other cases include kfence / vmalloc / memfd_secret which - * may call `set_direct_map_invalid_noflush()`. - * - * Before marking this entry valid, check the pfn should - * be mapped. - */ - BUG_ON(!pfn_valid(pte_pfn(pte))); - - __set_pte(dst_ptep, pte_mkvalid(pte_mkwrite_novma(pte))); - } -} - static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long start, unsigned long end) { @@ -76,7 +46,11 @@ static int copy_pte(struct trans_pgd_info *info, pmd_t *dst_pmdp, src_ptep = pte_offset_kernel(src_pmdp, start); do { - _copy_pte(dst_ptep, src_ptep, addr); + pte_t pte = __ptep_get(src_ptep); + + if (pte_none(pte)) + continue; + __set_pte(dst_ptep, pte_mkvalid_k(pte_mkwrite_novma(pte))); } while (dst_ptep++, src_ptep++, addr += PAGE_SIZE, addr != end); return 0; @@ -109,8 +83,7 @@ static int copy_pmd(struct trans_pgd_info *info, pud_t *dst_pudp, if (copy_pte(info, dst_pmdp, src_pmdp, addr, next)) return -ENOMEM; } else { - set_pmd(dst_pmdp, - __pmd(pmd_val(pmd) & ~PMD_SECT_RDONLY)); + set_pmd(dst_pmdp, pmd_mkvalid_k(pmd_mkwrite_novma(pmd))); } } while (dst_pmdp++, src_pmdp++, addr = next, addr != end); @@ -145,8 +118,7 @@ static int copy_pud(struct trans_pgd_info *info, p4d_t *dst_p4dp, if (copy_pmd(info, dst_pudp, src_pudp, addr, next)) return -ENOMEM; } else { - set_pud(dst_pudp, - __pud(pud_val(pud) & ~PUD_SECT_RDONLY)); + set_pud(dst_pudp, pud_mkvalid_k(pud_mkwrite_novma(pud))); } } while (dst_pudp++, src_pudp++, addr = next, addr != end); diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index adf84962d579d0..524b67c0867e75 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -18,7 +18,6 @@ #include #include -#include #include #include #include @@ -35,8 +34,8 @@ #define ARENA_VM_START (MAX_BPF_JIT_REG + 5) #define check_imm(bits, imm) do { \ - if ((((imm) > 0) && ((imm) >> (bits))) || \ - (((imm) < 0) && (~(imm) >> (bits)))) { \ + if ((((imm) > 0) && ((imm) >> ((bits) - 1))) || \ + (((imm) < 0) && (~(imm) >> ((bits) - 1)))) { \ pr_info("[%2d] imm=%d(0x%x) out of range\n", \ i, imm, imm); \ return -EINVAL; \ @@ -1961,11 +1960,6 @@ static int validate_ctx(struct jit_ctx *ctx) return 0; } -static inline void bpf_flush_icache(void *start, void *end) -{ - flush_icache_range((unsigned long)start, (unsigned long)end); -} - static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size) { int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3; @@ -2204,12 +2198,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog = orig_prog; goto out_off; } - /* - * The instructions have now been copied to the ROX region from - * where they will execute. Now the data cache has to be cleaned to - * the PoU and the I-cache has to be invalidated for the VAs. - */ - bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx); } else { jit_data->ctx = ctx; jit_data->ro_image = ro_image_ptr; diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 7261553b644b2b..488ffccbfc98eb 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -8,6 +8,8 @@ BTI # Unreliable: use system_supports_32bit_el0() instead. HAS_32BIT_EL0_DO_NOT_USE HAS_32BIT_EL1 +HAS_ACTLR_VIRT +HAS_ACTLR_VIRT_APPLE HAS_ADDRESS_AUTH HAS_ADDRESS_AUTH_ARCH_QARMA3 HAS_ADDRESS_AUTH_ARCH_QARMA5 @@ -64,6 +66,8 @@ HAS_STAGE2_FWB HAS_TCR2 HAS_TIDCP1 HAS_TLB_RANGE +HAS_TSO_APPLE +HAS_TSO_FIXED HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT diff --git a/arch/loongarch/Kbuild b/arch/loongarch/Kbuild index beb8499dd8ed84..1c7a0dbe5e72f2 100644 --- a/arch/loongarch/Kbuild +++ b/arch/loongarch/Kbuild @@ -3,7 +3,7 @@ obj-y += mm/ obj-y += net/ obj-y += vdso/ -obj-$(CONFIG_KVM) += kvm/ +obj-$(subst m,y,$(CONFIG_KVM)) += kvm/ # for cleaning subdir- += boot diff --git a/arch/loongarch/include/asm/asm-prototypes.h b/arch/loongarch/include/asm/asm-prototypes.h index 704066b4f7368b..de0c17f3f49c2c 100644 --- a/arch/loongarch/include/asm/asm-prototypes.h +++ b/arch/loongarch/include/asm/asm-prototypes.h @@ -20,3 +20,23 @@ asmlinkage void noinstr __no_stack_protector ret_from_kernel_thread(struct task_ struct pt_regs *regs, int (*fn)(void *), void *fn_arg); + +struct kvm_run; +struct kvm_vcpu; +struct loongarch_fpu; + +void kvm_exc_entry(void); +int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu); + +void kvm_save_fpu(struct loongarch_fpu *fpu); +void kvm_restore_fpu(struct loongarch_fpu *fpu); + +#ifdef CONFIG_CPU_HAS_LSX +void kvm_save_lsx(struct loongarch_fpu *fpu); +void kvm_restore_lsx(struct loongarch_fpu *fpu); +#endif + +#ifdef CONFIG_CPU_HAS_LASX +void kvm_save_lasx(struct loongarch_fpu *fpu); +void kvm_restore_lasx(struct loongarch_fpu *fpu); +#endif diff --git a/arch/loongarch/include/asm/irq_work.h b/arch/loongarch/include/asm/irq_work.h index d63076e9160d56..63aee0335d1a36 100644 --- a/arch/loongarch/include/asm/irq_work.h +++ b/arch/loongarch/include/asm/irq_work.h @@ -4,7 +4,7 @@ static inline bool arch_irq_work_has_interrupt(void) { - return IS_ENABLED(CONFIG_SMP); + return IS_ENABLED(CONFIG_SMP) && cpu_opt(LOONGARCH_CPU_CSRIPI); } #endif /* _ASM_LOONGARCH_IRQ_WORK_H */ diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 19eb5e5c39841a..0bcdffc14c5fb1 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -86,7 +86,6 @@ struct kvm_context { struct kvm_world_switch { int (*exc_entry)(void); int (*enter_guest)(struct kvm_run *run, struct kvm_vcpu *vcpu); - unsigned long page_order; }; #define MAX_PGTABLE_LEVELS 4 @@ -356,8 +355,6 @@ void kvm_exc_entry(void); int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu); extern unsigned long vpid_mask; -extern const unsigned long kvm_exception_size; -extern const unsigned long kvm_enter_guest_size; extern struct kvm_world_switch *kvm_loongarch_ops; #define SW_GCSR (1 << 0) diff --git a/arch/loongarch/include/asm/linkage.h b/arch/loongarch/include/asm/linkage.h index a1bd6a3ee03a19..ae937d1708b247 100644 --- a/arch/loongarch/include/asm/linkage.h +++ b/arch/loongarch/include/asm/linkage.h @@ -69,7 +69,7 @@ 9, 10, 11, 12, 13, 14, 15, 16, \ 17, 18, 19, 20, 21, 22, 23, 24, \ 25, 26, 27, 28, 29, 30, 31; \ - .cfi_offset \num, SC_REGS + \num * SZREG; \ + .cfi_offset \num, SC_REGS + \num * 8; \ .endr; \ \ nop; \ diff --git a/arch/loongarch/kernel/cpu-probe.c b/arch/loongarch/kernel/cpu-probe.c index 657bbae6c1c7ea..82cf426faafda5 100644 --- a/arch/loongarch/kernel/cpu-probe.c +++ b/arch/loongarch/kernel/cpu-probe.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -402,3 +403,9 @@ void cpu_probe(void) cpu_report(); } + +ssize_t cpu_show_spectre_v1(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n"); +} diff --git a/arch/loongarch/kernel/syscall.c b/arch/loongarch/kernel/syscall.c index 1249d82c1cd0ac..dac435c3274337 100644 --- a/arch/loongarch/kernel/syscall.c +++ b/arch/loongarch/kernel/syscall.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -74,7 +75,7 @@ void noinstr __no_stack_protector do_syscall(struct pt_regs *regs) add_random_kstack_offset(); if (nr < NR_syscalls) { - syscall_fn = sys_call_table[nr]; + syscall_fn = sys_call_table[array_index_nospec(nr, NR_syscalls)]; regs->regs[4] = syscall_fn(regs->orig_a0, regs->regs[5], regs->regs[6], regs->regs[7], regs->regs[8], regs->regs[9]); } diff --git a/arch/loongarch/kvm/Makefile b/arch/loongarch/kvm/Makefile index cb41d9265662f4..f32a170c18388b 100644 --- a/arch/loongarch/kvm/Makefile +++ b/arch/loongarch/kvm/Makefile @@ -7,11 +7,12 @@ include $(srctree)/virt/kvm/Makefile.kvm obj-$(CONFIG_KVM) += kvm.o +obj-y += switch.o + kvm-y += exit.o kvm-y += interrupt.o kvm-y += main.o kvm-y += mmu.o -kvm-y += switch.o kvm-y += timer.o kvm-y += tlb.o kvm-y += vcpu.o diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index da0ad89f2eb746..3b95cd0f989b08 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -390,6 +390,7 @@ int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) run->mmio.len = 8; break; default: + ret = EMULATE_FAIL; break; } break; diff --git a/arch/loongarch/kvm/interrupt.c b/arch/loongarch/kvm/interrupt.c index fb704f4c8ac593..656092e19062a5 100644 --- a/arch/loongarch/kvm/interrupt.c +++ b/arch/loongarch/kvm/interrupt.c @@ -27,6 +27,7 @@ static unsigned int priority_to_irq[EXCCODE_INT_NUM] = { static int kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority) { unsigned int irq = 0; + unsigned long old, new; clear_bit(priority, &vcpu->arch.irq_pending); if (priority < EXCCODE_INT_NUM) @@ -42,7 +43,13 @@ static int kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority) case INT_IPI: case INT_SWI0: case INT_SWI1: + old = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL); set_gcsr_estat(irq); + new = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL); + + /* Inject TI if TVAL inverted */ + if (new > old) + set_gcsr_estat(CPU_TIMER); break; case INT_HWI0 ... INT_HWI7: @@ -59,6 +66,7 @@ static int kvm_irq_deliver(struct kvm_vcpu *vcpu, unsigned int priority) static int kvm_irq_clear(struct kvm_vcpu *vcpu, unsigned int priority) { unsigned int irq = 0; + unsigned long old, new; clear_bit(priority, &vcpu->arch.irq_clear); if (priority < EXCCODE_INT_NUM) @@ -74,7 +82,13 @@ static int kvm_irq_clear(struct kvm_vcpu *vcpu, unsigned int priority) case INT_IPI: case INT_SWI0: case INT_SWI1: + old = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL); clear_gcsr_estat(irq); + new = kvm_read_hw_gcsr(LOONGARCH_CSR_TVAL); + + /* Inject TI if TVAL inverted */ + if (new > old) + set_gcsr_estat(CPU_TIMER); break; case INT_HWI0 ... INT_HWI7: diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 2c593ac7892f6c..18800a38b150b2 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -348,8 +348,7 @@ void kvm_arch_disable_virtualization_cpu(void) static int kvm_loongarch_env_init(void) { - int cpu, order, ret; - void *addr; + int cpu, ret; struct kvm_context *context; vmcs = alloc_percpu(struct kvm_context); @@ -365,30 +364,8 @@ static int kvm_loongarch_env_init(void) return -ENOMEM; } - /* - * PGD register is shared between root kernel and kvm hypervisor. - * So world switch entry should be in DMW area rather than TLB area - * to avoid page fault reenter. - * - * In future if hardware pagetable walking is supported, we won't - * need to copy world switch code to DMW area. - */ - order = get_order(kvm_exception_size + kvm_enter_guest_size); - addr = (void *)__get_free_pages(GFP_KERNEL, order); - if (!addr) { - free_percpu(vmcs); - vmcs = NULL; - kfree(kvm_loongarch_ops); - kvm_loongarch_ops = NULL; - return -ENOMEM; - } - - memcpy(addr, kvm_exc_entry, kvm_exception_size); - memcpy(addr + kvm_exception_size, kvm_enter_guest, kvm_enter_guest_size); - flush_icache_range((unsigned long)addr, (unsigned long)addr + kvm_exception_size + kvm_enter_guest_size); - kvm_loongarch_ops->exc_entry = addr; - kvm_loongarch_ops->enter_guest = addr + kvm_exception_size; - kvm_loongarch_ops->page_order = order; + kvm_loongarch_ops->exc_entry = (void *)kvm_exc_entry; + kvm_loongarch_ops->enter_guest = (void *)kvm_enter_guest; vpid_mask = read_csr_gstat(); vpid_mask = (vpid_mask & CSR_GSTAT_GIDBIT) >> CSR_GSTAT_GIDBIT_SHIFT; @@ -422,16 +399,10 @@ static int kvm_loongarch_env_init(void) static void kvm_loongarch_env_exit(void) { - unsigned long addr; - if (vmcs) free_percpu(vmcs); if (kvm_loongarch_ops) { - if (kvm_loongarch_ops->exc_entry) { - addr = (unsigned long)kvm_loongarch_ops->exc_entry; - free_pages(addr, kvm_loongarch_ops->page_order); - } kfree(kvm_loongarch_ops); } diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index a7fa458e33605e..e104897aa53285 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -95,7 +95,7 @@ static int kvm_flush_pte(kvm_pte_t *pte, phys_addr_t addr, kvm_ptw_ctx *ctx) else kvm->stat.pages--; - *pte = ctx->invalid_entry; + kvm_set_pte(pte, ctx->invalid_entry); return 1; } diff --git a/arch/loongarch/kvm/switch.S b/arch/loongarch/kvm/switch.S index f1768b7a619497..936e4ae3e40859 100644 --- a/arch/loongarch/kvm/switch.S +++ b/arch/loongarch/kvm/switch.S @@ -4,9 +4,11 @@ */ #include +#include #include #include #include +#include #include #include @@ -100,11 +102,16 @@ * - is still in guest mode, such as pgd table/vmid registers etc, * - will fix with hw page walk enabled in future * load kvm_vcpu from reserved CSR KVM_VCPU_KS, and save a2 to KVM_TEMP_KS + * + * PGD register is shared between root kernel and kvm hypervisor. + * So world switch entry should be in DMW area rather than TLB area + * to avoid page fault re-enter. */ .text + .p2align PAGE_SHIFT .cfi_sections .debug_frame SYM_CODE_START(kvm_exc_entry) - UNWIND_HINT_UNDEFINED + UNWIND_HINT_END_OF_STACK csrwr a2, KVM_TEMP_KS csrrd a2, KVM_VCPU_KS addi.d a2, a2, KVM_VCPU_ARCH @@ -190,8 +197,8 @@ ret_to_host: kvm_restore_host_gpr a2 jr ra -SYM_INNER_LABEL(kvm_exc_entry_end, SYM_L_LOCAL) SYM_CODE_END(kvm_exc_entry) +EXPORT_SYMBOL_FOR_KVM(kvm_exc_entry) /* * int kvm_enter_guest(struct kvm_run *run, struct kvm_vcpu *vcpu) @@ -215,8 +222,8 @@ SYM_FUNC_START(kvm_enter_guest) /* Save kvm_vcpu to kscratch */ csrwr a1, KVM_VCPU_KS kvm_switch_to_guest -SYM_INNER_LABEL(kvm_enter_guest_end, SYM_L_LOCAL) SYM_FUNC_END(kvm_enter_guest) +EXPORT_SYMBOL_FOR_KVM(kvm_enter_guest) SYM_FUNC_START(kvm_save_fpu) fpu_save_csr a0 t1 @@ -224,6 +231,7 @@ SYM_FUNC_START(kvm_save_fpu) fpu_save_cc a0 t1 t2 jr ra SYM_FUNC_END(kvm_save_fpu) +EXPORT_SYMBOL_FOR_KVM(kvm_save_fpu) SYM_FUNC_START(kvm_restore_fpu) fpu_restore_double a0 t1 @@ -231,6 +239,7 @@ SYM_FUNC_START(kvm_restore_fpu) fpu_restore_cc a0 t1 t2 jr ra SYM_FUNC_END(kvm_restore_fpu) +EXPORT_SYMBOL_FOR_KVM(kvm_restore_fpu) #ifdef CONFIG_CPU_HAS_LSX SYM_FUNC_START(kvm_save_lsx) @@ -239,6 +248,7 @@ SYM_FUNC_START(kvm_save_lsx) lsx_save_data a0 t1 jr ra SYM_FUNC_END(kvm_save_lsx) +EXPORT_SYMBOL_FOR_KVM(kvm_save_lsx) SYM_FUNC_START(kvm_restore_lsx) lsx_restore_data a0 t1 @@ -246,6 +256,7 @@ SYM_FUNC_START(kvm_restore_lsx) fpu_restore_csr a0 t1 t2 jr ra SYM_FUNC_END(kvm_restore_lsx) +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lsx) #endif #ifdef CONFIG_CPU_HAS_LASX @@ -255,6 +266,7 @@ SYM_FUNC_START(kvm_save_lasx) lasx_save_data a0 t1 jr ra SYM_FUNC_END(kvm_save_lasx) +EXPORT_SYMBOL_FOR_KVM(kvm_save_lasx) SYM_FUNC_START(kvm_restore_lasx) lasx_restore_data a0 t1 @@ -262,10 +274,8 @@ SYM_FUNC_START(kvm_restore_lasx) fpu_restore_csr a0 t1 t2 jr ra SYM_FUNC_END(kvm_restore_lasx) +EXPORT_SYMBOL_FOR_KVM(kvm_restore_lasx) #endif - .section ".rodata" -SYM_DATA(kvm_exception_size, .quad kvm_exc_entry_end - kvm_exc_entry) -SYM_DATA(kvm_enter_guest_size, .quad kvm_enter_guest_end - kvm_enter_guest) #ifdef CONFIG_CPU_HAS_LBT STACK_FRAME_NON_STANDARD kvm_restore_fpu diff --git a/arch/loongarch/kvm/timer.c b/arch/loongarch/kvm/timer.c index 29c2aaba63c33b..8356fce0043f60 100644 --- a/arch/loongarch/kvm/timer.c +++ b/arch/loongarch/kvm/timer.c @@ -96,15 +96,21 @@ void kvm_restore_timer(struct kvm_vcpu *vcpu) * and set CSR TVAL with -1 */ write_gcsr_timertick(0); - __delay(2); /* Wait cycles until timer interrupt injected */ /* * Writing CSR_TINTCLR_TI to LOONGARCH_CSR_TINTCLR will clear * timer interrupt, and CSR TVAL keeps unchanged with -1, it * avoids spurious timer interrupt */ - if (!(estat & CPU_TIMER)) + if (!(estat & CPU_TIMER)) { + __delay(2); /* Wait cycles until timer interrupt injected */ + + /* Write TVAL with max value if no TI shot */ + estat = kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT); + if (!(estat & CPU_TIMER)) + write_gcsr_timertick(CSR_TCFG_VAL); gcsr_write(CSR_TINTCLR_TI, LOONGARCH_CSR_TINTCLR); + } return; } diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 831f381a8fd121..ed2cfcd76f6073 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -402,7 +402,7 @@ bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) val = gcsr_read(LOONGARCH_CSR_CRMD); preempt_enable(); - return (val & CSR_PRMD_PPLV) == PLV_KERN; + return (val & CSR_CRMD_PLV) == PLV_KERN; } #ifdef CONFIG_GUEST_PERF_EVENTS diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index 8cc5ee1c53efbe..1317c718f896af 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -125,7 +125,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = 1; break; case KVM_CAP_NR_VCPUS: - r = num_online_cpus(); + r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS); break; case KVM_CAP_MAX_VCPUS: r = KVM_MAX_VCPUS; diff --git a/arch/loongarch/pci/acpi.c b/arch/loongarch/pci/acpi.c index 0dde3ddcd54436..b02698a338eefe 100644 --- a/arch/loongarch/pci/acpi.c +++ b/arch/loongarch/pci/acpi.c @@ -61,11 +61,16 @@ static void acpi_release_root_info(struct acpi_pci_root_info *ci) static int acpi_prepare_root_resources(struct acpi_pci_root_info *ci) { int status; + unsigned long long pci_h = 0; struct resource_entry *entry, *tmp; struct acpi_device *device = ci->bridge; status = acpi_pci_probe_root_resources(ci); if (status > 0) { + acpi_evaluate_integer(device->handle, "PCIH", NULL, &pci_h); + if (pci_h) + return status; + resource_list_for_each_entry_safe(entry, tmp, &ci->resources) { if (entry->res->flags & IORESOURCE_MEM) { entry->offset = ci->root->mcfg_addr & GENMASK_ULL(63, 40); diff --git a/arch/loongarch/pci/pci.c b/arch/loongarch/pci/pci.c index d233ea2218fe0a..f33c7ea1443d94 100644 --- a/arch/loongarch/pci/pci.c +++ b/arch/loongarch/pci/pci.c @@ -132,6 +132,9 @@ static void loongson_gpu_fixup_dma_hang(struct pci_dev *pdev, bool on) crtc_reg = regbase; crtc_offset = 0x400; break; + default: + iounmap(regbase); + return; } for (i = 0; i < CRTC_NUM_MAX; i++, crtc_reg += crtc_offset) { diff --git a/arch/mips/kernel/linux32.c b/arch/mips/kernel/linux32.c index a0c0a7a654e941..fe9a787db56941 100644 --- a/arch/mips/kernel/linux32.c +++ b/arch/mips/kernel/linux32.c @@ -60,7 +60,7 @@ SYSCALL_DEFINE4(32_truncate64, const char __user *, path, SYSCALL_DEFINE4(32_ftruncate64, unsigned long, fd, unsigned long, __dummy, unsigned long, a2, unsigned long, a3) { - return ksys_ftruncate(fd, merge_64(a2, a3)); + return ksys_ftruncate(fd, merge_64(a2, a3), FTRUNCATE_LFS); } SYSCALL_DEFINE5(32_llseek, unsigned int, fd, unsigned int, offset_high, diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 62d5a89d5c7bcd..450c9283951109 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -8,6 +8,7 @@ config PARISC select HAVE_FUNCTION_GRAPH_TRACER select HAVE_SYSCALL_TRACEPOINTS select ARCH_WANT_FRAME_POINTERS + select ARCH_WANTS_MODULES_TEXT_SECTIONS if !64BIT select ARCH_HAS_CPU_CACHE_ALIASING select ARCH_HAS_DMA_ALLOC if PA11 select ARCH_HAS_DMA_OPS @@ -130,6 +131,9 @@ config GENERIC_BUG config GENERIC_BUG_RELATIVE_POINTERS bool +config GENERIC_CSUM + def_bool y + config GENERIC_HWEIGHT bool default y diff --git a/arch/parisc/include/asm/checksum.h b/arch/parisc/include/asm/checksum.h index 2aceebcd695c80..382758808726a0 100644 --- a/arch/parisc/include/asm/checksum.h +++ b/arch/parisc/include/asm/checksum.h @@ -4,73 +4,7 @@ #include -/* - * computes the checksum of a memory block at buff, length len, - * and adds in "sum" (32-bit) - * - * returns a 32-bit number suitable for feeding into itself - * or csum_tcpudp_magic - * - * this function must be called with even lengths, except - * for the last fragment, which may be odd - * - * it's best to have buff aligned on a 32-bit boundary - */ -extern __wsum csum_partial(const void *, int, __wsum); - -/* - * Optimized for IP headers, which always checksum on 4 octet boundaries. - * - * Written by Randolph Chung , and then mucked with by - * LaMont Jones - */ -static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) -{ - unsigned int sum; - unsigned long t0, t1, t2; - - __asm__ __volatile__ ( -" ldws,ma 4(%1), %0\n" -" addib,<= -4, %2, 2f\n" -"\n" -" ldws 4(%1), %4\n" -" ldws 8(%1), %5\n" -" add %0, %4, %0\n" -" ldws,ma 12(%1), %3\n" -" addc %0, %5, %0\n" -" addc %0, %3, %0\n" -"1: ldws,ma 4(%1), %3\n" -" addib,> -1, %2, 1b\n" -" addc %0, %3, %0\n" -"\n" -" extru %0, 31, 16, %4\n" -" extru %0, 15, 16, %5\n" -" addc %4, %5, %0\n" -" extru %0, 15, 16, %5\n" -" add %0, %5, %0\n" -" subi -1, %0, %0\n" -"2:\n" - : "=r" (sum), "=r" (iph), "=r" (ihl), "=r" (t0), "=r" (t1), "=r" (t2) - : "1" (iph), "2" (ihl) - : "memory"); - - return (__force __sum16)sum; -} - -/* - * Fold a partial checksum - */ -static inline __sum16 csum_fold(__wsum csum) -{ - u32 sum = (__force u32)csum; - /* add the swapped two 16-bit halves of sum, - a possible carry from adding the two 16-bit halves, - will carry from the lower half into the upper half, - giving us the correct sum in the upper half. */ - sum += (sum << 16) + (sum >> 16); - return (__force __sum16)(~sum >> 16); -} - +#define csum_tcpudp_nofold csum_tcpudp_nofold static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, __u8 proto, __wsum sum) @@ -85,26 +19,7 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, return sum; } -/* - * computes the checksum of the TCP/UDP pseudo-header - * returns a 16-bit checksum, already complemented - */ -static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - __u32 len, __u8 proto, - __wsum sum) -{ - return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); -} - -/* - * this routine is used for miscellaneous IP-like checksums, mainly - * in icmp.c - */ -static inline __sum16 ip_compute_csum(const void *buf, int len) -{ - return csum_fold (csum_partial(buf, len, 0)); -} - +#include #define _HAVE_ARCH_IPV6_CSUM static __inline__ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index b2cdbb8a12b16e..fcb0d806913925 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -216,7 +216,7 @@ asmlinkage long parisc_truncate64(const char __user * path, asmlinkage long parisc_ftruncate64(unsigned int fd, unsigned int high, unsigned int low) { - return ksys_ftruncate(fd, (long)high << 32 | low); + return ksys_ftruncate(fd, (long)high << 32 | low, FTRUNCATE_LFS); } /* stubs for the benefit of the syscall_table since truncate64 and truncate @@ -227,7 +227,7 @@ asmlinkage long sys_truncate64(const char __user * path, unsigned long length) } asmlinkage long sys_ftruncate64(unsigned int fd, unsigned long length) { - return ksys_ftruncate(fd, length); + return ksys_ftruncate(fd, length, FTRUNCATE_LFS); } asmlinkage long sys_fcntl64(unsigned int fd, unsigned int cmd, unsigned long arg) { diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index f6e2d0379d57c9..c6331dad94613b 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -154,7 +154,7 @@ # 137 was afs_syscall 138 common setfsuid sys_setfsuid 139 common setfsgid sys_setfsgid -140 common _llseek sys_llseek +140 32 _llseek sys_llseek 141 common getdents sys_getdents compat_sys_getdents 142 common _newselect sys_select compat_sys_select 143 common flock sys_flock diff --git a/arch/parisc/lib/Makefile b/arch/parisc/lib/Makefile index 7b197667faf6cd..d5975d1fb40688 100644 --- a/arch/parisc/lib/Makefile +++ b/arch/parisc/lib/Makefile @@ -3,7 +3,7 @@ # Makefile for parisc-specific library files # -lib-y := lusercopy.o bitops.o checksum.o io.o memset.o memcpy.o \ +lib-y := lusercopy.o bitops.o io.o memset.o memcpy.o \ ucmpdi2.o delay.o obj-y := iomap.o diff --git a/arch/parisc/lib/checksum.c b/arch/parisc/lib/checksum.c deleted file mode 100644 index 59d8c15d81bd09..00000000000000 --- a/arch/parisc/lib/checksum.c +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * INET An implementation of the TCP/IP protocol suite for the LINUX - * operating system. INET is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * MIPS specific IP/TCP/UDP checksumming routines - * - * Authors: Ralf Baechle, - * Lots of code moved from tcp.c and ip.c; see those files - * for more names. - */ -#include -#include - -#include -#include -#include -#include - -#define addc(_t,_r) \ - __asm__ __volatile__ ( \ -" add %0, %1, %0\n" \ -" addc %0, %%r0, %0\n" \ - : "=r"(_t) \ - : "r"(_r), "0"(_t)); - -static inline unsigned int do_csum(const unsigned char * buff, int len) -{ - int odd, count; - unsigned int result = 0; - - if (len <= 0) - goto out; - odd = 1 & (unsigned long) buff; - if (odd) { - result = be16_to_cpu(*buff); - len--; - buff++; - } - count = len >> 1; /* nr of 16-bit words.. */ - if (count) { - if (2 & (unsigned long) buff) { - result += *(unsigned short *) buff; - count--; - len -= 2; - buff += 2; - } - count >>= 1; /* nr of 32-bit words.. */ - if (count) { - while (count >= 4) { - unsigned int r1, r2, r3, r4; - r1 = *(unsigned int *)(buff + 0); - r2 = *(unsigned int *)(buff + 4); - r3 = *(unsigned int *)(buff + 8); - r4 = *(unsigned int *)(buff + 12); - addc(result, r1); - addc(result, r2); - addc(result, r3); - addc(result, r4); - count -= 4; - buff += 16; - } - while (count) { - unsigned int w = *(unsigned int *) buff; - count--; - buff += 4; - addc(result, w); - } - result = (result & 0xffff) + (result >> 16); - } - if (len & 2) { - result += *(unsigned short *) buff; - buff += 2; - } - } - if (len & 1) - result += le16_to_cpu(*buff); - result = csum_from32to16(result); - if (odd) - result = swab16(result); -out: - return result; -} - -/* - * computes a partial checksum, e.g. for TCP/UDP fragments - */ -/* - * why bother folding? - */ -__wsum csum_partial(const void *buff, int len, __wsum sum) -{ - unsigned int result = do_csum(buff, len); - addc(result, sum); - return (__force __wsum)csum_from32to16(result); -} - -EXPORT_SYMBOL(csum_partial); diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 1a91762b455d93..66a953046a49a9 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -1313,12 +1313,27 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, { pmd_t old_pmd; + /* + * Non-present PMDs can be migration entries or device-private THP + * entries. This can happen at 2 places: + * - When the address space is being unmapped zap_huge_pmd(), and we + * encounter non-present pmds. + * - migrate_vma_collect_huge_pmd() could calls this during migration + * of device-private pmd entries. + */ + if (!pmd_present(*pmdp)) { + old_pmd = READ_ONCE(*pmdp); + pmd_clear(pmdp); + goto out; + } + if (radix_enabled()) { old_pmd = radix__pmdp_huge_get_and_clear(mm, addr, pmdp); } else { old_pmd = hash__pmdp_huge_get_and_clear(mm, addr, pmdp); } +out: page_table_check_pmd_clear(mm, addr, old_pmd); return old_pmd; diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index bd4a6c42a5f38f..e02710d6a2e1a3 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -66,11 +66,9 @@ void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_co unsigned long start_address) __noreturn; void kexec_copy_flush(struct kimage *image); -#ifdef CONFIG_KEXEC_FILE -extern const struct kexec_file_ops kexec_elf64_ops; +#if defined(CONFIG_KEXEC_FILE) || defined(CONFIG_CRASH_DUMP) #define ARCH_HAS_KIMAGE_ARCH - struct kimage_arch { struct crash_mem *exclude_ranges; @@ -78,6 +76,10 @@ struct kimage_arch { void *backup_buf; void *fdt; }; +#endif + +#ifdef CONFIG_KEXEC_FILE +extern const struct kexec_file_ops kexec_elf64_ops; char *setup_kdump_cmdline(struct kimage *image, char *cmdline, unsigned long cmdline_len); @@ -145,6 +147,10 @@ int arch_crash_hotplug_support(struct kimage *image, unsigned long kexec_flags); unsigned int arch_crash_get_elfcorehdr_size(void); #define crash_get_elfcorehdr_size arch_crash_get_elfcorehdr_size + +int machine_kexec_post_load(struct kimage *image); +#define machine_kexec_post_load machine_kexec_post_load + #endif /* CONFIG_CRASH_HOTPLUG */ extern int crashing_cpu; @@ -159,6 +165,8 @@ extern void default_machine_crash_shutdown(struct pt_regs *regs); extern void crash_kexec_prepare(void); extern void crash_kexec_secondary(struct pt_regs *regs); +extern void sync_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr, + bool phdr_to_kimage); static inline bool kdump_in_progress(void) { return crashing_cpu >= 0; diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 17e63244e88552..e98c628e389938 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -434,8 +434,7 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n) } #endif -extern long __copy_from_user_flushcache(void *dst, const void __user *src, - unsigned size); +extern size_t copy_from_user_flushcache(void *dst, const void __user *src, size_t size); static __must_check __always_inline bool __user_access_begin(const void __user *ptr, size_t len, unsigned long dir) diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c index d451a8229223a3..03fa487f26147c 100644 --- a/arch/powerpc/kernel/sys_ppc32.c +++ b/arch/powerpc/kernel/sys_ppc32.c @@ -101,7 +101,7 @@ PPC32_SYSCALL_DEFINE4(ppc_ftruncate64, unsigned int, fd, u32, reg4, unsigned long, len1, unsigned long, len2) { - return ksys_ftruncate(fd, merge_64(len1, len2)); + return ksys_ftruncate(fd, merge_64(len1, len2), FTRUNCATE_LFS); } PPC32_SYSCALL_DEFINE6(ppc32_fadvise64, diff --git a/arch/powerpc/kexec/Makefile b/arch/powerpc/kexec/Makefile index 470eb0453e17f3..ec7a0eed75dc59 100644 --- a/arch/powerpc/kexec/Makefile +++ b/arch/powerpc/kexec/Makefile @@ -16,4 +16,4 @@ GCOV_PROFILE_core_$(BITS).o := n KCOV_INSTRUMENT_core_$(BITS).o := n UBSAN_SANITIZE_core_$(BITS).o := n KASAN_SANITIZE_core.o := n -KASAN_SANITIZE_core_$(BITS) := n +KASAN_SANITIZE_core_$(BITS).o := n diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index a325c1c02f96dc..e6539f213b3d14 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -27,6 +27,7 @@ #include #include #include +#include /* * The primary CPU waits a while for all secondary CPUs to enter. This is to @@ -399,7 +400,68 @@ void default_machine_crash_shutdown(struct pt_regs *regs) ppc_md.kexec_cpu_down(1, 0); } +#ifdef CONFIG_CRASH_DUMP +/** + * sync_backup_region_phdr - synchronize backup region offset between + * kexec image and ELF core header. + * @image: Kexec image. + * @ehdr: ELF core header. + * @phdr_to_kimage: If true, read the offset from the ELF program header + * and update the kimage backup region. If false, update + * the ELF program header offset from the kimage backup + * region. + * + * Note: During kexec_load, this is called with phdr_to_kimage = true. For + * kexec_file_load and ELF core header recreation during memory hotplug + * events, it is called with phdr_to_kimage = false. + * + * Returns nothing. + */ +void sync_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr, bool phdr_to_kimage) +{ + Elf64_Phdr *phdr; + unsigned int i; + + phdr = (Elf64_Phdr *)(ehdr + 1); + for (i = 0; i < ehdr->e_phnum; i++, phdr++) { + if (phdr->p_paddr == BACKUP_SRC_START) { + if (phdr_to_kimage) + image->arch.backup_start = phdr->p_offset; + else + phdr->p_offset = image->arch.backup_start; + + kexec_dprintk("Backup region offset updated to 0x%lx\n", + image->arch.backup_start); + return; + } + } +} +#endif /* CONFIG_CRASH_DUMP */ + #ifdef CONFIG_CRASH_HOTPLUG + +int machine_kexec_post_load(struct kimage *image) +{ + int i; + unsigned long mem; + unsigned char *ptr; + + if (image->type != KEXEC_TYPE_CRASH) + return 0; + + if (image->file_mode) + return 0; + + for (i = 0; i < image->nr_segments; i++) { + mem = image->segment[i].mem; + ptr = (char *)__va(mem); + + if (ptr && memcmp(ptr, ELFMAG, SELFMAG) == 0) + sync_backup_region_phdr(image, (Elf64_Ehdr *) ptr, true); + } + return 0; +} + #undef pr_fmt #define pr_fmt(fmt) "crash hp: " fmt @@ -474,6 +536,8 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify * goto out; } + sync_backup_region_phdr(image, (Elf64_Ehdr *) elfbuf, false); + ptr = __va(mem); if (ptr) { /* Temporarily invalidate the crash image while it is replaced */ diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 5f6d50e4c3d450..8c72e12ea44e5a 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -374,33 +374,6 @@ static int load_backup_segment(struct kimage *image, struct kexec_buf *kbuf) return 0; } -/** - * update_backup_region_phdr - Update backup region's offset for the core to - * export the region appropriately. - * @image: Kexec image. - * @ehdr: ELF core header. - * - * Assumes an exclusive program header is setup for the backup region - * in the ELF headers - * - * Returns nothing. - */ -static void update_backup_region_phdr(struct kimage *image, Elf64_Ehdr *ehdr) -{ - Elf64_Phdr *phdr; - unsigned int i; - - phdr = (Elf64_Phdr *)(ehdr + 1); - for (i = 0; i < ehdr->e_phnum; i++) { - if (phdr->p_paddr == BACKUP_SRC_START) { - phdr->p_offset = image->arch.backup_start; - kexec_dprintk("Backup region offset updated to 0x%lx\n", - image->arch.backup_start); - return; - } - } -} - static unsigned int kdump_extra_elfcorehdr_size(struct crash_mem *cmem) { #if defined(CONFIG_CRASH_HOTPLUG) && defined(CONFIG_MEMORY_HOTPLUG) @@ -445,7 +418,7 @@ static int load_elfcorehdr_segment(struct kimage *image, struct kexec_buf *kbuf) } /* Fix the offset for backup region in the ELF header */ - update_backup_region_phdr(image, headers); + sync_backup_region_phdr(image, headers, false); kbuf->buffer = headers; kbuf->mem = KEXEC_BUF_MEM_UNKNOWN; diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c index 4e724c4c01add7..0f0f2d851ac670 100644 --- a/arch/powerpc/lib/pmem.c +++ b/arch/powerpc/lib/pmem.c @@ -66,15 +66,16 @@ EXPORT_SYMBOL_GPL(arch_invalidate_pmem); /* * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols */ -long __copy_from_user_flushcache(void *dest, const void __user *src, - unsigned size) +size_t copy_from_user_flushcache(void *dest, const void __user *src, + size_t size) { - unsigned long copied, start = (unsigned long) dest; + unsigned long not_copied, start = (unsigned long) dest; - copied = __copy_from_user(dest, src, size); + src = mask_user_address(src); + not_copied = __copy_from_user(dest, src, size); clean_pmem_range(start, start + size); - return copied; + return not_copied; } void memcpy_flushcache(void *dest, const void *src, size_t size) diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 4b09c04654a8f2..42c7906d0e4363 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -209,16 +209,21 @@ pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, int full) { pmd_t pmd; + bool was_present = pmd_present(*pmdp); + VM_BUG_ON(addr & ~HPAGE_PMD_MASK); - VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp)) || - !pmd_present(*pmdp)); + VM_BUG_ON(was_present && !pmd_trans_huge(*pmdp)); + /* + * Check pmdp_huge_get_and_clear() for non-present pmd case. + */ pmd = pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp); /* * if it not a fullmm flush, then we can possibly end up converting * this PMD pte entry to a regular level 0 PTE by a parallel page fault. - * Make sure we flush the tlb in this case. + * Make sure we flush the tlb in this case. TLB flush not needed for + * non-present case. */ - if (!full) + if (was_present && !full) flush_pmd_tlb_range(vma, addr, addr + HPAGE_PMD_SIZE); return pmd; } diff --git a/arch/powerpc/mm/pgtable-frag.c b/arch/powerpc/mm/pgtable-frag.c index 77e55eac16e42e..ae742564a3d56d 100644 --- a/arch/powerpc/mm/pgtable-frag.c +++ b/arch/powerpc/mm/pgtable-frag.c @@ -25,6 +25,7 @@ void pte_frag_destroy(void *pte_frag) count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT; /* We allow PTE_FRAG_NR fragments from a PTE page */ if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { + folio_clear_active(ptdesc_folio(ptdesc)); pagetable_dtor(ptdesc); pagetable_free(ptdesc); } diff --git a/arch/powerpc/platforms/44x/warp.c b/arch/powerpc/platforms/44x/warp.c index a5001d32f978d7..6f674f86dc853c 100644 --- a/arch/powerpc/platforms/44x/warp.c +++ b/arch/powerpc/platforms/44x/warp.c @@ -293,6 +293,8 @@ static int pika_dtm_thread(void __iomem *fpga) schedule_timeout(HZ); } + put_device(&client->dev); + return 0; } diff --git a/arch/powerpc/platforms/pseries/papr-hvpipe.c b/arch/powerpc/platforms/pseries/papr-hvpipe.c index 14ae480d060a4d..c007560d2d8ce9 100644 --- a/arch/powerpc/platforms/pseries/papr-hvpipe.c +++ b/arch/powerpc/platforms/pseries/papr-hvpipe.c @@ -206,10 +206,11 @@ static int hvpipe_rtas_recv_msg(char __user *buf, int size) bytes_written, size); bytes_written = size; } - ret = copy_to_user(buf, + if (copy_to_user(buf, rtas_work_area_raw_buf(work_area), - bytes_written); - if (!ret) + bytes_written)) + ret = -EFAULT; + else ret = bytes_written; } } else { @@ -327,8 +328,8 @@ static ssize_t papr_hvpipe_handle_read(struct file *file, { struct hvpipe_source_info *src_info = file->private_data; - struct papr_hvpipe_hdr hdr; - long ret; + struct papr_hvpipe_hdr hdr = {}; + ssize_t ret = 0; /* * Return -ENXIO during migration @@ -376,7 +377,7 @@ static ssize_t papr_hvpipe_handle_read(struct file *file, ret = copy_to_user(buf, &hdr, HVPIPE_HDR_LEN); if (ret) - return ret; + return -EFAULT; /* * Message event has payload, so get the payload with @@ -385,19 +386,23 @@ static ssize_t papr_hvpipe_handle_read(struct file *file, if (hdr.flags & HVPIPE_MSG_AVAILABLE) { ret = hvpipe_rtas_recv_msg(buf + HVPIPE_HDR_LEN, size - HVPIPE_HDR_LEN); - if (ret > 0) { + /* + * Always clear MSG_AVAILABLE once the RTAS call has drained + * the message, regardless of whether copy_to_user succeeded. + */ + if (ret >= 0 || ret == -EFAULT) src_info->hvpipe_status &= ~HVPIPE_MSG_AVAILABLE; - ret += HVPIPE_HDR_LEN; - } } else if (hdr.flags & HVPIPE_LOST_CONNECTION) { /* * Hypervisor is closing the pipe for the specific * source. So notify user space. */ src_info->hvpipe_status &= ~HVPIPE_LOST_CONNECTION; - ret = HVPIPE_HDR_LEN; } + if (ret >= 0) + ret += HVPIPE_HDR_LEN; + return ret; } @@ -444,13 +449,14 @@ static int papr_hvpipe_handle_release(struct inode *inode, struct file *file) { struct hvpipe_source_info *src_info; + unsigned long flags; /* * Hold the lock, remove source from src_list, reset the * hvpipe status and release the lock to prevent any race * with message event IRQ. */ - spin_lock(&hvpipe_src_list_lock); + spin_lock_irqsave(&hvpipe_src_list_lock, flags); src_info = file->private_data; list_del(&src_info->list); file->private_data = NULL; @@ -461,10 +467,10 @@ static int papr_hvpipe_handle_release(struct inode *inode, */ if (src_info->hvpipe_status & HVPIPE_MSG_AVAILABLE) { src_info->hvpipe_status = 0; - spin_unlock(&hvpipe_src_list_lock); + spin_unlock_irqrestore(&hvpipe_src_list_lock, flags); hvpipe_rtas_recv_msg(NULL, 0); } else - spin_unlock(&hvpipe_src_list_lock); + spin_unlock_irqrestore(&hvpipe_src_list_lock, flags); kfree(src_info); return 0; @@ -479,21 +485,9 @@ static const struct file_operations papr_hvpipe_handle_ops = { static int papr_hvpipe_dev_create_handle(u32 srcID) { - struct hvpipe_source_info *src_info __free(kfree) = NULL; - - spin_lock(&hvpipe_src_list_lock); - /* - * Do not allow more than one process communicates with - * each source. - */ - src_info = hvpipe_find_source(srcID); - if (src_info) { - spin_unlock(&hvpipe_src_list_lock); - pr_err("pid(%d) is already using the source(%d)\n", - src_info->tsk->pid, srcID); - return -EALREADY; - } - spin_unlock(&hvpipe_src_list_lock); + struct hvpipe_source_info *src_info; + int fd; + unsigned long flags; src_info = kzalloc_obj(*src_info, GFP_KERNEL_ACCOUNT); if (!src_info) @@ -503,26 +497,42 @@ static int papr_hvpipe_dev_create_handle(u32 srcID) src_info->tsk = current; init_waitqueue_head(&src_info->recv_wqh); - FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC, - anon_inode_getfile("[papr-hvpipe]", &papr_hvpipe_handle_ops, - (void *)src_info, O_RDWR)); - if (fdf.err) - return fdf.err; - - retain_and_null_ptr(src_info); - spin_lock(&hvpipe_src_list_lock); /* - * If two processes are executing ioctl() for the same - * source ID concurrently, prevent the second process to - * acquire FD. + * Do not allow more than one process communicates with + * each source. */ + spin_lock_irqsave(&hvpipe_src_list_lock, flags); if (hvpipe_find_source(srcID)) { - spin_unlock(&hvpipe_src_list_lock); + spin_unlock_irqrestore(&hvpipe_src_list_lock, flags); + pr_err("pid(%d) could not get the source(%d)\n", + src_info->tsk->pid, srcID); + kfree(src_info); return -EALREADY; } list_add(&src_info->list, &hvpipe_src_list); - spin_unlock(&hvpipe_src_list_lock); - return fd_publish(fdf); + spin_unlock_irqrestore(&hvpipe_src_list_lock, flags); + + fd = FD_ADD(O_RDONLY | O_CLOEXEC, + anon_inode_getfile("[papr-hvpipe]", &papr_hvpipe_handle_ops, + (void *)src_info, O_RDWR)); + if (fd < 0) { + spin_lock_irqsave(&hvpipe_src_list_lock, flags); + list_del(&src_info->list); + spin_unlock_irqrestore(&hvpipe_src_list_lock, flags); + /* + * if we fail to add FD, that means no userspace program is + * polling. In that case if there is a msg pending because the + * interrupt was fired after the src_info was added to the + * global list, then let's consume it here, to unblock the + * hvpipe + */ + if (src_info->hvpipe_status & HVPIPE_MSG_AVAILABLE) + hvpipe_rtas_recv_msg(NULL, 0); + kfree(src_info); + return fd; + } + + return fd; } /* @@ -775,23 +785,29 @@ static int __init papr_hvpipe_init(void) } ret = enable_hvpipe_IRQ(); - if (!ret) { - ret = set_hvpipe_sys_param(1); - if (!ret) - ret = misc_register(&papr_hvpipe_dev); - } + if (ret) + goto out_wq; - if (!ret) { - pr_info("hvpipe feature is enabled\n"); - hvpipe_feature = true; - return 0; - } + ret = misc_register(&papr_hvpipe_dev); + if (ret) + goto out_wq; - pr_err("hvpipe feature is not enabled %d\n", ret); + ret = set_hvpipe_sys_param(1); + if (ret) + goto out_misc; + + pr_info("hvpipe feature is enabled\n"); + hvpipe_feature = true; + return 0; + +out_misc: + misc_deregister(&papr_hvpipe_dev); +out_wq: destroy_workqueue(papr_hvpipe_wq); out: kfree(papr_hvpipe_work); papr_hvpipe_work = NULL; + pr_err("hvpipe feature is not enabled %d\n", ret); return ret; } machine_device_initcall(pseries, papr_hvpipe_init); diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index e1a4f8a97393f5..6b1b7541ca311e 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -1038,13 +1038,19 @@ static struct xive_irq_data *xive_irq_alloc_data(unsigned int virq, irq_hw_numbe return xd; } -static void xive_irq_free_data(unsigned int virq) +static void xive_irq_free_data(struct irq_domain *domain, unsigned int virq) { - struct xive_irq_data *xd = irq_get_chip_data(virq); + struct xive_irq_data *xd; + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + + if (!data) + return; + xd = irq_data_get_irq_chip_data(data); if (!xd) return; - irq_set_chip_data(virq, NULL); + + irq_domain_reset_irq_data(data); xive_cleanup_irq_data(xd); kfree(xd); } @@ -1305,7 +1311,7 @@ static int xive_irq_domain_map(struct irq_domain *h, unsigned int virq, static void xive_irq_domain_unmap(struct irq_domain *d, unsigned int virq) { - xive_irq_free_data(virq); + xive_irq_free_data(d, virq); } static int xive_irq_domain_xlate(struct irq_domain *h, struct device_node *ct, @@ -1443,7 +1449,7 @@ static void xive_irq_domain_free(struct irq_domain *domain, pr_debug("%s %d #%d\n", __func__, virq, nr_irqs); for (i = 0; i < nr_irqs; i++) - xive_irq_free_data(virq + i); + xive_irq_free_data(domain, virq + i); } #endif diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index 5971605754b353..48c034736aa5a0 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -81,8 +81,6 @@ }; &combo_phy { - pinctrl-names = "default"; - pinctrl-0 = <&pcie0_3_cfg>; status = "okay"; }; @@ -305,6 +303,7 @@ &pcie1_port { phys = <&pcie1_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; }; &pcie1 { @@ -320,6 +319,7 @@ &pcie2_port { phys = <&pcie2_phy>; + vpcie3v3-supply = <&pcie_vcc_3v3>; }; &pcie2 { diff --git a/arch/riscv/kvm/vcpu_vector.c b/arch/riscv/kvm/vcpu_vector.c index 05f3cc2d8e311a..5b6ad82d47beb7 100644 --- a/arch/riscv/kvm/vcpu_vector.c +++ b/arch/riscv/kvm/vcpu_vector.c @@ -80,8 +80,11 @@ int kvm_riscv_vcpu_alloc_vector_context(struct kvm_vcpu *vcpu) return -ENOMEM; vcpu->arch.host_context.vector.datap = kzalloc(riscv_v_vsize, GFP_KERNEL); - if (!vcpu->arch.host_context.vector.datap) + if (!vcpu->arch.host_context.vector.datap) { + kfree(vcpu->arch.guest_context.vector.datap); + vcpu->arch.guest_context.vector.datap = NULL; return -ENOMEM; + } return 0; } diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index 632ced07bca442..da027179024421 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -11,7 +11,6 @@ #include #include -#include /* verify runtime detection extension status */ #define rv_ext_enabled(ext) \ @@ -105,11 +104,6 @@ static inline void bpf_fill_ill_insns(void *area, unsigned int size) memset(area, 0, size); } -static inline void bpf_flush_icache(void *start, void *end) -{ - flush_icache_range((unsigned long)start, (unsigned long)end); -} - /* Emit a 4-byte riscv instruction. */ static inline void emit(const u32 insn, struct rv_jit_context *ctx) { diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index b3581e92643629..f7fd4afc3ca3f4 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -183,13 +183,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog = orig_prog; goto out_offset; } - /* - * The instructions have now been copied to the ROX region from - * where they will execute. - * Write any modified data cache blocks out to memory and - * invalidate the corresponding blocks in the instruction cache. - */ - bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns); for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 31430e9bcfdd9d..7650f2adb5cf86 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -1414,6 +1414,9 @@ static inline char *debug_get_user_string(const char __user *user_buf, { char *buffer; + if (!user_len) + return ERR_PTR(-EINVAL); + buffer = memdup_user_nul(user_buf, user_len); if (IS_ERR(buffer)) return buffer; @@ -1584,6 +1587,11 @@ static int debug_input_flush_fn(debug_info_t *id, struct debug_view *view, char input_buf[1]; int rc = user_len; + if (!user_len) { + rc = -EINVAL; + goto out; + } + if (user_len > 0x10000) user_len = 0x10000; if (*offset != 0) { diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 7cb8ce833b6254..f48f25c7dc8fe4 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3307,8 +3307,7 @@ static void aen_host_forward(unsigned long si) struct zpci_gaite *gaite; struct kvm *kvm; - gaite = (struct zpci_gaite *)aift->gait + - (si * sizeof(struct zpci_gaite)); + gaite = aift->gait + si; if (gaite->count == 0) return; if (gaite->aisb != 0) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 86d93e8dddae3e..eed45af1a92d48 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -290,8 +290,7 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, phys_to_virt(fib->fmt0.aibv)); spin_lock_irq(&aift->gait_lock); - gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb * - sizeof(struct zpci_gaite)); + gaite = aift->gait + zdev->aisb; /* If assist not requested, host will get all alerts */ if (assist) @@ -357,8 +356,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) if (zdev->kzdev->fib.fmt0.aibv == 0) goto out; spin_lock_irq(&aift->gait_lock); - gaite = (struct zpci_gaite *)aift->gait + (zdev->aisb * - sizeof(struct zpci_gaite)); + gaite = aift->gait + zdev->aisb; isc = gaite->gisc; gaite->count--; if (gaite->count == 0) { diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 191cc53caead39..028aeb9c48d6ff 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -438,7 +438,7 @@ void do_secure_storage_access(struct pt_regs *regs) panic("Unexpected PGM 0x3d with TEID bit 61=0"); } if (is_kernel_fault(regs)) { - folio = phys_to_folio(addr); + folio = virt_to_folio((void *)addr); if (unlikely(!folio_try_get(folio))) return; rc = uv_convert_from_secure(folio_to_phys(folio)); diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index bf92964246eb19..10ab247e1994cb 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -830,25 +830,34 @@ static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp, } /* - * Sign-extend the register if necessary + * Sign- or zero-extend the register if necessary */ -static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) +static int sign_zero_extend(struct bpf_jit *jit, int r, u8 size, u8 flags) { - if (!(flags & BTF_FMODEL_SIGNED_ARG)) - return 0; - switch (size) { case 1: - /* lgbr %r,%r */ - EMIT4(0xb9060000, r, r); + if (flags & BTF_FMODEL_SIGNED_ARG) + /* lgbr %r,%r */ + EMIT4(0xb9060000, r, r); + else + /* llgcr %r,%r */ + EMIT4(0xb9840000, r, r); return 0; case 2: - /* lghr %r,%r */ - EMIT4(0xb9070000, r, r); + if (flags & BTF_FMODEL_SIGNED_ARG) + /* lghr %r,%r */ + EMIT4(0xb9070000, r, r); + else + /* llghr %r,%r */ + EMIT4(0xb9850000, r, r); return 0; case 4: - /* lgfr %r,%r */ - EMIT4(0xb9140000, r, r); + if (flags & BTF_FMODEL_SIGNED_ARG) + /* lgfr %r,%r */ + EMIT4(0xb9140000, r, r); + else + /* llgfr %r,%r */ + EMIT4(0xb9160000, r, r); return 0; case 8: return 0; @@ -1798,9 +1807,9 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, return -1; for (j = 0; j < m->nr_args; j++) { - if (sign_extend(jit, BPF_REG_1 + j, - m->arg_size[j], - m->arg_flags[j])) + if (sign_zero_extend(jit, BPF_REG_1 + j, + m->arg_size[j], + m->arg_flags[j])) return -1; } } @@ -2555,7 +2564,7 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, p->bpf_func); /* stg %r2,retval_off(%r15) */ if (save_ret) { - if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags)) + if (sign_zero_extend(jit, REG_2, m->ret_size, m->ret_flags)) return -1; EMIT6_DISP_LH(0xe3000000, 0x0024, REG_2, REG_0, REG_15, tjit->retval_off); diff --git a/arch/sh/include/cpu-sh3/cpu/dac.h b/arch/sh/include/cpu-sh3/cpu/dac.h index fd02331608a8d6..323ec8570bcd1e 100644 --- a/arch/sh/include/cpu-sh3/cpu/dac.h +++ b/arch/sh/include/cpu-sh3/cpu/dac.h @@ -2,6 +2,8 @@ #ifndef __ASM_CPU_SH3_DAC_H #define __ASM_CPU_SH3_DAC_H +#include + /* * Copyright (C) 2003 Andriy Skulysh */ diff --git a/arch/sparc/kernel/sys_sparc32.c b/arch/sparc/kernel/sys_sparc32.c index f84a02ab6bf90f..04432b82b9e3ef 100644 --- a/arch/sparc/kernel/sys_sparc32.c +++ b/arch/sparc/kernel/sys_sparc32.c @@ -58,7 +58,7 @@ COMPAT_SYSCALL_DEFINE3(truncate64, const char __user *, path, u32, high, u32, lo COMPAT_SYSCALL_DEFINE3(ftruncate64, unsigned int, fd, u32, high, u32, low) { - return ksys_ftruncate(fd, ((u64)high << 32) | low); + return ksys_ftruncate(fd, ((u64)high << 32) | low, FTRUNCATE_LFS); } static int cp_compat_stat64(struct kstat *stat, diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile index 683b2d40822447..400529acd1c10e 100644 --- a/arch/sparc/vdso/Makefile +++ b/arch/sparc/vdso/Makefile @@ -104,4 +104,4 @@ quiet_cmd_vdso = VDSO $@ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ -T $(filter %.lds,$^) $(filter %.o,$^) -VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined +VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined -z noexecstack diff --git a/arch/um/drivers/cow_user.c b/arch/um/drivers/cow_user.c index 29b46581ddd137..dc1d1bcd85ec24 100644 --- a/arch/um/drivers/cow_user.c +++ b/arch/um/drivers/cow_user.c @@ -15,6 +15,12 @@ #include "cow.h" #include "cow_sys.h" +/* + * arch/um/Makefile remaps strrchr to kernel_strrchr; call the kernel + * name directly to avoid glibc >= 2.43's C23 strrchr macro. + */ +char *kernel_strrchr(const char *, int); + #define PATH_LEN_V1 256 /* unsigned time_t works until year 2106 */ @@ -153,7 +159,7 @@ static int absolutize(char *to, int size, char *from) errno); return -1; } - slash = strrchr(from, '/'); + slash = kernel_strrchr(from, '/'); if (slash != NULL) { *slash = '\0'; if (chdir(from)) { diff --git a/arch/um/kernel/tlb.c b/arch/um/kernel/tlb.c index 39608cccf2c690..5386ab2d0da505 100644 --- a/arch/um/kernel/tlb.c +++ b/arch/um/kernel/tlb.c @@ -165,6 +165,7 @@ int um_tlb_sync(struct mm_struct *mm) unsigned long addr, next; int ret = 0; + guard(spinlock_irqsave)(&mm->page_table_lock); guard(spinlock_irqsave)(&mm->context.sync_tlb_lock); if (mm->context.sync_tlb_range_to == 0) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index e2df1b147184a6..fe106da41ae587 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1889,6 +1889,7 @@ config X86_USER_SHADOW_STACK bool "X86 userspace shadow stack" depends on AS_WRUSS depends on X86_64 + depends on PER_VMA_LOCK select ARCH_USES_HIGH_VMA_FLAGS select ARCH_HAS_USER_SHADOW_STACK select X86_CET diff --git a/arch/x86/Makefile.um b/arch/x86/Makefile.um index c86cbd9cbba38f..19c13afa474e9d 100644 --- a/arch/x86/Makefile.um +++ b/arch/x86/Makefile.um @@ -60,4 +60,6 @@ ELF_FORMAT := elf64-x86-64 LINK-$(CONFIG_LD_SCRIPT_DYN_RPATH) += -Wl,-rpath,/lib64 LINK-y += -m64 +vdso-install-y += arch/x86/um/vdso/vdso.so.dbg + endif diff --git a/arch/x86/coco/tdx/debug.c b/arch/x86/coco/tdx/debug.c index cef847c8bb67f7..28990c2ab0a143 100644 --- a/arch/x86/coco/tdx/debug.c +++ b/arch/x86/coco/tdx/debug.c @@ -17,7 +17,7 @@ static __initdata const char *tdx_attributes[] = { DEF_TDX_ATTR_NAME(ICSSD), DEF_TDX_ATTR_NAME(LASS), DEF_TDX_ATTR_NAME(SEPT_VE_DISABLE), - DEF_TDX_ATTR_NAME(MIGRTABLE), + DEF_TDX_ATTR_NAME(MIGRATABLE), DEF_TDX_ATTR_NAME(PKS), DEF_TDX_ATTR_NAME(KL), DEF_TDX_ATTR_NAME(TPA), diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index aca89f23d2e002..7b8eea1d75c101 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -313,6 +313,9 @@ static int perf_ibs_init(struct perf_event *event) if (ret) return ret; + if (perf_allow_kernel()) + hwc->flags |= PERF_X86_EVENT_UNPRIVILEGED; + if (hwc->sample_period) { if (config & perf_ibs->cnt_mask) /* raw max_cnt may not be set */ @@ -1214,12 +1217,10 @@ static void perf_ibs_phyaddr_clear(struct perf_ibs *perf_ibs, struct perf_ibs_data *ibs_data) { if (perf_ibs == &perf_ibs_op) { - ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)] &= ~(1ULL << 18); ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)] = 0; return; } - ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHCTL)] &= ~(1ULL << 52); ibs_data->regs[ibs_fetch_msr_idx(MSR_AMD64_IBSFETCHPHYSAD)] = 0; } @@ -1293,8 +1294,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) * within [128, 2048] range. */ if (!op_data3.ld_op || !op_data3.dc_miss || - op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) + op_data3.dc_miss_lat <= (event->attr.config1 & 0xFFF)) { + throttle = perf_event_account_interrupt(event); goto out; + } } /* @@ -1326,8 +1329,10 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) regs.flags &= ~PERF_EFLAGS_EXACT; } else { /* Workaround for erratum #1197 */ - if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) + if (perf_ibs->fetch_ignore_if_zero_rip && !(ibs_data.regs[1])) { + throttle = perf_event_account_interrupt(event); goto out; + } set_linear_ip(®s, ibs_data.regs[1]); regs.flags |= PERF_EFLAGS_EXACT; @@ -1344,7 +1349,7 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) * unprivileged users. */ if ((event->attr.sample_type & PERF_SAMPLE_RAW) && - perf_allow_kernel()) { + (hwc->flags & PERF_X86_EVENT_UNPRIVILEGED)) { perf_ibs_phyaddr_clear(perf_ibs, &ibs_data); } diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index 810ab21ffd9913..4b9e105309c6a9 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -1294,13 +1294,16 @@ int x86_perf_rdpmc_index(struct perf_event *event) return event->hw.event_base_rdpmc; } -static inline int match_prev_assignment(struct hw_perf_event *hwc, +static inline int match_prev_assignment(struct perf_event *event, struct cpu_hw_events *cpuc, int i) { + struct hw_perf_event *hwc = &event->hw; + return hwc->idx == cpuc->assign[i] && - hwc->last_cpu == smp_processor_id() && - hwc->last_tag == cpuc->tags[i]; + hwc->last_cpu == smp_processor_id() && + hwc->last_tag == cpuc->tags[i] && + !is_acr_event_group(event); } static void x86_pmu_start(struct perf_event *event, int flags); @@ -1346,7 +1349,7 @@ static void x86_pmu_enable(struct pmu *pmu) * - no other event has used the counter since */ if (hwc->idx == -1 || - match_prev_assignment(hwc, cpuc, i)) + match_prev_assignment(event, cpuc, i)) continue; /* @@ -1367,7 +1370,7 @@ static void x86_pmu_enable(struct pmu *pmu) event = cpuc->event_list[i]; hwc = &event->hw; - if (!match_prev_assignment(hwc, cpuc, i)) + if (!match_prev_assignment(event, cpuc, i)) x86_assign_hw_event(event, cpuc, i); else if (i < n_running) continue; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 793335c3ce787d..d8ac015f5fdbcf 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3118,11 +3118,11 @@ static void intel_pmu_enable_fixed(struct perf_event *event) intel_set_masks(event, idx); /* - * Enable IRQ generation (0x8), if not PEBS, - * and enable ring-3 counting (0x2) and ring-0 counting (0x1) - * if requested: + * Enable IRQ generation (0x8), if not PEBS or self-reloaded + * ACR event, and enable ring-3 counting (0x2) and ring-0 + * counting (0x1) if requested: */ - if (!event->attr.precise_ip) + if (!event->attr.precise_ip && !is_acr_self_reload_event(event)) bits |= INTEL_FIXED_0_ENABLE_PMI; if (hwc->config & ARCH_PERFMON_EVENTSEL_USR) bits |= INTEL_FIXED_0_USER; @@ -3306,6 +3306,15 @@ static void intel_pmu_enable_event(struct perf_event *event) intel_set_masks(event, idx); static_call_cond(intel_pmu_enable_acr_event)(event); static_call_cond(intel_pmu_enable_event_ext)(event); + /* + * For self-reloaded ACR event, don't enable PMI since + * HW won't set overflow bit in GLOBAL_STATUS. Otherwise, + * the PMI would be recognized as a suspicious NMI. + */ + if (is_acr_self_reload_event(event)) + hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT; + else if (!event->attr.precise_ip) + hwc->config |= ARCH_PERFMON_EVENTSEL_INT; __x86_pmu_enable_event(hwc, enable_mask); break; case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1: @@ -3332,23 +3341,41 @@ static void intel_pmu_enable_event(struct perf_event *event) static void intel_pmu_acr_late_setup(struct cpu_hw_events *cpuc) { struct perf_event *event, *leader; - int i, j, idx; + int i, j, k, bit, idx; + /* + * FIXME: ACR mask parsing relies on cpuc->event_list[] (active events only). + * Disabling an ACR event causes bit-shifting errors in the acr_mask of + * remaining group members. As ACR sampling requires all events to be active, + * this limitation is acceptable for now. Revisit if independent event toggling + * is required. + */ for (i = 0; i < cpuc->n_events; i++) { leader = cpuc->event_list[i]; if (!is_acr_event_group(leader)) continue; - /* The ACR events must be contiguous. */ + /* Find the last event of the ACR group. */ for (j = i; j < cpuc->n_events; j++) { event = cpuc->event_list[j]; if (event->group_leader != leader->group_leader) break; - for_each_set_bit(idx, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) { - if (i + idx >= cpuc->n_events || - !is_acr_event_group(cpuc->event_list[i + idx])) - return; - __set_bit(cpuc->assign[i + idx], (unsigned long *)&event->hw.config1); + } + + /* + * Translate the user-space ACR mask (attr.config2) into the physical + * counter bitmask (hw.config1) for each ACR event in the group. + * NOTE: ACR event contiguity is guaranteed by intel_pmu_hw_config(). + */ + for (k = i; k < j; k++) { + event = cpuc->event_list[k]; + event->hw.config1 = 0; + for_each_set_bit(bit, (unsigned long *)&event->attr.config2, X86_PMC_IDX_MAX) { + idx = i + bit; + /* Event index of ACR group must locate in [i, j). */ + if (idx >= j || !is_acr_event_group(cpuc->event_list[idx])) + continue; + __set_bit(cpuc->assign[idx], (unsigned long *)&event->hw.config1); } } i = j - 1; @@ -7498,6 +7525,7 @@ static __always_inline void intel_pmu_init_pnc(struct pmu *pmu) hybrid(pmu, event_constraints) = intel_pnc_event_constraints; hybrid(pmu, pebs_constraints) = intel_pnc_pebs_event_constraints; hybrid(pmu, extra_regs) = intel_pnc_extra_regs; + static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr); } static __always_inline void intel_pmu_init_skt(struct pmu *pmu) diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index fad87d3c8b2caa..524668dcf4cc10 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -137,6 +137,16 @@ static inline bool is_acr_event_group(struct perf_event *event) return check_leader_group(event->group_leader, PERF_X86_EVENT_ACR); } +static inline bool is_acr_self_reload_event(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + if (hwc->idx < 0) + return false; + + return test_bit(hwc->idx, (unsigned long *)&hwc->config1); +} + struct amd_nb { int nb_id; /* NorthBridge id */ int refcnt; /* reference count */ diff --git a/arch/x86/events/perf_event_flags.h b/arch/x86/events/perf_event_flags.h index 70078334e4a338..47f84ee8f5409c 100644 --- a/arch/x86/events/perf_event_flags.h +++ b/arch/x86/events/perf_event_flags.h @@ -23,3 +23,4 @@ PERF_ARCH(PEBS_LAT_HYBRID, 0x0020000) /* ld and st lat for hybrid */ PERF_ARCH(NEEDS_BRANCH_STACK, 0x0040000) /* require branch stack setup */ PERF_ARCH(BRANCH_COUNTERS, 0x0080000) /* logs the counters in the extra space of each branch */ PERF_ARCH(ACR, 0x0100000) /* Auto counter reload */ +PERF_ARCH(UNPRIVILEGED, 0x0200000) /* Unprivileged event (wrt perf_allow_kernel()) */ diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index 51b4cdbea061ae..f5932705f4b070 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -137,7 +137,8 @@ extern void __init efi_dump_pagetable(void); extern void __init efi_apply_memmap_quirks(void); extern int __init efi_reuse_config(u64 tables, int nr_tables); extern void efi_delete_dummy_variable(void); -extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr); +extern void efi_crash_gracefully_on_page_fault(unsigned long phys_addr, + const struct pt_regs *regs); extern void efi_unmap_boot_services(void); void arch_efi_call_virt_setup(void); diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index 462754b0bf8ac1..6f25de05ed58f5 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -96,11 +96,11 @@ static __always_inline void halt(void) native_halt(); } #endif /* __ASSEMBLER__ */ +#else +#include #endif /* CONFIG_PARAVIRT */ -#ifdef CONFIG_PARAVIRT_XXL -#include -#else +#ifndef CONFIG_PARAVIRT_XXL #ifndef __ASSEMBLER__ #include diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 6673601246b382..4efbbf9d117b06 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -674,6 +674,9 @@ #define MSR_AMD64_DC_CFG 0xc0011022 #define MSR_AMD64_TW_CFG 0xc0011023 +#define MSR_AMD64_FP_CFG 0xc0011028 +#define MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT 9 + #define MSR_AMD64_DE_CFG 0xc0011029 #define MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT 1 #define MSR_AMD64_DE_CFG_LFENCE_SERIALIZE BIT_ULL(MSR_AMD64_DE_CFG_LFENCE_SERIALIZE_BIT) @@ -793,9 +796,10 @@ #define MSR_AMD64_LBR_SELECT 0xc000010e /* Zen4 */ -#define MSR_ZEN4_BP_CFG 0xc001102e +#define MSR_ZEN4_BP_CFG 0xc001102e #define MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT 4 #define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5 +#define MSR_ZEN2_BP_CFG_BUG_FIX_BIT 33 /* Fam 19h MSRs */ #define MSR_F19H_UMC_PERF_CTL 0xc0010800 diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h index 8bc074c8d7c6a1..11f3cf30b1ac8a 100644 --- a/arch/x86/include/asm/shared/tdx.h +++ b/arch/x86/include/asm/shared/tdx.h @@ -35,8 +35,8 @@ #define TDX_ATTR_LASS BIT_ULL(TDX_ATTR_LASS_BIT) #define TDX_ATTR_SEPT_VE_DISABLE_BIT 28 #define TDX_ATTR_SEPT_VE_DISABLE BIT_ULL(TDX_ATTR_SEPT_VE_DISABLE_BIT) -#define TDX_ATTR_MIGRTABLE_BIT 29 -#define TDX_ATTR_MIGRTABLE BIT_ULL(TDX_ATTR_MIGRTABLE_BIT) +#define TDX_ATTR_MIGRATABLE_BIT 29 +#define TDX_ATTR_MIGRATABLE BIT_ULL(TDX_ATTR_MIGRATABLE_BIT) #define TDX_ATTR_PKS_BIT 30 #define TDX_ATTR_PKS BIT_ULL(TDX_ATTR_PKS_BIT) #define TDX_ATTR_KL_BIT 31 diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 367297b188c39c..3a0dd3c2b233e4 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -507,7 +507,7 @@ extern struct movsl_mask { } ____cacheline_aligned_in_smp movsl_mask; #endif -#define ARCH_HAS_NOCACHE_UACCESS 1 +#define ARCH_HAS_NONTEMPORAL_UACCESS 1 /* * The "unsafe" user accesses aren't really "unsafe", but the naming diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h index 40379a1adbb846..fff19e73ccb333 100644 --- a/arch/x86/include/asm/uaccess_32.h +++ b/arch/x86/include/asm/uaccess_32.h @@ -26,13 +26,7 @@ raw_copy_from_user(void *to, const void __user *from, unsigned long n) return __copy_user_ll(to, (__force const void *)from, n); } -static __always_inline unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, - unsigned long n) -{ - return __copy_from_user_ll_nocache_nozero(to, from, n); -} - +unsigned long __must_check copy_from_user_inatomic_nontemporal(void *, const void __user *, unsigned long n); unsigned long __must_check clear_user(void __user *mem, unsigned long len); unsigned long __must_check __clear_user(void __user *mem, unsigned long len); diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 915124011c2764..20de34cc9aa6e7 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -147,26 +147,28 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) return copy_user_generic((__force void *)dst, src, size); } -extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size); -extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size); +#define copy_to_nontemporal copy_to_nontemporal +extern size_t copy_to_nontemporal(void *dst, const void *src, size_t size); +extern size_t copy_user_flushcache(void *dst, const void __user *src, size_t size); static inline int -__copy_from_user_inatomic_nocache(void *dst, const void __user *src, +copy_from_user_inatomic_nontemporal(void *dst, const void __user *src, unsigned size) { long ret; kasan_check_write(dst, size); + src = mask_user_address(src); stac(); - ret = __copy_user_nocache(dst, src, size); + ret = copy_to_nontemporal(dst, (__force const void *)src, size); clac(); return ret; } -static inline int -__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size) +static inline size_t +copy_from_user_flushcache(void *dst, const void __user *src, size_t size) { kasan_check_write(dst, size); - return __copy_user_flushcache(dst, src, size); + return copy_user_flushcache(dst, src, size); } /* diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index e8afbe9faa5b9c..f2d49212ae9027 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -18,7 +18,6 @@ struct vdso_image { unsigned long extable_base, extable_len; const void *extable; - long sym_VDSO32_NOTE_MASK; long sym___kernel_sigreturn; long sym___kernel_rt_sigreturn; long sym___kernel_vsyscall; diff --git a/arch/x86/kernel/acpi/cppc.c b/arch/x86/kernel/acpi/cppc.c index d7c8ef1e354d30..be4c5e9e5ff6f9 100644 --- a/arch/x86/kernel/acpi/cppc.c +++ b/arch/x86/kernel/acpi/cppc.c @@ -88,19 +88,19 @@ static void amd_set_max_freq_ratio(void) rc = cppc_get_perf_caps(0, &perf_caps); if (rc) { - pr_warn("Could not retrieve perf counters (%d)\n", rc); + pr_debug("Could not retrieve perf counters (%d)\n", rc); return; } rc = amd_get_boost_ratio_numerator(0, &numerator); if (rc) { - pr_warn("Could not retrieve highest performance (%d)\n", rc); + pr_debug("Could not retrieve highest performance (%d)\n", rc); return; } nominal_perf = perf_caps.nominal_perf; if (!nominal_perf) { - pr_warn("Could not retrieve nominal performance\n"); + pr_debug("Could not retrieve nominal performance\n"); return; } diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 09de584e4c8fa9..820fee2658c6ae 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -943,6 +943,9 @@ static void init_amd_zen1(struct cpuinfo_x86 *c) msr_clear_bit(MSR_K7_HWCR, MSR_K7_HWCR_IRPERF_EN_BIT); clear_cpu_cap(c, X86_FEATURE_IRPERF); } + + pr_notice_once("AMD Zen1 FPDSS bug detected, enabling mitigation.\n"); + msr_set_bit(MSR_AMD64_FP_CFG, MSR_AMD64_FP_CFG_ZEN1_DENORM_FIX_BIT); } static const struct x86_cpu_id amd_zenbleed_microcode[] = { @@ -985,6 +988,9 @@ static void init_amd_zen2(struct cpuinfo_x86 *c) /* Correct misconfigured CPUID on some clients. */ clear_cpu_cap(c, X86_FEATURE_INVLPGB); + + if (!cpu_has(c, X86_FEATURE_HYPERVISOR)) + msr_set_bit(MSR_ZEN4_BP_CFG, MSR_ZEN2_BP_CFG_BUG_FIX_BIT); } static void init_amd_zen3(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 2a999275893369..eb72537bc0b195 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -450,6 +450,10 @@ __init static int append_e820_table(struct boot_e820_entry *entries, u32 nr_entr { struct boot_e820_entry *entry = entries; + /* If there aren't any entries, we'll want to fall back to another source: */ + if (!nr_entries) + return -ENOENT; + while (nr_entries) { u64 start = entry->addr; u64 size = entry->size; @@ -458,7 +462,7 @@ __init static int append_e820_table(struct boot_e820_entry *entries, u32 nr_entr /* Ignore the remaining entries on 64-bit overflow: */ if (start > end && likely(size)) - return -1; + return -EINVAL; e820__range_add(start, size, type); diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S index 4ffba68dc57b29..eaeb77464c066e 100644 --- a/arch/x86/kernel/relocate_kernel_64.S +++ b/arch/x86/kernel/relocate_kernel_64.S @@ -136,6 +136,14 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped) * %r13 original CR4 when relocate_kernel() was invoked */ + /* + * Set return address to 0 if not preserving context. The purgatory + * shipped in kexec-tools will unconditionally look for the return + * address on the stack and set a kexec_jump_back_entry= command + * line option if it's non-zero. There's no other way that it can + * tell a preserve-context (kjump) kexec from a normal one. + */ + pushq $0 /* store the start address on the stack */ pushq %rdx diff --git a/arch/x86/kernel/shstk.c b/arch/x86/kernel/shstk.c index ff8edea8511b48..63775693a3bb45 100644 --- a/arch/x86/kernel/shstk.c +++ b/arch/x86/kernel/shstk.c @@ -334,10 +334,8 @@ static int shstk_push_sigframe(unsigned long *ssp) static int shstk_pop_sigframe(unsigned long *ssp) { - struct vm_area_struct *vma; unsigned long token_addr; - bool need_to_check_vma; - int err = 1; + unsigned int seq; /* * It is possible for the SSP to be off the end of a shadow stack by 4 @@ -348,25 +346,35 @@ static int shstk_pop_sigframe(unsigned long *ssp) if (!IS_ALIGNED(*ssp, 8)) return -EINVAL; - need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp; + do { + struct vm_area_struct *vma; + bool valid_vma; + int err; - if (need_to_check_vma) if (mmap_read_lock_killable(current->mm)) return -EINTR; - err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp); - if (unlikely(err)) - goto out_err; - - if (need_to_check_vma) { vma = find_vma(current->mm, *ssp); - if (!vma || !(vma->vm_flags & VM_SHADOW_STACK)) { - err = -EFAULT; - goto out_err; - } - + valid_vma = vma && (vma->vm_flags & VM_SHADOW_STACK); + + /* + * VMAs can change between get_shstk_data() and find_vma(). + * Watch for changes and ensure that 'token_addr' comes from + * 'vma' by recording a seqcount. + * + * Ignore the return value of mmap_lock_speculate_try_begin() + * because the mmap lock excludes the possibility of writers. + */ + mmap_lock_speculate_try_begin(current->mm, &seq); mmap_read_unlock(current->mm); - } + + if (!valid_vma) + return -EINVAL; + + err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp); + if (err) + return err; + } while (mmap_lock_speculate_retry(current->mm, seq)); /* Restore SSP aligned? */ if (unlikely(!IS_ALIGNED(token_addr, 8))) @@ -379,10 +387,6 @@ static int shstk_pop_sigframe(unsigned long *ssp) *ssp = token_addr; return 0; -out_err: - if (need_to_check_vma) - mmap_read_unlock(current->mm); - return err; } int setup_signal_shadow_stack(struct ksignal *ksig) diff --git a/arch/x86/kernel/sys_ia32.c b/arch/x86/kernel/sys_ia32.c index 6cf65397d2257a..610a1c2f451919 100644 --- a/arch/x86/kernel/sys_ia32.c +++ b/arch/x86/kernel/sys_ia32.c @@ -61,7 +61,8 @@ SYSCALL_DEFINE3(ia32_truncate64, const char __user *, filename, SYSCALL_DEFINE3(ia32_ftruncate64, unsigned int, fd, unsigned long, offset_low, unsigned long, offset_high) { - return ksys_ftruncate(fd, ((loff_t) offset_high << 32) | offset_low); + return ksys_ftruncate(fd, ((loff_t) offset_high << 32) | offset_low, + FTRUNCATE_LFS); } /* warning: next two assume little endian */ diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index c8e292e9a24dfe..292f1dded78f28 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -4469,7 +4469,7 @@ static const struct opcode opcode_map_0f_38[256] = { X16(N), X16(N), /* 0x20 - 0x2f */ X8(N), - X2(N), GP(SrcReg | DstMem | ModRM | Mov | Aligned, &pfx_0f_e7_0f_38_2a), N, N, N, N, N, + X2(N), GP(SrcMem | DstReg | ModRM | Mov | Aligned, &pfx_0f_e7_0f_38_2a), N, N, N, N, N, /* 0x30 - 0x7f */ X16(N), X16(N), X16(N), X16(N), X16(N), /* 0x80 - 0xef */ diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 9b140bbdc1d83b..4438ecac9a89bb 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -2040,7 +2040,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) * flush). Translate the address here so the memory can be uniformly * read with kvm_read_guest(). */ - if (!hc->fast && is_guest_mode(vcpu)) { + if (!hc->fast && mmu_is_nested(vcpu)) { hc->ingpa = translate_nested_gpa(vcpu, hc->ingpa, 0, NULL); if (unlikely(hc->ingpa == INVALID_GPA)) return HV_STATUS_INVALID_HYPERCALL_INPUT; diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 6ce160ffa67865..6301f79fcbae7c 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -305,14 +305,6 @@ static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu) { return false; } -static inline bool kvm_hv_is_tlb_flush_hcall(struct kvm_vcpu *vcpu) -{ - return false; -} -static inline bool guest_hv_cpuid_has_l2_tlb_flush(struct kvm_vcpu *vcpu) -{ - return false; -} static inline int kvm_hv_verify_vp_assist(struct kvm_vcpu *vcpu) { return 0; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 9381c58d4c85c9..e9f1e545116011 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -669,12 +669,14 @@ bool __kvm_apic_update_irr(unsigned long *pir, void *regs, int *max_irr) u32 irr_val, prev_irr_val; int max_updated_irr; + if (!pi_harvest_pir(pir, pir_vals)) { + *max_irr = apic_find_highest_vector(regs + APIC_IRR); + return false; + } + max_updated_irr = -1; *max_irr = -1; - if (!pi_harvest_pir(pir, pir_vals)) - return false; - for (i = vec = 0; i <= 7; i++, vec += 32) { u32 *p_irr = (u32 *)(regs + APIC_IRR + i * 0x10); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index dd06453d5b72c1..729240bc00a269 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -182,6 +182,8 @@ static struct kmem_cache *pte_list_desc_cache; struct kmem_cache *mmu_page_header_cache; static void mmu_spte_set(u64 *sptep, u64 spte); +static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp, + u64 *spte, struct list_head *invalid_list); struct kvm_mmu_role_regs { const unsigned long cr0; @@ -1287,19 +1289,6 @@ static void drop_spte(struct kvm *kvm, u64 *sptep) rmap_remove(kvm, sptep); } -static void drop_large_spte(struct kvm *kvm, u64 *sptep, bool flush) -{ - struct kvm_mmu_page *sp; - - sp = sptep_to_sp(sptep); - WARN_ON_ONCE(sp->role.level == PG_LEVEL_4K); - - drop_spte(kvm, sptep); - - if (flush) - kvm_flush_remote_tlbs_sptep(kvm, sptep); -} - /* * Write-protect on the specified @sptep, @pt_protect indicates whether * spte write-protection is caused by protecting shadow page table. @@ -2466,7 +2455,8 @@ static struct kvm_mmu_page *kvm_mmu_get_child_sp(struct kvm_vcpu *vcpu, { union kvm_mmu_page_role role; - if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) + if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep) && + spte_to_child_sp(*sptep) && spte_to_child_sp(*sptep)->gfn == gfn) return ERR_PTR(-EEXIST); role = kvm_mmu_child_role(sptep, direct, access); @@ -2544,13 +2534,16 @@ static void __link_shadow_page(struct kvm *kvm, BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK); - /* - * If an SPTE is present already, it must be a leaf and therefore - * a large one. Drop it, and flush the TLB if needed, before - * installing sp. - */ - if (is_shadow_present_pte(*sptep)) - drop_large_spte(kvm, sptep, flush); + if (is_shadow_present_pte(*sptep)) { + struct kvm_mmu_page *parent_sp; + LIST_HEAD(invalid_list); + + parent_sp = sptep_to_sp(sptep); + WARN_ON_ONCE(parent_sp->role.level == PG_LEVEL_4K); + + mmu_page_zap_pte(kvm, parent_sp, sptep, &invalid_list); + kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, true); + } spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp)); diff --git a/arch/x86/kvm/svm/hyperv.h b/arch/x86/kvm/svm/hyperv.h index d3f8bfc05832ee..f70d076911a63d 100644 --- a/arch/x86/kvm/svm/hyperv.h +++ b/arch/x86/kvm/svm/hyperv.h @@ -41,10 +41,17 @@ static inline bool nested_svm_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu) return hv_vcpu->vp_assist_page.nested_control.features.directhypercall; } +static inline bool nested_svm_is_l2_tlb_flush_hcall(struct kvm_vcpu *vcpu) +{ + return guest_hv_cpuid_has_l2_tlb_flush(vcpu) && + nested_svm_l2_tlb_flush_enabled(vcpu) && + kvm_hv_is_tlb_flush_hcall(vcpu); +} + void svm_hv_inject_synthetic_vmexit_post_tlb_flush(struct kvm_vcpu *vcpu); #else /* CONFIG_KVM_HYPERV */ static inline void nested_svm_hv_update_vm_vp_ids(struct kvm_vcpu *vcpu) {} -static inline bool nested_svm_l2_tlb_flush_enabled(struct kvm_vcpu *vcpu) +static inline bool nested_svm_is_l2_tlb_flush_hcall(struct kvm_vcpu *vcpu) { return false; } diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index b36c33255bed66..b3565ebdc7decb 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -128,11 +128,13 @@ void recalc_intercepts(struct vcpu_svm *svm) struct vmcb_ctrl_area_cached *g; unsigned int i; - vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); + vmcb_mark_dirty(svm->vmcb01.ptr, VMCB_INTERCEPTS); if (!is_guest_mode(&svm->vcpu)) return; + vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); + c = &svm->vmcb->control; h = &svm->vmcb01.ptr->control; g = &svm->nested.ctl; @@ -156,13 +158,6 @@ void recalc_intercepts(struct vcpu_svm *svm) vmcb_clr_intercept(c, INTERCEPT_VINTR); } - /* - * We want to see VMMCALLs from a nested guest only when Hyper-V L2 TLB - * flush feature is enabled. - */ - if (!nested_svm_l2_tlb_flush_enabled(&svm->vcpu)) - vmcb_clr_intercept(c, INTERCEPT_VMMCALL); - for (i = 0; i < MAX_INTERCEPT; i++) c->intercepts[i] |= g->intercepts[i]; @@ -348,7 +343,8 @@ static bool __nested_vmcb_check_controls(struct kvm_vcpu *vcpu, if (CC(control->asid == 0)) return false; - if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && !npt_enabled)) + if (CC((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && + !kvm_vcpu_is_legal_gpa(vcpu, control->nested_cr3))) return false; if (CC(!nested_svm_check_bitmap_pa(vcpu, control->msrpm_base_pa, @@ -390,6 +386,10 @@ static bool __nested_vmcb_check_save(struct kvm_vcpu *vcpu, CC(!(save->cr0 & X86_CR0_PE)) || CC(!kvm_vcpu_is_legal_cr3(vcpu, save->cr3))) return false; + + if (CC((save->cs.attrib & SVM_SELECTOR_L_MASK) && + (save->cs.attrib & SVM_SELECTOR_DB_MASK))) + return false; } /* Note, SVM doesn't have any additional restrictions on CR4. */ @@ -456,6 +456,11 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, nested_svm_sanitize_intercept(vcpu, to, SKINIT); nested_svm_sanitize_intercept(vcpu, to, RDPRU); + /* Always clear SVM_NESTED_CTL_NP_ENABLE if the guest cannot use NPTs */ + to->nested_ctl = from->nested_ctl; + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT)) + to->nested_ctl &= ~SVM_NESTED_CTL_NP_ENABLE; + to->iopm_base_pa = from->iopm_base_pa; to->msrpm_base_pa = from->msrpm_base_pa; to->tsc_offset = from->tsc_offset; @@ -469,7 +474,6 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu, to->exit_info_2 = from->exit_info_2; to->exit_int_info = from->exit_int_info; to->exit_int_info_err = from->exit_int_info_err; - to->nested_ctl = from->nested_ctl; to->event_inj = from->event_inj; to->event_inj_err = from->event_inj_err; to->next_rip = from->next_rip; @@ -506,6 +510,8 @@ static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to, * Copy only fields that are validated, as we need them * to avoid TOC/TOU races. */ + to->cs = from->cs; + to->efer = from->efer; to->cr0 = from->cr0; to->cr3 = from->cr3; @@ -530,6 +536,7 @@ void nested_sync_control_from_vmcb02(struct vcpu_svm *svm) u32 mask; svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; + svm->nested.ctl.int_state = svm->vmcb->control.int_state; /* Only a few fields of int_ctl are written by the processor. */ mask = V_IRQ_MASK | V_TPR_MASK; @@ -648,6 +655,12 @@ void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm) svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat; } +static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu) +{ + return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && + (to_svm(vcpu)->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK); +} + static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12) { bool new_vmcb12 = false; @@ -712,17 +725,17 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 vmcb_mark_dirty(vmcb02, VMCB_DR); } - if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) { + if (nested_vmcb12_has_lbrv(vcpu)) { /* * Reserved bits of DEBUGCTL are ignored. Be consistent with * svm_set_msr's definition of reserved bits. */ - svm_copy_lbrs(vmcb02, vmcb12); + svm_copy_lbrs(&vmcb02->save, &vmcb12->save); vmcb02->save.dbgctl &= ~DEBUGCTL_RESERVED_BITS; } else { - svm_copy_lbrs(vmcb02, vmcb01); + svm_copy_lbrs(&vmcb02->save, &vmcb01->save); } + vmcb_mark_dirty(vmcb02, VMCB_LBR); svm_update_lbrv(&svm->vcpu); } @@ -750,9 +763,7 @@ static bool is_evtinj_nmi(u32 evtinj) return type == SVM_EVTINJ_TYPE_NMI; } -static void nested_vmcb02_prepare_control(struct vcpu_svm *svm, - unsigned long vmcb12_rip, - unsigned long vmcb12_csbase) +static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) { u32 int_ctl_vmcb01_bits = V_INTR_MASKING_MASK; u32 int_ctl_vmcb12_bits = V_TPR_MASK | V_IRQ_INJECTION_BITS_MASK; @@ -853,27 +864,27 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm, vmcb02->control.event_inj_err = svm->nested.ctl.event_inj_err; /* - * next_rip is consumed on VMRUN as the return address pushed on the - * stack for injected soft exceptions/interrupts. If nrips is exposed - * to L1, take it verbatim from vmcb12. If nrips is supported in - * hardware but not exposed to L1, stuff the actual L2 RIP to emulate - * what a nrips=0 CPU would do (L1 is responsible for advancing RIP - * prior to injecting the event). + * If nrips is exposed to L1, take NextRIP as-is. Otherwise, L1 + * advances L2's RIP before VMRUN instead of using NextRIP. KVM will + * stuff the current RIP as vmcb02's NextRIP before L2 is run. After + * the first run of L2 (e.g. after save+restore), NextRIP is updated by + * the CPU and/or KVM and should be used regardless of L1's support. */ - if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) - vmcb02->control.next_rip = svm->nested.ctl.next_rip; - else if (boot_cpu_has(X86_FEATURE_NRIPS)) - vmcb02->control.next_rip = vmcb12_rip; + if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) || + !svm->nested.nested_run_pending) + vmcb02->control.next_rip = svm->nested.ctl.next_rip; svm->nmi_l1_to_l2 = is_evtinj_nmi(vmcb02->control.event_inj); + + /* + * soft_int_csbase, soft_int_old_rip, and soft_int_next_rip (if L1 + * doesn't have NRIPS) are initialized later, before the vCPU is run. + */ if (is_evtinj_soft(vmcb02->control.event_inj)) { svm->soft_int_injected = true; - svm->soft_int_csbase = vmcb12_csbase; - svm->soft_int_old_rip = vmcb12_rip; - if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) + if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS) || + !svm->nested.nested_run_pending) svm->soft_int_next_rip = svm->nested.ctl.next_rip; - else - svm->soft_int_next_rip = vmcb12_rip; } /* LBR_CTL_ENABLE_MASK is controlled by svm_update_lbrv() */ @@ -971,7 +982,7 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, nested_svm_copy_common_state(svm->vmcb01.ptr, svm->nested.vmcb02.ptr); svm_switch_vmcb(svm, &svm->nested.vmcb02); - nested_vmcb02_prepare_control(svm, vmcb12->save.rip, vmcb12->save.cs.base); + nested_vmcb02_prepare_control(svm); nested_vmcb02_prepare_save(svm, vmcb12); ret = nested_svm_load_cr3(&svm->vcpu, svm->nested.save.cr3, @@ -1019,12 +1030,9 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) } vmcb12_gpa = svm->vmcb->save.rax; - ret = kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map); - if (ret == -EINVAL) { + if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) { kvm_inject_gp(vcpu, 0); return 1; - } else if (ret) { - return kvm_skip_emulated_instruction(vcpu); } ret = kvm_skip_emulated_instruction(vcpu); @@ -1041,6 +1049,9 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) vmcb12->control.exit_code = SVM_EXIT_ERR; vmcb12->control.exit_info_1 = 0; vmcb12->control.exit_info_2 = 0; + vmcb12->control.event_inj = 0; + vmcb12->control.event_inj_err = 0; + svm_set_gif(svm, false); goto out; } @@ -1067,8 +1078,6 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) out_exit_err: svm->nested.nested_run_pending = 0; - svm->nmi_l1_to_l2 = false; - svm->soft_int_injected = false; svm->vmcb->control.exit_code = SVM_EXIT_ERR; svm->vmcb->control.exit_info_1 = 0; @@ -1107,6 +1116,11 @@ void svm_copy_vmrun_state(struct vmcb_save_area *to_save, to_save->isst_addr = from_save->isst_addr; to_save->ssp = from_save->ssp; } + + if (kvm_cpu_cap_has(X86_FEATURE_LBRV)) { + svm_copy_lbrs(to_save, from_save); + to_save->dbgctl &= ~DEBUGCTL_RESERVED_BITS; + } } void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) @@ -1125,36 +1139,20 @@ void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb) to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; } -int nested_svm_vmexit(struct vcpu_svm *svm) +static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) { - struct kvm_vcpu *vcpu = &svm->vcpu; - struct vmcb *vmcb01 = svm->vmcb01.ptr; + struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; - struct vmcb *vmcb12; struct kvm_host_map map; + struct vmcb *vmcb12; int rc; rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); - if (rc) { - if (rc == -EINVAL) - kvm_inject_gp(vcpu, 0); - return 1; - } + if (rc) + return rc; vmcb12 = map.hva; - /* Exit Guest-Mode */ - leave_guest_mode(vcpu); - svm->nested.vmcb12_gpa = 0; - WARN_ON_ONCE(svm->nested.nested_run_pending); - - kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); - - /* in case we halted in L2 */ - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - - /* Give the current vmcb to the guest */ - vmcb12->save.es = vmcb02->save.es; vmcb12->save.cs = vmcb02->save.cs; vmcb12->save.ss = vmcb02->save.ss; @@ -1164,7 +1162,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) vmcb12->save.efer = svm->vcpu.arch.efer; vmcb12->save.cr0 = kvm_read_cr0(vcpu); vmcb12->save.cr3 = kvm_read_cr3(vcpu); - vmcb12->save.cr2 = vmcb02->save.cr2; + vmcb12->save.cr2 = vcpu->arch.cr2; vmcb12->save.cr4 = svm->vcpu.arch.cr4; vmcb12->save.rflags = kvm_get_rflags(vcpu); vmcb12->save.rip = kvm_rip_read(vcpu); @@ -1191,9 +1189,42 @@ int nested_svm_vmexit(struct vcpu_svm *svm) if (guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) vmcb12->control.next_rip = vmcb02->control.next_rip; + if (nested_vmcb12_has_lbrv(vcpu)) + svm_copy_lbrs(&vmcb12->save, &vmcb02->save); + + vmcb12->control.event_inj = 0; + vmcb12->control.event_inj_err = 0; vmcb12->control.int_ctl = svm->nested.ctl.int_ctl; - vmcb12->control.event_inj = svm->nested.ctl.event_inj; - vmcb12->control.event_inj_err = svm->nested.ctl.event_inj_err; + + trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, + vmcb12->control.exit_info_1, + vmcb12->control.exit_info_2, + vmcb12->control.exit_int_info, + vmcb12->control.exit_int_info_err, + KVM_ISA_SVM); + + kvm_vcpu_unmap(vcpu, &map); + return 0; +} + +void nested_svm_vmexit(struct vcpu_svm *svm) +{ + struct kvm_vcpu *vcpu = &svm->vcpu; + struct vmcb *vmcb01 = svm->vmcb01.ptr; + struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; + + if (nested_svm_vmexit_update_vmcb12(vcpu)) + kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); + + /* Exit Guest-Mode */ + leave_guest_mode(vcpu); + svm->nested.vmcb12_gpa = 0; + WARN_ON_ONCE(svm->nested.nested_run_pending); + + kvm_clear_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); + + /* in case we halted in L2 */ + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); if (!kvm_pause_in_guest(vcpu->kvm)) { vmcb01->control.pause_filter_count = vmcb02->control.pause_filter_count; @@ -1239,11 +1270,10 @@ int nested_svm_vmexit(struct vcpu_svm *svm) if (!nested_exit_on_intr(svm)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - if (unlikely(guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) && - (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK))) - svm_copy_lbrs(vmcb12, vmcb02); - else - svm_copy_lbrs(vmcb01, vmcb02); + if (!nested_vmcb12_has_lbrv(vcpu)) { + svm_copy_lbrs(&vmcb01->save, &vmcb02->save); + vmcb_mark_dirty(vmcb01, VMCB_LBR); + } svm_update_lbrv(vcpu); @@ -1296,22 +1326,16 @@ int nested_svm_vmexit(struct vcpu_svm *svm) svm->vcpu.arch.dr7 = DR7_FIXED_1; kvm_update_dr7(&svm->vcpu); - trace_kvm_nested_vmexit_inject(vmcb12->control.exit_code, - vmcb12->control.exit_info_1, - vmcb12->control.exit_info_2, - vmcb12->control.exit_int_info, - vmcb12->control.exit_int_info_err, - KVM_ISA_SVM); - - kvm_vcpu_unmap(vcpu, &map); - nested_svm_transition_tlb_flush(vcpu); nested_svm_uninit_mmu_context(vcpu); - rc = nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true); - if (rc) - return 1; + if (nested_svm_load_cr3(vcpu, vmcb01->save.cr3, false, true)) + kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu); + + /* Drop tracking for L1->L2 injected NMIs and soft IRQs */ + svm->nmi_l1_to_l2 = false; + svm->soft_int_injected = false; /* * Drop what we picked up for L2 via svm_complete_interrupts() so it @@ -1336,8 +1360,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm) */ if (kvm_apicv_activated(vcpu->kvm)) __kvm_vcpu_update_apicv(vcpu); - - return 0; } static void nested_svm_triple_fault(struct kvm_vcpu *vcpu) @@ -1682,9 +1704,7 @@ int nested_svm_exit_special(struct vcpu_svm *svm) } case SVM_EXIT_VMMCALL: /* Hyper-V L2 TLB flush hypercall is handled by L0 */ - if (guest_hv_cpuid_has_l2_tlb_flush(vcpu) && - nested_svm_l2_tlb_flush_enabled(vcpu) && - kvm_hv_is_tlb_flush_hcall(vcpu)) + if (nested_svm_is_l2_tlb_flush_hcall(vcpu)) return NESTED_EXIT_HOST; break; default: @@ -1915,7 +1935,13 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, nested_copy_vmcb_control_to_cache(svm, ctl); svm_switch_vmcb(svm, &svm->nested.vmcb02); - nested_vmcb02_prepare_control(svm, svm->vmcb->save.rip, svm->vmcb->save.cs.base); + nested_vmcb02_prepare_control(svm); + + /* + * Any previously restored state (e.g. KVM_SET_SREGS) would mark fields + * dirty in vmcb01 instead of vmcb02, so mark all of vmcb02 dirty here. + */ + vmcb_mark_all_dirty(svm->vmcb); /* * While the nested guest CR3 is already checked and set by @@ -1930,6 +1956,9 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, svm->nested.force_msr_bitmap_recalc = true; + if (kvm_vcpu_apicv_active(vcpu)) + kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); + kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); ret = 0; out_free: diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 3f9c1aa39a0af5..fb278fb1652d8d 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -690,10 +690,16 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, if (ulen == 0 || uaddr + ulen < uaddr) return ERR_PTR(-EINVAL); - /* Calculate number of pages. */ + /* + * Calculate the number of pages that need to be pinned to cover the + * entire range. Note! This isn't simply ulen >> PAGE_SHIFT, as KVM + * doesn't require the incoming address+size to be page aligned! + */ first = (uaddr & PAGE_MASK) >> PAGE_SHIFT; last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT; npages = (last - first + 1); + if (npages > INT_MAX) + return ERR_PTR(-EINVAL); locked = sev->pages_locked + npages; lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; @@ -702,9 +708,6 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, return ERR_PTR(-ENOMEM); } - if (WARN_ON_ONCE(npages > INT_MAX)) - return ERR_PTR(-EINVAL); - /* Avoid using vmalloc for smaller buffers. */ size = npages * sizeof(struct page *); if (size > PAGE_SIZE) @@ -882,6 +885,11 @@ static int sev_es_sync_vmsa(struct vcpu_svm *svm) u8 *d; int i; + lockdep_assert_held(&vcpu->mutex); + + if (vcpu->arch.guest_state_protected) + return -EINVAL; + /* Check some debug related fields before encrypting the VMSA */ if (svm->vcpu.guest_debug || (svm->vmcb->save.dr7 & ~DR7_FIXED_1)) return -EINVAL; @@ -1027,6 +1035,9 @@ static int sev_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (!sev_es_guest(kvm)) return -ENOTTY; + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + kvm_for_each_vcpu(i, vcpu, kvm) { ret = mutex_lock_killable(&vcpu->mutex); if (ret) @@ -2047,8 +2058,8 @@ static int sev_check_source_vcpus(struct kvm *dst, struct kvm *src) struct kvm_vcpu *src_vcpu; unsigned long i; - if (src->created_vcpus != atomic_read(&src->online_vcpus) || - dst->created_vcpus != atomic_read(&dst->online_vcpus)) + if (kvm_is_vcpu_creation_in_progress(src) || + kvm_is_vcpu_creation_in_progress(dst)) return -EBUSY; if (!sev_es_guest(src)) @@ -2447,6 +2458,13 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) unsigned long i; int ret; + if (kvm_is_vcpu_creation_in_progress(kvm)) + return -EBUSY; + + ret = kvm_lock_all_vcpus(kvm); + if (ret) + return ret; + data.gctx_paddr = __psp_pa(sev->snp_context); data.page_type = SNP_PAGE_TYPE_VMSA; @@ -2456,12 +2474,12 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) ret = sev_es_sync_vmsa(svm); if (ret) - return ret; + goto out; /* Transition the VMSA page to a firmware state. */ ret = rmp_make_private(pfn, INITIAL_VMSA_GPA, PG_LEVEL_4K, sev->asid, true); if (ret) - return ret; + goto out; /* Issue the SNP command to encrypt the VMSA */ data.address = __sme_pa(svm->sev_es.vmsa); @@ -2470,7 +2488,7 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) if (ret) { snp_page_reclaim(kvm, pfn); - return ret; + goto out; } svm->vcpu.arch.guest_state_protected = true; @@ -2484,7 +2502,9 @@ static int snp_launch_update_vmsa(struct kvm *kvm, struct kvm_sev_cmd *argp) svm_enable_lbrv(vcpu); } - return 0; +out: + kvm_unlock_all_vcpus(kvm); + return ret; } static int snp_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp) @@ -2701,6 +2721,8 @@ int sev_mem_enc_register_region(struct kvm *kvm, struct enc_region *region; int ret = 0; + guard(mutex)(&kvm->lock); + if (!sev_guest(kvm)) return -ENOTTY; @@ -2715,12 +2737,10 @@ int sev_mem_enc_register_region(struct kvm *kvm, if (!region) return -ENOMEM; - mutex_lock(&kvm->lock); region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, FOLL_WRITE | FOLL_LONGTERM); if (IS_ERR(region->pages)) { ret = PTR_ERR(region->pages); - mutex_unlock(&kvm->lock); goto e_free; } @@ -2738,8 +2758,6 @@ int sev_mem_enc_register_region(struct kvm *kvm, region->size = range->size; list_add_tail(®ion->list, &sev->regions_list); - mutex_unlock(&kvm->lock); - return ret; e_free: diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e6477affac9a04..cb0a89da99d097 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -52,6 +52,7 @@ #include "svm.h" #include "svm_ops.h" +#include "hyperv.h" #include "kvm_onhyperv.h" #include "svm_onhyperv.h" @@ -841,17 +842,6 @@ static void svm_recalc_msr_intercepts(struct kvm_vcpu *vcpu) */ } -void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb) -{ - to_vmcb->save.dbgctl = from_vmcb->save.dbgctl; - to_vmcb->save.br_from = from_vmcb->save.br_from; - to_vmcb->save.br_to = from_vmcb->save.br_to; - to_vmcb->save.last_excp_from = from_vmcb->save.last_excp_from; - to_vmcb->save.last_excp_to = from_vmcb->save.last_excp_to; - - vmcb_mark_dirty(to_vmcb, VMCB_LBR); -} - static void __svm_enable_lbrv(struct kvm_vcpu *vcpu) { to_svm(vcpu)->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; @@ -2244,13 +2234,9 @@ static int emulate_svm_instr(struct kvm_vcpu *vcpu, int opcode) [SVM_INSTR_VMSAVE] = vmsave_interception, }; struct vcpu_svm *svm = to_svm(vcpu); - int ret; if (is_guest_mode(vcpu)) { - /* Returns '1' or -errno on failure, '0' on success. */ - ret = nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]); - if (ret) - return ret; + nested_svm_simple_vmexit(svm, guest_mode_exit_codes[opcode]); return 1; } return svm_instr_handlers[opcode](vcpu); @@ -2366,6 +2352,9 @@ static int invlpga_interception(struct kvm_vcpu *vcpu) gva_t gva = kvm_rax_read(vcpu); u32 asid = kvm_rcx_read(vcpu); + if (nested_svm_check_permissions(vcpu)) + return 1; + /* FIXME: Handle an address size prefix. */ if (!is_long_mode(vcpu)) gva = (u32)gva; @@ -2796,19 +2785,19 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = svm->tsc_aux; break; case MSR_IA32_DEBUGCTLMSR: - msr_info->data = svm->vmcb->save.dbgctl; + msr_info->data = lbrv ? svm->vmcb->save.dbgctl : 0; break; case MSR_IA32_LASTBRANCHFROMIP: - msr_info->data = svm->vmcb->save.br_from; + msr_info->data = lbrv ? svm->vmcb->save.br_from : 0; break; case MSR_IA32_LASTBRANCHTOIP: - msr_info->data = svm->vmcb->save.br_to; + msr_info->data = lbrv ? svm->vmcb->save.br_to : 0; break; case MSR_IA32_LASTINTFROMIP: - msr_info->data = svm->vmcb->save.last_excp_from; + msr_info->data = lbrv ? svm->vmcb->save.last_excp_from : 0; break; case MSR_IA32_LASTINTTOIP: - msr_info->data = svm->vmcb->save.last_excp_to; + msr_info->data = lbrv ? svm->vmcb->save.last_excp_to : 0; break; case MSR_VM_HSAVE_PA: msr_info->data = svm->nested.hsave_msr; @@ -3083,6 +3072,38 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr) vmcb_mark_dirty(svm->vmcb, VMCB_LBR); svm_update_lbrv(vcpu); break; + case MSR_IA32_LASTBRANCHFROMIP: + if (!lbrv) + return KVM_MSR_RET_UNSUPPORTED; + if (!msr->host_initiated) + return 1; + svm->vmcb->save.br_from = data; + vmcb_mark_dirty(svm->vmcb, VMCB_LBR); + break; + case MSR_IA32_LASTBRANCHTOIP: + if (!lbrv) + return KVM_MSR_RET_UNSUPPORTED; + if (!msr->host_initiated) + return 1; + svm->vmcb->save.br_to = data; + vmcb_mark_dirty(svm->vmcb, VMCB_LBR); + break; + case MSR_IA32_LASTINTFROMIP: + if (!lbrv) + return KVM_MSR_RET_UNSUPPORTED; + if (!msr->host_initiated) + return 1; + svm->vmcb->save.last_excp_from = data; + vmcb_mark_dirty(svm->vmcb, VMCB_LBR); + break; + case MSR_IA32_LASTINTTOIP: + if (!lbrv) + return KVM_MSR_RET_UNSUPPORTED; + if (!msr->host_initiated) + return 1; + svm->vmcb->save.last_excp_to = data; + vmcb_mark_dirty(svm->vmcb, VMCB_LBR); + break; case MSR_VM_HSAVE_PA: /* * Old kernels did not validate the value written to @@ -3229,6 +3250,22 @@ static int bus_lock_exit(struct kvm_vcpu *vcpu) return 0; } +static int vmmcall_interception(struct kvm_vcpu *vcpu) +{ + /* + * Inject a #UD if L2 is active and the VMMCALL isn't a Hyper-V TLB + * hypercall, as VMMCALL #UDs if it's not intercepted, and this path is + * reachable if and only if L1 doesn't want to intercept VMMCALL or has + * enabled L0 (KVM) handling of Hyper-V L2 TLB flush hypercalls. + */ + if (is_guest_mode(vcpu) && !nested_svm_is_l2_tlb_flush_hcall(vcpu)) { + kvm_queue_exception(vcpu, UD_VECTOR); + return 1; + } + + return kvm_emulate_hypercall(vcpu); +} + static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_READ_CR0] = cr_interception, [SVM_EXIT_READ_CR3] = cr_interception, @@ -3279,7 +3316,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_TASK_SWITCH] = task_switch_interception, [SVM_EXIT_SHUTDOWN] = shutdown_interception, [SVM_EXIT_VMRUN] = vmrun_interception, - [SVM_EXIT_VMMCALL] = kvm_emulate_hypercall, + [SVM_EXIT_VMMCALL] = vmmcall_interception, [SVM_EXIT_VMLOAD] = vmload_interception, [SVM_EXIT_VMSAVE] = vmsave_interception, [SVM_EXIT_STGI] = stgi_interception, @@ -3638,6 +3675,16 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return svm_invoke_exit_handler(vcpu, svm->vmcb->control.exit_code); } +static void svm_set_nested_run_soft_int_state(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + svm->soft_int_csbase = svm->vmcb->save.cs.base; + svm->soft_int_old_rip = kvm_rip_read(vcpu); + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) + svm->soft_int_next_rip = kvm_rip_read(vcpu); +} + static int pre_svm_run(struct kvm_vcpu *vcpu) { struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); @@ -3739,6 +3786,36 @@ static void svm_inject_irq(struct kvm_vcpu *vcpu, bool reinjected) svm->vmcb->control.event_inj = intr->nr | SVM_EVTINJ_VALID | type; } +static void svm_fixup_nested_rips(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + + if (!is_guest_mode(vcpu) || !svm->nested.nested_run_pending) + return; + + /* + * If nrips is supported in hardware but not exposed to L1, stuff the + * actual L2 RIP to emulate what a nrips=0 CPU would do (L1 is + * responsible for advancing RIP prior to injecting the event). Once L2 + * runs after L1 executes VMRUN, NextRIP is updated by the CPU and/or + * KVM, and this is no longer needed. + * + * This is done here (as opposed to when preparing vmcb02) to use the + * most up-to-date value of RIP regardless of the order of restoring + * registers and nested state in the vCPU save+restore path. + */ + if (boot_cpu_has(X86_FEATURE_NRIPS) && + !guest_cpu_cap_has(vcpu, X86_FEATURE_NRIPS)) + svm->vmcb->control.next_rip = kvm_rip_read(vcpu); + + /* + * Simiarly, initialize the soft int metadata here to use the most + * up-to-date values of RIP and CS base, regardless of restore order. + */ + if (svm->soft_int_injected) + svm_set_nested_run_soft_int_state(vcpu); +} + void svm_complete_interrupt_delivery(struct kvm_vcpu *vcpu, int delivery_mode, int trig_mode, int vector) { @@ -4106,6 +4183,18 @@ static void svm_complete_soft_interrupt(struct kvm_vcpu *vcpu, u8 vector, bool is_soft = (type == SVM_EXITINTINFO_TYPE_SOFT); struct vcpu_svm *svm = to_svm(vcpu); + /* + * Initialize the soft int fields *before* reading them below if KVM + * aborted entry to the guest with a nested VMRUN pending. To ensure + * KVM uses up-to-date values for RIP and CS base across save/restore, + * regardless of restore order, KVM waits to set the soft int fields + * until VMRUN is imminent. But when canceling injection, KVM requeues + * the soft int and will reinject it via the standard injection flow, + * and so KVM needs to grab the state from the pending nested VMRUN. + */ + if (is_guest_mode(vcpu) && svm->nested.nested_run_pending) + svm_set_nested_run_soft_int_state(vcpu); + /* * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's * associated with the original soft exception/interrupt. next_rip is @@ -4335,6 +4424,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS)) svm->vmcb->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP; + svm_fixup_nested_rips(vcpu); + svm_hv_update_vp_id(svm->vmcb, vcpu); /* @@ -4436,6 +4527,16 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags) svm_complete_interrupts(vcpu); + /* + * Update the cache after completing interrupts to get an accurate + * NextRIP, e.g. when re-injecting a soft interrupt. + * + * FIXME: Rework svm_get_nested_state() to not pull data from the + * cache (except for maybe int_ctl). + */ + if (is_guest_mode(vcpu)) + svm->nested.ctl.next_rip = svm->vmcb->control.next_rip; + return svm_exit_handlers_fastpath(vcpu); } @@ -4784,7 +4885,6 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_host_map map_save; - int ret; if (!is_guest_mode(vcpu)) return 0; @@ -4804,9 +4904,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; - ret = nested_svm_simple_vmexit(svm, SVM_EXIT_SW); - if (ret) - return ret; + nested_svm_simple_vmexit(svm, SVM_EXIT_SW); /* * KVM uses VMCB01 to store L1 host state while L2 runs but diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 6942e6b0eda67e..521787ad1224ac 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -140,6 +140,7 @@ struct kvm_vmcb_info { }; struct vmcb_save_area_cached { + struct vmcb_seg cs; u64 efer; u64 cr4; u64 cr3; @@ -713,8 +714,16 @@ static inline void *svm_vcpu_alloc_msrpm(void) return svm_alloc_permissions_map(MSRPM_SIZE, GFP_KERNEL_ACCOUNT); } +#define svm_copy_lbrs(to, from) \ +do { \ + (to)->dbgctl = (from)->dbgctl; \ + (to)->br_from = (from)->br_from; \ + (to)->br_to = (from)->br_to; \ + (to)->last_excp_from = (from)->last_excp_from; \ + (to)->last_excp_to = (from)->last_excp_to; \ +} while (0) + void svm_vcpu_free_msrpm(void *msrpm); -void svm_copy_lbrs(struct vmcb *to_vmcb, struct vmcb *from_vmcb); void svm_enable_lbrv(struct kvm_vcpu *vcpu); void svm_update_lbrv(struct kvm_vcpu *vcpu); @@ -785,14 +794,14 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu); void svm_copy_vmrun_state(struct vmcb_save_area *to_save, struct vmcb_save_area *from_save); void svm_copy_vmloadsave_state(struct vmcb *to_vmcb, struct vmcb *from_vmcb); -int nested_svm_vmexit(struct vcpu_svm *svm); +void nested_svm_vmexit(struct vcpu_svm *svm); -static inline int nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code) +static inline void nested_svm_simple_vmexit(struct vcpu_svm *svm, u32 exit_code) { svm->vmcb->control.exit_code = exit_code; svm->vmcb->control.exit_info_1 = 0; svm->vmcb->control.exit_info_2 = 0; - return nested_svm_vmexit(svm); + nested_svm_vmexit(svm); } int nested_svm_exit_handled(struct vcpu_svm *svm); diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index e7fdbe9efc904c..0db25bba17f6e8 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -154,7 +154,7 @@ TRACE_EVENT(kvm_xen_hypercall, __entry->a2 = a2; __entry->a3 = a3; __entry->a4 = a4; - __entry->a4 = a5; + __entry->a5 = a5; ), TP_printk("cpl %d nr 0x%lx a0 0x%lx a1 0x%lx a2 0x%lx a3 0x%lx a4 0x%lx a5 %lx", diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index fd1c4a36b59369..ccc7b2e34a11e8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -351,6 +351,9 @@ static const u32 msrs_to_save_base[] = { MSR_IA32_U_CET, MSR_IA32_S_CET, MSR_IA32_PL0_SSP, MSR_IA32_PL1_SSP, MSR_IA32_PL2_SSP, MSR_IA32_PL3_SSP, MSR_IA32_INT_SSP_TAB, + MSR_IA32_DEBUGCTLMSR, + MSR_IA32_LASTBRANCHFROMIP, MSR_IA32_LASTBRANCHTOIP, + MSR_IA32_LASTINTFROMIP, MSR_IA32_LASTINTTOIP, }; static const u32 msrs_to_save_pmu[] = { @@ -864,9 +867,6 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr, vcpu->arch.exception.error_code = error_code; vcpu->arch.exception.has_payload = has_payload; vcpu->arch.exception.payload = payload; - if (!is_guest_mode(vcpu)) - kvm_deliver_exception_payload(vcpu, - &vcpu->arch.exception); return; } @@ -5531,18 +5531,8 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, return 0; } -static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, - struct kvm_vcpu_events *events) +static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *vcpu) { - struct kvm_queued_exception *ex; - - process_nmi(vcpu); - -#ifdef CONFIG_KVM_SMM - if (kvm_check_request(KVM_REQ_SMI, vcpu)) - process_smi(vcpu); -#endif - /* * KVM's ABI only allows for one exception to be migrated. Luckily, * the only time there can be two queued exceptions is if there's a @@ -5553,21 +5543,46 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, if (vcpu->arch.exception_vmexit.pending && !vcpu->arch.exception.pending && !vcpu->arch.exception.injected) - ex = &vcpu->arch.exception_vmexit; - else - ex = &vcpu->arch.exception; + return &vcpu->arch.exception_vmexit; + + return &vcpu->arch.exception; +} + +static void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) +{ + struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); /* - * In guest mode, payload delivery should be deferred if the exception - * will be intercepted by L1, e.g. KVM should not modifying CR2 if L1 - * intercepts #PF, ditto for DR6 and #DBs. If the per-VM capability, - * KVM_CAP_EXCEPTION_PAYLOAD, is not set, userspace may or may not - * propagate the payload and so it cannot be safely deferred. Deliver - * the payload if the capability hasn't been requested. + * If KVM_CAP_EXCEPTION_PAYLOAD is disabled, then (prematurely) deliver + * the pending exception payload when userspace saves *any* vCPU state + * that interacts with exception payloads to avoid breaking userspace. + * + * Architecturally, KVM must not deliver an exception payload until the + * exception is actually injected, e.g. to avoid losing pending #DB + * information (which VMX tracks in the VMCS), and to avoid clobbering + * state if the exception is never injected for whatever reason. But + * if KVM_CAP_EXCEPTION_PAYLOAD isn't enabled, then userspace may or + * may not propagate the payload across save+restore, and so KVM can't + * safely defer delivery of the payload. */ if (!vcpu->kvm->arch.exception_payload_enabled && ex->pending && ex->has_payload) kvm_deliver_exception_payload(vcpu, ex); +} + +static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, + struct kvm_vcpu_events *events) +{ + struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); + + process_nmi(vcpu); + +#ifdef CONFIG_KVM_SMM + if (kvm_check_request(KVM_REQ_SMI, vcpu)) + process_smi(vcpu); +#endif + + kvm_handle_exception_payload_quirk(vcpu); memset(events, 0, sizeof(*events)); @@ -5746,6 +5761,8 @@ static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, vcpu->arch.guest_state_protected) return -EINVAL; + kvm_handle_exception_payload_quirk(vcpu); + memset(dbgregs, 0, sizeof(*dbgregs)); BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); @@ -8225,7 +8242,13 @@ static int emulator_read_write_onepage(unsigned long addr, void *val, WARN_ON(vcpu->mmio_nr_fragments >= KVM_MAX_MMIO_FRAGMENTS); frag = &vcpu->mmio_fragments[vcpu->mmio_nr_fragments++]; frag->gpa = gpa; - frag->data = val; + if (write && bytes <= 8u) { + frag->val = 0; + frag->data = &frag->val; + memcpy(&frag->val, val, bytes); + } else { + frag->data = val; + } frag->len = bytes; return X86EMUL_CONTINUE; } @@ -8240,6 +8263,9 @@ static int emulator_read_write(struct x86_emulate_ctxt *ctxt, gpa_t gpa; int rc; + if (WARN_ON_ONCE((bytes > 8u || !ops->write) && object_is_on_stack(val))) + return X86EMUL_UNHANDLEABLE; + if (ops->read_write_prepare && ops->read_write_prepare(vcpu, val, bytes)) return X86EMUL_CONTINUE; @@ -11846,6 +11872,9 @@ static int complete_emulated_mmio(struct kvm_vcpu *vcpu) frag++; vcpu->mmio_cur_fragment++; } else { + if (WARN_ON_ONCE(frag->data == &frag->val)) + return -EIO; + /* Go forward to the next mmio piece. */ frag->data += len; frag->gpa += len; @@ -12136,6 +12165,8 @@ static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) if (vcpu->arch.guest_state_protected) goto skip_protected_regs; + kvm_handle_exception_payload_quirk(vcpu); + kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); diff --git a/arch/x86/lib/copy_user_uncached_64.S b/arch/x86/lib/copy_user_uncached_64.S index 18350b343c2abb..8ed0ce3ad2271f 100644 --- a/arch/x86/lib/copy_user_uncached_64.S +++ b/arch/x86/lib/copy_user_uncached_64.S @@ -27,7 +27,7 @@ * Output: * rax uncopied bytes or 0 if successful. */ -SYM_FUNC_START(__copy_user_nocache) +SYM_FUNC_START(copy_to_nontemporal) ANNOTATE_NOENDBR /* If destination is not 7-byte aligned, we'll have to align it */ testb $7,%dil @@ -240,5 +240,5 @@ _ASM_EXTABLE_UA(95b, .Ldone) _ASM_EXTABLE_UA(52b, .Ldone0) _ASM_EXTABLE_UA(53b, .Ldone0) -SYM_FUNC_END(__copy_user_nocache) -EXPORT_SYMBOL(__copy_user_nocache) +SYM_FUNC_END(copy_to_nontemporal) +EXPORT_SYMBOL(copy_to_nontemporal) diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c index f6f436f1d57375..ac27e39fc993d9 100644 --- a/arch/x86/lib/usercopy_32.c +++ b/arch/x86/lib/usercopy_32.c @@ -322,10 +322,11 @@ unsigned long __copy_user_ll(void *to, const void *from, unsigned long n) } EXPORT_SYMBOL(__copy_user_ll); -unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, +unsigned long copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { - __uaccess_begin_nospec(); + if (!user_access_begin(from, n)) + return n; #ifdef CONFIG_X86_INTEL_USERCOPY if (n > 64 && static_cpu_has(X86_FEATURE_XMM2)) n = __copy_user_intel_nocache(to, from, n); @@ -334,7 +335,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr #else __copy_user(to, from, n); #endif - __uaccess_end(); + user_access_end(); return n; } -EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); +EXPORT_SYMBOL(copy_from_user_inatomic_nontemporal); diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c index 654280aaa3e9e0..c47d8cd0e243b1 100644 --- a/arch/x86/lib/usercopy_64.c +++ b/arch/x86/lib/usercopy_64.c @@ -43,17 +43,17 @@ void arch_wb_cache_pmem(void *addr, size_t size) } EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); -long __copy_user_flushcache(void *dst, const void __user *src, unsigned size) +size_t copy_user_flushcache(void *dst, const void __user *src, size_t size) { unsigned long flushed, dest = (unsigned long) dst; - long rc; + unsigned long rc; - stac(); - rc = __copy_user_nocache(dst, src, size); - clac(); + src = masked_user_access_begin(src); + rc = copy_to_nontemporal(dst, (__force const void *)src, size); + user_access_end(); /* - * __copy_user_nocache() uses non-temporal stores for the bulk + * copy_to_nontemporal() uses non-temporal stores for the bulk * of the transfer, but we need to manually flush if the * transfer is unaligned. A cached memory copy is used when * destination or size is not naturally aligned. That is: diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index b83a06739b5118..b33a52a3c515de 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -686,7 +686,7 @@ page_fault_oops(struct pt_regs *regs, unsigned long error_code, * avoid hanging the system. */ if (IS_ENABLED(CONFIG_EFI)) - efi_crash_gracefully_on_page_fault(address); + efi_crash_gracefully_on_page_fault(address, regs); /* Only not-present faults should be handled by KFENCE. */ if (!(error_code & X86_PF_PROT) && diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index f7546e9e8e8965..631f0375bd4270 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -105,6 +105,11 @@ void __init pti_check_boottime_disable(void) pr_debug("PTI enabled, disabling INVLPGB\n"); setup_clear_cpu_cap(X86_FEATURE_INVLPGB); } + + if (cpu_feature_enabled(X86_FEATURE_FRED)) { + pr_debug("PTI enabled, disabling FRED\n"); + setup_clear_cpu_cap(X86_FEATURE_FRED); + } } static int __init pti_parse_cmdline(char *arg) diff --git a/arch/x86/platform/efi/quirks.c b/arch/x86/platform/efi/quirks.c index 79f0818131e830..1f234c33c85a7a 100644 --- a/arch/x86/platform/efi/quirks.c +++ b/arch/x86/platform/efi/quirks.c @@ -761,7 +761,8 @@ int efi_capsule_setup_info(struct capsule_info *cap_info, void *kbuff, * @return: Returns, if the page fault is not handled. This function * will never return if the page fault is handled successfully. */ -void efi_crash_gracefully_on_page_fault(unsigned long phys_addr) +void efi_crash_gracefully_on_page_fault(unsigned long phys_addr, + const struct pt_regs *regs) { if (!IS_ENABLED(CONFIG_X86_64)) return; @@ -770,7 +771,7 @@ void efi_crash_gracefully_on_page_fault(unsigned long phys_addr) * If we get an interrupt/NMI while processing an EFI runtime service * then this is a regular OOPS, not an EFI failure. */ - if (in_interrupt()) + if (!in_task()) return; /* @@ -810,6 +811,14 @@ void efi_crash_gracefully_on_page_fault(unsigned long phys_addr) return; } + /* + * The API does not permit entering a kernel mode FPU section with + * interrupts enabled and leaving it with interrupts disabled. So + * re-enable interrupts now if they were enabled when the page fault + * occurred. + */ + local_irq_restore(regs->flags); + /* * Before calling EFI Runtime Service, the kernel has switched the * calling process to efi_mm. Hence, switch back to task_mm. diff --git a/arch/x86/tools/vdso2c.c b/arch/x86/tools/vdso2c.c index f84e8f8fa5fe6d..b8a555763f4376 100644 --- a/arch/x86/tools/vdso2c.c +++ b/arch/x86/tools/vdso2c.c @@ -75,7 +75,6 @@ struct vdso_sym { }; struct vdso_sym required_syms[] = { - {"VDSO32_NOTE_MASK", true}, {"__kernel_vsyscall", true}, {"__kernel_sigreturn", true}, {"__kernel_rt_sigreturn", true}, diff --git a/arch/x86/um/vdso/Makefile b/arch/x86/um/vdso/Makefile index 8a7c8b37cb6eb6..7664cbedbe30fb 100644 --- a/arch/x86/um/vdso/Makefile +++ b/arch/x86/um/vdso/Makefile @@ -3,8 +3,6 @@ # Building vDSO images for x86. # -vdso-install-y += vdso.so - # files to link into the vdso vobjs-y := vdso-note.o um_vdso.o diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 20f5d301d32dd3..a3193622170377 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -167,10 +167,10 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, if (bip->bip_vcnt > 0) { struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1]; - if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) + if (!zone_device_pages_compatible(bv->bv_page, page)) return 0; - - if (bvec_try_merge_hw_page(q, bv, page, len, offset)) { + if (zone_device_pages_have_same_pgmap(bv->bv_page, page) && + bvec_try_merge_hw_page(q, bv, page, len, offset)) { bip->bip_iter.bi_size += len; return len; } diff --git a/block/bio.c b/block/bio.c index d80d5d26804e32..3acd216bbc2eee 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1070,10 +1070,10 @@ int bio_add_page(struct bio *bio, struct page *page, if (bio->bi_vcnt > 0) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; - if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) + if (!zone_device_pages_compatible(bv->bv_page, page)) return 0; - - if (bvec_try_merge_page(bv, page, len, offset)) { + if (zone_device_pages_have_same_pgmap(bv->bv_page, page) && + bvec_try_merge_page(bv, page, len, offset)) { bio->bi_iter.bi_size += len; return len; } diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index b70096497d389a..554c87bb4a8651 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -611,6 +612,8 @@ static void blkg_destroy_all(struct gendisk *disk) q->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); + + wake_up_var(&q->root_blkg); } static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src) @@ -1498,6 +1501,18 @@ int blkcg_init_disk(struct gendisk *disk) struct blkcg_gq *new_blkg, *blkg; bool preloaded; + /* + * If the queue is shared across disk rebind (e.g., SCSI), the + * previous disk's blkcg state is cleaned up asynchronously via + * disk_release() -> blkcg_exit_disk(). Wait for that cleanup to + * finish (indicated by root_blkg becoming NULL) before setting up + * new blkcg state. Otherwise, we may overwrite q->root_blkg while + * the old one is still alive, and radix_tree_insert() in + * blkg_create() will fail with -EEXIST because the old entries + * still occupy the same queue id slot in blkcg->blkg_tree. + */ + wait_var_event(&q->root_blkg, !READ_ONCE(q->root_blkg)); + new_blkg = blkg_alloc(&blkcg_root, disk, GFP_KERNEL); if (!new_blkg) return -ENOMEM; @@ -2022,6 +2037,7 @@ void blkcg_maybe_throttle_current(void) return; out: rcu_read_unlock(); + put_disk(disk); } /** diff --git a/block/blk-wbt.c b/block/blk-wbt.c index 33006edfccd44a..dcc2438ca16dc7 100644 --- a/block/blk-wbt.c +++ b/block/blk-wbt.c @@ -782,10 +782,11 @@ void wbt_init_enable_default(struct gendisk *disk) return; rwb = wbt_alloc(); - if (WARN_ON_ONCE(!rwb)) + if (!rwb) return; - if (WARN_ON_ONCE(wbt_init(disk, rwb))) { + if (wbt_init(disk, rwb)) { + pr_warn("%s: failed to enable wbt\n", disk->disk_name); wbt_free(rwb); return; } diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 9d1dd6ccfad723..a4d82342e37acf 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -99,17 +99,17 @@ static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk) * being executed or the zone write plug bio list is not empty. * - BLK_ZONE_WPLUG_NEED_WP_UPDATE: Indicates that we lost track of a zone * write pointer offset and need to update it. - * - BLK_ZONE_WPLUG_UNHASHED: Indicates that the zone write plug was removed - * from the disk hash table and that the initial reference to the zone - * write plug set when the plug was first added to the hash table has been - * dropped. This flag is set when a zone is reset, finished or become full, - * to prevent new references to the zone write plug to be taken for - * newly incoming BIOs. A zone write plug flagged with this flag will be - * freed once all remaining references from BIOs or functions are dropped. + * - BLK_ZONE_WPLUG_DEAD: Indicates that the zone write plug will be + * removed from the disk hash table of zone write plugs when the last + * reference on the zone write plug is dropped. If set, this flag also + * indicates that the initial extra reference on the zone write plug was + * dropped, meaning that the reference count indicates the current number of + * active users (code context or BIOs and requests in flight). This flag is + * set when a zone is reset, finished or becomes full. */ #define BLK_ZONE_WPLUG_PLUGGED (1U << 0) #define BLK_ZONE_WPLUG_NEED_WP_UPDATE (1U << 1) -#define BLK_ZONE_WPLUG_UNHASHED (1U << 2) +#define BLK_ZONE_WPLUG_DEAD (1U << 2) /** * blk_zone_cond_str - Return a zone condition name string @@ -587,64 +587,15 @@ static void disk_free_zone_wplug_rcu(struct rcu_head *rcu_head) mempool_free(zwplug, zwplug->disk->zone_wplugs_pool); } -static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug) -{ - if (refcount_dec_and_test(&zwplug->ref)) { - WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list)); - WARN_ON_ONCE(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED); - WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_UNHASHED)); - - call_rcu(&zwplug->rcu_head, disk_free_zone_wplug_rcu); - } -} - -static inline bool disk_should_remove_zone_wplug(struct gendisk *disk, - struct blk_zone_wplug *zwplug) -{ - lockdep_assert_held(&zwplug->lock); - - /* If the zone write plug was already removed, we are done. */ - if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) - return false; - - /* If the zone write plug is still plugged, it cannot be removed. */ - if (zwplug->flags & BLK_ZONE_WPLUG_PLUGGED) - return false; - - /* - * Completions of BIOs with blk_zone_write_plug_bio_endio() may - * happen after handling a request completion with - * blk_zone_write_plug_finish_request() (e.g. with split BIOs - * that are chained). In such case, disk_zone_wplug_unplug_bio() - * should not attempt to remove the zone write plug until all BIO - * completions are seen. Check by looking at the zone write plug - * reference count, which is 2 when the plug is unused (one reference - * taken when the plug was allocated and another reference taken by the - * caller context). - */ - if (refcount_read(&zwplug->ref) > 2) - return false; - - /* We can remove zone write plugs for zones that are empty or full. */ - return !zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug); -} - -static void disk_remove_zone_wplug(struct gendisk *disk, - struct blk_zone_wplug *zwplug) +static void disk_free_zone_wplug(struct blk_zone_wplug *zwplug) { + struct gendisk *disk = zwplug->disk; unsigned long flags; - /* If the zone write plug was already removed, we have nothing to do. */ - if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) - return; + WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_DEAD)); + WARN_ON_ONCE(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED); + WARN_ON_ONCE(!bio_list_empty(&zwplug->bio_list)); - /* - * Mark the zone write plug as unhashed and drop the extra reference we - * took when the plug was inserted in the hash table. Also update the - * disk zone condition array with the current condition of the zone - * write plug. - */ - zwplug->flags |= BLK_ZONE_WPLUG_UNHASHED; spin_lock_irqsave(&disk->zone_wplugs_lock, flags); blk_zone_set_cond(rcu_dereference_check(disk->zones_cond, lockdep_is_held(&disk->zone_wplugs_lock)), @@ -652,7 +603,29 @@ static void disk_remove_zone_wplug(struct gendisk *disk, hlist_del_init_rcu(&zwplug->node); atomic_dec(&disk->nr_zone_wplugs); spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags); - disk_put_zone_wplug(zwplug); + + call_rcu(&zwplug->rcu_head, disk_free_zone_wplug_rcu); +} + +static inline void disk_put_zone_wplug(struct blk_zone_wplug *zwplug) +{ + if (refcount_dec_and_test(&zwplug->ref)) + disk_free_zone_wplug(zwplug); +} + +/* + * Flag the zone write plug as dead and drop the initial reference we got when + * the zone write plug was added to the hash table. The zone write plug will be + * unhashed when its last reference is dropped. + */ +static void disk_mark_zone_wplug_dead(struct blk_zone_wplug *zwplug) +{ + lockdep_assert_held(&zwplug->lock); + + if (!(zwplug->flags & BLK_ZONE_WPLUG_DEAD)) { + zwplug->flags |= BLK_ZONE_WPLUG_DEAD; + disk_put_zone_wplug(zwplug); + } } static void blk_zone_wplug_bio_work(struct work_struct *work); @@ -672,18 +645,7 @@ static struct blk_zone_wplug *disk_get_and_lock_zone_wplug(struct gendisk *disk, again: zwplug = disk_get_zone_wplug(disk, sector); if (zwplug) { - /* - * Check that a BIO completion or a zone reset or finish - * operation has not already removed the zone write plug from - * the hash table and dropped its reference count. In such case, - * we need to get a new plug so start over from the beginning. - */ spin_lock_irqsave(&zwplug->lock, *flags); - if (zwplug->flags & BLK_ZONE_WPLUG_UNHASHED) { - spin_unlock_irqrestore(&zwplug->lock, *flags); - disk_put_zone_wplug(zwplug); - goto again; - } return zwplug; } @@ -788,14 +750,8 @@ static void disk_zone_wplug_set_wp_offset(struct gendisk *disk, disk_zone_wplug_update_cond(disk, zwplug); disk_zone_wplug_abort(zwplug); - - /* - * The zone write plug now has no BIO plugged: remove it from the - * hash table so that it cannot be seen. The plug will be freed - * when the last reference is dropped. - */ - if (disk_should_remove_zone_wplug(disk, zwplug)) - disk_remove_zone_wplug(disk, zwplug); + if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug)) + disk_mark_zone_wplug_dead(zwplug); } static unsigned int blk_zone_wp_offset(struct blk_zone *zone) @@ -1198,13 +1154,17 @@ static void disk_zone_wplug_schedule_bio_work(struct gendisk *disk, lockdep_assert_held(&zwplug->lock); /* - * Take a reference on the zone write plug and schedule the submission - * of the next plugged BIO. blk_zone_wplug_bio_work() will release the - * reference we take here. + * Schedule the submission of the next plugged BIO. Taking a reference + * to the zone write plug is required as the bio_work belongs to the + * plug, and thus we must ensure that the write plug does not go away + * while the work is being scheduled but has not run yet. + * blk_zone_wplug_bio_work() will release the reference we take here, + * and we also drop this reference if the work is already scheduled. */ WARN_ON_ONCE(!(zwplug->flags & BLK_ZONE_WPLUG_PLUGGED)); refcount_inc(&zwplug->ref); - queue_work(disk->zone_wplugs_wq, &zwplug->bio_work); + if (!queue_work(disk->zone_wplugs_wq, &zwplug->bio_work)) + disk_put_zone_wplug(zwplug); } static inline void disk_zone_wplug_add_bio(struct gendisk *disk, @@ -1447,6 +1407,19 @@ static bool blk_zone_wplug_handle_write(struct bio *bio, unsigned int nr_segs) return true; } + /* + * If we got a zone write plug marked as dead, then the user is issuing + * writes to a full zone, or without synchronizing with zone reset or + * zone finish operations. In such case, fail the BIO to signal this + * invalid usage. + */ + if (zwplug->flags & BLK_ZONE_WPLUG_DEAD) { + spin_unlock_irqrestore(&zwplug->lock, flags); + disk_put_zone_wplug(zwplug); + bio_io_error(bio); + return true; + } + /* Indicate that this BIO is being handled using zone write plugging. */ bio_set_flag(bio, BIO_ZONE_WRITE_PLUGGING); @@ -1527,7 +1500,7 @@ static void blk_zone_wplug_handle_native_zone_append(struct bio *bio) disk->disk_name, zwplug->zone_no); disk_zone_wplug_abort(zwplug); } - disk_remove_zone_wplug(disk, zwplug); + disk_mark_zone_wplug_dead(zwplug); spin_unlock_irqrestore(&zwplug->lock, flags); disk_put_zone_wplug(zwplug); @@ -1630,14 +1603,8 @@ static void disk_zone_wplug_unplug_bio(struct gendisk *disk, } zwplug->flags &= ~BLK_ZONE_WPLUG_PLUGGED; - - /* - * If the zone is full (it was fully written or finished, or empty - * (it was reset), remove its zone write plug from the hash table. - */ - if (disk_should_remove_zone_wplug(disk, zwplug)) - disk_remove_zone_wplug(disk, zwplug); - + if (!zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug)) + disk_mark_zone_wplug_dead(zwplug); spin_unlock_irqrestore(&zwplug->lock, flags); } @@ -1848,9 +1815,9 @@ static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk) while (!hlist_empty(&disk->zone_wplugs_hash[i])) { zwplug = hlist_entry(disk->zone_wplugs_hash[i].first, struct blk_zone_wplug, node); - refcount_inc(&zwplug->ref); - disk_remove_zone_wplug(disk, zwplug); - disk_put_zone_wplug(zwplug); + spin_lock_irq(&zwplug->lock); + disk_mark_zone_wplug_dead(zwplug); + spin_unlock_irq(&zwplug->lock); } } @@ -1910,6 +1877,7 @@ static int disk_revalidate_zone_resources(struct gendisk *disk, { struct queue_limits *lim = &disk->queue->limits; unsigned int pool_size; + int ret = 0; args->disk = disk; args->nr_zones = @@ -1932,10 +1900,13 @@ static int disk_revalidate_zone_resources(struct gendisk *disk, pool_size = min(BLK_ZONE_WPLUG_DEFAULT_POOL_SIZE, args->nr_zones); - if (!disk->zone_wplugs_hash) - return disk_alloc_zone_resources(disk, pool_size); + if (!disk->zone_wplugs_hash) { + ret = disk_alloc_zone_resources(disk, pool_size); + if (ret) + kfree(args->zones_cond); + } - return 0; + return ret; } /* @@ -1967,6 +1938,7 @@ static int disk_update_zone_resources(struct gendisk *disk, disk->zone_capacity = args->zone_capacity; disk->last_zone_capacity = args->last_zone_capacity; disk_set_zones_cond_array(disk, args->zones_cond); + args->zones_cond = NULL; /* * Some devices can advertise zone resource limits that are larger than @@ -2249,21 +2221,30 @@ int blk_revalidate_disk_zones(struct gendisk *disk) } memalloc_noio_restore(noio_flag); + if (ret <= 0) + goto free_resources; + /* * If zones where reported, make sure that the entire disk capacity * has been checked. */ - if (ret > 0 && args.sector != capacity) { + if (args.sector != capacity) { pr_warn("%s: Missing zones from sector %llu\n", disk->disk_name, args.sector); ret = -ENODEV; + goto free_resources; } - if (ret > 0) - return disk_update_zone_resources(disk, &args); + ret = disk_update_zone_resources(disk, &args); + if (ret) + goto free_resources; + + return 0; +free_resources: pr_warn("%s: failed to revalidate zones\n", disk->disk_name); + kfree(args.zones_cond); memflags = blk_mq_freeze_queue(q); disk_free_zone_resources(disk); blk_mq_unfreeze_queue(q, memflags); diff --git a/block/blk.h b/block/blk.h index f6053e9dd2aac1..a7abf3be34ef8a 100644 --- a/block/blk.h +++ b/block/blk.h @@ -132,6 +132,8 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, if (addr1 + vec1->bv_len != addr2) return false; + if (!zone_device_pages_have_same_pgmap(vec1->bv_page, vec2->bv_page)) + return false; if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) return false; if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) @@ -139,6 +141,25 @@ static inline bool biovec_phys_mergeable(struct request_queue *q, return true; } +/* + * Check if two pages from potentially different zone device pgmaps can + * coexist as separate bvec entries in the same bio. + * + * The block DMA iterator (blk_dma_map_iter_start) caches the P2PDMA mapping + * state from the first segment and applies it to all subsequent segments, so + * P2PDMA pages from different pgmaps must not be mixed in the same bio. + * + * Other zone device types (FS_DAX, GENERIC) use the same dma_map_phys() path + * as normal RAM. PRIVATE and COHERENT pages never appear in bios. + */ +static inline bool zone_device_pages_compatible(const struct page *a, + const struct page *b) +{ + if (is_pci_p2pdma_page(a) || is_pci_p2pdma_page(b)) + return zone_device_pages_have_same_pgmap(a, b); + return true; +} + static inline bool __bvec_gap_to_prev(const struct queue_limits *lim, struct bio_vec *bprv, unsigned int offset) { diff --git a/block/disk-events.c b/block/disk-events.c index 9f9f9f8a2d6b15..074731ecc3d258 100644 --- a/block/disk-events.c +++ b/block/disk-events.c @@ -290,13 +290,14 @@ EXPORT_SYMBOL(disk_check_media_change); * Should be called when the media changes for @disk. Generates a uevent * and attempts to free all dentries and inodes and invalidates all block * device page cache entries in that case. + * + * Callers that need a partition re-scan should arrange for one explicitly. */ void disk_force_media_change(struct gendisk *disk) { disk_event_uevent(disk, DISK_EVENT_MEDIA_CHANGE); inc_diskseq(disk); bdev_mark_dead(disk->part0, true); - set_bit(GD_NEED_PART_SCAN, &disk->state); } EXPORT_SYMBOL_GPL(disk_force_media_change); diff --git a/block/ioctl.c b/block/ioctl.c index 0b04661ac80982..6fc63c79489220 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -864,6 +864,8 @@ long compat_blkdev_ioctl(struct file *file, unsigned cmd, unsigned long arg) #endif struct blk_iou_cmd { + u64 start; + u64 len; int res; bool nowait; }; @@ -953,23 +955,27 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { struct block_device *bdev = I_BDEV(cmd->file->f_mapping->host); struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); - const struct io_uring_sqe *sqe = cmd->sqe; u32 cmd_op = cmd->cmd_op; - uint64_t start, len; - if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len || - sqe->rw_flags || sqe->file_index)) - return -EINVAL; + /* Read what we need from the SQE on the first issue */ + if (!(issue_flags & IORING_URING_CMD_REISSUE)) { + const struct io_uring_sqe *sqe = cmd->sqe; + + if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len || + sqe->rw_flags || sqe->file_index)) + return -EINVAL; + + bic->start = READ_ONCE(sqe->addr); + bic->len = READ_ONCE(sqe->addr3); + } bic->res = 0; bic->nowait = issue_flags & IO_URING_F_NONBLOCK; - start = READ_ONCE(sqe->addr); - len = READ_ONCE(sqe->addr3); - switch (cmd_op) { case BLOCK_URING_CMD_DISCARD: - return blkdev_cmd_discard(cmd, bdev, start, len, bic->nowait); + return blkdev_cmd_discard(cmd, bdev, bic->start, bic->len, + bic->nowait); } return -EINVAL; } diff --git a/certs/extract-cert.c b/certs/extract-cert.c index 7d6d468ed6129d..54ecd10242746c 100644 --- a/certs/extract-cert.c +++ b/certs/extract-cert.c @@ -43,7 +43,9 @@ void format(void) exit(2); } +#ifdef USE_PKCS11_ENGINE static const char *key_pass; +#endif static BIO *wb; static char *cert_dst; static bool verbose; @@ -135,7 +137,9 @@ int main(int argc, char **argv) if (verbose_env && strchr(verbose_env, '1')) verbose = true; - key_pass = getenv("KBUILD_SIGN_PIN"); +#ifdef USE_PKCS11_ENGINE + key_pass = getenv("KBUILD_SIGN_PIN"); +#endif if (argc != 3) format(); diff --git a/crypto/acompress.c b/crypto/acompress.c index 1f9cb04b447f59..6025c1acce4910 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -169,15 +169,13 @@ static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) state->compl = req->base.complete; state->data = req->base.data; req->base.complete = cplt; - req->base.data = state; + req->base.data = req; } static void acomp_restore_req(struct acomp_req *req) { - struct acomp_req_chain *state = req->base.data; - - req->base.complete = state->compl; - req->base.data = state->data; + req->base.complete = req->chain.compl; + req->base.data = req->chain.data; } static void acomp_reqchain_virt(struct acomp_req *req) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index dd0e5be4d8c07a..53d2655913634a 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -586,6 +586,8 @@ static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) return -EINVAL; con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg); + if (con->aead_assoclen >= 0x80000000u) + return -EINVAL; break; default: diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c index f8bd45f7dc8392..cb651ab58d6293 100644 --- a/crypto/algif_aead.c +++ b/crypto/algif_aead.c @@ -72,8 +72,10 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, struct af_alg_ctx *ctx = ask->private; struct crypto_aead *tfm = pask->private; unsigned int as = crypto_aead_authsize(tfm); + unsigned int ivsize = crypto_aead_ivsize(tfm); struct af_alg_async_req *areq; struct scatterlist *rsgl_src, *tsgl_src = NULL; + void *iv; int err = 0; size_t used = 0; /* [in] TX bufs to be en/decrypted */ size_t outlen = 0; /* [out] RX bufs produced by kernel */ @@ -125,10 +127,14 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, /* Allocate cipher request for current operation. */ areq = af_alg_alloc_areq(sk, sizeof(struct af_alg_async_req) + - crypto_aead_reqsize(tfm)); + crypto_aead_reqsize(tfm) + ivsize); if (IS_ERR(areq)) return PTR_ERR(areq); + iv = (u8 *)aead_request_ctx(&areq->cra_u.aead_req) + + crypto_aead_reqsize(tfm); + memcpy(iv, ctx->iv, ivsize); + /* convert iovecs of output buffers into RX SGL */ err = af_alg_get_rsgl(sk, msg, flags, areq, outlen, &usedpages); if (err) @@ -187,7 +193,7 @@ static int _aead_recvmsg(struct socket *sock, struct msghdr *msg, /* Initialize the crypto operation */ aead_request_set_crypt(&areq->cra_u.aead_req, tsgl_src, - areq->first_rsgl.sgl.sgt.sgl, used, ctx->iv); + areq->first_rsgl.sgl.sgt.sgl, used, iv); aead_request_set_ad(&areq->cra_u.aead_req, ctx->aead_assoclen); aead_request_set_tfm(&areq->cra_u.aead_req, tfm); diff --git a/crypto/authencesn.c b/crypto/authencesn.c index c0a01d738d9bc3..522df41365d8f9 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -228,9 +228,11 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req, decrypt: - if (src != dst) - src = scatterwalk_ffwd(areq_ctx->src, src, assoclen); dst = scatterwalk_ffwd(areq_ctx->dst, dst, assoclen); + if (req->src == req->dst) + src = dst; + else + src = scatterwalk_ffwd(areq_ctx->src, src, assoclen); skcipher_request_set_tfm(skreq, ctx->enc); skcipher_request_set_callback(skreq, flags, @@ -388,6 +390,11 @@ static int crypto_authenc_esn_create(struct crypto_template *tmpl, auth = crypto_spawn_ahash_alg(&ctx->auth); auth_base = &auth->base; + if (auth->digestsize > 0 && auth->digestsize < 4) { + err = -EINVAL; + goto err_free_inst; + } + err = crypto_grab_skcipher(&ctx->enc, aead_crypto_instance(inst), crypto_attr_alg_name(tb[2]), 0, mask); if (err) diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 7c880cf34c5233..5edc6d285aa149 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -193,7 +194,7 @@ int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) ***************************************************************************/ struct jitterentropy { - spinlock_t jent_lock; + struct mutex jent_lock; struct rand_data *entropy_collector; struct crypto_shash *tfm; struct shash_desc *sdesc; @@ -203,7 +204,7 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) { struct jitterentropy *rng = crypto_tfm_ctx(tfm); - spin_lock(&rng->jent_lock); + mutex_lock(&rng->jent_lock); if (rng->sdesc) { shash_desc_zero(rng->sdesc); @@ -218,7 +219,7 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) if (rng->entropy_collector) jent_entropy_collector_free(rng->entropy_collector); rng->entropy_collector = NULL; - spin_unlock(&rng->jent_lock); + mutex_unlock(&rng->jent_lock); } static int jent_kcapi_init(struct crypto_tfm *tfm) @@ -228,7 +229,7 @@ static int jent_kcapi_init(struct crypto_tfm *tfm) struct shash_desc *sdesc; int size, ret = 0; - spin_lock_init(&rng->jent_lock); + mutex_init(&rng->jent_lock); /* Use SHA3-256 as conditioner */ hash = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); @@ -257,7 +258,6 @@ static int jent_kcapi_init(struct crypto_tfm *tfm) goto err; } - spin_lock_init(&rng->jent_lock); return 0; err: @@ -272,7 +272,7 @@ static int jent_kcapi_random(struct crypto_rng *tfm, struct jitterentropy *rng = crypto_rng_ctx(tfm); int ret = 0; - spin_lock(&rng->jent_lock); + mutex_lock(&rng->jent_lock); ret = jent_read_entropy(rng->entropy_collector, rdata, dlen); @@ -298,7 +298,7 @@ static int jent_kcapi_random(struct crypto_rng *tfm, ret = -EINVAL; } - spin_unlock(&rng->jent_lock); + mutex_unlock(&rng->jent_lock); return ret; } diff --git a/crypto/krb5enc.c b/crypto/krb5enc.c index a1de55994d922b..fefa8d2c753225 100644 --- a/crypto/krb5enc.c +++ b/crypto/krb5enc.c @@ -39,12 +39,6 @@ struct krb5enc_request_ctx { char tail[]; }; -static void krb5enc_request_complete(struct aead_request *req, int err) -{ - if (err != -EINPROGRESS) - aead_request_complete(req, err); -} - /** * crypto_krb5enc_extractkeys - Extract Ke and Ki keys from the key blob. * @keys: Where to put the key sizes and pointers @@ -127,7 +121,7 @@ static void krb5enc_encrypt_done(void *data, int err) { struct aead_request *req = data; - krb5enc_request_complete(req, err); + aead_request_complete(req, err); } /* @@ -154,7 +148,7 @@ static int krb5enc_dispatch_encrypt(struct aead_request *req, dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); skcipher_request_set_tfm(skreq, enc); - skcipher_request_set_callback(skreq, aead_request_flags(req), + skcipher_request_set_callback(skreq, flags, krb5enc_encrypt_done, req); skcipher_request_set_crypt(skreq, src, dst, req->cryptlen, req->iv); @@ -188,13 +182,16 @@ static void krb5enc_encrypt_ahash_done(void *data, int err) struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); if (err) - return krb5enc_request_complete(req, err); + goto out; krb5enc_insert_checksum(req, ahreq->result); err = krb5enc_dispatch_encrypt(req, 0); - if (err != -EINPROGRESS) - aead_request_complete(req, err); + if (err == -EINPROGRESS) + return; + +out: + aead_request_complete(req, err); } /* @@ -264,17 +261,16 @@ static void krb5enc_decrypt_hash_done(void *data, int err) { struct aead_request *req = data; - if (err) - return krb5enc_request_complete(req, err); - - err = krb5enc_verify_hash(req); - krb5enc_request_complete(req, err); + if (!err) + err = krb5enc_verify_hash(req); + aead_request_complete(req, err); } /* * Dispatch the hashing of the plaintext after we've done the decryption. */ -static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) +static int krb5enc_dispatch_decrypt_hash(struct aead_request *req, + unsigned int flags) { struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); struct aead_instance *inst = aead_alg_instance(krb5enc); @@ -290,7 +286,7 @@ static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) ahash_request_set_tfm(ahreq, auth); ahash_request_set_crypt(ahreq, req->dst, hash, req->assoclen + req->cryptlen - authsize); - ahash_request_set_callback(ahreq, aead_request_flags(req), + ahash_request_set_callback(ahreq, flags, krb5enc_decrypt_hash_done, req); err = crypto_ahash_digest(ahreq); @@ -300,6 +296,21 @@ static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) return krb5enc_verify_hash(req); } +static void krb5enc_decrypt_done(void *data, int err) +{ + struct aead_request *req = data; + + if (err) + goto out; + + err = krb5enc_dispatch_decrypt_hash(req, 0); + if (err == -EINPROGRESS) + return; + +out: + aead_request_complete(req, err); +} + /* * Dispatch the decryption of the ciphertext. */ @@ -323,7 +334,7 @@ static int krb5enc_dispatch_decrypt(struct aead_request *req) skcipher_request_set_tfm(skreq, ctx->enc); skcipher_request_set_callback(skreq, aead_request_flags(req), - req->base.complete, req->base.data); + krb5enc_decrypt_done, req); skcipher_request_set_crypt(skreq, src, dst, req->cryptlen - authsize, req->iv); @@ -338,7 +349,7 @@ static int krb5enc_decrypt(struct aead_request *req) if (err < 0) return err; - return krb5enc_dispatch_decrypt_hash(req); + return krb5enc_dispatch_decrypt_hash(req, aead_request_flags(req)); } static int krb5enc_init_tfm(struct crypto_aead *tfm) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index c3a9d4f2995c7a..ed0feaba238323 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -69,6 +69,9 @@ static void pcrypt_aead_done(void *data, int err) struct pcrypt_request *preq = aead_request_ctx(req); struct padata_priv *padata = pcrypt_request_padata(preq); + if (err == -EINPROGRESS) + return; + padata->info = err; padata_do_serial(padata); @@ -82,7 +85,7 @@ static void pcrypt_aead_enc(struct padata_priv *padata) ret = crypto_aead_encrypt(req); - if (ret == -EINPROGRESS) + if (ret == -EINPROGRESS || ret == -EBUSY) return; padata->info = ret; @@ -133,7 +136,7 @@ static void pcrypt_aead_dec(struct padata_priv *padata) ret = crypto_aead_decrypt(req); - if (ret == -EINPROGRESS) + if (ret == -EINPROGRESS || ret == -EBUSY) return; padata->info = ret; diff --git a/crypto/simd.c b/crypto/simd.c index f71c4a334c7d0e..4e6f437e9e778a 100644 --- a/crypto/simd.c +++ b/crypto/simd.c @@ -214,13 +214,17 @@ int simd_register_skciphers_compat(struct skcipher_alg *algs, int count, const char *basename; struct simd_skcipher_alg *simd; + for (i = 0; i < count; i++) { + if (WARN_ON(strncmp(algs[i].base.cra_name, "__", 2) || + strncmp(algs[i].base.cra_driver_name, "__", 2))) + return -EINVAL; + } + err = crypto_register_skciphers(algs, count); if (err) return err; for (i = 0; i < count; i++) { - WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); - WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); algname = algs[i].base.cra_name + 2; drvname = algs[i].base.cra_driver_name + 2; basename = algs[i].base.cra_driver_name; @@ -437,13 +441,17 @@ int simd_register_aeads_compat(struct aead_alg *algs, int count, const char *basename; struct simd_aead_alg *simd; + for (i = 0; i < count; i++) { + if (WARN_ON(strncmp(algs[i].base.cra_name, "__", 2) || + strncmp(algs[i].base.cra_driver_name, "__", 2))) + return -EINVAL; + } + err = crypto_register_aeads(algs, count); if (err) return err; for (i = 0; i < count; i++) { - WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); - WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); algname = algs[i].base.cra_name + 2; drvname = algs[i].base.cra_driver_name + 2; basename = algs[i].base.cra_driver_name; diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c index 46d844a73a948a..e681a090752df4 100644 --- a/drivers/accel/amdxdna/amdxdna_mailbox.c +++ b/drivers/accel/amdxdna/amdxdna_mailbox.c @@ -499,7 +499,7 @@ xdna_mailbox_start_channel(struct mailbox_channel *mb_chann, int ret; if (!is_power_of_2(x2i->rb_size) || !is_power_of_2(i2x->rb_size)) { - pr_err("Ring buf size must be power of 2"); + pr_err("Ring buf size must be power of 2\n"); return -EINVAL; } diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c index 5900a40c7a7865..328c4fc468ba6f 100644 --- a/drivers/accel/ivpu/ivpu_drv.c +++ b/drivers/accel/ivpu/ivpu_drv.c @@ -460,6 +460,26 @@ static const struct file_operations ivpu_fops = { #endif }; +static int ivpu_gem_prime_handle_to_fd(struct drm_device *dev, struct drm_file *file_priv, + u32 handle, u32 flags, int *prime_fd) +{ + struct drm_gem_object *obj; + + obj = drm_gem_object_lookup(file_priv, handle); + if (!obj) + return -ENOENT; + + if (drm_gem_is_imported(obj)) { + /* Do not allow re-exporting */ + drm_gem_object_put(obj); + return -EOPNOTSUPP; + } + + drm_gem_object_put(obj); + + return drm_gem_prime_handle_to_fd(dev, file_priv, handle, flags, prime_fd); +} + static const struct drm_driver driver = { .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL, @@ -468,6 +488,7 @@ static const struct drm_driver driver = { .gem_create_object = ivpu_gem_create_object, .gem_prime_import = ivpu_gem_prime_import, + .prime_handle_to_fd = ivpu_gem_prime_handle_to_fd, .ioctls = ivpu_drm_ioctls, .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls), diff --git a/drivers/accel/rocket/rocket_gem.c b/drivers/accel/rocket/rocket_gem.c index b6a385d2edfc30..c8084719208a2a 100644 --- a/drivers/accel/rocket/rocket_gem.c +++ b/drivers/accel/rocket/rocket_gem.c @@ -145,6 +145,8 @@ int rocket_ioctl_prep_bo(struct drm_device *dev, void *data, struct drm_file *fi ret = dma_resv_wait_timeout(gem_obj->resv, DMA_RESV_USAGE_WRITE, true, timeout); if (!ret) ret = timeout ? -ETIMEDOUT : -EBUSY; + else if (ret > 0) + ret = 0; shmem_obj = &to_rocket_bo(gem_obj)->base; diff --git a/drivers/acpi/apei/einj-core.c b/drivers/acpi/apei/einj-core.c index a9248af078f69b..1f3fa2278584b1 100644 --- a/drivers/acpi/apei/einj-core.c +++ b/drivers/acpi/apei/einj-core.c @@ -401,8 +401,18 @@ static struct acpi_generic_address *einj_get_trigger_parameter_region( return NULL; } + +static bool is_memory_injection(u32 type, u32 flags) +{ + if (flags & SETWA_FLAGS_EINJV2) + return !!(type & ACPI_EINJV2_MEMORY); + if (type & ACPI5_VENDOR_BIT) + return !!(vendor_flags & SETWA_FLAGS_MEM); + return !!(type & MEM_ERROR_MASK) || !!(flags & SETWA_FLAGS_MEM); +} + /* Execute instructions in trigger error action table */ -static int __einj_error_trigger(u64 trigger_paddr, u32 type, +static int __einj_error_trigger(u64 trigger_paddr, u32 type, u32 flags, u64 param1, u64 param2) { struct acpi_einj_trigger trigger_tab; @@ -480,7 +490,7 @@ static int __einj_error_trigger(u64 trigger_paddr, u32 type, * This will cause resource conflict with regular memory. So * remove it from trigger table resources. */ - if ((param_extension || acpi5) && (type & MEM_ERROR_MASK) && param2) { + if ((param_extension || acpi5) && is_memory_injection(type, flags)) { struct apei_resources addr_resources; apei_resources_init(&addr_resources); @@ -660,7 +670,7 @@ static int __einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, return rc; trigger_paddr = apei_exec_ctx_get_output(&ctx); if (notrigger == 0) { - rc = __einj_error_trigger(trigger_paddr, type, param1, param2); + rc = __einj_error_trigger(trigger_paddr, type, flags, param1, param2); if (rc) return rc; } @@ -718,35 +728,30 @@ int einj_error_inject(u32 type, u32 flags, u64 param1, u64 param2, u64 param3, SETWA_FLAGS_PCIE_SBDF | SETWA_FLAGS_EINJV2))) return -EINVAL; + /* + * Injections targeting a CXL 1.0/1.1 port have to be injected + * via the einj_cxl_rch_error_inject() path as that does the proper + * validation of the given RCRB base (MMIO) address. + */ + if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM)) + return -EINVAL; + /* check if type is a valid EINJv2 error type */ if (is_v2) { if (!(type & available_error_type_v2)) return -EINVAL; } - /* - * We need extra sanity checks for memory errors. - * Other types leap directly to injection. - */ /* ensure param1/param2 existed */ if (!(param_extension || acpi5)) goto inject; - /* ensure injection is memory related */ - if (type & ACPI5_VENDOR_BIT) { - if (vendor_flags != SETWA_FLAGS_MEM) - goto inject; - } else if (!(type & MEM_ERROR_MASK) && !(flags & SETWA_FLAGS_MEM)) { - goto inject; - } - /* - * Injections targeting a CXL 1.0/1.1 port have to be injected - * via the einj_cxl_rch_error_inject() path as that does the proper - * validation of the given RCRB base (MMIO) address. + * We need extra sanity checks for memory errors. + * Other types leap directly to injection. */ - if (einj_is_cxl_error_type(type) && (flags & SETWA_FLAGS_MEM)) - return -EINVAL; + if (!is_memory_injection(type, flags)) + goto inject; /* * Disallow crazy address masks that give BIOS leeway to pick diff --git a/drivers/acpi/arm64/agdi.c b/drivers/acpi/arm64/agdi.c index feb4b2cb4618e7..0c2d9d6c160be4 100644 --- a/drivers/acpi/arm64/agdi.c +++ b/drivers/acpi/arm64/agdi.c @@ -36,7 +36,7 @@ static int agdi_sdei_probe(struct platform_device *pdev, err = sdei_event_register(adata->sdei_event, agdi_sdei_handler, pdev); if (err) { - dev_err(&pdev->dev, "Failed to register for SDEI event %d", + dev_err(&pdev->dev, "Failed to register for SDEI event %d\n", adata->sdei_event); return err; } diff --git a/drivers/acpi/arm64/cpuidle.c b/drivers/acpi/arm64/cpuidle.c index 801f9c45014255..c68a5db8ebba85 100644 --- a/drivers/acpi/arm64/cpuidle.c +++ b/drivers/acpi/arm64/cpuidle.c @@ -16,7 +16,7 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu) { - int i, count; + int i; struct acpi_lpi_state *lpi; struct acpi_processor *pr = per_cpu(processors, cpu); @@ -30,14 +30,10 @@ static int psci_acpi_cpu_init_idle(unsigned int cpu) if (!psci_ops.cpu_suspend) return -EOPNOTSUPP; - count = pr->power.count - 1; - if (count <= 0) - return -ENODEV; - - for (i = 0; i < count; i++) { + for (i = 1; i < pr->power.count; i++) { u32 state; - lpi = &pr->power.lpi_states[i + 1]; + lpi = &pr->power.lpi_states[i]; /* * Only bits[31:0] represent a PSCI power_state while * bits[63:32] must be 0x0 as per ARM ACPI FFH Specification diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index f0e513e9ed5d3b..bcfe2e6b8445f9 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -362,7 +362,7 @@ static int send_pcc_cmd(int pcc_ss_id, u16 cmd) end: if (cmd == CMD_WRITE) { if (unlikely(ret)) { - for_each_online_cpu(i) { + for_each_possible_cpu(i) { struct cpc_desc *desc = per_cpu(cpc_desc_ptr, i); if (!desc) @@ -524,13 +524,13 @@ int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data) else if (pdomain->coord_type == DOMAIN_COORD_TYPE_SW_ANY) cpu_data->shared_type = CPUFREQ_SHARED_TYPE_ANY; - for_each_online_cpu(i) { + for_each_possible_cpu(i) { if (i == cpu) continue; match_cpc_ptr = per_cpu(cpc_desc_ptr, i); if (!match_cpc_ptr) - goto err_fault; + continue; match_pdomain = &(match_cpc_ptr->domain_info); if (match_pdomain->domain != pdomain->domain) diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 4611159ee734d4..cf535966b9f778 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -991,7 +991,7 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle) return device; err: - acpi_release_power_resource(&device->dev); + acpi_dev_put(device); return NULL; } diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index e8cdbdb46fdb46..530547cda8b286 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -1900,7 +1900,7 @@ static int acpi_add_single_object(struct acpi_device **child, result = acpi_device_add(device); if (result) { - acpi_device_release(&device->dev); + acpi_dev_put(device); return result; } diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 4cf74f173c785d..2c120ade8f51a5 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -878,6 +878,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7760 AIO"), }, }, + { + .callback = video_detect_force_native, + /* Dell OptiPlex 7770 AIO */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "OptiPlex 7770 AIO"), + }, + }, /* * Models which have nvidia-ec-wmi support, but should not use it. @@ -899,6 +907,14 @@ static const struct dmi_system_id video_detect_dmi_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Vostro 15 3535"), }, }, + { + .callback = video_detect_force_native, + /* HP OMEN Gaming Laptop 16-n0xxx */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "OMEN by HP Gaming Laptop 16-n0xxx"), + }, + }, /* * x86 android tablets which directly control the backlight through diff --git a/drivers/acpi/x86/cmos_rtc.c b/drivers/acpi/x86/cmos_rtc.c index 51643ff6fe5fc4..45db7e51cbe607 100644 --- a/drivers/acpi/x86/cmos_rtc.c +++ b/drivers/acpi/x86/cmos_rtc.c @@ -24,72 +24,91 @@ static const struct acpi_device_id acpi_cmos_rtc_ids[] = { {} }; -static acpi_status -acpi_cmos_rtc_space_handler(u32 function, acpi_physical_address address, - u32 bits, u64 *value64, - void *handler_context, void *region_context) +static bool cmos_rtc_space_handler_present __read_mostly; + +static acpi_status acpi_cmos_rtc_space_handler(u32 function, + acpi_physical_address address, + u32 bits, u64 *value64, + void *handler_context, + void *region_context) { - int i; + unsigned int i, bytes = DIV_ROUND_UP(bits, 8); u8 *value = (u8 *)value64; if (address > 0xff || !value64) return AE_BAD_PARAMETER; - if (function != ACPI_WRITE && function != ACPI_READ) - return AE_BAD_PARAMETER; + guard(spinlock_irq)(&rtc_lock); + + if (function == ACPI_WRITE) { + for (i = 0; i < bytes; i++, address++, value++) + CMOS_WRITE(*value, address); - spin_lock_irq(&rtc_lock); + return AE_OK; + } - for (i = 0; i < DIV_ROUND_UP(bits, 8); ++i, ++address, ++value) - if (function == ACPI_READ) + if (function == ACPI_READ) { + for (i = 0; i < bytes; i++, address++, value++) *value = CMOS_READ(address); - else - CMOS_WRITE(*value, address); - spin_unlock_irq(&rtc_lock); + return AE_OK; + } - return AE_OK; + return AE_BAD_PARAMETER; } int acpi_install_cmos_rtc_space_handler(acpi_handle handle) { acpi_status status; + if (cmos_rtc_space_handler_present) + return 0; + status = acpi_install_address_space_handler(handle, - ACPI_ADR_SPACE_CMOS, - &acpi_cmos_rtc_space_handler, - NULL, NULL); + ACPI_ADR_SPACE_CMOS, + acpi_cmos_rtc_space_handler, + NULL, NULL); if (ACPI_FAILURE(status)) { - pr_err("Error installing CMOS-RTC region handler\n"); + pr_err("Failed to install CMOS-RTC address space handler\n"); return -ENODEV; } + cmos_rtc_space_handler_present = true; + return 1; } EXPORT_SYMBOL_GPL(acpi_install_cmos_rtc_space_handler); void acpi_remove_cmos_rtc_space_handler(acpi_handle handle) { - if (ACPI_FAILURE(acpi_remove_address_space_handler(handle, - ACPI_ADR_SPACE_CMOS, &acpi_cmos_rtc_space_handler))) - pr_err("Error removing CMOS-RTC region handler\n"); + acpi_status status; + + if (cmos_rtc_space_handler_present) + return; + + status = acpi_remove_address_space_handler(handle, + ACPI_ADR_SPACE_CMOS, + acpi_cmos_rtc_space_handler); + if (ACPI_FAILURE(status)) + pr_err("Failed to remove CMOS-RTC address space handler\n"); } EXPORT_SYMBOL_GPL(acpi_remove_cmos_rtc_space_handler); -static int acpi_cmos_rtc_attach_handler(struct acpi_device *adev, const struct acpi_device_id *id) +static int acpi_cmos_rtc_attach(struct acpi_device *adev, + const struct acpi_device_id *id) { - return acpi_install_cmos_rtc_space_handler(adev->handle); -} + int ret; -static void acpi_cmos_rtc_detach_handler(struct acpi_device *adev) -{ - acpi_remove_cmos_rtc_space_handler(adev->handle); + ret = acpi_install_cmos_rtc_space_handler(adev->handle); + if (ret < 0) + return ret; + + return 1; } static struct acpi_scan_handler cmos_rtc_handler = { .ids = acpi_cmos_rtc_ids, - .attach = acpi_cmos_rtc_attach_handler, - .detach = acpi_cmos_rtc_detach_handler, + .attach = acpi_cmos_rtc_attach, }; void __init acpi_cmos_rtc_init(void) diff --git a/drivers/android/binder/range_alloc/array.rs b/drivers/android/binder/range_alloc/array.rs index ada1d1b4302e53..081d19b09d4bb4 100644 --- a/drivers/android/binder/range_alloc/array.rs +++ b/drivers/android/binder/range_alloc/array.rs @@ -204,7 +204,6 @@ impl ArrayRangeAllocator { // caller will mark them as unused, which means that they can be freed if the system comes // under memory pressure. let mut freed_range = FreedRange::interior_pages(offset, size); - #[expect(clippy::collapsible_if)] // reads better like this if offset % PAGE_SIZE != 0 { if i == 0 || self.ranges[i - 1].endpoint() <= (offset & PAGE_MASK) { freed_range.start_page_idx -= 1; diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 3b65df914ebbe7..cd607911d72484 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -1692,7 +1692,7 @@ void ata_scsi_requeue_deferred_qc(struct ata_port *ap) /* * If we have a deferred qc when a reset occurs or NCQ commands fail, * do not try to be smart about what to do with this deferred command - * and simply retry it by completing it with DID_SOFT_ERROR. + * and simply requeue it by completing it with DID_REQUEUE. */ if (!qc) return; @@ -1701,7 +1701,7 @@ void ata_scsi_requeue_deferred_qc(struct ata_port *ap) ap->deferred_qc = NULL; cancel_work(&ap->deferred_qc_work); ata_qc_free(qc); - scmd->result = (DID_SOFT_ERROR << 16); + scmd->result = (DID_REQUEUE << 16); scsi_done(scmd); } diff --git a/drivers/base/core.c b/drivers/base/core.c index 0613de0fbe44dc..d213908c34f726 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -182,7 +182,7 @@ void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode) if (fwnode->dev) return; - fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_set_flag(fwnode, FWNODE_FLAG_NOT_DEVICE); fwnode_links_purge_consumers(fwnode); fwnode_for_each_available_child_node(fwnode, child) @@ -228,7 +228,7 @@ static void __fw_devlink_pickup_dangling_consumers(struct fwnode_handle *fwnode, if (fwnode->dev && fwnode->dev->bus) return; - fwnode->flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_set_flag(fwnode, FWNODE_FLAG_NOT_DEVICE); __fwnode_links_move_consumers(fwnode, new_sup); fwnode_for_each_available_child_node(fwnode, child) @@ -1003,6 +1003,7 @@ static void device_links_missing_supplier(struct device *dev) if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) { WRITE_ONCE(link->status, DL_STATE_AVAILABLE); } else { + dev_err(dev, "devices misses supplier %s\n", dev_name(link->supplier)); WARN_ON(!device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)); WRITE_ONCE(link->status, DL_STATE_DORMANT); } @@ -1012,7 +1013,7 @@ static void device_links_missing_supplier(struct device *dev) static bool dev_is_best_effort(struct device *dev) { return (fw_devlink_best_effort && dev->can_match) || - (dev->fwnode && (dev->fwnode->flags & FWNODE_FLAG_BEST_EFFORT)); + (dev->fwnode && fwnode_test_flag(dev->fwnode, FWNODE_FLAG_BEST_EFFORT)); } static struct fwnode_handle *fwnode_links_check_suppliers( @@ -1723,11 +1724,11 @@ bool fw_devlink_is_strict(void) static void fw_devlink_parse_fwnode(struct fwnode_handle *fwnode) { - if (fwnode->flags & FWNODE_FLAG_LINKS_ADDED) + if (fwnode_test_flag(fwnode, FWNODE_FLAG_LINKS_ADDED)) return; fwnode_call_int_op(fwnode, add_links); - fwnode->flags |= FWNODE_FLAG_LINKS_ADDED; + fwnode_set_flag(fwnode, FWNODE_FLAG_LINKS_ADDED); } static void fw_devlink_parse_fwtree(struct fwnode_handle *fwnode) @@ -1885,7 +1886,7 @@ static bool fwnode_init_without_drv(struct fwnode_handle *fwnode) struct device *dev; bool ret; - if (!(fwnode->flags & FWNODE_FLAG_INITIALIZED)) + if (!fwnode_test_flag(fwnode, FWNODE_FLAG_INITIALIZED)) return false; dev = get_dev_from_fwnode(fwnode); @@ -2001,10 +2002,10 @@ static bool __fw_devlink_relax_cycles(struct fwnode_handle *con_handle, * We aren't trying to find all cycles. Just a cycle between con and * sup_handle. */ - if (sup_handle->flags & FWNODE_FLAG_VISITED) + if (fwnode_test_flag(sup_handle, FWNODE_FLAG_VISITED)) return false; - sup_handle->flags |= FWNODE_FLAG_VISITED; + fwnode_set_flag(sup_handle, FWNODE_FLAG_VISITED); /* Termination condition. */ if (sup_handle == con_handle) { @@ -2074,7 +2075,7 @@ static bool __fw_devlink_relax_cycles(struct fwnode_handle *con_handle, } out: - sup_handle->flags &= ~FWNODE_FLAG_VISITED; + fwnode_clear_flag(sup_handle, FWNODE_FLAG_VISITED); put_device(sup_dev); put_device(con_dev); put_device(par_dev); @@ -2127,7 +2128,7 @@ static int fw_devlink_create_devlink(struct device *con, * When such a flag is set, we can't create device links where P is the * supplier of C as that would delay the probe of C. */ - if (sup_handle->flags & FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD && + if (fwnode_test_flag(sup_handle, FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD) && fwnode_is_ancestor_of(sup_handle, con->fwnode)) return -EINVAL; @@ -2150,7 +2151,7 @@ static int fw_devlink_create_devlink(struct device *con, else flags = FW_DEVLINK_FLAGS_PERMISSIVE; - if (sup_handle->flags & FWNODE_FLAG_NOT_DEVICE) + if (fwnode_test_flag(sup_handle, FWNODE_FLAG_NOT_DEVICE)) sup_dev = fwnode_get_next_parent_dev(sup_handle); else sup_dev = get_dev_from_fwnode(sup_handle); @@ -2162,7 +2163,7 @@ static int fw_devlink_create_devlink(struct device *con, * supplier device indefinitely. */ if (sup_dev->links.status == DL_DEV_NO_DRIVER && - sup_handle->flags & FWNODE_FLAG_INITIALIZED) { + fwnode_test_flag(sup_handle, FWNODE_FLAG_INITIALIZED)) { dev_dbg(con, "Not linking %pfwf - dev might never probe\n", sup_handle); @@ -2328,6 +2329,32 @@ static void fw_devlink_link_device(struct device *dev) __fw_devlink_link_to_suppliers(dev, fwnode); } +/** + * fw_devlink_count_absent_consumers - Return how many consumers have + * either not been created yet, or do not yet have a driver attached. + * @fwnode: fwnode of the supplier + */ +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode) +{ + struct fwnode_link *link, *tmp; + struct device_link *dlink, *dtmp; + struct device *sup_dev = get_dev_from_fwnode(fwnode); + int count = 0; + + list_for_each_entry_safe(link, tmp, &fwnode->consumers, s_hook) + count++; + + if (!sup_dev) + return count; + + list_for_each_entry_safe(dlink, dtmp, &sup_dev->links.consumers, s_node) + if (dlink->consumer->links.status != DL_DEV_DRIVER_BOUND) + count++; + + return count; +} +EXPORT_SYMBOL_GPL(fw_devlink_count_absent_consumers); + /* Device links support end. */ static struct kobject *dev_kobj; @@ -3687,6 +3714,21 @@ int device_add(struct device *dev) fw_devlink_link_device(dev); } + /* + * The moment the device was linked into the bus's "klist_devices" in + * bus_add_device() then it's possible that probe could have been + * attempted in a different thread via userspace loading a driver + * matching the device. "ready_to_probe" being unset would have + * blocked those attempts. Now that all of the above initialization has + * happened, unblock probe. If probe happens through another thread + * after this point but before bus_probe_device() runs then it's fine. + * bus_probe_device() -> device_initial_probe() -> __device_attach() + * will notice (under device_lock) that the device is already bound. + */ + device_lock(dev); + dev_set_ready_to_probe(dev); + device_unlock(dev); + bus_probe_device(dev); /* diff --git a/drivers/base/dd.c b/drivers/base/dd.c index 37c7e54e0e4c74..ec7ef9c5d62e1a 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -848,6 +848,26 @@ static int __driver_probe_device(const struct device_driver *drv, struct device if (dev->driver) return -EBUSY; + /* + * In device_add(), the "struct device" gets linked into the subsystem's + * list of devices and broadcast to userspace (via uevent) before we're + * quite ready to probe. Those open pathways to driver probe before + * we've finished enough of device_add() to reliably support probe. + * Detect this and tell other pathways to try again later. device_add() + * itself will also try to probe immediately after setting + * "ready_to_probe". + */ + if (!dev_ready_to_probe(dev)) + return dev_err_probe(dev, -EPROBE_DEFER, "Device not ready to probe\n"); + + /* + * Set can_match = true after calling dev_ready_to_probe(), so + * driver_deferred_probe_add() won't actually add the device to the + * deferred probe list when dev_ready_to_probe() returns false. + * + * When dev_ready_to_probe() returns false, it means that device_add() + * will do another probe() attempt for us. + */ dev->can_match = true; dev_dbg(dev, "bus: '%s': %s: matched device with driver %s\n", drv->bus->name, __func__, drv->name); diff --git a/drivers/base/devres.c b/drivers/base/devres.c index 171750c1f69186..ce519b98a18984 100644 --- a/drivers/base/devres.c +++ b/drivers/base/devres.c @@ -940,6 +940,8 @@ void *devm_krealloc(struct device *dev, void *ptr, size_t new_size, gfp_t gfp) if (!new_dr) return NULL; + set_node_dbginfo(&new_dr->node, "devm_krealloc_release", new_size); + /* * The spinlock protects the linked list against concurrent * modifications but not the resource itself. diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index a11b30dda23be5..744d070c2ae13f 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -471,6 +471,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv, static char fw_path_para[256]; static const char * const fw_path[] = { fw_path_para, + "/lib/firmware/vendor/" UTS_RELEASE, + "/lib/firmware/vendor", "/lib/firmware/updates/" UTS_RELEASE, "/lib/firmware/updates", "/lib/firmware/" UTS_RELEASE, diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index e201f0087a0f78..728ecc431b38d7 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -3378,8 +3378,10 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb) if (resource_filter) { retcode = ERR_RES_NOT_KNOWN; resource = drbd_find_resource(nla_data(resource_filter)); - if (!resource) + if (!resource) { + rcu_read_lock(); goto put_result; + } cb->args[0] = (long)resource; } } @@ -3628,8 +3630,10 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb) if (resource_filter) { retcode = ERR_RES_NOT_KNOWN; resource = drbd_find_resource(nla_data(resource_filter)); - if (!resource) + if (!resource) { + rcu_read_lock(); goto put_result; + } } cb->args[0] = (long)resource; } diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index e7da06200c1e1c..4065336ebd1f1a 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -7165,7 +7165,7 @@ static ssize_t do_rbd_add(const char *buf, size_t count) rc = device_add_disk(&rbd_dev->dev, rbd_dev->disk, NULL); if (rc) - goto err_out_cleanup_disk; + goto err_out_device; spin_lock(&rbd_dev_list_lock); list_add_tail(&rbd_dev->node, &rbd_dev_list); @@ -7179,8 +7179,8 @@ static ssize_t do_rbd_add(const char *buf, size_t count) module_put(THIS_MODULE); return rc; -err_out_cleanup_disk: - rbd_free_disk(rbd_dev); +err_out_device: + device_del(&rbd_dev->dev); err_out_image_lock: rbd_dev_image_unlock(rbd_dev); rbd_dev_device_release(rbd_dev); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 63aeb7a76a8c9b..0bdb804fca839a 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2910,22 +2910,26 @@ static void ublk_stop_dev(struct ublk_device *ub) ublk_cancel_dev(ub); } +static void ublk_reset_io_flags(struct ublk_queue *ubq, struct ublk_io *io) +{ + /* UBLK_IO_FLAG_CANCELED can be cleared now */ + spin_lock(&ubq->cancel_lock); + io->flags &= ~UBLK_IO_FLAG_CANCELED; + spin_unlock(&ubq->cancel_lock); +} + /* reset per-queue io flags */ static void ublk_queue_reset_io_flags(struct ublk_queue *ubq) { - int j; - - /* UBLK_IO_FLAG_CANCELED can be cleared now */ spin_lock(&ubq->cancel_lock); - for (j = 0; j < ubq->q_depth; j++) - ubq->ios[j].flags &= ~UBLK_IO_FLAG_CANCELED; ubq->canceling = false; spin_unlock(&ubq->cancel_lock); ubq->fail_io = false; } /* device can only be started after all IOs are ready */ -static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id) +static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id, + struct ublk_io *io) __must_hold(&ub->mutex) { struct ublk_queue *ubq = ublk_get_queue(ub, q_id); @@ -2934,6 +2938,7 @@ static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id) ub->unprivileged_daemons = true; ubq->nr_io_ready++; + ublk_reset_io_flags(ubq, io); /* Check if this specific queue is now fully ready */ if (ublk_queue_ready(ubq)) { @@ -3196,7 +3201,7 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, if (!ret) ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); if (!ret) - ublk_mark_io_ready(ub, q_id); + ublk_mark_io_ready(ub, q_id, io); mutex_unlock(&ub->mutex); return ret; } @@ -3604,7 +3609,7 @@ static int ublk_batch_prep_io(struct ublk_queue *ubq, ublk_io_unlock(io); if (!ret) - ublk_mark_io_ready(data->ub, ubq->q_id); + ublk_mark_io_ready(data->ub, ubq->q_id, io); return ret; } diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index af679375b19359..aaaef8dd82538b 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -1748,6 +1748,10 @@ static ssize_t algorithm_params_store(struct device *dev, } } + guard(rwsem_write)(&zram->dev_lock); + if (init_done(zram)) + return -EBUSY; + /* Lookup priority by algorithm name */ if (algo) { s32 p; @@ -2697,7 +2701,7 @@ static void zram_bio_discard(struct zram *zram, struct bio *bio) */ if (offset) { if (n <= (PAGE_SIZE - offset)) - return; + goto end_bio; n -= (PAGE_SIZE - offset); index++; @@ -2712,6 +2716,7 @@ static void zram_bio_discard(struct zram *zram, struct bio *bio) n -= PAGE_SIZE; } +end_bio: bio_endio(bio); } diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c index fa7533578f85c9..a4b4dacfd2ad35 100644 --- a/drivers/bluetooth/btmtk.c +++ b/drivers/bluetooth/btmtk.c @@ -654,8 +654,13 @@ static int btmtk_usb_hci_wmt_sync(struct hci_dev *hdev, if (data->evt_skb == NULL) goto err_free_wc; - /* Parse and handle the return WMT event */ - wmt_evt = (struct btmtk_hci_wmt_evt *)data->evt_skb->data; + wmt_evt = skb_pull_data(data->evt_skb, sizeof(*wmt_evt)); + if (!wmt_evt) { + bt_dev_err(hdev, "WMT event too short (%u bytes)", + data->evt_skb->len); + err = -EINVAL; + goto err_free_skb; + } if (wmt_evt->whdr.op != hdr->op) { bt_dev_err(hdev, "Wrong op received %d expected %d", wmt_evt->whdr.op, hdr->op); @@ -671,6 +676,12 @@ static int btmtk_usb_hci_wmt_sync(struct hci_dev *hdev, status = BTMTK_WMT_PATCH_DONE; break; case BTMTK_WMT_FUNC_CTRL: + if (!skb_pull_data(data->evt_skb, + sizeof(wmt_evt_funcc->status))) { + status = BTMTK_WMT_ON_UNDONE; + break; + } + wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt; if (be16_to_cpu(wmt_evt_funcc->status) == 0x404) status = BTMTK_WMT_ON_DONE; diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 925d0a6359453e..5f79920c030681 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2397,6 +2397,8 @@ static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (bcm4377->hw->broken_le_ext_adv_report_phy) hci_set_quirk(hdev, HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY); + hci_set_brcm_capable(hdev); + pci_set_drvdata(pdev, bcm4377); hci_set_drvdata(hdev, bcm4377); SET_HCIDEV_DEV(hdev, &pdev->dev); diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 71c1997a0f7344..275ea865bc297b 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -692,6 +692,9 @@ static int hci_uart_register_dev(struct hci_uart *hu) if (hci_register_dev(hdev) < 0) { BT_ERR("Can't register HCI device"); + percpu_down_write(&hu->proto_lock); + clear_bit(HCI_UART_PROTO_INIT, &hu->flags); + percpu_up_write(&hu->proto_lock); hu->proto->close(hu); hu->hdev = NULL; hci_free_dev(hdev); diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 76d61af8a275e9..140ab55c9fc5a9 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -12,6 +12,7 @@ #include #define VERSION "0.1" +#define VIRTBT_RX_BUF_SIZE 1000 enum { VIRTBT_VQ_TX, @@ -33,11 +34,11 @@ static int virtbt_add_inbuf(struct virtio_bluetooth *vbt) struct sk_buff *skb; int err; - skb = alloc_skb(1000, GFP_KERNEL); + skb = alloc_skb(VIRTBT_RX_BUF_SIZE, GFP_KERNEL); if (!skb) return -ENOMEM; - sg_init_one(sg, skb->data, 1000); + sg_init_one(sg, skb->data, VIRTBT_RX_BUF_SIZE); err = virtqueue_add_inbuf(vq, sg, 1, skb, GFP_KERNEL); if (err < 0) { @@ -197,6 +198,7 @@ static int virtbt_shutdown_generic(struct hci_dev *hdev) static void virtbt_rx_handle(struct virtio_bluetooth *vbt, struct sk_buff *skb) { + size_t min_hdr; __u8 pkt_type; pkt_type = *((__u8 *) skb->data); @@ -204,16 +206,32 @@ static void virtbt_rx_handle(struct virtio_bluetooth *vbt, struct sk_buff *skb) switch (pkt_type) { case HCI_EVENT_PKT: + min_hdr = sizeof(struct hci_event_hdr); + break; case HCI_ACLDATA_PKT: + min_hdr = sizeof(struct hci_acl_hdr); + break; case HCI_SCODATA_PKT: + min_hdr = sizeof(struct hci_sco_hdr); + break; case HCI_ISODATA_PKT: - hci_skb_pkt_type(skb) = pkt_type; - hci_recv_frame(vbt->hdev, skb); + min_hdr = sizeof(struct hci_iso_hdr); break; default: kfree_skb(skb); - break; + return; + } + + if (skb->len < min_hdr) { + bt_dev_err_ratelimited(vbt->hdev, + "rx pkt_type 0x%02x payload %u < hdr %zu\n", + pkt_type, skb->len, min_hdr); + kfree_skb(skb); + return; } + + hci_skb_pkt_type(skb) = pkt_type; + hci_recv_frame(vbt->hdev, skb); } static void virtbt_rx_work(struct work_struct *work) @@ -227,8 +245,15 @@ static void virtbt_rx_work(struct work_struct *work) if (!skb) return; - skb_put(skb, len); - virtbt_rx_handle(vbt, skb); + if (!len || len > VIRTBT_RX_BUF_SIZE) { + bt_dev_err_ratelimited(vbt->hdev, + "rx reply len %u outside [1, %u]\n", + len, VIRTBT_RX_BUF_SIZE); + kfree_skb(skb); + } else { + skb_put(skb, len); + virtbt_rx_handle(vbt, skb); + } if (virtbt_add_inbuf(vbt) < 0) return; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index c117745cf20654..221146e4860be7 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -86,12 +86,16 @@ static int fsl_mc_bus_match(struct device *dev, const struct device_driver *drv) struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); const struct fsl_mc_driver *mc_drv = to_fsl_mc_driver(drv); bool found = false; + int ret; /* When driver_override is set, only bind to the matching driver */ - if (mc_dev->driver_override) { - found = !strcmp(mc_dev->driver_override, mc_drv->driver.name); + ret = device_match_driver_override(dev, drv); + if (ret > 0) { + found = true; goto out; } + if (ret == 0) + goto out; if (!mc_drv->match_id_table) goto out; @@ -210,39 +214,8 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, } static DEVICE_ATTR_RO(modalias); -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); - int ret; - - if (WARN_ON(dev->bus != &fsl_mc_bus_type)) - return -EINVAL; - - ret = driver_set_override(dev, &mc_dev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", mc_dev->driver_override); - device_unlock(dev); - return len; -} -static DEVICE_ATTR_RW(driver_override); - static struct attribute *fsl_mc_dev_attrs[] = { &dev_attr_modalias.attr, - &dev_attr_driver_override.attr, NULL, }; @@ -345,6 +318,7 @@ ATTRIBUTE_GROUPS(fsl_mc_bus); const struct bus_type fsl_mc_bus_type = { .name = "fsl-mc", + .driver_override = true, .match = fsl_mc_bus_match, .uevent = fsl_mc_bus_uevent, .probe = fsl_mc_probe, @@ -910,9 +884,6 @@ static struct notifier_block fsl_mc_nb; */ void fsl_mc_device_remove(struct fsl_mc_device *mc_dev) { - kfree(mc_dev->driver_override); - mc_dev->driver_override = NULL; - /* * The device-specific remove callback will get invoked by device_del() */ diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c index 83d623d97f5f28..f735e0462c55ee 100644 --- a/drivers/bus/imx-weim.c +++ b/drivers/bus/imx-weim.c @@ -332,7 +332,7 @@ static int of_weim_notify(struct notifier_block *nb, unsigned long action, * fw_devlink doesn't skip adding consumers to this * device. */ - rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE; + fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); if (!of_platform_device_create(rd->dn, NULL, &pdev->dev)) { dev_err(&pdev->dev, "Failed to create child device '%pOF'\n", diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index 0884a384b77fc3..51456cee70a21f 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -1393,7 +1393,7 @@ static int mhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_unregister; } - err = mhi_sync_power_up(mhi_cntrl); + err = mhi_async_power_up(mhi_cntrl); if (err) { dev_err(&pdev->dev, "failed to power up MHI controller\n"); goto err_unprepare; diff --git a/drivers/bus/simple-pm-bus.c b/drivers/bus/simple-pm-bus.c index c920bd6fbaafd4..6d351ca51aa4e6 100644 --- a/drivers/bus/simple-pm-bus.c +++ b/drivers/bus/simple-pm-bus.c @@ -142,6 +142,15 @@ static const struct of_device_id simple_pm_bus_of_match[] = { { .compatible = "simple-mfd", .data = ONLY_BUS }, { .compatible = "isa", .data = ONLY_BUS }, { .compatible = "arm,amba-bus", .data = ONLY_BUS }, + { .compatible = "apple,s5l8960x-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t7000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,s8000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8010-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8015-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8103-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t8112-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t6000-pmgr", .data = ONLY_BUS }, + { .compatible = "apple,t6020-pmgr", .data = ONLY_BUS },\ { .compatible = "fsl,ls1021a-scfg", }, { .compatible = "fsl,ls1043a-scfg", }, { .compatible = "fsl,ls1046a-scfg", }, diff --git a/drivers/bus/stm32_rifsc.c b/drivers/bus/stm32_rifsc.c index debeaf8ea1bd2b..5682c086ba1e85 100644 --- a/drivers/bus/stm32_rifsc.c +++ b/drivers/bus/stm32_rifsc.c @@ -688,34 +688,6 @@ static int stm32_rifsc_grant_access(struct stm32_firewall_controller *ctrl, u32 sec_reg_value = readl(rifsc_controller->mmio + RIFSC_RISC_SECCFGR0 + 0x4 * reg_id); cid_reg_value = readl(rifsc_controller->mmio + RIFSC_RISC_PER0_CIDCFGR + 0x8 * firewall_id); - /* First check conditions for semaphore mode, which doesn't take into account static CID. */ - if ((cid_reg_value & CIDCFGR_SEMEN) && (cid_reg_value & CIDCFGR_CFEN)) { - if (cid_reg_value & BIT(RIF_CID1 + SEMWL_SHIFT)) { - /* Static CID is irrelevant if semaphore mode */ - goto skip_cid_check; - } else { - dev_dbg(rifsc_controller->dev, - "Invalid bus semaphore configuration: index %d\n", firewall_id); - return -EACCES; - } - } - - /* - * Skip CID check if CID filtering isn't enabled or filtering is enabled on CID0, which - * corresponds to whatever CID. - */ - if (!(cid_reg_value & CIDCFGR_CFEN) || - FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) == RIF_CID0) - goto skip_cid_check; - - /* Coherency check with the CID configuration */ - if (FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) != RIF_CID1) { - dev_dbg(rifsc_controller->dev, "Invalid CID configuration for peripheral: %d\n", - firewall_id); - return -EACCES; - } - -skip_cid_check: /* Check security configuration */ if (sec_reg_value & BIT(reg_offset)) { dev_dbg(rifsc_controller->dev, @@ -723,19 +695,31 @@ static int stm32_rifsc_grant_access(struct stm32_firewall_controller *ctrl, u32 return -EACCES; } - /* - * If the peripheral is in semaphore mode, take the semaphore so that - * the CID1 has the ownership. - */ - if ((cid_reg_value & CIDCFGR_SEMEN) && (cid_reg_value & CIDCFGR_CFEN)) { + /* Skip CID check if CID filtering isn't enabled */ + if (!(cid_reg_value & CIDCFGR_CFEN)) + goto skip_cid_check; + + /* First check conditions for semaphore mode, which doesn't take into account static CID. */ + if (cid_reg_value & CIDCFGR_SEMEN) { + if (!(cid_reg_value & BIT(RIF_CID1 + SEMWL_SHIFT))) { + dev_dbg(rifsc_controller->dev, + "Invalid bus semaphore configuration: index %d\n", firewall_id); + return -EACCES; + } + rc = stm32_rif_acquire_semaphore(rifsc_controller, firewall_id); if (rc) { - dev_err(rifsc_controller->dev, + dev_dbg(rifsc_controller->dev, "Couldn't acquire semaphore for peripheral: %d\n", firewall_id); return rc; } + } else if (FIELD_GET(RIFSC_RISC_SCID_MASK, cid_reg_value) != RIF_CID1) { + dev_dbg(rifsc_controller->dev, "Invalid CID configuration for peripheral: %d\n", + firewall_id); + return -EACCES; } +skip_cid_check: return 0; } diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index fc049612d6dc1a..62934cf4b10de8 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -631,6 +631,16 @@ int register_cdrom(struct gendisk *disk, struct cdrom_device_info *cdi) WARN_ON(!cdo->generic_packet); + /* + * Propagate the drive's write support to the block layer so BLKROGET + * reflects actual write capability. Drivers that use GET CONFIGURATION + * features (CDC_MRW_W, CDC_RAM) must have called + * cdrom_probe_write_features() before register_cdrom() so the mask is + * complete here. + */ + set_disk_ro(disk, !CDROM_CAN(CDC_DVD_RAM | CDC_MRW_W | CDC_RAM | + CDC_CD_RW)); + cd_dbg(CD_REG_UNREG, "drive \"/dev/%s\" registered\n", cdi->name); mutex_lock(&cdrom_mutex); list_add(&cdi->list, &cdrom_list); @@ -742,6 +752,44 @@ static int cdrom_is_random_writable(struct cdrom_device_info *cdi, int *write) return 0; } +/* + * Probe write-related MMC features via GET CONFIGURATION and update + * cdi->mask accordingly. Drivers that populate cdi->mask from the MODE SENSE + * capabilities page (e.g. sr) should call this after those MODE SENSE bits + * have been set but before register_cdrom(), so that the full set of + * write-capability bits is known by the time register_cdrom() decides on the + * initial read-only state of the disk. + */ +void cdrom_probe_write_features(struct cdrom_device_info *cdi) +{ + int mrw, mrw_write, ram_write; + + mrw = 0; + if (!cdrom_is_mrw(cdi, &mrw_write)) + mrw = 1; + + if (CDROM_CAN(CDC_MO_DRIVE)) + ram_write = 1; + else + (void) cdrom_is_random_writable(cdi, &ram_write); + + if (mrw) + cdi->mask &= ~CDC_MRW; + else + cdi->mask |= CDC_MRW; + + if (mrw_write) + cdi->mask &= ~CDC_MRW_W; + else + cdi->mask |= CDC_MRW_W; + + if (ram_write) + cdi->mask &= ~CDC_RAM; + else + cdi->mask |= CDC_RAM; +} +EXPORT_SYMBOL(cdrom_probe_write_features); + static int cdrom_media_erasable(struct cdrom_device_info *cdi) { disc_information di; @@ -894,33 +942,8 @@ static int cdrom_is_dvd_rw(struct cdrom_device_info *cdi) */ static int cdrom_open_write(struct cdrom_device_info *cdi) { - int mrw, mrw_write, ram_write; int ret = 1; - mrw = 0; - if (!cdrom_is_mrw(cdi, &mrw_write)) - mrw = 1; - - if (CDROM_CAN(CDC_MO_DRIVE)) - ram_write = 1; - else - (void) cdrom_is_random_writable(cdi, &ram_write); - - if (mrw) - cdi->mask &= ~CDC_MRW; - else - cdi->mask |= CDC_MRW; - - if (mrw_write) - cdi->mask &= ~CDC_MRW_W; - else - cdi->mask |= CDC_MRW_W; - - if (ram_write) - cdi->mask &= ~CDC_RAM; - else - cdi->mask |= CDC_RAM; - if (CDROM_CAN(CDC_MRW_W)) ret = cdrom_mrw_open_write(cdi); else if (CDROM_CAN(CDC_DVD_RAM)) diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 4a9e9de4d684f9..9a9d12be9bf743 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -168,6 +168,10 @@ struct smi_info { OEM2_DATA_AVAIL) unsigned char msg_flags; + /* When requesting events and messages, don't do it forever. */ + unsigned int num_requests_in_a_row; + bool last_was_flag_fetch; + /* Does the BMC have an event buffer? */ bool has_event_buffer; @@ -410,7 +414,10 @@ static void start_getting_msg_queue(struct smi_info *smi_info) start_new_msg(smi_info, smi_info->curr_msg->data, smi_info->curr_msg->data_size); - smi_info->si_state = SI_GETTING_MESSAGES; + if (smi_info->si_state != SI_GETTING_MESSAGES) { + smi_info->num_requests_in_a_row = 0; + smi_info->si_state = SI_GETTING_MESSAGES; + } } static void start_getting_events(struct smi_info *smi_info) @@ -421,7 +428,10 @@ static void start_getting_events(struct smi_info *smi_info) start_new_msg(smi_info, smi_info->curr_msg->data, smi_info->curr_msg->data_size); - smi_info->si_state = SI_GETTING_EVENTS; + if (smi_info->si_state != SI_GETTING_EVENTS) { + smi_info->num_requests_in_a_row = 0; + smi_info->si_state = SI_GETTING_EVENTS; + } } /* @@ -487,15 +497,19 @@ static void handle_flags(struct smi_info *smi_info) } else if (smi_info->msg_flags & RECEIVE_MSG_AVAIL) { /* Messages available. */ smi_info->curr_msg = alloc_msg_handle_irq(smi_info); - if (!smi_info->curr_msg) + if (!smi_info->curr_msg) { + smi_info->si_state = SI_NORMAL; return; + } start_getting_msg_queue(smi_info); } else if (smi_info->msg_flags & EVENT_MSG_BUFFER_FULL) { /* Events available. */ smi_info->curr_msg = alloc_msg_handle_irq(smi_info); - if (!smi_info->curr_msg) + if (!smi_info->curr_msg) { + smi_info->si_state = SI_NORMAL; return; + } start_getting_events(smi_info); } else if (smi_info->msg_flags & OEM_DATA_AVAIL && @@ -595,6 +609,7 @@ static void handle_transaction_done(struct smi_info *smi_info) smi_info->si_state = SI_NORMAL; } else { smi_info->msg_flags = msg[3]; + smi_info->last_was_flag_fetch = true; handle_flags(smi_info); } break; @@ -630,7 +645,13 @@ static void handle_transaction_done(struct smi_info *smi_info) */ msg = smi_info->curr_msg; smi_info->curr_msg = NULL; - if (msg->rsp[2] != 0) { + /* + * It appears some BMCs, with no event data, return no + * data in the message and not a 0x80 error as the + * spec says they should. Shut down processing if + * the data is not the right length. + */ + if (msg->rsp[2] != 0 || msg->rsp_size != 19) { /* Error getting event, probably done. */ msg->done(msg); @@ -640,6 +661,11 @@ static void handle_transaction_done(struct smi_info *smi_info) } else { smi_inc_stat(smi_info, events); + smi_info->num_requests_in_a_row++; + if (smi_info->num_requests_in_a_row > 10) + /* Stop if we do this too many times. */ + smi_info->msg_flags &= ~EVENT_MSG_BUFFER_FULL; + /* * Do this before we deliver the message * because delivering the message releases the @@ -678,6 +704,11 @@ static void handle_transaction_done(struct smi_info *smi_info) } else { smi_inc_stat(smi_info, incoming_messages); + smi_info->num_requests_in_a_row++; + if (smi_info->num_requests_in_a_row > 10) + /* Stop if we do this too many times. */ + smi_info->msg_flags &= ~RECEIVE_MSG_AVAIL; + /* * Do this before we deliver the message * because delivering the message releases the @@ -819,6 +850,26 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info, goto out; } + /* + * If we are currently idle, or if the last thing that was + * done was a flag fetch and there is a message pending, try + * to start the next message. + * + * We do the waiting message check to avoid a stuck flag + * completely wedging the driver. Let a message through + * in between flag operations if that happens. + */ + if (si_sm_result == SI_SM_IDLE || + (si_sm_result == SI_SM_ATTN && smi_info->waiting_msg && + smi_info->last_was_flag_fetch)) { + smi_info->last_was_flag_fetch = false; + smi_inc_stat(smi_info, idles); + + si_sm_result = start_next_msg(smi_info); + if (si_sm_result != SI_SM_IDLE) + goto restart; + } + /* * We prefer handling attn over new messages. But don't do * this if there is not yet an upper layer to handle anything. @@ -846,15 +897,6 @@ static enum si_sm_result smi_event_handler(struct smi_info *smi_info, } } - /* If we are currently idle, try to start the next message. */ - if (si_sm_result == SI_SM_IDLE) { - smi_inc_stat(smi_info, idles); - - si_sm_result = start_next_msg(smi_info); - if (si_sm_result != SI_SM_IDLE) - goto restart; - } - if ((si_sm_result == SI_SM_IDLE) && (atomic_read(&smi_info->req_events))) { /* diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 37a5cb5c53f1fc..f419b46bf00207 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -225,6 +225,9 @@ struct ssif_info { bool has_event_buffer; bool supports_alert; + /* When requesting events and messages, don't do it forever. */ + unsigned int num_requests_in_a_row; + /* * Used to tell what we should do with alerts. If we are * waiting on a response, read the data immediately. @@ -413,7 +416,10 @@ static void start_event_fetch(struct ssif_info *ssif_info, unsigned long *flags) } ssif_info->curr_msg = msg; - ssif_info->ssif_state = SSIF_GETTING_EVENTS; + if (ssif_info->ssif_state != SSIF_GETTING_EVENTS) { + ssif_info->num_requests_in_a_row = 0; + ssif_info->ssif_state = SSIF_GETTING_EVENTS; + } ipmi_ssif_unlock_cond(ssif_info, flags); msg->data[0] = (IPMI_NETFN_APP_REQUEST << 2); @@ -436,7 +442,10 @@ static void start_recv_msg_fetch(struct ssif_info *ssif_info, } ssif_info->curr_msg = msg; - ssif_info->ssif_state = SSIF_GETTING_MESSAGES; + if (ssif_info->ssif_state != SSIF_GETTING_MESSAGES) { + ssif_info->num_requests_in_a_row = 0; + ssif_info->ssif_state = SSIF_GETTING_MESSAGES; + } ipmi_ssif_unlock_cond(ssif_info, flags); msg->data[0] = (IPMI_NETFN_APP_REQUEST << 2); @@ -843,6 +852,11 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result, ssif_info->msg_flags &= ~EVENT_MSG_BUFFER_FULL; handle_flags(ssif_info, flags); } else { + ssif_info->num_requests_in_a_row++; + if (ssif_info->num_requests_in_a_row > 10) + /* Stop if we do this too many times. */ + ssif_info->msg_flags &= ~EVENT_MSG_BUFFER_FULL; + handle_flags(ssif_info, flags); ssif_inc_stat(ssif_info, events); deliver_recv_msg(ssif_info, msg); @@ -876,6 +890,11 @@ static void msg_done_handler(struct ssif_info *ssif_info, int result, ssif_info->msg_flags &= ~RECEIVE_MSG_AVAIL; handle_flags(ssif_info, flags); } else { + ssif_info->num_requests_in_a_row++; + if (ssif_info->num_requests_in_a_row > 10) + /* Stop if we do this too many times. */ + ssif_info->msg_flags &= ~RECEIVE_MSG_AVAIL; + ssif_inc_stat(ssif_info, incoming_messages); handle_flags(ssif_info, flags); deliver_recv_msg(ssif_info, msg); @@ -1268,8 +1287,10 @@ static void shutdown_ssif(void *send_info) ssif_info->stopping = true; timer_delete_sync(&ssif_info->watch_timer); timer_delete_sync(&ssif_info->retry_timer); - if (ssif_info->thread) + if (ssif_info->thread) { kthread_stop(ssif_info->thread); + ssif_info->thread = NULL; + } } static void ssif_remove(struct i2c_client *client) @@ -1658,6 +1679,7 @@ static int ssif_probe(struct i2c_client *client) int len = 0; int i; u8 slave_addr = 0; + unsigned int thread_num; struct ssif_addr_info *addr_info = NULL; mutex_lock(&ssif_infos_mutex); @@ -1876,22 +1898,18 @@ static int ssif_probe(struct i2c_client *client) ssif_info->handlers.request_events = request_events; ssif_info->handlers.set_need_watch = ssif_set_need_watch; - { - unsigned int thread_num; - - thread_num = ((i2c_adapter_id(ssif_info->client->adapter) - << 8) | - ssif_info->client->addr); - init_completion(&ssif_info->wake_thread); - ssif_info->thread = kthread_run(ipmi_ssif_thread, ssif_info, - "kssif%4.4x", thread_num); - if (IS_ERR(ssif_info->thread)) { - rv = PTR_ERR(ssif_info->thread); - dev_notice(&ssif_info->client->dev, - "Could not start kernel thread: error %d\n", - rv); - goto out; - } + thread_num = ((i2c_adapter_id(ssif_info->client->adapter) << 8) | + ssif_info->client->addr); + init_completion(&ssif_info->wake_thread); + ssif_info->thread = kthread_run(ipmi_ssif_thread, ssif_info, + "kssif%4.4x", thread_num); + if (IS_ERR(ssif_info->thread)) { + rv = PTR_ERR(ssif_info->thread); + ssif_info->thread = NULL; + dev_notice(&ssif_info->client->dev, + "Could not start kernel thread: error %d\n", + rv); + goto out; } dev_set_drvdata(&ssif_info->client->dev, ssif_info); @@ -1916,6 +1934,15 @@ static int ssif_probe(struct i2c_client *client) out: if (rv) { + /* + * If ipmi_register_smi() starts the interface, it will + * call shutdown and that will free the thread and set + * it to NULL. Otherwise it must be freed here. + */ + if (ssif_info->thread) { + kthread_stop(ssif_info->thread); + ssif_info->thread = NULL; + } if (addr_info) addr_info->client = NULL; diff --git a/drivers/char/ipmi/ssif_bmc.c b/drivers/char/ipmi/ssif_bmc.c index 7a52e3ea49ed8b..a45e80d13e10e1 100644 --- a/drivers/char/ipmi/ssif_bmc.c +++ b/drivers/char/ipmi/ssif_bmc.c @@ -163,6 +163,8 @@ static ssize_t ssif_bmc_read(struct file *file, char __user *buf, size_t count, spin_unlock_irqrestore(&ssif_bmc->lock, flags); ret = copy_to_user(buf, &msg, count); + if (ret > 0) + ret = -EFAULT; } return (ret < 0) ? ret : count; @@ -456,6 +458,15 @@ static bool supported_write_cmd(u8 cmd) return false; } +static bool supported_write_start_cmd(u8 cmd) +{ + if (cmd == SSIF_IPMI_SINGLEPART_WRITE || + cmd == SSIF_IPMI_MULTIPART_WRITE_START) + return true; + + return false; +} + /* Process the IPMI response that will be read by master */ static void handle_read_processed(struct ssif_bmc_ctx *ssif_bmc, u8 *val) { @@ -558,7 +569,7 @@ static void process_request_part(struct ssif_bmc_ctx *ssif_bmc) len = ssif_bmc->request.len + part->length; /* Do the bound check here, not allow the request len exceed 254 bytes */ if (len > IPMI_SSIF_PAYLOAD_MAX) { - dev_warn(&ssif_bmc->client->dev, + dev_dbg(&ssif_bmc->client->dev, "Warn: Request exceeded 254 bytes, aborting"); /* Request too long, aborting */ ssif_bmc->aborting = true; @@ -604,7 +615,7 @@ static void on_read_requested_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) ssif_bmc->state == SSIF_START || ssif_bmc->state == SSIF_REQ_RECVING || ssif_bmc->state == SSIF_RES_SENDING) { - dev_warn(&ssif_bmc->client->dev, + dev_dbg(&ssif_bmc->client->dev, "Warn: %s unexpected READ REQUESTED in state=%s\n", __func__, state_to_string(ssif_bmc->state)); ssif_bmc->state = SSIF_ABORTING; @@ -613,7 +624,7 @@ static void on_read_requested_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) } else if (ssif_bmc->state == SSIF_SMBUS_CMD) { if (!supported_read_cmd(ssif_bmc->part_buf.smbus_cmd)) { - dev_warn(&ssif_bmc->client->dev, "Warn: Unknown SMBus read command=0x%x", + dev_dbg(&ssif_bmc->client->dev, "Warn: Unknown SMBus read command=0x%x", ssif_bmc->part_buf.smbus_cmd); ssif_bmc->aborting = true; } @@ -648,7 +659,7 @@ static void on_read_processed_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) ssif_bmc->state == SSIF_START || ssif_bmc->state == SSIF_REQ_RECVING || ssif_bmc->state == SSIF_SMBUS_CMD) { - dev_warn(&ssif_bmc->client->dev, + dev_dbg(&ssif_bmc->client->dev, "Warn: %s unexpected READ PROCESSED in state=%s\n", __func__, state_to_string(ssif_bmc->state)); ssif_bmc->state = SSIF_ABORTING; @@ -673,7 +684,7 @@ static void on_write_requested_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) } else if (ssif_bmc->state == SSIF_START || ssif_bmc->state == SSIF_REQ_RECVING || ssif_bmc->state == SSIF_RES_SENDING) { - dev_warn(&ssif_bmc->client->dev, + dev_dbg(&ssif_bmc->client->dev, "Warn: %s unexpected WRITE REQUEST in state=%s\n", __func__, state_to_string(ssif_bmc->state)); ssif_bmc->state = SSIF_ABORTING; @@ -688,7 +699,7 @@ static void on_write_received_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) { if (ssif_bmc->state == SSIF_READY || ssif_bmc->state == SSIF_RES_SENDING) { - dev_warn(&ssif_bmc->client->dev, + dev_dbg(&ssif_bmc->client->dev, "Warn: %s unexpected WRITE RECEIVED in state=%s\n", __func__, state_to_string(ssif_bmc->state)); ssif_bmc->state = SSIF_ABORTING; @@ -698,7 +709,7 @@ static void on_write_received_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) } else if (ssif_bmc->state == SSIF_SMBUS_CMD) { if (!supported_write_cmd(ssif_bmc->part_buf.smbus_cmd)) { - dev_warn(&ssif_bmc->client->dev, "Warn: Unknown SMBus write command=0x%x", + dev_dbg(&ssif_bmc->client->dev, "Warn: Unknown SMBus write command=0x%x", ssif_bmc->part_buf.smbus_cmd); ssif_bmc->aborting = true; } @@ -707,6 +718,11 @@ static void on_write_received_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) ssif_bmc->state = SSIF_ABORTING; else ssif_bmc->state = SSIF_REQ_RECVING; + } else if (ssif_bmc->state == SSIF_ABORTING) { + if (supported_write_start_cmd(*val)) { + ssif_bmc->state = SSIF_SMBUS_CMD; + ssif_bmc->aborting = false; + } } /* This is response sending state */ @@ -722,7 +738,7 @@ static void on_stop_event(struct ssif_bmc_ctx *ssif_bmc, u8 *val) ssif_bmc->state == SSIF_START || ssif_bmc->state == SSIF_SMBUS_CMD || ssif_bmc->state == SSIF_ABORTING) { - dev_warn(&ssif_bmc->client->dev, + dev_dbg(&ssif_bmc->client->dev, "Warn: %s unexpected SLAVE STOP in state=%s\n", __func__, state_to_string(ssif_bmc->state)); ssif_bmc->state = SSIF_READY; @@ -789,7 +805,7 @@ static int ssif_bmc_cb(struct i2c_client *client, enum i2c_slave_event event, u8 break; default: - dev_warn(&ssif_bmc->client->dev, "Warn: Unknown i2c slave event\n"); + dev_dbg(&ssif_bmc->client->dev, "Warn: Unknown i2c slave event\n"); break; } diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index 0719577e584dc4..12b7394b34bdce 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -247,7 +247,7 @@ static void tpm_dev_release(struct device *dev) kfree(chip->work_space.context_buf); kfree(chip->work_space.session_buf); #ifdef CONFIG_TCG_TPM2_HMAC - kfree(chip->auth); + kfree_sensitive(chip->auth); #endif kfree(chip); } diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index 3a77be7ebf4aa1..a2b952538ede0d 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -295,10 +295,8 @@ int tpm2_get_random(struct tpm_chip *chip, u8 *dest, size_t max) } tpm_buf_append_u16(&buf, num_bytes); err = tpm_buf_fill_hmac_session(chip, &buf); - if (err) { - tpm_buf_destroy(&buf); - return err; - } + if (err) + goto out; err = tpm_transmit_cmd(chip, &buf, offsetof(struct tpm2_get_random_out, diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c index 3b1cf1ca042005..c4da6fde748f41 100644 --- a/drivers/char/tpm/tpm2-sessions.c +++ b/drivers/char/tpm/tpm2-sessions.c @@ -203,8 +203,10 @@ static int tpm2_read_public(struct tpm_chip *chip, u32 handle, void *name) rc = tpm_buf_read_u16(&buf, &offset); name_size_alg = name_size(&buf.data[offset]); - if (name_size_alg < 0) + if (name_size_alg < 0) { + tpm_buf_destroy(&buf); return name_size_alg; + } if (rc != name_size_alg) { tpm_buf_destroy(&buf); @@ -217,6 +219,7 @@ static int tpm2_read_public(struct tpm_chip *chip, u32 handle, void *name) } memcpy(name, &buf.data[offset], rc); + tpm_buf_destroy(&buf); return name_size_alg; } #endif /* CONFIG_TCG_TPM2_HMAC */ diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index e2a1769081b1af..21d79ad3b164e7 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -471,6 +471,8 @@ static int tpm_tis_send_data(struct tpm_chip *chip, const u8 *buf, size_t len) status = tpm_tis_status(chip); if (!itpm && (status & TPM_STS_DATA_EXPECT) == 0) { rc = -EIO; + dev_err(&chip->dev, "TPM_STS_DATA_EXPECT should be set. sts = 0x%08x\n", + status); goto out_err; } } @@ -491,6 +493,8 @@ static int tpm_tis_send_data(struct tpm_chip *chip, const u8 *buf, size_t len) status = tpm_tis_status(chip); if (!itpm && (status & TPM_STS_DATA_EXPECT) != 0) { rc = -EIO; + dev_err(&chip->dev, "TPM_STS_DATA_EXPECT should be unset. sts = 0x%08x\n", + status); goto out_err; } @@ -552,11 +556,16 @@ static int tpm_tis_send_main(struct tpm_chip *chip, const u8 *buf, size_t len) break; else if (rc != -EAGAIN && rc != -EIO) /* Data transfer failed, not recoverable */ - return rc; + goto out_err; usleep_range(priv->timeout_min, priv->timeout_max); } + if (rc == -EAGAIN || rc == -EIO) { + dev_err(&chip->dev, "Exhausted %d tpm_tis_send_data retries\n", TPM_RETRY); + goto out_err; + } + /* go and do it */ rc = tpm_tis_write8(priv, TPM_STS(priv->locality), TPM_STS_GO); if (rc < 0) diff --git a/drivers/clk/clk-qoriq.c b/drivers/clk/clk-qoriq.c index f05631e5531061..2524c5c0eb4600 100644 --- a/drivers/clk/clk-qoriq.c +++ b/drivers/clk/clk-qoriq.c @@ -907,13 +907,11 @@ static const struct clockgen_pll_div *get_pll_div(struct clockgen *cg, return &cg->pll[pll].div[div]; } -static struct clk * __init create_mux_common(struct clockgen *cg, - struct mux_hwclock *hwc, - const struct clk_ops *ops, - unsigned long min_rate, - unsigned long max_rate, - unsigned long pct80_rate, - const char *fmt, int idx) +static struct clk * __init __printf(7, 8) +create_mux_common(struct clockgen *cg, struct mux_hwclock *hwc, + const struct clk_ops *ops, unsigned long min_rate, + unsigned long max_rate, unsigned long pct80_rate, + const char *fmt, ...) { struct clk_init_data init = {}; struct clk *clk; @@ -921,8 +919,11 @@ static struct clk * __init create_mux_common(struct clockgen *cg, const char *parent_names[NUM_MUX_PARENTS]; char name[32]; int i, j; + va_list args; - snprintf(name, sizeof(name), fmt, idx); + va_start(args, fmt); + vsnprintf(name, sizeof(name), fmt, args); + va_end(args); for (i = 0, j = 0; i < NUM_MUX_PARENTS; i++) { unsigned long rate; diff --git a/drivers/clk/clk-rk808.c b/drivers/clk/clk-rk808.c index f7412b137e5ef4..5a75b5c9155519 100644 --- a/drivers/clk/clk-rk808.c +++ b/drivers/clk/clk-rk808.c @@ -153,7 +153,7 @@ static int rk808_clkout_probe(struct platform_device *pdev) struct rk808_clkout *rk808_clkout; int ret; - dev->of_node = pdev->dev.parent->of_node; + device_set_of_node_from_dev(dev, dev->parent); rk808_clkout = devm_kzalloc(dev, sizeof(*rk808_clkout), GFP_KERNEL); diff --git a/drivers/clk/clk-xgene.c b/drivers/clk/clk-xgene.c index ba3b1057e4f0c6..abb6c8fcdc91fa 100644 --- a/drivers/clk/clk-xgene.c +++ b/drivers/clk/clk-xgene.c @@ -188,6 +188,8 @@ static void xgene_pllclk_init(struct device_node *np, enum xgene_pll_type pll_ty of_clk_add_provider(np, of_clk_src_simple_get, clk); clk_register_clkdev(clk, clk_name, NULL); pr_debug("Add %s clock PLL\n", clk_name); + } else { + iounmap(reg); } } diff --git a/drivers/clk/imx/clk-imx6q.c b/drivers/clk/imx/clk-imx6q.c index f726c00aba721f..35e6b59c01dbef 100644 --- a/drivers/clk/imx/clk-imx6q.c +++ b/drivers/clk/imx/clk-imx6q.c @@ -188,9 +188,11 @@ static void of_assigned_ldb_sels(struct device_node *node, } if (clkspec.np != node || clkspec.args[0] >= IMX6QDL_CLK_END) { pr_err("ccm: parent clock %d not in ccm\n", index); + of_node_put(clkspec.np); return; } parent = clkspec.args[0]; + of_node_put(clkspec.np); rc = of_parse_phandle_with_args(node, "assigned-clocks", "#clock-cells", index, &clkspec); @@ -198,9 +200,11 @@ static void of_assigned_ldb_sels(struct device_node *node, return; if (clkspec.np != node || clkspec.args[0] >= IMX6QDL_CLK_END) { pr_err("ccm: child clock %d not in ccm\n", index); + of_node_put(clkspec.np); return; } child = clkspec.args[0]; + of_node_put(clkspec.np); if (child != IMX6QDL_CLK_LDB_DI0_SEL && child != IMX6QDL_CLK_LDB_DI1_SEL) @@ -238,8 +242,11 @@ static bool pll6_bypassed(struct device_node *node) return false; if (clkspec.np == node && - clkspec.args[0] == IMX6QDL_PLL6_BYPASS) + clkspec.args[0] == IMX6QDL_PLL6_BYPASS) { + of_node_put(clkspec.np); break; + } + of_node_put(clkspec.np); } /* PLL6 bypass is not part of the assigned clock list */ @@ -249,6 +256,9 @@ static bool pll6_bypassed(struct device_node *node) ret = of_parse_phandle_with_args(node, "assigned-clock-parents", "#clock-cells", index, &clkspec); + if (!ret) + of_node_put(clkspec.np); + if (clkspec.args[0] != IMX6QDL_CLK_PLL6) return true; diff --git a/drivers/clk/imx/clk-imx8-acm.c b/drivers/clk/imx/clk-imx8-acm.c index 790f7e44b11e20..07dca6f31cf8ee 100644 --- a/drivers/clk/imx/clk-imx8-acm.c +++ b/drivers/clk/imx/clk-imx8-acm.c @@ -371,7 +371,8 @@ static int imx8_acm_clk_probe(struct platform_device *pdev) for (i = 0; i < priv->soc_data->num_sels; i++) { hws[sels[i].clkid] = devm_clk_hw_register_mux_parent_data_table(dev, sels[i].name, sels[i].parents, - sels[i].num_parents, 0, + sels[i].num_parents, + CLK_SET_RATE_NO_REPARENT, base + sels[i].reg, sels[i].shift, sels[i].width, 0, NULL, NULL); diff --git a/drivers/clk/imx/clk-imx8mq.c b/drivers/clk/imx/clk-imx8mq.c index f70ed231b92d63..cedc8a02aa1f0a 100644 --- a/drivers/clk/imx/clk-imx8mq.c +++ b/drivers/clk/imx/clk-imx8mq.c @@ -237,7 +237,7 @@ static const char * const imx8mq_dsi_esc_sels[] = {"osc_25m", "sys2_pll_100m", " static const char * const imx8mq_csi1_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll_out", "audio_pll2_out", "video_pll1_out", }; -static const char * const imx8mq_csi1_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", +static const char * const imx8mq_csi1_phy_sels[] = {"osc_25m", "sys2_pll_333m", "sys2_pll_100m", "sys1_pll_800m", "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; static const char * const imx8mq_csi1_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", @@ -246,7 +246,7 @@ static const char * const imx8mq_csi1_esc_sels[] = {"osc_25m", "sys2_pll_100m", static const char * const imx8mq_csi2_core_sels[] = {"osc_25m", "sys1_pll_266m", "sys2_pll_250m", "sys1_pll_800m", "sys2_pll_1000m", "sys3_pll_out", "audio_pll2_out", "video_pll1_out", }; -static const char * const imx8mq_csi2_phy_sels[] = {"osc_25m", "sys2_pll_125m", "sys2_pll_100m", "sys1_pll_800m", +static const char * const imx8mq_csi2_phy_sels[] = {"osc_25m", "sys2_pll_333m", "sys2_pll_100m", "sys1_pll_800m", "sys2_pll_1000m", "clk_ext2", "audio_pll2_out", "video_pll1_out", }; static const char * const imx8mq_csi2_esc_sels[] = {"osc_25m", "sys2_pll_100m", "sys1_pll_80m", "sys1_pll_800m", diff --git a/drivers/clk/microchip/clk-mpfs-ccc.c b/drivers/clk/microchip/clk-mpfs-ccc.c index 3a3ea2d142f8a2..0a76a1aaa50f7f 100644 --- a/drivers/clk/microchip/clk-mpfs-ccc.c +++ b/drivers/clk/microchip/clk-mpfs-ccc.c @@ -178,7 +178,7 @@ static int mpfs_ccc_register_outputs(struct device *dev, struct mpfs_ccc_out_hw_ return dev_err_probe(dev, ret, "failed to register clock id: %d\n", out_hw->id); - data->hw_data.hws[out_hw->id] = &out_hw->divider.hw; + data->hw_data.hws[out_hw->id - 2] = &out_hw->divider.hw; } return 0; @@ -234,6 +234,10 @@ static int mpfs_ccc_probe(struct platform_device *pdev) unsigned int num_clks; int ret; + /* + * If DLLs get added here, mpfs_ccc_register_outputs() currently packs + * sparse clock IDs in the hws array + */ num_clks = ARRAY_SIZE(mpfs_ccc_pll_clks) + ARRAY_SIZE(mpfs_ccc_pll0out_clks) + ARRAY_SIZE(mpfs_ccc_pll1out_clks); diff --git a/drivers/clk/qcom/dispcc-glymur.c b/drivers/clk/qcom/dispcc-glymur.c index 5203fa6383f6ac..bef74f58405baf 100644 --- a/drivers/clk/qcom/dispcc-glymur.c +++ b/drivers/clk/qcom/dispcc-glymur.c @@ -417,7 +417,7 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_aux_clk_src = { .parent_data = disp_cc_parent_data_1, .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_dp_ops, + .ops = &clk_rcg2_shared_ops, }, }; @@ -747,7 +747,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { &disp_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -762,7 +761,6 @@ static struct clk_regmap_div disp_cc_mdss_byte1_div_clk_src = { &disp_cc_mdss_byte1_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-kaanapali.c b/drivers/clk/qcom/dispcc-kaanapali.c index baae2ec1f72aac..c1578cd07041b0 100644 --- a/drivers/clk/qcom/dispcc-kaanapali.c +++ b/drivers/clk/qcom/dispcc-kaanapali.c @@ -800,7 +800,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { &disp_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -815,7 +814,6 @@ static struct clk_regmap_div disp_cc_mdss_byte1_div_clk_src = { &disp_cc_mdss_byte1_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-milos.c b/drivers/clk/qcom/dispcc-milos.c index 95b6dd89d9ae3d..339cb1c63ba77c 100644 --- a/drivers/clk/qcom/dispcc-milos.c +++ b/drivers/clk/qcom/dispcc-milos.c @@ -394,7 +394,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { &disp_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sc8280xp.c b/drivers/clk/qcom/dispcc-sc8280xp.c index 5903a759d4af49..e91dfed0f37e9b 100644 --- a/drivers/clk/qcom/dispcc-sc8280xp.c +++ b/drivers/clk/qcom/dispcc-sc8280xp.c @@ -1160,7 +1160,6 @@ static struct clk_regmap_div disp0_cc_mdss_byte0_div_clk_src = { &disp0_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -1175,7 +1174,6 @@ static struct clk_regmap_div disp1_cc_mdss_byte0_div_clk_src = { &disp1_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -1190,7 +1188,6 @@ static struct clk_regmap_div disp0_cc_mdss_byte1_div_clk_src = { &disp0_cc_mdss_byte1_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -1205,7 +1202,6 @@ static struct clk_regmap_div disp1_cc_mdss_byte1_div_clk_src = { &disp1_cc_mdss_byte1_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm4450.c b/drivers/clk/qcom/dispcc-sm4450.c index e8752d01c8e629..2fdacc26df698d 100644 --- a/drivers/clk/qcom/dispcc-sm4450.c +++ b/drivers/clk/qcom/dispcc-sm4450.c @@ -335,7 +335,6 @@ static struct clk_regmap_div disp_cc_mdss_byte0_div_clk_src = { &disp_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm8250.c b/drivers/clk/qcom/dispcc-sm8250.c index 8f433e1e702839..e59cdadd564795 100644 --- a/drivers/clk/qcom/dispcc-sm8250.c +++ b/drivers/clk/qcom/dispcc-sm8250.c @@ -578,7 +578,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk0_clk_src = { .name = "disp_cc_mdss_pclk0_clk_src", .parent_data = disp_cc_parent_data_6, .num_parents = ARRAY_SIZE(disp_cc_parent_data_6), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -592,7 +592,7 @@ static struct clk_rcg2 disp_cc_mdss_pclk1_clk_src = { .name = "disp_cc_mdss_pclk1_clk_src", .parent_data = disp_cc_parent_data_6, .num_parents = ARRAY_SIZE(disp_cc_parent_data_6), - .flags = CLK_SET_RATE_PARENT, + .flags = CLK_SET_RATE_PARENT | CLK_OPS_PARENT_ENABLE, .ops = &clk_pixel_ops, }, }; @@ -632,7 +632,7 @@ static struct clk_rcg2 disp_cc_mdss_vsync_clk_src = { .parent_data = disp_cc_parent_data_1, .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; diff --git a/drivers/clk/qcom/dispcc-sm8450.c b/drivers/clk/qcom/dispcc-sm8450.c index 9ce9fd28e55b23..2e91332dd92ab8 100644 --- a/drivers/clk/qcom/dispcc-sm8450.c +++ b/drivers/clk/qcom/dispcc-sm8450.c @@ -409,7 +409,7 @@ static struct clk_rcg2 disp_cc_mdss_dptx1_aux_clk_src = { .parent_data = disp_cc_parent_data_1, .num_parents = ARRAY_SIZE(disp_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_dp_ops, + .ops = &clk_rcg2_ops, }, }; diff --git a/drivers/clk/qcom/dispcc0-sa8775p.c b/drivers/clk/qcom/dispcc0-sa8775p.c index aeda9cf4bfee89..b248fa97058735 100644 --- a/drivers/clk/qcom/dispcc0-sa8775p.c +++ b/drivers/clk/qcom/dispcc0-sa8775p.c @@ -591,7 +591,6 @@ static struct clk_regmap_div mdss_0_disp_cc_mdss_byte0_div_clk_src = { &mdss_0_disp_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -606,7 +605,6 @@ static struct clk_regmap_div mdss_0_disp_cc_mdss_byte1_div_clk_src = { &mdss_0_disp_cc_mdss_byte1_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/dispcc1-sa8775p.c b/drivers/clk/qcom/dispcc1-sa8775p.c index cd55d1c1190241..9882edbb79f9e3 100644 --- a/drivers/clk/qcom/dispcc1-sa8775p.c +++ b/drivers/clk/qcom/dispcc1-sa8775p.c @@ -591,7 +591,6 @@ static struct clk_regmap_div mdss_1_disp_cc_mdss_byte0_div_clk_src = { &mdss_1_disp_cc_mdss_byte0_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; @@ -606,7 +605,6 @@ static struct clk_regmap_div mdss_1_disp_cc_mdss_byte1_div_clk_src = { &mdss_1_disp_cc_mdss_byte1_clk_src.clkr.hw, }, .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_regmap_div_ops, }, }; diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index 238e205735ed59..cd11470a75f3fe 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -8507,6 +8507,7 @@ static const struct qcom_reset_map gcc_glymur_resets[] = { [GCC_VIDEO_AXI0_CLK_ARES] = { 0x3201c, 2 }, [GCC_VIDEO_AXI1_CLK_ARES] = { 0x32044, 2 }, [GCC_VIDEO_BCR] = { 0x32000 }, + [GCC_VIDEO_AXI0C_CLK_ARES] = { 0x32030, 2 }, }; static const struct clk_rcg_dfs_data gcc_dfs_clocks[] = { diff --git a/drivers/clk/qcom/gcc-sc8180x.c b/drivers/clk/qcom/gcc-sc8180x.c index 31e788e22ab4ad..4095a1f54a0990 100644 --- a/drivers/clk/qcom/gcc-sc8180x.c +++ b/drivers/clk/qcom/gcc-sc8180x.c @@ -4172,7 +4172,7 @@ static struct gdsc usb30_sec_gdsc = { .pd = { .name = "usb30_sec_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; @@ -4190,7 +4190,7 @@ static struct gdsc usb30_prim_gdsc = { .pd = { .name = "usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; @@ -4199,7 +4199,7 @@ static struct gdsc pcie_0_gdsc = { .pd = { .name = "pcie_0_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; @@ -4226,7 +4226,7 @@ static struct gdsc pcie_1_gdsc = { .pd = { .name = "pcie_1_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; @@ -4235,7 +4235,7 @@ static struct gdsc pcie_2_gdsc = { .pd = { .name = "pcie_2_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; @@ -4253,7 +4253,7 @@ static struct gdsc pcie_3_gdsc = { .pd = { .name = "pcie_3_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; @@ -4262,10 +4262,55 @@ static struct gdsc usb30_mp_gdsc = { .pd = { .name = "usb30_mp_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = POLL_CFG_GDSCR, }; +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc = { + .gdscr = 0x7d050, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc = { + .gdscr = 0x7d058, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_mmnoc_mmu_tbu_sf_gdsc = { + .gdscr = 0x7d054, + .pd = { + .name = "hlos1_vote_mmnoc_mmu_tbu_sf_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu0_gdsc = { + .gdscr = 0x7d05c, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu0_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + +static struct gdsc hlos1_vote_turing_mmu_tbu1_gdsc = { + .gdscr = 0x7d060, + .pd = { + .name = "hlos1_vote_turing_mmu_tbu1_gdsc", + }, + .pwrsts = PWRSTS_OFF_ON, + .flags = VOTABLE, +}; + static struct clk_regmap *gcc_sc8180x_clocks[] = { [GCC_AGGRE_NOC_PCIE_TBU_CLK] = &gcc_aggre_noc_pcie_tbu_clk.clkr, [GCC_AGGRE_UFS_CARD_AXI_CLK] = &gcc_aggre_ufs_card_axi_clk.clkr, @@ -4595,6 +4640,11 @@ static struct gdsc *gcc_sc8180x_gdscs[] = { [USB30_MP_GDSC] = &usb30_mp_gdsc, [USB30_PRIM_GDSC] = &usb30_prim_gdsc, [USB30_SEC_GDSC] = &usb30_sec_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf0_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_hf1_gdsc, + [HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC] = &hlos1_vote_mmnoc_mmu_tbu_sf_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU0_GDSC] = &hlos1_vote_turing_mmu_tbu0_gdsc, + [HLOS1_VOTE_TURING_MMU_TBU1_GDSC] = &hlos1_vote_turing_mmu_tbu1_gdsc, }; static const struct regmap_config gcc_sc8180x_regmap_config = { diff --git a/drivers/clk/qcom/gcc-x1e80100.c b/drivers/clk/qcom/gcc-x1e80100.c index 74afd12c158c01..73a2a5112623e5 100644 --- a/drivers/clk/qcom/gcc-x1e80100.c +++ b/drivers/clk/qcom/gcc-x1e80100.c @@ -7480,6 +7480,7 @@ static int gcc_x1e80100_probe(struct platform_device *pdev) qcom_branch_set_clk_en(regmap, 0x32004); /* GCC_VIDEO_AHB_CLK */ qcom_branch_set_clk_en(regmap, 0x32030); /* GCC_VIDEO_XO_CLK */ qcom_branch_set_clk_en(regmap, 0x71004); /* GCC_GPU_CFG_AHB_CLK */ + qcom_branch_set_clk_en(regmap, 0x7d01c); /* GCC_HLOS1_VOTE_AGGRE_NOC_MMU_USB_QTB_CLK */ /* Clear GDSC_SLEEP_ENA_VOTE to stop votes being auto-removed in sleep. */ regmap_write(regmap, 0x52224, 0x0); diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index 7deabf8400cf66..95aa0712024554 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -518,10 +518,20 @@ static int gdsc_add_subdomain_list(struct dev_pm_domain_list *pd_list, ret = pm_genpd_add_subdomain(genpd, subdomain); if (ret) - return ret; + goto remove_added_subdomains; } return 0; + +remove_added_subdomains: + for (i--; i >= 0; i--) { + struct device *dev = pd_list->pd_devs[i]; + struct generic_pm_domain *genpd = pd_to_genpd(dev->pm_domain); + + pm_genpd_remove_subdomain(genpd, subdomain); + } + + return ret; } static void gdsc_remove_subdomain_list(struct dev_pm_domain_list *pd_list, diff --git a/drivers/clk/renesas/r9a09g056-cpg.c b/drivers/clk/renesas/r9a09g056-cpg.c index fead173cae8b27..70de6bb929b912 100644 --- a/drivers/clk/renesas/r9a09g056-cpg.c +++ b/drivers/clk/renesas/r9a09g056-cpg.c @@ -289,6 +289,24 @@ static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = { BUS_MSTOP(5, BIT(13))), DEF_MOD("wdt_3_clk_loco", CLK_QEXTAL, 5, 2, 2, 18, BUS_MSTOP(5, BIT(13))), + DEF_MOD("rspi_0_pclk", CLK_PLLCLN_DIV8, 5, 4, 2, 20, + BUS_MSTOP(11, BIT(0))), + DEF_MOD("rspi_0_pclk_sfr", CLK_PLLCLN_DIV8, 5, 5, 2, 21, + BUS_MSTOP(11, BIT(0))), + DEF_MOD("rspi_0_tclk", CLK_PLLCLN_DIV8, 5, 6, 2, 22, + BUS_MSTOP(11, BIT(0))), + DEF_MOD("rspi_1_pclk", CLK_PLLCLN_DIV8, 5, 7, 2, 23, + BUS_MSTOP(11, BIT(1))), + DEF_MOD("rspi_1_pclk_sfr", CLK_PLLCLN_DIV8, 5, 8, 2, 24, + BUS_MSTOP(11, BIT(1))), + DEF_MOD("rspi_1_tclk", CLK_PLLCLN_DIV8, 5, 9, 2, 25, + BUS_MSTOP(11, BIT(1))), + DEF_MOD("rspi_2_pclk", CLK_PLLCLN_DIV8, 5, 10, 2, 26, + BUS_MSTOP(11, BIT(2))), + DEF_MOD("rspi_2_pclk_sfr", CLK_PLLCLN_DIV8, 5, 11, 2, 27, + BUS_MSTOP(11, BIT(2))), + DEF_MOD("rspi_2_tclk", CLK_PLLCLN_DIV8, 5, 12, 2, 28, + BUS_MSTOP(11, BIT(2))), DEF_MOD("rsci0_pclk", CLK_PLLCLN_DIV16, 5, 13, 2, 29, BUS_MSTOP(11, BIT(3))), DEF_MOD("rsci0_tclk", CLK_PLLCLN_DIV16, 5, 14, 2, 30, @@ -389,24 +407,6 @@ static const struct rzv2h_mod_clk r9a09g056_mod_clks[] __initconst = { BUS_MSTOP(11, BIT(12))), DEF_MOD("rsci9_ps_ps1_n", CLK_PLLCLN_DIV64, 8, 14, 4, 14, BUS_MSTOP(11, BIT(12))), - DEF_MOD("rspi_0_pclk", CLK_PLLCLN_DIV8, 5, 4, 2, 20, - BUS_MSTOP(11, BIT(0))), - DEF_MOD("rspi_0_pclk_sfr", CLK_PLLCLN_DIV8, 5, 5, 2, 21, - BUS_MSTOP(11, BIT(0))), - DEF_MOD("rspi_0_tclk", CLK_PLLCLN_DIV8, 5, 6, 2, 22, - BUS_MSTOP(11, BIT(0))), - DEF_MOD("rspi_1_pclk", CLK_PLLCLN_DIV8, 5, 7, 2, 23, - BUS_MSTOP(11, BIT(1))), - DEF_MOD("rspi_1_pclk_sfr", CLK_PLLCLN_DIV8, 5, 8, 2, 24, - BUS_MSTOP(11, BIT(1))), - DEF_MOD("rspi_1_tclk", CLK_PLLCLN_DIV8, 5, 9, 2, 25, - BUS_MSTOP(11, BIT(1))), - DEF_MOD("rspi_2_pclk", CLK_PLLCLN_DIV8, 5, 10, 2, 26, - BUS_MSTOP(11, BIT(2))), - DEF_MOD("rspi_2_pclk_sfr", CLK_PLLCLN_DIV8, 5, 11, 2, 27, - BUS_MSTOP(11, BIT(2))), - DEF_MOD("rspi_2_tclk", CLK_PLLCLN_DIV8, 5, 12, 2, 28, - BUS_MSTOP(11, BIT(2))), DEF_MOD("scif_0_clk_pck", CLK_PLLCM33_DIV16, 8, 15, 4, 15, BUS_MSTOP(3, BIT(14))), DEF_MOD("i3c_0_pclkrw", CLK_PLLCLN_DIV16, 9, 0, 4, 16, diff --git a/drivers/clk/renesas/r9a09g057-cpg.c b/drivers/clk/renesas/r9a09g057-cpg.c index 6943cad318b5d7..c3174f40fdb406 100644 --- a/drivers/clk/renesas/r9a09g057-cpg.c +++ b/drivers/clk/renesas/r9a09g057-cpg.c @@ -280,22 +280,30 @@ static const struct rzv2h_mod_clk r9a09g057_mod_clks[] __initconst = { BUS_MSTOP(11, BIT(15))), DEF_MOD("gtm_7_pclk", CLK_PLLCLN_DIV16, 4, 10, 2, 10, BUS_MSTOP(12, BIT(0))), - DEF_MOD("wdt_0_clkp", CLK_PLLCM33_DIV16, 4, 11, 2, 11, - BUS_MSTOP(3, BIT(10))), - DEF_MOD("wdt_0_clk_loco", CLK_QEXTAL, 4, 12, 2, 12, - BUS_MSTOP(3, BIT(10))), DEF_MOD("wdt_1_clkp", CLK_PLLCLN_DIV16, 4, 13, 2, 13, BUS_MSTOP(1, BIT(0))), DEF_MOD("wdt_1_clk_loco", CLK_QEXTAL, 4, 14, 2, 14, BUS_MSTOP(1, BIT(0))), - DEF_MOD("wdt_2_clkp", CLK_PLLCLN_DIV16, 4, 15, 2, 15, - BUS_MSTOP(5, BIT(12))), - DEF_MOD("wdt_2_clk_loco", CLK_QEXTAL, 5, 0, 2, 16, - BUS_MSTOP(5, BIT(12))), - DEF_MOD("wdt_3_clkp", CLK_PLLCLN_DIV16, 5, 1, 2, 17, - BUS_MSTOP(5, BIT(13))), - DEF_MOD("wdt_3_clk_loco", CLK_QEXTAL, 5, 2, 2, 18, - BUS_MSTOP(5, BIT(13))), + DEF_MOD("rtc_0_clk_rtc", CLK_PLLCM33_DIV16, 5, 3, 2, 19, + BUS_MSTOP(3, BIT(11) | BIT(12))), + DEF_MOD("rspi_0_pclk", CLK_PLLCLN_DIV8, 5, 4, 2, 20, + BUS_MSTOP(11, BIT(0))), + DEF_MOD("rspi_0_pclk_sfr", CLK_PLLCLN_DIV8, 5, 5, 2, 21, + BUS_MSTOP(11, BIT(0))), + DEF_MOD("rspi_0_tclk", CLK_PLLCLN_DIV8, 5, 6, 2, 22, + BUS_MSTOP(11, BIT(0))), + DEF_MOD("rspi_1_pclk", CLK_PLLCLN_DIV8, 5, 7, 2, 23, + BUS_MSTOP(11, BIT(1))), + DEF_MOD("rspi_1_pclk_sfr", CLK_PLLCLN_DIV8, 5, 8, 2, 24, + BUS_MSTOP(11, BIT(1))), + DEF_MOD("rspi_1_tclk", CLK_PLLCLN_DIV8, 5, 9, 2, 25, + BUS_MSTOP(11, BIT(1))), + DEF_MOD("rspi_2_pclk", CLK_PLLCLN_DIV8, 5, 10, 2, 26, + BUS_MSTOP(11, BIT(2))), + DEF_MOD("rspi_2_pclk_sfr", CLK_PLLCLN_DIV8, 5, 11, 2, 27, + BUS_MSTOP(11, BIT(2))), + DEF_MOD("rspi_2_tclk", CLK_PLLCLN_DIV8, 5, 12, 2, 28, + BUS_MSTOP(11, BIT(2))), DEF_MOD("rsci0_pclk", CLK_PLLCLN_DIV16, 5, 13, 2, 29, BUS_MSTOP(11, BIT(3))), DEF_MOD("rsci0_tclk", CLK_PLLCLN_DIV16, 5, 14, 2, 30, @@ -396,26 +404,6 @@ static const struct rzv2h_mod_clk r9a09g057_mod_clks[] __initconst = { BUS_MSTOP(11, BIT(12))), DEF_MOD("rsci9_ps_ps1_n", CLK_PLLCLN_DIV64, 8, 14, 4, 14, BUS_MSTOP(11, BIT(12))), - DEF_MOD("rtc_0_clk_rtc", CLK_PLLCM33_DIV16, 5, 3, 2, 19, - BUS_MSTOP(3, BIT(11) | BIT(12))), - DEF_MOD("rspi_0_pclk", CLK_PLLCLN_DIV8, 5, 4, 2, 20, - BUS_MSTOP(11, BIT(0))), - DEF_MOD("rspi_0_pclk_sfr", CLK_PLLCLN_DIV8, 5, 5, 2, 21, - BUS_MSTOP(11, BIT(0))), - DEF_MOD("rspi_0_tclk", CLK_PLLCLN_DIV8, 5, 6, 2, 22, - BUS_MSTOP(11, BIT(0))), - DEF_MOD("rspi_1_pclk", CLK_PLLCLN_DIV8, 5, 7, 2, 23, - BUS_MSTOP(11, BIT(1))), - DEF_MOD("rspi_1_pclk_sfr", CLK_PLLCLN_DIV8, 5, 8, 2, 24, - BUS_MSTOP(11, BIT(1))), - DEF_MOD("rspi_1_tclk", CLK_PLLCLN_DIV8, 5, 9, 2, 25, - BUS_MSTOP(11, BIT(1))), - DEF_MOD("rspi_2_pclk", CLK_PLLCLN_DIV8, 5, 10, 2, 26, - BUS_MSTOP(11, BIT(2))), - DEF_MOD("rspi_2_pclk_sfr", CLK_PLLCLN_DIV8, 5, 11, 2, 27, - BUS_MSTOP(11, BIT(2))), - DEF_MOD("rspi_2_tclk", CLK_PLLCLN_DIV8, 5, 12, 2, 28, - BUS_MSTOP(11, BIT(2))), DEF_MOD("scif_0_clk_pck", CLK_PLLCM33_DIV16, 8, 15, 4, 15, BUS_MSTOP(3, BIT(14))), DEF_MOD("i3c_0_pclkrw", CLK_PLLCLN_DIV16, 9, 0, 4, 16, @@ -598,10 +586,7 @@ static const struct rzv2h_reset r9a09g057_resets[] __initconst = { DEF_RST(7, 2, 3, 3), /* GTM_5_PRESETZ */ DEF_RST(7, 3, 3, 4), /* GTM_6_PRESETZ */ DEF_RST(7, 4, 3, 5), /* GTM_7_PRESETZ */ - DEF_RST(7, 5, 3, 6), /* WDT_0_RESET */ DEF_RST(7, 6, 3, 7), /* WDT_1_RESET */ - DEF_RST(7, 7, 3, 8), /* WDT_2_RESET */ - DEF_RST(7, 8, 3, 9), /* WDT_3_RESET */ DEF_RST(8, 1, 3, 18), /* RSCI0_PRESETN */ DEF_RST(8, 2, 3, 19), /* RSCI0_TRESETN */ DEF_RST(8, 3, 3, 20), /* RSCI1_PRESETN */ diff --git a/drivers/clk/samsung/clk-acpm.c b/drivers/clk/samsung/clk-acpm.c index b90809ce3f882c..d8944160793a07 100644 --- a/drivers/clk/samsung/clk-acpm.c +++ b/drivers/clk/samsung/clk-acpm.c @@ -20,7 +20,7 @@ struct acpm_clk { u32 id; struct clk_hw hw; unsigned int mbox_chan_id; - const struct acpm_handle *handle; + struct acpm_handle *handle; }; struct acpm_clk_variant { @@ -113,7 +113,7 @@ static int acpm_clk_register(struct device *dev, struct acpm_clk *aclk, static int acpm_clk_probe(struct platform_device *pdev) { - const struct acpm_handle *acpm_handle; + struct acpm_handle *acpm_handle; struct clk_hw_onecell_data *clk_data; struct clk_hw **hws; struct device *dev = &pdev->dev; diff --git a/drivers/clk/spacemit/ccu_mix.c b/drivers/clk/spacemit/ccu_mix.c index 9578366e97466e..a8b407049bf4da 100644 --- a/drivers/clk/spacemit/ccu_mix.c +++ b/drivers/clk/spacemit/ccu_mix.c @@ -73,7 +73,7 @@ static int ccu_mix_trigger_fc(struct clk_hw *hw) struct ccu_common *common = hw_to_ccu_common(hw); unsigned int val; - if (common->reg_fc) + if (!common->reg_fc) return 0; ccu_update(common, fc, common->mask_fc, common->mask_fc); diff --git a/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c b/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c index 0339c4af0fe5b5..db0e36d8838e74 100644 --- a/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c +++ b/drivers/clk/sunxi-ng/ccu-sun55i-a523-r.c @@ -83,9 +83,22 @@ static SUNXI_CCU_MUX_DATA_WITH_GATE(r_pwmctrl_clk, "r-pwmctrl", static SUNXI_CCU_GATE_HW(bus_r_pwmctrl_clk, "bus-r-pwmctrl", &r_apb0_clk.common.hw, 0x13c, BIT(0), 0); -/* SPI clock is /M/N (same as new MMC?) */ +static const struct clk_parent_data r_spi_parents[] = { + { .fw_name = "hosc" }, + { .fw_name = "pll-periph" }, + { .name = "pll-periph0-300M" }, + { .name = "pll-periph1-300M" }, + { .name = "pll-audio" }, +}; +static SUNXI_CCU_DUALDIV_MUX_GATE(r_spi_clk, "r-spi", r_spi_parents, 0x150, + 0, 5, /* M */ + 8, 5, /* P */ + 24, 3, /* mux */ + BIT(31), /* gate */ + 0); static SUNXI_CCU_GATE_HW(bus_r_spi_clk, "bus-r-spi", &r_ahb_clk.common.hw, 0x15c, BIT(0), 0); + static SUNXI_CCU_GATE_HW(bus_r_spinlock_clk, "bus-r-spinlock", &r_ahb_clk.common.hw, 0x16c, BIT(0), 0); static SUNXI_CCU_GATE_HW(bus_r_msgbox_clk, "bus-r-msgbox", @@ -138,6 +151,7 @@ static struct ccu_common *sun55i_a523_r_ccu_clks[] = { &bus_r_twd_clk.common, &r_pwmctrl_clk.common, &bus_r_pwmctrl_clk.common, + &r_spi_clk.common, &bus_r_spi_clk.common, &bus_r_spinlock_clk.common, &bus_r_msgbox_clk.common, @@ -169,6 +183,7 @@ static struct clk_hw_onecell_data sun55i_a523_r_hw_clks = { [CLK_BUS_R_TWD] = &bus_r_twd_clk.common.hw, [CLK_R_PWMCTRL] = &r_pwmctrl_clk.common.hw, [CLK_BUS_R_PWMCTRL] = &bus_r_pwmctrl_clk.common.hw, + [CLK_R_SPI] = &r_spi_clk.common.hw, [CLK_BUS_R_SPI] = &bus_r_spi_clk.common.hw, [CLK_BUS_R_SPINLOCK] = &bus_r_spinlock_clk.common.hw, [CLK_BUS_R_MSGBOX] = &bus_r_msgbox_clk.common.hw, diff --git a/drivers/clk/visconti/pll.c b/drivers/clk/visconti/pll.c index 6fd02c4b641ed5..805b9548128176 100644 --- a/drivers/clk/visconti/pll.c +++ b/drivers/clk/visconti/pll.c @@ -249,7 +249,7 @@ static struct clk_hw *visconti_register_pll(struct visconti_pll_provider *ctx, const struct visconti_pll_rate_table *rate_table, spinlock_t *lock) { - struct clk_init_data init; + struct clk_init_data init = {}; struct visconti_pll *pll; struct clk_hw *pll_hw_clk; size_t len; diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index 5aa9fcd80cf519..78b70bdcb008ff 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -261,7 +261,6 @@ static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf, if (fast_switch) { wrmsrq(MSR_AMD_CPPC_REQ, value); - return 0; } else { int ret = wrmsrq_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); @@ -713,13 +712,12 @@ static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, return policy->cur; } -static void amd_pstate_adjust_perf(unsigned int cpu, +static void amd_pstate_adjust_perf(struct cpufreq_policy *policy, unsigned long _min_perf, unsigned long target_perf, unsigned long capacity) { u8 max_perf, min_perf, des_perf, cap_perf; - struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpu); struct amd_cpudata *cpudata; union perf_cached perf; @@ -1533,7 +1531,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) ret = amd_pstate_set_epp(policy, cpudata->epp_default); if (ret) - return ret; + goto free_cpudata1; current_pstate_driver->adjust_perf = NULL; diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 1f794524a1d923..44441ceedb7689 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2228,7 +2228,7 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); /** * cpufreq_driver_adjust_perf - Adjust CPU performance level in one go. - * @cpu: Target CPU. + * @policy: cpufreq policy object of the target CPU. * @min_perf: Minimum (required) performance level (units of @capacity). * @target_perf: Target (desired) performance level (units of @capacity). * @capacity: Capacity of the target CPU. @@ -2247,12 +2247,12 @@ EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch); * parallel with either ->target() or ->target_index() or ->fast_switch() for * the same CPU. */ -void cpufreq_driver_adjust_perf(unsigned int cpu, +void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity) { - cpufreq_driver->adjust_perf(cpu, min_perf, target_perf, capacity); + cpufreq_driver->adjust_perf(policy, min_perf, target_perf, capacity); } /** diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 11c58af4190064..0f50034e4b6808 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -3239,12 +3239,12 @@ static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, return target_pstate * cpu->pstate.scaling; } -static void intel_cpufreq_adjust_perf(unsigned int cpunum, +static void intel_cpufreq_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity) { - struct cpudata *cpu = all_cpu_data[cpunum]; + struct cpudata *cpu = all_cpu_data[policy->cpu]; u64 hwp_cap = READ_ONCE(cpu->hwp_cap_cached); int old_pstate = cpu->pstate.current_pstate; int cap_pstate, min_pstate, max_pstate, target_pstate; diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index a1ee475d180dac..c6870f08457632 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -130,3 +130,11 @@ config ARM_QCOM_SPM_CPUIDLE The Subsystem Power Manager (SPM) controls low power modes for the CPU and L2 cores. It interface with various system drivers to put the cores in low power modes. + +config ARM_APPLE_CPUIDLE + bool "Apple SoC CPU idle driver" + depends on ARM64 + default ARCH_APPLE + select CPU_IDLE_MULTIPLE_DRIVERS + help + Select this to enable cpuidle on Apple SoCs. diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index 1de9e92c5b0fc9..f9e7a71d52c13f 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_ARM_PSCI_CPUIDLE) += cpuidle-psci.o obj-$(CONFIG_ARM_PSCI_CPUIDLE_DOMAIN) += cpuidle-psci-domain.o obj-$(CONFIG_ARM_TEGRA_CPUIDLE) += cpuidle-tegra.o obj-$(CONFIG_ARM_QCOM_SPM_CPUIDLE) += cpuidle-qcom-spm.o +obj-$(CONFIG_ARM_APPLE_CPUIDLE) += cpuidle-apple.o ############################################################################### # MIPS drivers diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c new file mode 100644 index 00000000000000..b7504066d75aa8 --- /dev/null +++ b/drivers/cpuidle/cpuidle-apple.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright The Asahi Linux Contributors + * + * CPU idle support for Apple SoCs + */ + +#include +#include +#include +#include +#include +#include + +enum idle_state { + STATE_WFI, + STATE_PWRDOWN, + STATE_COUNT +}; + +asm( + ".pushsection .cpuidle.text, \"ax\"\n" + ".type apple_cpu_deep_wfi, @function\n" + "apple_cpu_deep_wfi:\n" + "str x30, [sp, #-16]!\n" + "stp x28, x29, [sp, #-16]!\n" + "stp x26, x27, [sp, #-16]!\n" + "stp x24, x25, [sp, #-16]!\n" + "stp x22, x23, [sp, #-16]!\n" + "stp x20, x21, [sp, #-16]!\n" + "stp x18, x19, [sp, #-16]!\n" + + "mrs x0, s3_5_c15_c5_0\n" + "orr x0, x0, #(3L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "1:\n" + "dsb sy\n" + "wfi\n" + + "mrs x0, ISR_EL1\n" + "cbz x0, 1b\n" + + "mrs x0, s3_5_c15_c5_0\n" + "bic x0, x0, #(1L << 24)\n" + "msr s3_5_c15_c5_0, x0\n" + + "ldp x18, x19, [sp], #16\n" + "ldp x20, x21, [sp], #16\n" + "ldp x22, x23, [sp], #16\n" + "ldp x24, x25, [sp], #16\n" + "ldp x26, x27, [sp], #16\n" + "ldp x28, x29, [sp], #16\n" + "ldr x30, [sp], #16\n" + + "ret\n" + ".popsection\n" +); + +void apple_cpu_deep_wfi(void); + +static __cpuidle int apple_enter_wfi(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + cpu_do_idle(); + return index; +} + +static __cpuidle int apple_enter_idle(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index) +{ + /* + * Deep WFI will clobber FP state, among other things. + * The CPU PM notifier will take care of saving that and anything else + * that needs to be notified of the CPU powering down. + */ + if (cpu_pm_enter()) + return -1; + + ct_cpuidle_enter(); + + switch(index) { + case STATE_PWRDOWN: + apple_cpu_deep_wfi(); + break; + default: + WARN_ON(1); + break; + } + + ct_cpuidle_exit(); + + cpu_pm_exit(); + + return index; +} + +static struct cpuidle_driver apple_idle_driver = { + .name = "apple_idle", + .owner = THIS_MODULE, + .states = { + [STATE_WFI] = { + .enter = apple_enter_wfi, + .enter_s2idle = apple_enter_wfi, + .exit_latency = 1, + .target_residency = 1, + .power_usage = UINT_MAX, + .name = "WFI", + .desc = "CPU clock-gated", + .flags = 0, + }, + [STATE_PWRDOWN] = { + .enter = apple_enter_idle, + .enter_s2idle = apple_enter_idle, + .exit_latency = 10, + .target_residency = 10000, + .power_usage = 0, + .name = "CPU PD", + .desc = "CPU/cluster powered down", + .flags = CPUIDLE_FLAG_RCU_IDLE, + }, + }, + .safe_state_index = STATE_WFI, + .state_count = STATE_COUNT, +}; + +static int apple_cpuidle_probe(struct platform_device *pdev) +{ + return cpuidle_register(&apple_idle_driver, NULL); +} + +static struct platform_driver apple_cpuidle_driver = { + .driver = { + .name = "cpuidle-apple", + }, + .probe = apple_cpuidle_probe, +}; + +static int __init apple_cpuidle_init(void) +{ + struct platform_device *pdev; + int ret; + + ret = platform_driver_register(&apple_cpuidle_driver); + if (ret) + return ret; + + if (!of_machine_is_compatible("apple,arm-platform")) + return 0; + + if (!(of_machine_is_compatible("apple,t8103") || + of_machine_is_compatible("apple,t8112") || + of_machine_is_compatible("apple,t6000") || + of_machine_is_compatible("apple,t6001") || + of_machine_is_compatible("apple,t6002") || + of_machine_is_compatible("apple,t6020") || + of_machine_is_compatible("apple,t6021") || + of_machine_is_compatible("apple,t6022"))) + return 0; + + pdev = platform_device_register_simple("cpuidle-apple", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&apple_cpuidle_driver); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(apple_cpuidle_init); diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index 9ebedd972df0bb..b89e7111e7b8c6 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -95,7 +95,10 @@ static int snooze_loop(struct cpuidle_device *dev, HMT_medium(); ppc64_runlatch_on(); - clear_thread_flag(TIF_POLLING_NRFLAG); + + /* Avoid double clear when breaking */ + if (!dev->poll_time_limit) + clear_thread_flag(TIF_POLLING_NRFLAG); local_irq_disable(); diff --git a/drivers/cpuidle/cpuidle-pseries.c b/drivers/cpuidle/cpuidle-pseries.c index f68c65f1d023f2..864dd5d6e627b6 100644 --- a/drivers/cpuidle/cpuidle-pseries.c +++ b/drivers/cpuidle/cpuidle-pseries.c @@ -64,7 +64,10 @@ int snooze_loop(struct cpuidle_device *dev, struct cpuidle_driver *drv, } HMT_medium(); - clear_thread_flag(TIF_POLLING_NRFLAG); + + /* Avoid double clear when breaking */ + if (!dev->poll_time_limit) + clear_thread_flag(TIF_POLLING_NRFLAG); raw_local_irq_disable(); diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index bc0c40f10944ca..b393689400b4c1 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -2131,7 +2131,7 @@ static int atmel_aes_buff_init(struct atmel_aes_dev *dd) static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) { - free_page((unsigned long)dd->buf); + free_pages((unsigned long)dd->buf, ATMEL_AES_BUFFER_ORDER); } static int atmel_aes_dma_init(struct atmel_aes_dev *dd) @@ -2270,10 +2270,12 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd) /* i = ARRAY_SIZE(aes_authenc_algs); */ err_aes_authenc_alg: crypto_unregister_aeads(aes_authenc_algs, i); - crypto_unregister_skcipher(&aes_xts_alg); + if (dd->caps.has_xts) + crypto_unregister_skcipher(&aes_xts_alg); #endif err_aes_xts_alg: - crypto_unregister_aead(&aes_gcm_alg); + if (dd->caps.has_gcm) + crypto_unregister_aead(&aes_gcm_alg); err_aes_gcm_alg: i = ARRAY_SIZE(aes_algs); err_aes_algs: diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index b6a77c8d439cc5..9c380351d2f9fa 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -261,6 +261,7 @@ static int atmel_ecdh_init_tfm(struct crypto_kpp *tfm) if (IS_ERR(fallback)) { dev_err(&ctx->client->dev, "Failed to allocate transformation for '%s': %ld\n", alg, PTR_ERR(fallback)); + atmel_ecc_i2c_client_free(ctx->client); return PTR_ERR(fallback); } diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index da3cd986b1eb7c..59d11fa5caebc8 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -72,8 +72,8 @@ EXPORT_SYMBOL(atmel_i2c_init_read_config_cmd); int atmel_i2c_init_read_otp_cmd(struct atmel_i2c_cmd *cmd, u16 addr) { - if (addr < 0 || addr > OTP_ZONE_SIZE) - return -1; + if (addr >= OTP_ZONE_SIZE / 4) + return -EINVAL; cmd->word_addr = COMMAND; cmd->opcode = OPCODE_READ; diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 98d1023007e31e..570bc8b3f26af0 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "atmel-i2c.h" @@ -95,19 +96,24 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, static int atmel_sha204a_otp_read(struct i2c_client *client, u16 addr, u8 *otp) { struct atmel_i2c_cmd cmd; - int ret = -1; + int ret; - if (atmel_i2c_init_read_otp_cmd(&cmd, addr) < 0) { + ret = atmel_i2c_init_read_otp_cmd(&cmd, addr); + if (ret < 0) { dev_err(&client->dev, "failed, invalid otp address %04X\n", addr); return ret; } ret = atmel_i2c_send_receive(client, &cmd); + if (ret < 0) { + dev_err(&client->dev, "failed to read otp at %04X\n", addr); + return ret; + } if (cmd.data[0] == 0xff) { dev_err(&client->dev, "failed, device not ready\n"); - return -EINVAL; + return -EIO; } memcpy(otp, cmd.data+1, 4); @@ -120,21 +126,22 @@ static ssize_t otp_show(struct device *dev, { u16 addr; u8 otp[OTP_ZONE_SIZE]; - char *str = buf; struct i2c_client *client = to_i2c_client(dev); - int i; + ssize_t len = 0; + int i, ret; - for (addr = 0; addr < OTP_ZONE_SIZE/4; addr++) { - if (atmel_sha204a_otp_read(client, addr, otp + addr * 4) < 0) { + for (addr = 0; addr < OTP_ZONE_SIZE / 4; addr++) { + ret = atmel_sha204a_otp_read(client, addr, otp + addr * 4); + if (ret < 0) { dev_err(dev, "failed to read otp zone\n"); - break; + return ret; } } - for (i = 0; i < addr*2; i++) - str += sprintf(str, "%02X", otp[i]); - str += sprintf(str, "\n"); - return str - buf; + for (i = 0; i < OTP_ZONE_SIZE; i++) + len += sysfs_emit_at(buf, len, "%02X", otp[i]); + len += sysfs_emit_at(buf, len, "\n"); + return len; } static DEVICE_ATTR_RO(otp); @@ -191,10 +198,8 @@ static void atmel_sha204a_remove(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); - if (atomic_read(&i2c_priv->tfm_count)) { - dev_emerg(&client->dev, "Device is busy, will remove it anyhow\n"); - return; - } + devm_hwrng_unregister(&client->dev, &i2c_priv->hwrng); + atmel_i2c_flush_queue(); sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 278c0df3c92f26..643e507f9c0209 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -294,8 +294,8 @@ static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd) dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); } else { - dma_sync_single_for_device(dd->dev, dd->dma_addr_out, - dd->dma_size, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); /* copy data */ count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, @@ -619,8 +619,8 @@ static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd) dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); } else { - dma_sync_single_for_device(dd->dev, dd->dma_addr_out, - dd->dma_size, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); /* copy data */ count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset, diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 78964e1712e581..3343ddc30076f3 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -3269,7 +3269,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, dpaa2_fl_set_addr(out_fle, key_dma); dpaa2_fl_set_len(out_fle, digestsize); - print_hex_dump_debug("key_in@" __stringify(__LINE__)": ", + print_hex_dump_devel("key_in@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); print_hex_dump_debug("shdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), @@ -3289,7 +3289,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, /* in progress */ wait_for_completion(&result.completion); ret = result.err; - print_hex_dump_debug("digested key@" __stringify(__LINE__)": ", + print_hex_dump_devel("digested key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); } diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 44122208f70cb9..a0c417b7b8059c 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -393,7 +393,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); - print_hex_dump_debug("key_in@"__stringify(__LINE__)": ", + print_hex_dump_devel("key_in@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), @@ -408,7 +408,7 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, wait_for_completion(&result.completion); ret = result.err; - print_hex_dump_debug("digested key@"__stringify(__LINE__)": ", + print_hex_dump_devel("digested key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); } diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index 01d298350b9253..3ad6bb7666f625 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -30,8 +30,11 @@ static int ccp_aes_complete(struct crypto_async_request *async_req, int ret) if (ret) return ret; - if (ctx->u.aes.mode != CCP_AES_MODE_ECB) - memcpy(req->iv, rctx->iv, AES_BLOCK_SIZE); + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) { + size_t ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req)); + + memcpy(req->iv, rctx->iv, ivsize); + } return 0; } diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index aebf4dad545e2c..c16bb7d7067bea 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1860,7 +1860,10 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PEK_CSR, &data, &argp->error); - /* If we query the CSR length, FW responded with expected data. */ + /* + * Firmware will returns the length of the CSR blob (either the minimum + * required length or the actual length written), return it to the user. + */ input.length = data.len; if (copy_to_user((void __user *)argp->data, &input, sizeof(input))) { @@ -1868,6 +1871,9 @@ static int sev_ioctl_do_pek_csr(struct sev_issue_cmd *argp, bool writable) goto e_free_blob; } + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free_blob; + if (blob) { if (copy_to_user(input_address, blob, input.length)) ret = -EFAULT; @@ -2218,6 +2224,9 @@ static int sev_ioctl_do_get_id2(struct sev_issue_cmd *argp) goto e_free; } + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free; + if (id_blob) { if (copy_to_user(input_address, id_blob, data.len)) { ret = -EFAULT; @@ -2334,7 +2343,10 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) ret = __sev_do_cmd_locked(SEV_CMD_PDH_CERT_EXPORT, &data, &argp->error); - /* If we query the length, FW responded with expected data. */ + /* + * Firmware will return the length of the blobs (either the minimum + * required length or the actual length written), return 'em to the user. + */ input.cert_chain_len = data.cert_chain_len; input.pdh_cert_len = data.pdh_cert_len; @@ -2343,6 +2355,9 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable) goto e_free_cert; } + if (ret || WARN_ON_ONCE(argp->error)) + goto e_free_cert; + if (pdh_blob) { if (copy_to_user(input_pdh_cert_address, pdh_blob, input.pdh_cert_len)) { diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c index c6d085c8ff797e..73179bf725a712 100644 --- a/drivers/crypto/ccree/cc_hash.c +++ b/drivers/crypto/ccree/cc_hash.c @@ -1448,6 +1448,7 @@ static int cc_mac_digest(struct ahash_request *req) if (cc_map_hash_request_final(ctx->drvdata, state, req->src, req->nbytes, 1, flags)) { dev_err(dev, "map_ahash_request_final() failed\n"); + cc_unmap_result(dev, state, digestsize, req->result); cc_unmap_req(dev, state, ctx); return -ENOMEM; } diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c index 54e24fd7b9be9d..85eecbb40e7e79 100644 --- a/drivers/crypto/hisilicon/sec/sec_algs.c +++ b/drivers/crypto/hisilicon/sec/sec_algs.c @@ -844,7 +844,7 @@ static int sec_alg_skcipher_crypto(struct skcipher_request *skreq, if (crypto_skcipher_ivsize(atfm)) dma_unmap_single(info->dev, sec_req->dma_iv, crypto_skcipher_ivsize(atfm), - DMA_BIDIRECTIONAL); + DMA_TO_DEVICE); err_unmap_out_sg: if (split) sec_unmap_sg_on_err(skreq->dst, steps, splits_out, diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 15174216d8c413..2471a4dd0b508a 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -230,7 +230,7 @@ static int qp_send_message(struct sec_req *req) spin_unlock_bh(&qp_ctx->req_lock); - atomic64_inc(&req->ctx->sec->debug.dfx.send_cnt); + atomic64_inc(&qp_ctx->ctx->sec->debug.dfx.send_cnt); return -EINPROGRESS; } diff --git a/drivers/crypto/inside-secure/eip93/eip93-common.c b/drivers/crypto/inside-secure/eip93/eip93-common.c index f4ad6beff15e0c..259714a4ee4d34 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-common.c +++ b/drivers/crypto/inside-secure/eip93/eip93-common.c @@ -731,7 +731,7 @@ int eip93_hmac_setkey(u32 ctx_flags, const u8 *key, unsigned int keylen, return -EINVAL; } - ahash_tfm = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_ASYNC); + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); if (IS_ERR(ahash_tfm)) return PTR_ERR(ahash_tfm); diff --git a/drivers/crypto/inside-secure/eip93/eip93-main.c b/drivers/crypto/inside-secure/eip93/eip93-main.c index b7fd9795062d4c..76858bb4fcc22b 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-main.c +++ b/drivers/crypto/inside-secure/eip93/eip93-main.c @@ -36,6 +36,14 @@ static struct eip93_alg_template *eip93_algs[] = { &eip93_alg_cbc_aes, &eip93_alg_ctr_aes, &eip93_alg_rfc3686_aes, + &eip93_alg_md5, + &eip93_alg_sha1, + &eip93_alg_sha224, + &eip93_alg_sha256, + &eip93_alg_hmac_md5, + &eip93_alg_hmac_sha1, + &eip93_alg_hmac_sha224, + &eip93_alg_hmac_sha256, &eip93_alg_authenc_hmac_md5_cbc_des, &eip93_alg_authenc_hmac_sha1_cbc_des, &eip93_alg_authenc_hmac_sha224_cbc_des, @@ -52,14 +60,6 @@ static struct eip93_alg_template *eip93_algs[] = { &eip93_alg_authenc_hmac_sha1_rfc3686_aes, &eip93_alg_authenc_hmac_sha224_rfc3686_aes, &eip93_alg_authenc_hmac_sha256_rfc3686_aes, - &eip93_alg_md5, - &eip93_alg_sha1, - &eip93_alg_sha224, - &eip93_alg_sha256, - &eip93_alg_hmac_md5, - &eip93_alg_hmac_sha1, - &eip93_alg_hmac_sha224, - &eip93_alg_hmac_sha256, }; inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask) diff --git a/drivers/crypto/inside-secure/eip93/eip93-regs.h b/drivers/crypto/inside-secure/eip93/eip93-regs.h index 0490b8d151311c..116b3fbb6ad791 100644 --- a/drivers/crypto/inside-secure/eip93/eip93-regs.h +++ b/drivers/crypto/inside-secure/eip93/eip93-regs.h @@ -109,7 +109,7 @@ #define EIP93_REG_PE_BUF_THRESH 0x10c #define EIP93_PE_OUTBUF_THRESH GENMASK(23, 16) #define EIP93_PE_INBUF_THRESH GENMASK(7, 0) -#define EIP93_REG_PE_INBUF_COUNT 0x100 +#define EIP93_REG_PE_INBUF_COUNT 0x110 #define EIP93_REG_PE_OUTBUF_COUNT 0x114 #define EIP93_REG_PE_BUF_RW_PNTR 0x118 /* BUF_PNTR */ diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 547abf453d4a2c..f62b994e18e580 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -906,8 +906,8 @@ static void rebalance_wq_table(void) return; } + cpu = 0; for_each_node_with_cpus(node) { - cpu = 0; node_cpus = cpumask_of_node(node); for_each_cpu(node_cpu, node_cpus) { diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 35105213d40c09..0002122219bcbc 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -97,9 +97,25 @@ static struct adf_hw_device_class adf_420xx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses[ADF_FUSECTL4]; + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_420XX_ACCELENGINES_MASK; - return ~me_disable & ADF_420XX_ACCELENGINES_MASK; + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + if (test_bit(12, &fuses)) + mask &= ~ADF_AE_GROUP_3; + + if (test_bit(16, &fuses)) + mask &= ~ADF_AE_GROUP_4; + + return mask; } static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 740f68a36ac517..900f19b90b2dc4 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -100,9 +100,19 @@ static struct adf_hw_device_class adf_4xxx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses[ADF_FUSECTL4]; + unsigned long fuses = self->fuses[ADF_FUSECTL4]; + u32 mask = ADF_4XXX_ACCELENGINES_MASK; - return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; + if (test_bit(0, &fuses)) + mask &= ~ADF_AE_GROUP_0; + + if (test_bit(4, &fuses)) + mask &= ~ADF_AE_GROUP_1; + + if (test_bit(8, &fuses)) + mask &= ~ADF_AE_GROUP_2; + + return mask; } static u32 get_accel_cap(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index c1dc9c56fdf549..f0d112e4b56c3c 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -182,8 +182,10 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return ret; ret = adf_dev_up(accel_dev, true); - if (ret) + if (ret) { + adf_dev_down(accel_dev); return ret; + } ret = devm_add_action_or_reset(dev, adf_device_down, accel_dev); if (ret) diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c index f9f1018a28236d..09d4f547e082c3 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_engine.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ +#include #include #include #include "adf_cfg.h" @@ -162,8 +163,14 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev) static int adf_ae_reset(struct adf_accel_dev *accel_dev, int ae) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + unsigned long reset_delay; qat_hal_reset(loader_data->fw_loader); + + reset_delay = loader_data->fw_loader->chip_info->reset_delay_us; + if (reset_delay) + fsleep(reset_delay); + if (qat_hal_clr_reset(loader_data->fw_loader)) return -EFAULT; diff --git a/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c index e97c67c87b3cf1..6abb57bfd32858 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sysfs_ras_counters.c @@ -13,14 +13,14 @@ static ssize_t errors_correctable_show(struct device *dev, char *buf) { struct adf_accel_dev *accel_dev; - unsigned long counter; + int counter; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) return -EINVAL; counter = ADF_RAS_ERR_CTR_READ(accel_dev->ras_errors, ADF_RAS_CORR); - return scnprintf(buf, PAGE_SIZE, "%ld\n", counter); + return scnprintf(buf, PAGE_SIZE, "%d\n", counter); } static ssize_t errors_nonfatal_show(struct device *dev, @@ -28,14 +28,14 @@ static ssize_t errors_nonfatal_show(struct device *dev, char *buf) { struct adf_accel_dev *accel_dev; - unsigned long counter; + int counter; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) return -EINVAL; counter = ADF_RAS_ERR_CTR_READ(accel_dev->ras_errors, ADF_RAS_UNCORR); - return scnprintf(buf, PAGE_SIZE, "%ld\n", counter); + return scnprintf(buf, PAGE_SIZE, "%d\n", counter); } static ssize_t errors_fatal_show(struct device *dev, @@ -43,14 +43,14 @@ static ssize_t errors_fatal_show(struct device *dev, char *buf) { struct adf_accel_dev *accel_dev; - unsigned long counter; + int counter; accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev)); if (!accel_dev) return -EINVAL; counter = ADF_RAS_ERR_CTR_READ(accel_dev->ras_errors, ADF_RAS_FATAL); - return scnprintf(buf, PAGE_SIZE, "%ld\n", counter); + return scnprintf(buf, PAGE_SIZE, "%d\n", counter); } static ssize_t reset_error_counters_store(struct device *dev, diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h index 6887930c7995ef..e74cafa95f1cc7 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h @@ -27,6 +27,7 @@ struct icp_qat_fw_loader_chip_info { int mmp_sram_size; bool nn; bool lm2lm3; + u16 reset_delay_us; u32 lm_size; u32 icp_rst_csr; u32 icp_rst_mask; diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h index 7ea8962272f2f4..d28732225c9e05 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_hw_20_comp.h @@ -3,6 +3,8 @@ #ifndef _ICP_QAT_HW_20_COMP_H_ #define _ICP_QAT_HW_20_COMP_H_ +#include + #include "icp_qat_hw_20_comp_defs.h" #include "icp_qat_fw.h" @@ -54,7 +56,7 @@ ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(struct icp_qat_hw_comp_20_config_csr_lower QAT_FIELD_SET(val32, csr.abd, ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_BITPOS, ICP_QAT_HW_COMP_20_CONFIG_CSR_ABD_MASK); - return __builtin_bswap32(val32); + return swab32(val32); } struct icp_qat_hw_comp_20_config_csr_upper { @@ -106,7 +108,7 @@ ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(struct icp_qat_hw_comp_20_config_csr_upper ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_BITPOS, ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_MASK); - return __builtin_bswap32(val32); + return swab32(val32); } struct icp_qat_hw_decomp_20_config_csr_lower { @@ -138,7 +140,7 @@ ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(struct icp_qat_hw_decomp_20_config_csr_l ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_BITPOS, ICP_QAT_HW_DECOMP_20_CONFIG_CSR_LZ4_BLOCK_CHECKSUM_PRESENT_MASK); - return __builtin_bswap32(val32); + return swab32(val32); } struct icp_qat_hw_decomp_20_config_csr_upper { @@ -158,7 +160,7 @@ ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_UPPER(struct icp_qat_hw_decomp_20_config_csr_u ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_BITPOS, ICP_QAT_HW_DECOMP_20_CONFIG_CSR_MINI_CAM_CONTROL_MASK); - return __builtin_bswap32(val32); + return swab32(val32); } #endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c index 8b123472b71cc4..4273a0ecb6c80f 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -133,7 +133,7 @@ static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm) struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm); struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_compression_instance *inst; - int node; + int node, ret; if (tfm->node == NUMA_NO_NODE) node = numa_node_id(); @@ -146,7 +146,13 @@ static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm) return -EINVAL; ctx->inst = inst; - return qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, QAT_DEFLATE); + ret = qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, QAT_DEFLATE); + if (ret) { + qat_compression_put_instance(inst); + memset(ctx, 0, sizeof(*ctx)); + } + + return ret; } static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm) diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index 7a6ba6f22e3e8e..1c3d1311f1c7ad 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -9,17 +9,18 @@ #include "icp_qat_hal.h" #include "icp_qat_uclo.h" -#define BAD_REGADDR 0xffff -#define MAX_RETRY_TIMES 10000 -#define INIT_CTX_ARB_VALUE 0x0 -#define INIT_CTX_ENABLE_VALUE 0x0 -#define INIT_PC_VALUE 0x0 -#define INIT_WAKEUP_EVENTS_VALUE 0x1 -#define INIT_SIG_EVENTS_VALUE 0x1 -#define INIT_CCENABLE_VALUE 0x2000 -#define RST_CSR_QAT_LSB 20 -#define RST_CSR_AE_LSB 0 -#define MC_TIMESTAMP_ENABLE (0x1 << 7) +#define BAD_REGADDR 0xffff +#define MAX_RETRY_TIMES 10000 +#define INIT_CTX_ARB_VALUE 0x0 +#define INIT_CTX_ENABLE_VALUE 0x0 +#define INIT_PC_VALUE 0x0 +#define INIT_WAKEUP_EVENTS_VALUE 0x1 +#define INIT_SIG_EVENTS_VALUE 0x1 +#define INIT_CCENABLE_VALUE 0x2000 +#define RST_CSR_QAT_LSB 20 +#define RST_CSR_AE_LSB 0 +#define MC_TIMESTAMP_ENABLE (0x1 << 7) +#define MIN_RESET_DELAY_US 3 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \ (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \ @@ -713,8 +714,10 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = 0x80000000; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = true; - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) { handle->chip_info->dual_sign = true; + handle->chip_info->reset_delay_us = MIN_RESET_DELAY_US; + } handle->chip_info->tgroup_share_ustore = true; handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index b61f2545e1652f..a61208cbcd2792 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -115,10 +115,7 @@ void *nx842_crypto_alloc_ctx(struct nx842_driver *driver) ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) { - kfree(ctx->wmem); - free_page((unsigned long)ctx->sbounce); - free_page((unsigned long)ctx->dbounce); - kfree(ctx); + nx842_crypto_free_ctx(ctx); return ERR_PTR(-ENOMEM); } @@ -131,8 +128,9 @@ void nx842_crypto_free_ctx(void *p) struct nx842_crypto_ctx *ctx = p; kfree(ctx->wmem); - free_page((unsigned long)ctx->sbounce); - free_page((unsigned long)ctx->dbounce); + free_pages((unsigned long)ctx->sbounce, BOUNCE_BUFFER_ORDER); + free_pages((unsigned long)ctx->dbounce, BOUNCE_BUFFER_ORDER); + kfree(ctx); } EXPORT_SYMBOL_GPL(nx842_crypto_free_ctx); diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index f5e2c82ba87683..cd3c1a433e8cb4 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -159,7 +159,7 @@ struct nx842_crypto_header_group { struct nx842_crypto_header { /* New members MUST be added within the struct_group() macro below. */ - struct_group_tagged(nx842_crypto_header_hdr, hdr, + __struct_group(nx842_crypto_header_hdr, hdr, __packed, __be16 magic; /* NX842_CRYPTO_MAGIC */ __be16 ignore; /* decompressed end bytes to ignore */ u8 groups; /* total groups in this header */ @@ -167,7 +167,7 @@ struct nx842_crypto_header { struct nx842_crypto_header_group group[]; } __packed; static_assert(offsetof(struct nx842_crypto_header, group) == sizeof(struct nx842_crypto_header_hdr), - "struct member likely outside of struct_group_tagged()"); + "struct member likely outside of __struct_group()"); #define NX842_CRYPTO_GROUP_MAX (0x20) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index df3defa1ef4b64..965a03d5b27ae4 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1744,13 +1744,13 @@ static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) { return sa_cra_init_aead(tfm, "sha1", - "authenc(hmac(sha1-ce),cbc(aes-ce))"); + "authenc(hmac(sha1),cbc(aes))"); } static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) { return sa_cra_init_aead(tfm, "sha256", - "authenc(hmac(sha256-ce),cbc(aes-ce))"); + "authenc(hmac(sha256),cbc(aes))"); } static void sa_exit_tfm_aead(struct crypto_aead *tfm) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index e8c0db687c57fd..bc61d0fe35140c 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -12,6 +12,7 @@ * All rights reserved. */ +#include #include #include #include @@ -868,20 +869,28 @@ struct talitos_ahash_req_ctx { u8 buf[2][HASH_MAX_BLOCK_SIZE]; int buf_idx; unsigned int swinit; - unsigned int first; - unsigned int last; + unsigned int first_desc; + unsigned int last_desc; + unsigned int last_request; unsigned int to_hash_later; unsigned int nbuf; struct scatterlist bufsl[2]; struct scatterlist *psrc; + + struct scatterlist request_bufsl[2]; + struct ahash_request *areq; + struct scatterlist *request_sl; + unsigned int remaining_ahash_request_bytes; + unsigned int current_ahash_request_bytes; + struct work_struct sec1_ahash_process_remaining; }; struct talitos_export_state { u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; u8 buf[HASH_MAX_BLOCK_SIZE]; unsigned int swinit; - unsigned int first; - unsigned int last; + unsigned int first_desc; + unsigned int last_desc; unsigned int to_hash_later; unsigned int nbuf; }; @@ -1713,7 +1722,7 @@ static void common_nonsnoop_hash_unmap(struct device *dev, if (desc->next_desc && desc->ptr[5].ptr != desc2->ptr[5].ptr) unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); - if (req_ctx->last) + if (req_ctx->last_desc) memcpy(areq->result, req_ctx->hw_context, crypto_ahash_digestsize(tfm)); @@ -1750,7 +1759,7 @@ static void ahash_done(struct device *dev, container_of(desc, struct talitos_edesc, desc); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - if (!req_ctx->last && req_ctx->to_hash_later) { + if (!req_ctx->last_desc && req_ctx->to_hash_later) { /* Position any partial block for next update/final/finup */ req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1; req_ctx->nbuf = req_ctx->to_hash_later; @@ -1759,7 +1768,20 @@ static void ahash_done(struct device *dev, kfree(edesc); - ahash_request_complete(areq, err); + if (err) { + ahash_request_complete(areq, err); + return; + } + + req_ctx->remaining_ahash_request_bytes -= + req_ctx->current_ahash_request_bytes; + + if (!req_ctx->remaining_ahash_request_bytes) { + ahash_request_complete(areq, 0); + return; + } + + schedule_work(&req_ctx->sec1_ahash_process_remaining); } /* @@ -1803,7 +1825,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* first DWORD empty */ /* hash context in */ - if (!req_ctx->first || req_ctx->swinit) { + if (!req_ctx->first_desc || req_ctx->swinit) { map_single_talitos_ptr_nosync(dev, &desc->ptr[1], req_ctx->hw_context_size, req_ctx->hw_context, @@ -1811,7 +1833,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, req_ctx->swinit = 0; } /* Indicate next op is not the first. */ - req_ctx->first = 0; + req_ctx->first_desc = 0; /* HMAC key */ if (ctx->keylen) @@ -1844,7 +1866,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, /* fifth DWORD empty */ /* hash/HMAC out -or- hash context out */ - if (req_ctx->last) + if (req_ctx->last_desc) map_single_talitos_ptr(dev, &desc->ptr[5], crypto_ahash_digestsize(tfm), req_ctx->hw_context, DMA_FROM_DEVICE); @@ -1886,7 +1908,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, if (sg_count > 1) sync_needed = true; copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); - if (req_ctx->last) + if (req_ctx->last_desc) map_single_talitos_ptr_nosync(dev, &desc->ptr[5], req_ctx->hw_context_size, req_ctx->hw_context, @@ -1925,60 +1947,7 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, nbytes, 0, 0, 0, areq->base.flags, false); } -static int ahash_init(struct ahash_request *areq) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); - struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); - struct device *dev = ctx->dev; - struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - unsigned int size; - dma_addr_t dma; - - /* Initialize the context */ - req_ctx->buf_idx = 0; - req_ctx->nbuf = 0; - req_ctx->first = 1; /* first indicates h/w must init its context */ - req_ctx->swinit = 0; /* assume h/w init of context */ - size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) - ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 - : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; - req_ctx->hw_context_size = size; - - dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, - DMA_TO_DEVICE); - dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); - - return 0; -} - -/* - * on h/w without explicit sha224 support, we initialize h/w context - * manually with sha224 constants, and tell it to run sha256. - */ -static int ahash_init_sha224_swinit(struct ahash_request *areq) -{ - struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - - req_ctx->hw_context[0] = SHA224_H0; - req_ctx->hw_context[1] = SHA224_H1; - req_ctx->hw_context[2] = SHA224_H2; - req_ctx->hw_context[3] = SHA224_H3; - req_ctx->hw_context[4] = SHA224_H4; - req_ctx->hw_context[5] = SHA224_H5; - req_ctx->hw_context[6] = SHA224_H6; - req_ctx->hw_context[7] = SHA224_H7; - - /* init 64-bit count */ - req_ctx->hw_context[8] = 0; - req_ctx->hw_context[9] = 0; - - ahash_init(areq); - req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ - - return 0; -} - -static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) +static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); @@ -1995,14 +1964,14 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) bool is_sec1 = has_ftr_sec1(priv); u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; - if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { + if (!req_ctx->last_desc && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ - nents = sg_nents_for_len(areq->src, nbytes); + nents = sg_nents_for_len(req_ctx->request_sl, nbytes); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_copy_to_buffer(areq->src, nents, + sg_copy_to_buffer(req_ctx->request_sl, nents, ctx_buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; @@ -2012,7 +1981,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) nbytes_to_hash = nbytes + req_ctx->nbuf; to_hash_later = nbytes_to_hash & (blocksize - 1); - if (req_ctx->last) + if (req_ctx->last_desc) to_hash_later = 0; else if (to_hash_later) /* There is a partial block. Hash the full block(s) now */ @@ -2029,7 +1998,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_init_table(req_ctx->bufsl, nsg); sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf); if (nsg > 1) - sg_chain(req_ctx->bufsl, 2, areq->src); + sg_chain(req_ctx->bufsl, 2, req_ctx->request_sl); req_ctx->psrc = req_ctx->bufsl; } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { int offset; @@ -2038,26 +2007,26 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) offset = blocksize - req_ctx->nbuf; else offset = nbytes_to_hash - req_ctx->nbuf; - nents = sg_nents_for_len(areq->src, offset); + nents = sg_nents_for_len(req_ctx->request_sl, offset); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_copy_to_buffer(areq->src, nents, + sg_copy_to_buffer(req_ctx->request_sl, nents, ctx_buf + req_ctx->nbuf, offset); req_ctx->nbuf += offset; - req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src, + req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, req_ctx->request_sl, offset); } else - req_ctx->psrc = areq->src; + req_ctx->psrc = req_ctx->request_sl; if (to_hash_later) { - nents = sg_nents_for_len(areq->src, nbytes); + nents = sg_nents_for_len(req_ctx->request_sl, nbytes); if (nents < 0) { dev_err(dev, "Invalid number of src SG.\n"); return nents; } - sg_pcopy_to_buffer(areq->src, nents, + sg_pcopy_to_buffer(req_ctx->request_sl, nents, req_ctx->buf[(req_ctx->buf_idx + 1) & 1], to_hash_later, nbytes - to_hash_later); @@ -2065,36 +2034,145 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) req_ctx->to_hash_later = to_hash_later; /* Allocate extended descriptor */ - edesc = ahash_edesc_alloc(areq, nbytes_to_hash); + edesc = ahash_edesc_alloc(req_ctx->areq, nbytes_to_hash); if (IS_ERR(edesc)) return PTR_ERR(edesc); edesc->desc.hdr = ctx->desc_hdr_template; /* On last one, request SEC to pad; otherwise continue */ - if (req_ctx->last) + if (req_ctx->last_desc) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD; else edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT; /* request SEC to INIT hash. */ - if (req_ctx->first && !req_ctx->swinit) + if (req_ctx->first_desc && !req_ctx->swinit) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT; /* When the tfm context has a keylen, it's an HMAC. * A first or last (ie. not middle) descriptor must request HMAC. */ - if (ctx->keylen && (req_ctx->first || req_ctx->last)) + if (ctx->keylen && (req_ctx->first_desc || req_ctx->last_desc)) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done); + return common_nonsnoop_hash(edesc, req_ctx->areq, nbytes_to_hash, ahash_done); +} + +static void sec1_ahash_process_remaining(struct work_struct *work) +{ + struct talitos_ahash_req_ctx *req_ctx = + container_of(work, struct talitos_ahash_req_ctx, + sec1_ahash_process_remaining); + int err = 0; + + req_ctx->request_sl = scatterwalk_ffwd(req_ctx->request_bufsl, + req_ctx->request_sl, TALITOS1_MAX_DATA_LEN); + + if (req_ctx->remaining_ahash_request_bytes > TALITOS1_MAX_DATA_LEN) + req_ctx->current_ahash_request_bytes = TALITOS1_MAX_DATA_LEN; + else { + req_ctx->current_ahash_request_bytes = + req_ctx->remaining_ahash_request_bytes; + + if (req_ctx->last_request) + req_ctx->last_desc = 1; + } + + err = ahash_process_req_one(req_ctx->areq, + req_ctx->current_ahash_request_bytes); + + if (err != -EINPROGRESS) + ahash_request_complete(req_ctx->areq, err); +} + +static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_private *priv = dev_get_drvdata(dev); + bool is_sec1 = has_ftr_sec1(priv); + + req_ctx->areq = areq; + req_ctx->request_sl = areq->src; + req_ctx->remaining_ahash_request_bytes = nbytes; + + if (is_sec1) { + if (nbytes > TALITOS1_MAX_DATA_LEN) + nbytes = TALITOS1_MAX_DATA_LEN; + else if (req_ctx->last_request) + req_ctx->last_desc = 1; + } + + req_ctx->current_ahash_request_bytes = nbytes; + + return ahash_process_req_one(req_ctx->areq, + req_ctx->current_ahash_request_bytes); +} + +static int ahash_init(struct ahash_request *areq) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); + struct device *dev = ctx->dev; + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + unsigned int size; + dma_addr_t dma; + + /* Initialize the context */ + req_ctx->buf_idx = 0; + req_ctx->nbuf = 0; + req_ctx->first_desc = 1; /* first_desc indicates h/w must init its context */ + req_ctx->swinit = 0; /* assume h/w init of context */ + size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + req_ctx->hw_context_size = size; + req_ctx->last_request = 0; + req_ctx->last_desc = 0; + INIT_WORK(&req_ctx->sec1_ahash_process_remaining, sec1_ahash_process_remaining); + + dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size, + DMA_TO_DEVICE); + dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE); + + return 0; +} + +/* + * on h/w without explicit sha224 support, we initialize h/w context + * manually with sha224 constants, and tell it to run sha256. + */ +static int ahash_init_sha224_swinit(struct ahash_request *areq) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + + req_ctx->hw_context[0] = SHA224_H0; + req_ctx->hw_context[1] = SHA224_H1; + req_ctx->hw_context[2] = SHA224_H2; + req_ctx->hw_context[3] = SHA224_H3; + req_ctx->hw_context[4] = SHA224_H4; + req_ctx->hw_context[5] = SHA224_H5; + req_ctx->hw_context[6] = SHA224_H6; + req_ctx->hw_context[7] = SHA224_H7; + + /* init 64-bit count */ + req_ctx->hw_context[8] = 0; + req_ctx->hw_context[9] = 0; + + ahash_init(areq); + req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/ + + return 0; } static int ahash_update(struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - req_ctx->last = 0; + req_ctx->last_request = 0; return ahash_process_req(areq, areq->nbytes); } @@ -2103,7 +2181,7 @@ static int ahash_final(struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - req_ctx->last = 1; + req_ctx->last_request = 1; return ahash_process_req(areq, 0); } @@ -2112,7 +2190,7 @@ static int ahash_finup(struct ahash_request *areq) { struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - req_ctx->last = 1; + req_ctx->last_request = 1; return ahash_process_req(areq, areq->nbytes); } @@ -2146,8 +2224,8 @@ static int ahash_export(struct ahash_request *areq, void *out) req_ctx->hw_context_size); memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf); export->swinit = req_ctx->swinit; - export->first = req_ctx->first; - export->last = req_ctx->last; + export->first_desc = req_ctx->first_desc; + export->last_desc = req_ctx->last_desc; export->to_hash_later = req_ctx->to_hash_later; export->nbuf = req_ctx->nbuf; @@ -2172,8 +2250,8 @@ static int ahash_import(struct ahash_request *areq, const void *in) memcpy(req_ctx->hw_context, export->hw_context, size); memcpy(req_ctx->buf[0], export->buf, export->nbuf); req_ctx->swinit = export->swinit; - req_ctx->first = export->first; - req_ctx->last = export->last; + req_ctx->first_desc = export->first_desc; + req_ctx->last_desc = export->last_desc; req_ctx->to_hash_later = export->to_hash_later; req_ctx->nbuf = export->nbuf; diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 9210cceb4b7b2c..30c78afe3dea63 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -4,6 +4,7 @@ * Crypto driver to handle block cipher algorithms using NVIDIA Security Engine. */ +#include #include #include #include @@ -333,7 +334,9 @@ static int tegra_aes_do_one_req(struct crypto_engine *engine, void *areq) tegra_key_invalidate_reserved(ctx->se, key2_id, ctx->alg); out_finalize: + local_bh_disable(); crypto_finalize_skcipher_request(se->engine, req, ret); + local_bh_enable(); return 0; } @@ -1262,7 +1265,9 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); out_finalize: + local_bh_disable(); crypto_finalize_aead_request(ctx->se->engine, req, ret); + local_bh_enable(); return 0; } @@ -1348,7 +1353,9 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); out_finalize: + local_bh_disable(); crypto_finalize_aead_request(ctx->se->engine, req, ret); + local_bh_enable(); return 0; } @@ -1746,7 +1753,9 @@ static int tegra_cmac_do_one_req(struct crypto_engine *engine, void *areq) if (tegra_key_is_reserved(rctx->key_id)) tegra_key_invalidate_reserved(ctx->se, rctx->key_id, ctx->alg); + local_bh_disable(); crypto_finalize_hash_request(se->engine, req, ret); + local_bh_enable(); return 0; } diff --git a/drivers/crypto/tegra/tegra-se-hash.c b/drivers/crypto/tegra/tegra-se-hash.c index 06bb5bf0fa335c..23d549801612e1 100644 --- a/drivers/crypto/tegra/tegra-se-hash.c +++ b/drivers/crypto/tegra/tegra-se-hash.c @@ -4,6 +4,7 @@ * Crypto driver to handle HASH algorithms using NVIDIA Security Engine. */ +#include #include #include #include @@ -546,7 +547,9 @@ static int tegra_sha_do_one_req(struct crypto_engine *engine, void *areq) } out: + local_bh_disable(); crypto_finalize_hash_request(se->engine, req, ret); + local_bh_enable(); return 0; } diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index fbb300a0183021..a5922116db2a8f 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -1043,6 +1043,9 @@ static void cxl_reset_done(struct pci_dev *pdev) * that no longer exists. */ guard(device)(&cxlmd->dev); + if (!cxlmd->dev.driver) + return; + if (cxlmd->endpoint && cxl_endpoint_decoder_reset_detected(cxlmd->endpoint)) { dev_crit(dev, "SBR happened without memory regions removal.\n"); diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 35afcfcac5910e..abb6d8f8f95d22 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -1133,9 +1133,9 @@ const char __rcu *dma_fence_driver_name(struct dma_fence *fence) "RCU protection is required for safe access to returned string"); if (!dma_fence_test_signaled_flag(fence)) - return fence->ops->get_driver_name(fence); + return (const char __rcu *)fence->ops->get_driver_name(fence); else - return "detached-driver"; + return (const char __rcu *)"detached-driver"; } EXPORT_SYMBOL(dma_fence_driver_name); @@ -1165,8 +1165,8 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence) "RCU protection is required for safe access to returned string"); if (!dma_fence_test_signaled_flag(fence)) - return fence->ops->get_timeline_name(fence); + return (const char __rcu *)fence->ops->get_driver_name(fence); else - return "signaled-timeline"; + return (const char __rcu *)"signaled-timeline"; } EXPORT_SYMBOL(dma_fence_timeline_name); diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 66cda7cc9f7ab9..acb526be2d01ff 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -89,9 +89,20 @@ config APPLE_ADMAC tristate "Apple ADMAC support" depends on ARCH_APPLE || COMPILE_TEST select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS help Enable support for Audio DMA Controller found on Apple Silicon SoCs. +config APPLE_SIO + tristate "Apple SIO support" + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on OF_ADDRESS + select DMA_ENGINE + help + Enable support for the SIO coprocessor found on Apple Silicon SoCs + where it provides DMA services. + config ARM_DMA350 tristate "Arm DMA-350 support" depends on ARM || ARM64 || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a54d7688392b1a..1c11fdc02692cc 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o +obj-$(CONFIG_APPLE_SIO) += apple-sio.o obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c new file mode 100644 index 00000000000000..511f91999ed3de --- /dev/null +++ b/drivers/dma/apple-sio.c @@ -0,0 +1,942 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Driver for SIO coprocessor on t8103 (M1) and other Apple SoCs + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +#define NCHANNELS_MAX 0x80 + +#define REG_CPU_CONTROL 0x44 +#define CPU_CONTROL_RUN BIT(4) + +#define SIOMSG_DATA GENMASK(63, 32) +#define SIOMSG_TYPE GENMASK(23, 16) +#define SIOMSG_PARAM GENMASK(31, 24) +#define SIOMSG_TAG GENMASK(13, 8) +#define SIOMSG_EP GENMASK(7, 0) + +#define EP_SIO 0x20 + +#define MSG_START 0x2 +#define MSG_SETUP 0x3 +#define MSG_CONFIGURE 0x5 +#define MSG_ISSUE 0x6 +#define MSG_TERMINATE 0x8 +#define MSG_ACK 0x65 +#define MSG_NACK 0x66 +#define MSG_STARTED 0x67 +#define MSG_REPORT 0x68 + +#define SIO_CALL_TIMEOUT_MS 100 +#define SIO_SHMEM_SIZE 0x1000 +#define SIO_NO_DESC_SLOTS 64 + +/* + * There are two kinds of 'transaction descriptors' in play here. + * + * There's the struct sio_tx, and the struct dma_async_tx_descriptor embedded + * inside, which jointly represent a transaction to the dmaengine subsystem. + * At this time we only support those transactions to be cyclic. + * + * Then there are the coprocessor descriptors, which is what the coprocessor + * knows and understands. These don't seem to have a cyclic regime, so we can't + * map the dmaengine transaction on an exact coprocessor counterpart. Instead + * we continually queue up many coprocessor descriptors to implement a cyclic + * transaction. + * + * The number below is the maximum of how far ahead (how many) coprocessor + * descriptors we should be queuing up, per channel, for a cyclic transaction. + * Basically it's a made-up number. + */ +#define SIO_MAX_NINFLIGHT 4 + +struct sio_coproc_desc { + u32 pad1; + u32 flag; + u64 unk; + u64 iova; + u64 size; + u64 pad2; + u64 pad3; +} __packed; +static_assert(sizeof(struct sio_coproc_desc) == 48); + +struct sio_shmem_chan_config { + u32 datashape; + u32 timeout; + u32 fifo; + u32 threshold; + u32 limit; +} __packed; + +struct sio_data; +struct sio_tx; + +struct sio_chan { + unsigned int no; + struct sio_data *host; + struct virt_dma_chan vc; + struct work_struct terminate_wq; + + bool configured; + struct sio_shmem_chan_config cfg; + + struct sio_tx *current_tx; +}; + +#define SIO_NTAGS 16 + +typedef void (*sio_ack_callback)(struct sio_chan *, void *, bool); + +struct sio_data { + void __iomem *base; + struct dma_device dma; + struct device *dev; + struct apple_rtkit *rtk; + void *shmem; + struct sio_coproc_desc *shmem_desc_base; + unsigned long *desc_allocated; + + struct sio_tagdata { + DECLARE_BITMAP(allocated, SIO_NTAGS); + int last_tag; + + struct completion completions[SIO_NTAGS]; + bool atomic[SIO_NTAGS]; + bool acked[SIO_NTAGS]; + + sio_ack_callback ack_callback[SIO_NTAGS]; + void *cookie[SIO_NTAGS]; + } tags; + + int nchannels; + struct sio_chan channels[]; +}; + +struct sio_tx { + struct virt_dma_desc vd; + struct completion done; + + bool terminated; + size_t period_len; + int nperiods; + int ninflight; + int next; + + struct sio_coproc_desc *siodesc[]; +}; + +static int sio_send_siomsg(struct sio_data *sio, u64 msg); +static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, + sio_ack_callback ack_callback, + void *cookie); +static int sio_call(struct sio_data *sio, u64 msg); + +static struct sio_chan *to_sio_chan(struct dma_chan *chan) +{ + return container_of(chan, struct sio_chan, vc.chan); +} + +static struct sio_tx *to_sio_tx(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct sio_tx, vd.tx); +} + +static int sio_alloc_tag(struct sio_data *sio) +{ + struct sio_tagdata *tags = &sio->tags; + int tag, i; + + /* + * Because tag number 0 is special, the usable tag range + * is 1...(SIO_NTAGS - 1). So, to pick the next usable tag, + * we do modulo (SIO_NTAGS - 1) *then* plus one. + */ + +#define SIO_USABLE_TAGS (SIO_NTAGS - 1) + tag = (READ_ONCE(tags->last_tag) % SIO_USABLE_TAGS) + 1; + + for (i = 0; i < SIO_USABLE_TAGS; i++) { + if (!test_and_set_bit(tag, tags->allocated)) + break; + + tag = (tag % SIO_USABLE_TAGS) + 1; + } + + WRITE_ONCE(tags->last_tag, tag); + + if (i < SIO_USABLE_TAGS) + return tag; + else + return -EBUSY; +#undef SIO_USABLE_TAGS +} + +static void sio_free_tag(struct sio_data *sio, int tag) +{ + struct sio_tagdata *tags = &sio->tags; + + if (WARN_ON(tag >= SIO_NTAGS)) + return; + + tags->atomic[tag] = false; + tags->ack_callback[tag] = NULL; + + WARN_ON(!test_and_clear_bit(tag, tags->allocated)); +} + +static void sio_set_tag_atomic(struct sio_data *sio, int tag, + sio_ack_callback ack_callback, + void *cookie) +{ + struct sio_tagdata *tags = &sio->tags; + + tags->atomic[tag] = true; + tags->ack_callback[tag] = ack_callback; + tags->cookie[tag] = cookie; +} + +static struct sio_coproc_desc *sio_alloc_desc(struct sio_data *sio) +{ + int i; + + for (i = 0; i < SIO_NO_DESC_SLOTS; i++) + if (!test_and_set_bit(i, sio->desc_allocated)) + return sio->shmem_desc_base + i; + + return NULL; +} + +static void sio_free_desc(struct sio_data *sio, struct sio_coproc_desc *desc) +{ + clear_bit(desc - sio->shmem_desc_base, sio->desc_allocated); +} + +static int sio_coproc_desc_slot(struct sio_data *sio, struct sio_coproc_desc *desc) +{ + return (desc - sio->shmem_desc_base) * 4; +} + +static enum dma_transfer_direction sio_chan_direction(int channo) +{ + /* Channel directions are fixed based on channel number */ + return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; +} + +static void sio_tx_free(struct virt_dma_desc *vd) +{ + struct sio_data *sio = to_sio_chan(vd->tx.chan)->host; + struct sio_tx *siotx = to_sio_tx(&vd->tx); + int i; + + for (i = 0; i < siotx->nperiods; i++) + if (siotx->siodesc[i]) + sio_free_desc(sio, siotx->siodesc[i]); + kfree(siotx); +} + +static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_tx *siotx = NULL; + int i, nperiods = buf_len / period_len; + + if (direction != sio_chan_direction(siochan->no)) + return NULL; + + siotx = kzalloc(struct_size(siotx, siodesc, nperiods), GFP_NOWAIT); + if (!siotx) + return NULL; + + init_completion(&siotx->done); + siotx->period_len = period_len; + siotx->nperiods = nperiods; + + for (i = 0; i < nperiods; i++) { + struct sio_coproc_desc *d; + + siotx->siodesc[i] = d = sio_alloc_desc(siochan->host); + if (!d) { + siotx->vd.tx.chan = &siochan->vc.chan; + sio_tx_free(&siotx->vd); + return NULL; + } + + d->flag = 1; /* not sure what's up with this */ + d->iova = buf_addr + period_len * i; + d->size = period_len; + } + dma_wmb(); + + return vchan_tx_prep(&siochan->vc, &siotx->vd, flags); +} + +static enum dma_status sio_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct virt_dma_desc *vd; + struct sio_tx *siotx; + enum dma_status ret; + unsigned long flags; + int periods_residue; + size_t residue; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + siotx = siochan->current_tx; + + if (siotx && siotx->vd.tx.cookie == cookie) { + ret = DMA_IN_PROGRESS; + periods_residue = siotx->next - siotx->ninflight; + while (periods_residue < 0) + periods_residue += siotx->nperiods; + residue = (siotx->nperiods - periods_residue) * siotx->period_len; + } else { + ret = DMA_IN_PROGRESS; + residue = 0; + vd = vchan_find_desc(&siochan->vc, cookie); + if (vd) { + siotx = to_sio_tx(&vd->tx); + residue = siotx->period_len * siotx->nperiods; + } + } + spin_unlock_irqrestore(&siochan->vc.lock, flags); + dma_set_residue(txstate, residue); + + return ret; +} + +static bool sio_fill_in_locked(struct sio_chan *siochan); + +static void sio_handle_issue_ack(struct sio_chan *siochan, void *cookie, bool ok) +{ + dma_cookie_t tx_cookie = (unsigned long) cookie; + unsigned long flags; + struct sio_tx *tx; + + if (!ok) { + dev_err(siochan->host->dev, "nacked issue on chan %d\n", siochan->no); + return; + } + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (!siochan->current_tx || tx_cookie != siochan->current_tx->vd.tx.cookie || + siochan->current_tx->terminated) + goto out; + + tx = siochan->current_tx; + tx->next = (tx->next + 1) % tx->nperiods; + tx->ninflight++; + sio_fill_in_locked(siochan); + +out: + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static bool sio_fill_in_locked(struct sio_chan *siochan) +{ + struct sio_data *sio = siochan->host; + struct sio_tx *tx = siochan->current_tx; + struct sio_coproc_desc *d = tx->siodesc[tx->next]; + int ret; + + if (tx->ninflight >= SIO_MAX_NINFLIGHT || tx->terminated) + return false; + + static_assert(sizeof(dma_cookie_t) <= sizeof(void *)); + ret = sio_send_siomsg_atomic(sio, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_ISSUE) | + FIELD_PREP(SIOMSG_DATA, sio_coproc_desc_slot(sio, d)), + sio_handle_issue_ack, (void *) (uintptr_t) tx->vd.tx.cookie); + if (ret < 0) + dev_err_ratelimited(sio->dev, "can't issue on chan %d ninflight %d: %d\n", + siochan->no, tx->ninflight, ret); + return true; +} + +static void sio_update_current_tx_locked(struct sio_chan *siochan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&siochan->vc); + + if (vd && !siochan->current_tx) { + list_del(&vd->node); + siochan->current_tx = to_sio_tx(&vd->tx); + sio_fill_in_locked(siochan); + } +} + +static void sio_issue_pending(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + vchan_issue_pending(&siochan->vc); + sio_update_current_tx_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static int sio_terminate_all(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + unsigned long flags; + LIST_HEAD(to_free); + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->current_tx && !siochan->current_tx->terminated) { + dma_cookie_complete(&siochan->current_tx->vd.tx); + siochan->current_tx->terminated = true; + schedule_work(&siochan->terminate_wq); + } + vchan_get_all_descriptors(&siochan->vc, &to_free); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + vchan_dma_desc_free_list(&siochan->vc, &to_free); + + return 0; +} + +static void sio_terminate_work(struct work_struct *wq) +{ + struct sio_chan *siochan = container_of(wq, struct sio_chan, terminate_wq); + struct sio_tx *tx; + unsigned long flags; + int ret; + + spin_lock_irqsave(&siochan->vc.lock, flags); + tx = siochan->current_tx; + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + if (WARN_ON(!tx)) + return; + + ret = sio_call(siochan->host, FIELD_PREP(SIOMSG_EP, siochan->no) | + FIELD_PREP(SIOMSG_TYPE, MSG_TERMINATE)); + if (ret < 0) + dev_err(siochan->host->dev, "terminate call on chan %d failed: %d\n", + siochan->no, ret); + + ret = wait_for_completion_timeout(&tx->done, msecs_to_jiffies(500)); + if (!ret) + dev_err(siochan->host->dev, "terminate descriptor wait timed out\n"); + + tasklet_kill(&siochan->vc.task); + + spin_lock_irqsave(&siochan->vc.lock, flags); + WARN_ON(siochan->current_tx != tx); + siochan->current_tx = NULL; + sio_update_current_tx_locked(siochan); + spin_unlock_irqrestore(&siochan->vc.lock, flags); + + sio_tx_free(&tx->vd); +} + +static void sio_synchronize(struct dma_chan *chan) +{ + struct sio_chan *siochan = to_sio_chan(chan); + + flush_work(&siochan->terminate_wq); +} + +static void sio_free_chan_resources(struct dma_chan *chan) +{ + sio_terminate_all(chan); + sio_synchronize(chan); + vchan_free_chan_resources(&to_sio_chan(chan)->vc); +} + +static struct dma_chan *sio_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct sio_data *sio = (struct sio_data *) ofdma->of_dma_data; + unsigned int index = dma_spec->args[0]; + + if (dma_spec->args_count != 1 || index >= sio->nchannels) + return ERR_PTR(-EINVAL); + + return dma_get_slave_channel(&sio->channels[index].vc.chan); +} + +static void sio_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct sio_data *sio = cookie; + + dev_err(sio->dev, "SIO down (crashed)"); +} + +static void sio_process_report(struct sio_chan *siochan) +{ + unsigned long flags; + + spin_lock_irqsave(&siochan->vc.lock, flags); + if (siochan->current_tx) { + struct sio_tx *tx = siochan->current_tx; + + if (tx->ninflight) + tx->ninflight--; + vchan_cyclic_callback(&tx->vd); + if (!sio_fill_in_locked(siochan) && !tx->ninflight) + complete(&tx->done); + } + spin_unlock_irqrestore(&siochan->vc.lock, flags); +} + +static void sio_recv_msg(void *cookie, u8 ep, u64 msg) +{ + struct sio_data *sio = cookie; + struct sio_tagdata *tags = &sio->tags; + u32 data; + u8 type, tag, sioep; + + if (ep != EP_SIO) + goto unknown; + + data = FIELD_GET(SIOMSG_DATA, msg); + // param = FIELD_GET(SIOMSG_PARAM, msg); + type = FIELD_GET(SIOMSG_TYPE, msg); + tag = FIELD_GET(SIOMSG_TAG, msg); + sioep = FIELD_GET(SIOMSG_EP, msg); + + switch (type) { + case MSG_STARTED: + dev_info(sio->dev, "SIO protocol v%u\n", data); + type = MSG_ACK; /* Pretend this is an ACK */ + fallthrough; + case MSG_ACK: + case MSG_NACK: + if (WARN_ON(tag >= SIO_NTAGS)) + break; + + if (tags->atomic[tag]) { + sio_ack_callback callback = tags->ack_callback[tag]; + + if (callback && !WARN_ON(sioep >= sio->nchannels)) + callback(&sio->channels[sioep], + tags->cookie[tag], type == MSG_ACK); + if (type == MSG_NACK) + dev_err(sio->dev, "got a NACK on channel %d\n", sioep); + sio_free_tag(sio, tag); + } else { + tags->acked[tag] = (type == MSG_ACK); + complete(&tags->completions[tag]); + } + break; + + case MSG_REPORT: + if (WARN_ON(sioep >= sio->nchannels)) + break; + + sio_process_report(&sio->channels[sioep]); + break; + + default: + goto unknown; + } + return; + +unknown: + dev_warn(sio->dev, "received unknown message: ep %x data %016llx\n", + ep, msg); +} + +static int _sio_send_siomsg(struct sio_data *sio, u64 msg, bool atomic, + sio_ack_callback ack_callback, void *cookie) +{ + int tag, ret; + + tag = sio_alloc_tag(sio); + if (tag < 0) + return tag; + + if (atomic) + sio_set_tag_atomic(sio, tag, ack_callback, cookie); + else + reinit_completion(&sio->tags.completions[tag]); + + msg &= ~SIOMSG_TAG; + msg |= FIELD_PREP(SIOMSG_TAG, tag); + ret = apple_rtkit_send_message(sio->rtk, EP_SIO, msg, NULL, + atomic); + if (ret < 0) { + sio_free_tag(sio, tag); + return ret; + } + + return tag; +} + +static int sio_send_siomsg(struct sio_data *sio, u64 msg) +{ + return _sio_send_siomsg(sio, msg, false, NULL, NULL); +} + +static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, + sio_ack_callback ack_callback, + void *cookie) +{ + return _sio_send_siomsg(sio, msg, true, ack_callback, cookie); +} + +static int sio_call(struct sio_data *sio, u64 msg) +{ + int tag, ret; + + tag = sio_send_siomsg(sio, msg); + if (tag < 0) + return tag; + + ret = wait_for_completion_timeout(&sio->tags.completions[tag], + msecs_to_jiffies(SIO_CALL_TIMEOUT_MS)); + if (!ret) { + dev_warn(sio->dev, "call %8llx timed out\n", msg); + sio_free_tag(sio, tag); + return -ETIME; + } + + ret = sio->tags.acked[tag]; + sio_free_tag(sio, tag); + + return ret; +} + +static const struct apple_rtkit_ops sio_rtkit_ops = { + .crashed = sio_rtk_crashed, + .recv_message = sio_recv_msg, +}; + +static int sio_device_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct sio_chan *siochan = to_sio_chan(chan); + struct sio_data *sio = siochan->host; + bool is_tx = sio_chan_direction(siochan->no) == DMA_MEM_TO_DEV; + struct sio_shmem_chan_config *cfg_shmem = sio->shmem; + struct sio_shmem_chan_config cfg; + int ret; + + switch (is_tx ? config->dst_addr_width : config->src_addr_width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + cfg.datashape = 0; + break; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + cfg.datashape = 1; + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + cfg.datashape = 2; + break; + default: + return -EINVAL; + } + + cfg.timeout = 0; + cfg.fifo = 0x800; + cfg.limit = 0x800; + cfg.threshold = 0x800; + + /* + * Dmaengine prescribes we ought to apply the new configuration only + * to newly-queued descriptors. + * + * To comply with dmaengine's interface we take the lazy path here: + * we apply the configuration right away, we only allow the channel + * to be configured once, which means subsequent calls to `device_config` + * either return -EBUSY if the configuration differs, or they are + * a no-op if the configuration is the same as the starting one. + * + * This is the reasonable thing to do given that these sio channels + * are tied to fixed peripherals, and what's more given that the + * only planned consumer of this dmaengine driver in the kernel is + * diplayport audio support, where the DMA configuration is fixed, + * and no more than a single descriptor (a cyclic one) gets ever issued + * at the same time. + * + * The code complexity cost of tracking to which descriptor + * the configuration relates would be significant here, especially + * since we need to do a non-atomic operation to apply it (a call to + * the coprocessor) and dmaengine has its bunch of atomicity + * restrictions. And this complexity would be for naught since it + * doesn't even get exercised by the only planned consumer. + */ + if (siochan->configured && memcmp(&siochan->cfg, &cfg, sizeof(cfg))) + return -EBUSY; + + *cfg_shmem = cfg; + dma_wmb(); + + ret = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_CONFIGURE) | + FIELD_PREP(SIOMSG_EP, siochan->no)); + + if (ret == 1) + ret = 0; + else if (ret == 0) + ret = -EINVAL; + + if (ret == 0) { + siochan->configured = true; + siochan->cfg = cfg; + } + + return ret; +} + +static int sio_alloc_shmem(struct sio_data *sio) +{ + dma_addr_t iova; + int err; + + sio->shmem = dma_alloc_coherent(sio->dev, SIO_SHMEM_SIZE, + &iova, GFP_KERNEL | __GFP_ZERO); + if (!sio->shmem) + return -ENOMEM; + + sio->shmem_desc_base = (struct sio_coproc_desc *) (sio->shmem + 56); + sio->desc_allocated = devm_kzalloc(sio->dev, SIO_NO_DESC_SLOTS / 32, + GFP_KERNEL); + if (!sio->desc_allocated) + return -ENOMEM; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, 1) | + FIELD_PREP(SIOMSG_DATA, iova >> 12)); + if (err != 1) { + if (err == 0) + err = -EINVAL; + return err; + } + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, 2) | + FIELD_PREP(SIOMSG_DATA, SIO_SHMEM_SIZE)); + if (err != 1) { + if (err == 0) + err = -EINVAL; + return err; + } + + return 0; +} + +static int sio_send_dt_params(struct sio_data *sio) +{ + struct device_node *np = sio->dev->of_node; + const char *propname = "apple,sio-firmware-params"; + int nparams, err, i; + + nparams = of_property_count_u32_elems(np, propname); + if (nparams < 0) { + err = nparams; + goto badprop; + } + + for (i = 0; i < nparams / 2; i++) { + u32 key, val; + + err = of_property_read_u32_index(np, propname, 2 * i, &key); + if (err) + goto badprop; + err = of_property_read_u32_index(np, propname, 2 * i + 1, &val); + if (err) + goto badprop; + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_SETUP) | + FIELD_PREP(SIOMSG_PARAM, key & 0xff) | + FIELD_PREP(SIOMSG_EP, key >> 8) | + FIELD_PREP(SIOMSG_DATA, val)); + if (err < 1) { + if (err == 0) + err = -ENXIO; + return dev_err_probe(sio->dev, err, "sending SIO parameter %#x value %#x\n", + key, val); + } + } + + return 0; + +badprop: + return dev_err_probe(sio->dev, err, "failed to read '%s'\n", propname); +} + +static int sio_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct sio_data *sio; + struct dma_device *dma; + int nchannels; + int err, i; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); + if (err) + return dev_err_probe(&pdev->dev, err, "Failed to set DMA mask\n"); + + err = of_property_read_u32(np, "dma-channels", &nchannels); + if (err || nchannels > NCHANNELS_MAX) + return dev_err_probe(&pdev->dev, -EINVAL, + "missing or invalid dma-channels property\n"); + + sio = devm_kzalloc(&pdev->dev, struct_size(sio, channels, nchannels), GFP_KERNEL); + if (!sio) + return -ENOMEM; + + platform_set_drvdata(pdev, sio); + sio->dev = &pdev->dev; + sio->nchannels = nchannels; + + sio->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(sio->base)) + return PTR_ERR(sio->base); + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + err = devm_pm_runtime_enable(&pdev->dev); + if (err < 0) + return dev_err_probe(&pdev->dev, err, + "pm_runtime_enable failed: %d\n", err); + + sio->rtk = devm_apple_rtkit_init(&pdev->dev, sio, NULL, 0, &sio_rtkit_ops); + if (IS_ERR(sio->rtk)) { + err = PTR_ERR(sio->rtk); + dev_err(&pdev->dev, "couldn't initialize rtkit\n"); + goto rpm_put; + } + for (i = 1; i < SIO_NTAGS; i++) + init_completion(&sio->tags.completions[i]); + + dma = &sio->dma; + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma_cap_set(DMA_CYCLIC, dma->cap_mask); + + dma->dev = &pdev->dev; + dma->device_free_chan_resources = sio_free_chan_resources; + dma->device_tx_status = sio_tx_status; + dma->device_issue_pending = sio_issue_pending; + dma->device_terminate_all = sio_terminate_all; + dma->device_synchronize = sio_synchronize; + dma->device_prep_dma_cyclic = sio_prep_dma_cyclic; + dma->device_config = sio_device_config; + + dma->directions = BIT(DMA_MEM_TO_DEV); + dma->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + + INIT_LIST_HEAD(&dma->channels); + for (i = 0; i < nchannels; i++) { + struct sio_chan *siochan = &sio->channels[i]; + + siochan->host = sio; + siochan->no = i; + siochan->vc.desc_free = sio_tx_free; + INIT_WORK(&siochan->terminate_wq, sio_terminate_work); + vchan_init(&siochan->vc, dma); + } + + writel(CPU_CONTROL_RUN, sio->base + REG_CPU_CONTROL); + + err = apple_rtkit_boot(sio->rtk); + if (err) + return dev_err_probe(&pdev->dev, err, "SIO did not boot\n"); + + err = apple_rtkit_start_ep(sio->rtk, EP_SIO); + if (err) + return dev_err_probe(&pdev->dev, err, "starting SIO endpoint\n"); + + err = sio_call(sio, FIELD_PREP(SIOMSG_TYPE, MSG_START)); + if (err < 1) { + if (err == 0) + err = -ENXIO; + return dev_err_probe(&pdev->dev, err, "starting SIO service\n"); + } + + err = sio_send_dt_params(sio); + if (err < 0) + return dev_err_probe(&pdev->dev, err, "failed to send boot-up parameters\n"); + + err = sio_alloc_shmem(sio); + if (err < 0) + return err; + + err = dma_async_device_register(&sio->dma); + if (err) + return dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); + + err = of_dma_controller_register(pdev->dev.of_node, sio_dma_of_xlate, sio); + if (err) { + dma_async_device_unregister(&sio->dma); + return dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); + } + +rpm_put: + pm_runtime_put(&pdev->dev); + + return err; +} + +static void sio_remove(struct platform_device *pdev) +{ + struct sio_data *sio = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&sio->dma); +} + +static const struct of_device_id sio_of_match[] = { + { .compatible = "apple,sio", }, + { } +}; +MODULE_DEVICE_TABLE(of, sio_of_match); + +static __maybe_unused int sio_suspend(struct device *dev) +{ + /* + * TODO: SIO coproc sleep state + */ + return 0; +} + +static __maybe_unused int sio_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(sio_pm_ops, sio_suspend, sio_resume, NULL); + +static struct platform_driver apple_sio_driver = { + .driver = { + .name = "apple-sio", + .of_match_table = sio_of_match, + .pm = pm_ptr(&sio_pm_ops), + }, + .probe = sio_probe, + .remove = sio_remove, +}; +module_platform_driver(apple_sio_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Driver for SIO coprocessor on Apple SoCs"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 5d74bc29cf896d..95e460422b2ad2 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -342,8 +342,8 @@ static void axi_desc_put(struct axi_dma_desc *desc) kfree(desc); atomic_sub(descs_put, &chan->descs_allocated); dev_vdbg(chan2dev(chan), "%s: %d descs put, %d still allocated\n", - axi_chan_name(chan), descs_put, - atomic_read(&chan->descs_allocated)); + axi_chan_name(chan), descs_put, + atomic_read(&chan->descs_allocated)); } static void vchan_desc_put(struct virt_dma_desc *vdesc) @@ -353,7 +353,7 @@ static void vchan_desc_put(struct virt_dma_desc *vdesc) static enum dma_status dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, - struct dma_tx_state *txstate) + struct dma_tx_state *txstate) { struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); struct virt_dma_desc *vdesc; @@ -491,7 +491,7 @@ static void axi_chan_start_first_queued(struct axi_dma_chan *chan) desc = vd_to_axi_desc(vd); dev_vdbg(chan2dev(chan), "%s: started %u\n", axi_chan_name(chan), - vd->tx.cookie); + vd->tx.cookie); axi_chan_block_xfer_start(chan, desc); } @@ -592,8 +592,6 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set) (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0); - - return; } /* @@ -1162,7 +1160,7 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) axi_chan_irq_clear(chan, status); dev_vdbg(chip->dev, "%s %u IRQ status: 0x%08x\n", - axi_chan_name(chan), i, status); + axi_chan_name(chan), i, status); if (status & DWAXIDMAC_IRQ_ALL_ERR) axi_chan_handle_err(chan, status); @@ -1451,7 +1449,7 @@ static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip) if (chip->irq[i] < 0) return chip->irq[i]; ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt, - IRQF_SHARED, KBUILD_MODNAME, chip); + IRQF_SHARED, KBUILD_MODNAME, chip); if (ret < 0) return ret; } @@ -1645,7 +1643,7 @@ static void dw_remove(struct platform_device *pdev) of_dma_controller_free(chip->dev->of_node); list_for_each_entry_safe(chan, _chan, &dw->dma.channels, - vc.chan.device_node) { + vc.chan.device_node) { list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); } diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index cfb9962417ef68..53f572b6b6fc62 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -824,6 +824,7 @@ static int mxs_dma_probe(struct platform_device *pdev) if (ret) { dev_err(mxs_dma->dma_device.dev, "failed to register controller\n"); + return ret; } dev_info(mxs_dma->dma_device.dev, "initialized\n"); diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index 83cbd64abf5a47..95ae786e98aab3 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -842,11 +842,21 @@ int dpll_pin_delete_ntf(struct dpll_pin *pin) return dpll_pin_event_send(DPLL_CMD_PIN_DELETE_NTF, pin); } +/** + * __dpll_pin_change_ntf - notify that the pin has been changed + * @pin: registered pin pointer + * + * Context: caller must hold dpll_lock. Suitable for use inside pin + * callbacks which are already invoked under dpll_lock. + * Return: 0 if succeeds, error code otherwise. + */ int __dpll_pin_change_ntf(struct dpll_pin *pin) { + lockdep_assert_held(&dpll_lock); dpll_pin_notify(pin, DPLL_PIN_CHANGED); return dpll_pin_event_send(DPLL_CMD_PIN_CHANGE_NTF, pin); } +EXPORT_SYMBOL_GPL(__dpll_pin_change_ntf); /** * dpll_pin_change_ntf - notify that the pin has been changed diff --git a/drivers/dpll/dpll_netlink.h b/drivers/dpll/dpll_netlink.h index dd28b56d27c56d..a9cfd55f57fc42 100644 --- a/drivers/dpll/dpll_netlink.h +++ b/drivers/dpll/dpll_netlink.h @@ -11,5 +11,3 @@ int dpll_device_delete_ntf(struct dpll_device *dpll); int dpll_pin_create_ntf(struct dpll_pin *pin); int dpll_pin_delete_ntf(struct dpll_pin *pin); - -int __dpll_pin_change_ntf(struct dpll_pin *pin); diff --git a/drivers/edac/versalnet_edac.c b/drivers/edac/versalnet_edac.c index 2cbc13d9bd00b8..97ec05d68bbbce 100644 --- a/drivers/edac/versalnet_edac.c +++ b/drivers/edac/versalnet_edac.c @@ -70,6 +70,8 @@ #define XDDR5_BUS_WIDTH_32 1 #define XDDR5_BUS_WIDTH_16 2 +#define MC_NAME_LEN 32 + /** * struct ecc_error_info - ECC error log information. * @burstpos: Burst position. @@ -760,120 +762,132 @@ static void versal_edac_release(struct device *dev) kfree(dev); } -static int init_versalnet(struct mc_priv *priv, struct platform_device *pdev) +static void remove_one_mc(struct mc_priv *priv, int i) +{ + struct mem_ctl_info *mci; + + mci = priv->mci[i]; + device_unregister(mci->pdev); + edac_mc_del_mc(mci->pdev); + edac_mc_free(mci); +} + +static int init_one_mc(struct mc_priv *priv, struct platform_device *pdev, int i) { u32 num_chans, rank, dwidth, config; struct edac_mc_layer layers[2]; struct mem_ctl_info *mci; + char name[MC_NAME_LEN]; struct device *dev; enum dev_type dt; - char *name; - int rc, i; - - for (i = 0; i < NUM_CONTROLLERS; i++) { - config = priv->adec[CONF + i * ADEC_NUM]; - num_chans = FIELD_GET(MC5_NUM_CHANS_MASK, config); - rank = 1 << FIELD_GET(MC5_RANK_MASK, config); - dwidth = FIELD_GET(MC5_BUS_WIDTH_MASK, config); - - switch (dwidth) { - case XDDR5_BUS_WIDTH_16: - dt = DEV_X16; - break; - case XDDR5_BUS_WIDTH_32: - dt = DEV_X32; - break; - case XDDR5_BUS_WIDTH_64: - dt = DEV_X64; - break; - default: - dt = DEV_UNKNOWN; - } + int rc; - if (dt == DEV_UNKNOWN) - continue; + config = priv->adec[CONF + i * ADEC_NUM]; + num_chans = FIELD_GET(MC5_NUM_CHANS_MASK, config); + rank = 1 << FIELD_GET(MC5_RANK_MASK, config); + dwidth = FIELD_GET(MC5_BUS_WIDTH_MASK, config); + + switch (dwidth) { + case XDDR5_BUS_WIDTH_16: + dt = DEV_X16; + break; + case XDDR5_BUS_WIDTH_32: + dt = DEV_X32; + break; + case XDDR5_BUS_WIDTH_64: + dt = DEV_X64; + break; + default: + dt = DEV_UNKNOWN; + } - /* Find the first enabled device and register that one. */ - layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; - layers[0].size = rank; - layers[0].is_virt_csrow = true; - layers[1].type = EDAC_MC_LAYER_CHANNEL; - layers[1].size = num_chans; - layers[1].is_virt_csrow = false; + if (dt == DEV_UNKNOWN) + return 0; - rc = -ENOMEM; - mci = edac_mc_alloc(i, ARRAY_SIZE(layers), layers, - sizeof(struct mc_priv)); - if (!mci) { - edac_printk(KERN_ERR, EDAC_MC, "Failed memory allocation for MC%d\n", i); - goto err_alloc; - } + /* Find the first enabled device and register that one. */ + layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; + layers[0].size = rank; + layers[0].is_virt_csrow = true; + layers[1].type = EDAC_MC_LAYER_CHANNEL; + layers[1].size = num_chans; + layers[1].is_virt_csrow = false; + + rc = -ENOMEM; + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return rc; + + mci = edac_mc_alloc(i, ARRAY_SIZE(layers), layers, sizeof(struct mc_priv)); + if (!mci) { + edac_printk(KERN_ERR, EDAC_MC, "Failed memory allocation for MC%d\n", i); + goto err_dev_free; + } - priv->mci[i] = mci; - priv->dwidth = dt; + sprintf(name, "versal-net-ddrmc5-edac-%d", i); - dev = kzalloc_obj(*dev); - dev->release = versal_edac_release; - name = kmalloc(32, GFP_KERNEL); - sprintf(name, "versal-net-ddrmc5-edac-%d", i); - dev->init_name = name; - rc = device_register(dev); - if (rc) - goto err_alloc; + dev->init_name = name; + dev->release = versal_edac_release; - mci->pdev = dev; + rc = device_register(dev); + if (rc) + goto err_mc_free; - platform_set_drvdata(pdev, priv); + mci->pdev = dev; + mc_init(mci, dev); - mc_init(mci, dev); - rc = edac_mc_add_mc(mci); - if (rc) { - edac_printk(KERN_ERR, EDAC_MC, "Failed to register MC%d with EDAC core\n", i); - goto err_alloc; - } + rc = edac_mc_add_mc(mci); + if (rc) { + edac_printk(KERN_ERR, EDAC_MC, "Failed to register MC%d with EDAC core\n", i); + goto err_unreg; } - return 0; -err_alloc: - while (i--) { - mci = priv->mci[i]; - if (!mci) - continue; - - if (mci->pdev) { - device_unregister(mci->pdev); - edac_mc_del_mc(mci->pdev); - } + priv->mci[i] = mci; + priv->dwidth = dt; - edac_mc_free(mci); - } + platform_set_drvdata(pdev, priv); + + return 0; + +err_unreg: + device_unregister(mci->pdev); +err_mc_free: + edac_mc_free(mci); +err_dev_free: + kfree(dev); return rc; } -static void remove_versalnet(struct mc_priv *priv) +static int init_versalnet(struct mc_priv *priv, struct platform_device *pdev) { - struct mem_ctl_info *mci; - int i; + int rc, i; for (i = 0; i < NUM_CONTROLLERS; i++) { - device_unregister(priv->mci[i]->pdev); - mci = edac_mc_del_mc(priv->mci[i]->pdev); - if (!mci) - return; + rc = init_one_mc(priv, pdev, i); + if (rc) { + while (i--) + remove_one_mc(priv, i); - edac_mc_free(mci); + return rc; + } } + return 0; +} + +static void remove_versalnet(struct mc_priv *priv) +{ + for (int i = 0; i < NUM_CONTROLLERS; i++) + remove_one_mc(priv, i); } static int mc_probe(struct platform_device *pdev) { - struct device_node *r5_core_node; struct mc_priv *priv; struct rproc *rp; int rc; - r5_core_node = of_parse_phandle(pdev->dev.of_node, "amd,rproc", 0); + struct device_node *r5_core_node __free(device_node) = + of_parse_phandle(pdev->dev.of_node, "amd,rproc", 0); if (!r5_core_node) { dev_err(&pdev->dev, "amd,rproc: invalid phandle\n"); return -EINVAL; @@ -917,6 +931,7 @@ static int mc_probe(struct platform_device *pdev) err_init: cdx_mcdi_finish(priv->mcdi); + kfree(priv->mcdi); err_unreg: unregister_rpmsg_driver(&amd_rpmsg_driver); @@ -938,6 +953,7 @@ static void mc_remove(struct platform_device *pdev) remove_versalnet(priv); rproc_shutdown(priv->mcdi->r5_rproc); cdx_mcdi_finish(priv->mcdi); + kfree(priv->mcdi); } static const struct of_device_id amd_edac_match[] = { diff --git a/drivers/extcon/extcon-ptn5150.c b/drivers/extcon/extcon-ptn5150.c index 78ad86c4a3bee2..31970fb34fcb2e 100644 --- a/drivers/extcon/extcon-ptn5150.c +++ b/drivers/extcon/extcon-ptn5150.c @@ -331,6 +331,19 @@ static int ptn5150_i2c_probe(struct i2c_client *i2c) return 0; } +static int ptn5150_resume(struct device *dev) +{ + struct i2c_client *i2c = to_i2c_client(dev); + struct ptn5150_info *info = i2c_get_clientdata(i2c); + + /* Need to check possible pending interrupt events */ + schedule_work(&info->irq_work); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(ptn5150_pm_ops, NULL, ptn5150_resume); + static const struct of_device_id ptn5150_dt_match[] = { { .compatible = "nxp,ptn5150" }, { }, @@ -346,6 +359,7 @@ MODULE_DEVICE_TABLE(i2c, ptn5150_i2c_id); static struct i2c_driver ptn5150_i2c_driver = { .driver = { .name = "ptn5150", + .pm = pm_sleep_ptr(&ptn5150_pm_ops), .of_match_table = ptn5150_dt_match, }, .probe = ptn5150_i2c_probe, diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index f2f94d4d533e8d..eb278284828376 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -2078,7 +2078,7 @@ static int __init ffa_init(void) ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer), virt_to_phys(drv_info->rx_buffer), - rxtx_bufsz / FFA_PAGE_SIZE); + PAGE_ALIGN(rxtx_bufsz) / FFA_PAGE_SIZE); if (ret) { pr_err("failed to register FFA RxTx buffers\n"); goto free_pages; diff --git a/drivers/firmware/efi/capsule-loader.c b/drivers/firmware/efi/capsule-loader.c index 2c628a12709196..8e8f81f0a5a0af 100644 --- a/drivers/firmware/efi/capsule-loader.c +++ b/drivers/firmware/efi/capsule-loader.c @@ -67,7 +67,7 @@ int __efi_capsule_setup_info(struct capsule_info *cap_info) cap_info->pages = temp_page; temp_page = krealloc(cap_info->phys, - pages_needed * sizeof(phys_addr_t *), + pages_needed * sizeof(phys_addr_t), GFP_KERNEL | __GFP_ZERO); if (!temp_page) return -ENOMEM; diff --git a/drivers/firmware/google/framebuffer-coreboot.c b/drivers/firmware/google/framebuffer-coreboot.c index c68c9f56370f28..f44183476ed73c 100644 --- a/drivers/firmware/google/framebuffer-coreboot.c +++ b/drivers/firmware/google/framebuffer-coreboot.c @@ -67,7 +67,7 @@ static int framebuffer_probe(struct coreboot_device *dev) return -ENODEV; memset(&res, 0, sizeof(res)); - res.flags = IORESOURCE_MEM | IORESOURCE_BUSY; + res.flags = IORESOURCE_MEM; res.name = "Coreboot Framebuffer"; res.start = fb->physical_address; length = PAGE_ALIGN(fb->y_resolution * fb->bytes_per_line); @@ -81,19 +81,10 @@ static int framebuffer_probe(struct coreboot_device *dev) sizeof(pdata)); if (IS_ERR(pdev)) pr_warn("coreboot: could not register framebuffer\n"); - else - dev_set_drvdata(&dev->dev, pdev); return PTR_ERR_OR_ZERO(pdev); } -static void framebuffer_remove(struct coreboot_device *dev) -{ - struct platform_device *pdev = dev_get_drvdata(&dev->dev); - - platform_device_unregister(pdev); -} - static const struct coreboot_device_id framebuffer_ids[] = { { .tag = CB_TAG_FRAMEBUFFER }, { /* sentinel */ } @@ -102,7 +93,6 @@ MODULE_DEVICE_TABLE(coreboot, framebuffer_ids); static struct coreboot_driver framebuffer_driver = { .probe = framebuffer_probe, - .remove = framebuffer_remove, .drv = { .name = "framebuffer", }, diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c index 8fbc96693a55f5..d439a9f5b62b8e 100644 --- a/drivers/firmware/qcom/qcom_scm.c +++ b/drivers/firmware/qcom/qcom_scm.c @@ -923,14 +923,13 @@ struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *c goto free_input_rt; } - tbl_ptr = kzalloc(size, GFP_KERNEL); + tbl_ptr = kmemdup(output_rt_tzm, size, GFP_KERNEL); if (!tbl_ptr) { qcom_tzmem_free(output_rt_tzm); ret = -ENOMEM; goto free_input_rt; } - memcpy(tbl_ptr, output_rt_tzm, size); *output_rt_size = size; qcom_tzmem_free(output_rt_tzm); diff --git a/drivers/firmware/samsung/exynos-acpm-dvfs.c b/drivers/firmware/samsung/exynos-acpm-dvfs.c index 1c5b2b143bcc3f..66448c8037aca6 100644 --- a/drivers/firmware/samsung/exynos-acpm-dvfs.c +++ b/drivers/firmware/samsung/exynos-acpm-dvfs.c @@ -42,7 +42,7 @@ static void acpm_dvfs_init_set_rate_cmd(u32 cmd[4], unsigned int clk_id, cmd[3] = ktime_to_ms(ktime_get()); } -int acpm_dvfs_set_rate(const struct acpm_handle *handle, +int acpm_dvfs_set_rate(struct acpm_handle *handle, unsigned int acpm_chan_id, unsigned int clk_id, unsigned long rate) { @@ -62,7 +62,7 @@ static void acpm_dvfs_init_get_rate_cmd(u32 cmd[4], unsigned int clk_id) cmd[3] = ktime_to_ms(ktime_get()); } -unsigned long acpm_dvfs_get_rate(const struct acpm_handle *handle, +unsigned long acpm_dvfs_get_rate(struct acpm_handle *handle, unsigned int acpm_chan_id, unsigned int clk_id) { struct acpm_xfer xfer; diff --git a/drivers/firmware/samsung/exynos-acpm-dvfs.h b/drivers/firmware/samsung/exynos-acpm-dvfs.h index 9f2778e649c9d8..b37b1542610295 100644 --- a/drivers/firmware/samsung/exynos-acpm-dvfs.h +++ b/drivers/firmware/samsung/exynos-acpm-dvfs.h @@ -11,10 +11,10 @@ struct acpm_handle; -int acpm_dvfs_set_rate(const struct acpm_handle *handle, +int acpm_dvfs_set_rate(struct acpm_handle *handle, unsigned int acpm_chan_id, unsigned int id, unsigned long rate); -unsigned long acpm_dvfs_get_rate(const struct acpm_handle *handle, +unsigned long acpm_dvfs_get_rate(struct acpm_handle *handle, unsigned int acpm_chan_id, unsigned int clk_id); diff --git a/drivers/firmware/samsung/exynos-acpm-pmic.c b/drivers/firmware/samsung/exynos-acpm-pmic.c index 961d7599e4224e..52e89d1b790f01 100644 --- a/drivers/firmware/samsung/exynos-acpm-pmic.c +++ b/drivers/firmware/samsung/exynos-acpm-pmic.c @@ -77,7 +77,7 @@ static void acpm_pmic_init_read_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan) cmd[3] = ktime_to_ms(ktime_get()); } -int acpm_pmic_read_reg(const struct acpm_handle *handle, +int acpm_pmic_read_reg(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 *buf) { @@ -107,7 +107,7 @@ static void acpm_pmic_init_bulk_read_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan, FIELD_PREP(ACPM_PMIC_VALUE, count); } -int acpm_pmic_bulk_read(const struct acpm_handle *handle, +int acpm_pmic_bulk_read(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 count, u8 *buf) { @@ -150,7 +150,7 @@ static void acpm_pmic_init_write_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan, cmd[3] = ktime_to_ms(ktime_get()); } -int acpm_pmic_write_reg(const struct acpm_handle *handle, +int acpm_pmic_write_reg(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 value) { @@ -187,7 +187,7 @@ static void acpm_pmic_init_bulk_write_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan, } } -int acpm_pmic_bulk_write(const struct acpm_handle *handle, +int acpm_pmic_bulk_write(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 count, const u8 *buf) { @@ -220,7 +220,7 @@ static void acpm_pmic_init_update_cmd(u32 cmd[4], u8 type, u8 reg, u8 chan, cmd[3] = ktime_to_ms(ktime_get()); } -int acpm_pmic_update_reg(const struct acpm_handle *handle, +int acpm_pmic_update_reg(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 value, u8 mask) { diff --git a/drivers/firmware/samsung/exynos-acpm-pmic.h b/drivers/firmware/samsung/exynos-acpm-pmic.h index 078421888a1409..88ae9aada2aea1 100644 --- a/drivers/firmware/samsung/exynos-acpm-pmic.h +++ b/drivers/firmware/samsung/exynos-acpm-pmic.h @@ -11,19 +11,19 @@ struct acpm_handle; -int acpm_pmic_read_reg(const struct acpm_handle *handle, +int acpm_pmic_read_reg(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 *buf); -int acpm_pmic_bulk_read(const struct acpm_handle *handle, +int acpm_pmic_bulk_read(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 count, u8 *buf); -int acpm_pmic_write_reg(const struct acpm_handle *handle, +int acpm_pmic_write_reg(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 value); -int acpm_pmic_bulk_write(const struct acpm_handle *handle, +int acpm_pmic_bulk_write(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 count, const u8 *buf); -int acpm_pmic_update_reg(const struct acpm_handle *handle, +int acpm_pmic_update_reg(struct acpm_handle *handle, unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, u8 value, u8 mask); #endif /* __EXYNOS_ACPM_PMIC_H__ */ diff --git a/drivers/firmware/samsung/exynos-acpm.c b/drivers/firmware/samsung/exynos-acpm.c index 0cb269c7046015..987b59778ffc4a 100644 --- a/drivers/firmware/samsung/exynos-acpm.c +++ b/drivers/firmware/samsung/exynos-acpm.c @@ -412,7 +412,7 @@ static int acpm_wait_for_message_response(struct acpm_chan *achan, * * Return: 0 on success, -errno otherwise. */ -int acpm_do_xfer(const struct acpm_handle *handle, const struct acpm_xfer *xfer) +int acpm_do_xfer(struct acpm_handle *handle, const struct acpm_xfer *xfer) { struct acpm_info *acpm = handle_to_acpm_info(handle); struct exynos_mbox_msg msg; @@ -674,7 +674,7 @@ static int acpm_probe(struct platform_device *pdev) * acpm_handle_put() - release the handle acquired by acpm_get_by_phandle. * @handle: Handle acquired by acpm_get_by_phandle. */ -static void acpm_handle_put(const struct acpm_handle *handle) +static void acpm_handle_put(struct acpm_handle *handle) { struct acpm_info *acpm = handle_to_acpm_info(handle); struct device *dev = acpm->dev; @@ -700,9 +700,11 @@ static void devm_acpm_release(struct device *dev, void *res) * @np: ACPM device tree node. * * Return: pointer to handle on success, ERR_PTR(-errno) otherwise. + * + * Note: handle CANNOT be pointer to const */ -static const struct acpm_handle *acpm_get_by_node(struct device *dev, - struct device_node *np) +static struct acpm_handle *acpm_get_by_node(struct device *dev, + struct device_node *np) { struct platform_device *pdev; struct device_link *link; @@ -743,10 +745,10 @@ static const struct acpm_handle *acpm_get_by_node(struct device *dev, * * Return: pointer to handle on success, ERR_PTR(-errno) otherwise. */ -const struct acpm_handle *devm_acpm_get_by_node(struct device *dev, - struct device_node *np) +struct acpm_handle *devm_acpm_get_by_node(struct device *dev, + struct device_node *np) { - const struct acpm_handle **ptr, *handle; + struct acpm_handle **ptr, *handle; ptr = devres_alloc(devm_acpm_release, sizeof(*ptr), GFP_KERNEL); if (!ptr) diff --git a/drivers/firmware/samsung/exynos-acpm.h b/drivers/firmware/samsung/exynos-acpm.h index 2d14cb58f98c93..6417550f89aa97 100644 --- a/drivers/firmware/samsung/exynos-acpm.h +++ b/drivers/firmware/samsung/exynos-acpm.h @@ -17,7 +17,7 @@ struct acpm_xfer { struct acpm_handle; -int acpm_do_xfer(const struct acpm_handle *handle, +int acpm_do_xfer(struct acpm_handle *handle, const struct acpm_xfer *xfer); #endif /* __EXYNOS_ACPM_H__ */ diff --git a/drivers/fwctl/main.c b/drivers/fwctl/main.c index bc6378506296cd..098c3824ad751f 100644 --- a/drivers/fwctl/main.c +++ b/drivers/fwctl/main.c @@ -415,7 +415,7 @@ static void __exit fwctl_exit(void) unregister_chrdev_region(fwctl_dev, FWCTL_MAX_DEVICES); } -module_init(fwctl_init); +subsys_initcall(fwctl_init); module_exit(fwctl_exit); MODULE_DESCRIPTION("fwctl device firmware access framework"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index ef1ac68b94b78f..08b7b662512b82 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -1210,7 +1210,14 @@ int of_gpiochip_add(struct gpio_chip *chip) void of_gpiochip_remove(struct gpio_chip *chip) { - of_node_put(dev_of_node(&chip->gpiodev->dev)); + struct device_node *np = dev_of_node(&chip->gpiodev->dev); + + for_each_child_of_node_scoped(np, child) { + if (of_property_present(child, "gpio-hog")) + of_node_clear_flag(child, OF_POPULATED); + } + + of_node_put(np); } bool of_gpiochip_instance_match(struct gpio_chip *gc, unsigned int index) diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index d3d52310c9cc15..743a5f993af904 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -257,6 +257,13 @@ config DRM_GEM_SHMEM_HELPER help Choose this if you need the GEM shmem helper functions +config RUST_DRM_GEM_SHMEM_HELPER + bool + depends on DRM && MMU + select DRM_GEM_SHMEM_HELPER + help + Choose this if you need the GEM shmem helper functions In Rust + config DRM_SUBALLOC_HELPER tristate depends on DRM @@ -280,8 +287,10 @@ config DRM_PRIVACY_SCREEN # DRM driver Kconfig files, sorted source "drivers/gpu/drm/adp/Kconfig" source "drivers/gpu/drm/amd/amdgpu/Kconfig" +source "drivers/gpu/drm/apple/Kconfig" source "drivers/gpu/drm/arm/Kconfig" source "drivers/gpu/drm/armada/Kconfig" +source "drivers/gpu/drm/asahi/Kconfig" source "drivers/gpu/drm/aspeed/Kconfig" source "drivers/gpu/drm/ast/Kconfig" source "drivers/gpu/drm/atmel-hlcdc/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index ec2c5ff8238263..8d368a7462e6db 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -220,12 +220,14 @@ obj-y += tiny/ obj-$(CONFIG_DRM_PL111) += pl111/ obj-$(CONFIG_DRM_TVE200) += tve200/ obj-$(CONFIG_DRM_ADP) += adp/ +obj-$(CONFIG_DRM_ASAHI) += asahi/ obj-$(CONFIG_DRM_XEN) += xen/ obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ obj-$(CONFIG_DRM_LIMA) += lima/ obj-$(CONFIG_DRM_PANFROST) += panfrost/ obj-$(CONFIG_DRM_PANTHOR) += panthor/ obj-$(CONFIG_DRM_TYR) += tyr/ +obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 29b400cdd6d5f9..72a5a29e63f6da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1735,7 +1735,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( alloc_domain = AMDGPU_GEM_DOMAIN_GTT; alloc_flags = 0; } else { - alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; + alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE | + AMDGPU_GEM_CREATE_VRAM_CLEARED; alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index d1bf2e150c1ad3..780a0078c91a43 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -1239,6 +1239,8 @@ static enum drm_mode_status amdgpu_connector_dvi_mode_valid(struct drm_connector case CONNECTOR_OBJECT_ID_HDMI_TYPE_B: max_digital_pixel_clock_khz = max_dvi_single_link_pixel_clock * 2; break; + default: + return MODE_BAD; } /* When the display EDID claims that it's an HDMI display, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 6d8531f9b882dd..2ec69fa05cb155 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3539,8 +3539,12 @@ static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) * that checks whether the PSP is running. A solution for those issues * in the APU is to trigger a GPU reset, but this should be done during * the unload phase to avoid adding boot latency and screen flicker. + * GFX V11 has GC block as default off IP. Every time AMDGPU driver sends + * a request to PMFW to unload MP1, PMFW will put GC in reset and power down + * the voltage. Hence, skipping reset for APUs with GFX V11 or later. */ - if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu) { + if ((adev->flags & AMD_IS_APU) && !adev->gmc.is_app_apu && + amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 0, 0)) { r = amdgpu_asic_reset(adev); if (r) dev_err(adev->dev, "asic reset on %s failed\n", __func__); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index af3d2fd61cf3fc..3459d356151ef3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -2986,10 +2986,8 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) case IP_VERSION(11, 5, 1): case IP_VERSION(11, 5, 2): case IP_VERSION(11, 5, 3): - adev->family = AMDGPU_FAMILY_GC_11_5_0; - break; case IP_VERSION(11, 5, 4): - adev->family = AMDGPU_FAMILY_GC_11_5_4; + adev->family = AMDGPU_FAMILY_GC_11_5_0; break; case IP_VERSION(12, 0, 0): case IP_VERSION(12, 0, 1): diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c index bc772ca3dab726..b6f849d51c2e77 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c @@ -262,12 +262,19 @@ void amdgpu_gart_table_ram_free(struct amdgpu_device *adev) */ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev) { + int r; + if (adev->gart.bo != NULL) return 0; - return amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo, - NULL, (void *)&adev->gart.ptr); + r = amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo, + NULL, (void *)&adev->gart.ptr); + if (r) + return r; + + memset_io(adev->gart.ptr, adev->gart.gart_pte_flags, adev->gart.table_size); + return 0; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 5179fa008626e0..19abb09f0e1974 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -313,7 +313,10 @@ void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, mc->gart_start = max_mc_address - mc->gart_size + 1; break; case AMDGPU_GART_PLACEMENT_LOW: - mc->gart_start = 0; + if (size_bf >= mc->gart_size) + mc->gart_start = 0; + else + mc->gart_start = ALIGN(mc->fb_end, four_gb); break; case AMDGPU_GART_PLACEMENT_BEST_FIT: default: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c index 569c5a89ff10b6..124fb38eb46532 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c @@ -22,7 +22,7 @@ */ #include "amdgpu_ids.h" -#include +#include #include @@ -40,8 +40,8 @@ * VMs are looked up from the PASID per amdgpu_device. */ -static DEFINE_IDR(amdgpu_pasid_idr); -static DEFINE_SPINLOCK(amdgpu_pasid_idr_lock); +static DEFINE_XARRAY_FLAGS(amdgpu_pasid_xa, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ALLOC1); +static u32 amdgpu_pasid_xa_next; /* Helper to free pasid from a fence callback */ struct amdgpu_pasid_cb { @@ -62,36 +62,37 @@ struct amdgpu_pasid_cb { */ int amdgpu_pasid_alloc(unsigned int bits) { - int pasid; + u32 pasid; + int r; if (bits == 0) return -EINVAL; - spin_lock(&amdgpu_pasid_idr_lock); - /* TODO: Need to replace the idr with an xarry, and then - * handle the internal locking with ATOMIC safe paths. - */ - pasid = idr_alloc_cyclic(&amdgpu_pasid_idr, NULL, 1, - 1U << bits, GFP_ATOMIC); - spin_unlock(&amdgpu_pasid_idr_lock); - - if (pasid >= 0) - trace_amdgpu_pasid_allocated(pasid); + r = xa_alloc_cyclic_irq(&amdgpu_pasid_xa, &pasid, xa_mk_value(0), + XA_LIMIT(1, (1U << bits) - 1), + &amdgpu_pasid_xa_next, GFP_KERNEL); + if (r < 0) + return r; + trace_amdgpu_pasid_allocated(pasid); return pasid; } /** * amdgpu_pasid_free - Free a PASID * @pasid: PASID to free + * + * Called in IRQ context. */ void amdgpu_pasid_free(u32 pasid) { + unsigned long flags; + trace_amdgpu_pasid_freed(pasid); - spin_lock(&amdgpu_pasid_idr_lock); - idr_remove(&amdgpu_pasid_idr, pasid); - spin_unlock(&amdgpu_pasid_idr_lock); + xa_lock_irqsave(&amdgpu_pasid_xa, flags); + __xa_erase(&amdgpu_pasid_xa, pasid); + xa_unlock_irqrestore(&amdgpu_pasid_xa, flags); } static void amdgpu_pasid_free_cb(struct dma_fence *fence, @@ -634,7 +635,5 @@ void amdgpu_vmid_mgr_fini(struct amdgpu_device *adev) */ void amdgpu_pasid_mgr_cleanup(void) { - spin_lock(&amdgpu_pasid_idr_lock); - idr_destroy(&amdgpu_pasid_idr); - spin_unlock(&amdgpu_pasid_idr_lock); + xa_destroy(&amdgpu_pasid_xa); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 7f19554b9ad11d..a50c3058f97f4b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -873,68 +873,59 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) ? -EFAULT : 0; } case AMDGPU_INFO_READ_MMR_REG: { - int ret = 0; - unsigned int n, alloc_size; - uint32_t *regs; unsigned int se_num = (info->read_mmr_reg.instance >> AMDGPU_INFO_MMR_SE_INDEX_SHIFT) & AMDGPU_INFO_MMR_SE_INDEX_MASK; unsigned int sh_num = (info->read_mmr_reg.instance >> AMDGPU_INFO_MMR_SH_INDEX_SHIFT) & AMDGPU_INFO_MMR_SH_INDEX_MASK; - - if (!down_read_trylock(&adev->reset_domain->sem)) - return -ENOENT; + unsigned int alloc_size; + uint32_t *regs; + int ret; /* set full masks if the userspace set all bits * in the bitfields */ - if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) { + if (se_num == AMDGPU_INFO_MMR_SE_INDEX_MASK) se_num = 0xffffffff; - } else if (se_num >= AMDGPU_GFX_MAX_SE) { - ret = -EINVAL; - goto out; - } + else if (se_num >= AMDGPU_GFX_MAX_SE) + return -EINVAL; - if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) { + if (sh_num == AMDGPU_INFO_MMR_SH_INDEX_MASK) sh_num = 0xffffffff; - } else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE) { - ret = -EINVAL; - goto out; - } + else if (sh_num >= AMDGPU_GFX_MAX_SH_PER_SE) + return -EINVAL; - if (info->read_mmr_reg.count > 128) { - ret = -EINVAL; - goto out; - } + if (info->read_mmr_reg.count > 128) + return -EINVAL; - regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), GFP_KERNEL); - if (!regs) { - ret = -ENOMEM; - goto out; - } + regs = kmalloc_array(info->read_mmr_reg.count, sizeof(*regs), + GFP_KERNEL); + if (!regs) + return -ENOMEM; + down_read(&adev->reset_domain->sem); alloc_size = info->read_mmr_reg.count * sizeof(*regs); - amdgpu_gfx_off_ctrl(adev, false); + ret = 0; for (i = 0; i < info->read_mmr_reg.count; i++) { if (amdgpu_asic_read_register(adev, se_num, sh_num, info->read_mmr_reg.dword_offset + i, ®s[i])) { DRM_DEBUG_KMS("unallowed offset %#x\n", info->read_mmr_reg.dword_offset + i); - kfree(regs); - amdgpu_gfx_off_ctrl(adev, true); ret = -EFAULT; - goto out; + break; } } amdgpu_gfx_off_ctrl(adev, true); - n = copy_to_user(out, regs, min(size, alloc_size)); - kfree(regs); - ret = (n ? -EFAULT : 0); -out: up_read(&adev->reset_domain->sem); + + if (!ret) { + ret = copy_to_user(out, regs, min(size, alloc_size)) + ? -EFAULT : 0; + } + kfree(regs); return ret; } case AMDGPU_INFO_DEV_INFO: { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 6fba9d5b29ea65..ee271f43d5ad08 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -1939,7 +1939,7 @@ void amdgpu_ras_check_bad_page_status(struct amdgpu_device *adev) if (!control || amdgpu_bad_page_threshold == 0) return; - if (control->ras_num_bad_pages >= ras->bad_page_cnt_threshold) { + if (control->ras_num_bad_pages > ras->bad_page_cnt_threshold) { if (amdgpu_dpm_send_rma_reason(adev)) dev_warn(adev->dev, "Unable to send out-of-band RMA CPER"); else diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index cb0fb1a989d2f5..14fd31d3437a85 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -553,15 +553,18 @@ void amdgpu_debugfs_ring_init(struct amdgpu_device *adev, int amdgpu_ring_init_mqd(struct amdgpu_ring *ring); -static inline u32 amdgpu_ib_get_value(struct amdgpu_ib *ib, int idx) +static inline u32 amdgpu_ib_get_value(struct amdgpu_ib *ib, uint32_t idx) { - return ib->ptr[idx]; + if (idx < ib->length_dw) + return ib->ptr[idx]; + return 0; } -static inline void amdgpu_ib_set_value(struct amdgpu_ib *ib, int idx, +static inline void amdgpu_ib_set_value(struct amdgpu_ib *ib, uint32_t idx, uint32_t value) { - ib->ptr[idx] = value; + if (idx < ib->length_dw) + ib->ptr[idx] = value; } int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 0ccb31788b20bc..95060217e3b024 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -75,6 +75,9 @@ static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev, unsigned int type, uint64_t size_in_page) { + if (!size_in_page) + return 0; + return ttm_range_man_init(&adev->mman.bdev, type, false, size_in_page); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 0a1b93259887ab..0e015741ab24e3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -156,7 +156,7 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work) struct dma_fence *fence; struct amdgpu_userq_mgr *uq_mgr; - if (!queue || !queue->userq_mgr) + if (!queue->userq_mgr) return; uq_mgr = queue->userq_mgr; @@ -1231,7 +1231,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr) bo = range->bo; ret = amdgpu_ttm_tt_get_user_pages(bo, range); if (ret) - goto unlock_all; + goto free_ranges; } invalidated = true; @@ -1258,6 +1258,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr) unlock_all: drm_exec_fini(&exec); +free_ranges: xa_for_each(&xa, tmp_key, range) { if (!range) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index a7d8f1ce6ac2d7..97cf2f4c58a639 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -698,6 +698,9 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, struct amdgpu_ib *ib, uint64_t addr; int r; + if (lo >= ib->length_dw || hi >= ib->length_dw) + return -EINVAL; + if (index == 0xffffffff) index = 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 275745aa582929..1e284ecad2170a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -950,11 +950,6 @@ int amdgpu_virt_init_critical_region(struct amdgpu_device *adev) if (adev->virt.req_init_data_ver != GPU_CRIT_REGION_V2) return 0; - if (init_hdr_offset < 0) { - dev_err(adev->dev, "Invalid init header offset\n"); - return -EINVAL; - } - vram_size = RREG32(mmRCC_CONFIG_MEMSIZE); if (!vram_size || vram_size == U32_MAX) return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c index 22e2e5b4734154..f078db3fef79e1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c @@ -21,6 +21,8 @@ */ #include "amdgpu_vm.h" +#include "amdgpu.h" +#include "amdgpu_reset.h" #include "amdgpu_object.h" #include "amdgpu_trace.h" @@ -108,11 +110,19 @@ static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p, static int amdgpu_vm_cpu_commit(struct amdgpu_vm_update_params *p, struct dma_fence **fence) { + struct amdgpu_device *adev = p->adev; + if (p->needs_flush) atomic64_inc(&p->vm->tlb_seq); mb(); - amdgpu_device_flush_hdp(p->adev, NULL); + /* A reset flushed the HDP anyway, so that here can be skipped when a reset is ongoing */ + if (!down_read_trylock(&adev->reset_domain->sem)) + return 0; + + amdgpu_device_flush_hdp(adev, NULL); + up_read(&adev->reset_domain->sem); + return 0; } diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 1893ceeeb26c82..8b60299b73ef7b 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -6752,7 +6752,7 @@ static void gfx_v10_0_gfx_mqd_set_priority(struct amdgpu_device *adev, /* set up default queue priority level * 0x0 = low priority, 0x1 = high priority */ - if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH) + if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM) priority = 1; tmp = RREG32_SOC15(GC, 0, mmCP_GFX_HQD_QUEUE_PRIORITY); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 427975b5a1d975..0f783adb933512 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -64,6 +64,11 @@ #define regPC_CONFIG_CNTL_1 0x194d #define regPC_CONFIG_CNTL_1_BASE_IDX 1 +#define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0 0x0030 +#define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0_BASE_IDX 1 +#define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0 0x0031 +#define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0_BASE_IDX 1 + #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000 @@ -4088,7 +4093,7 @@ static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev, /* set up default queue priority level * 0x0 = low priority, 0x1 = high priority */ - if (prop->hqd_pipe_priority == AMDGPU_GFX_PIPE_PRIO_HIGH) + if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM) priority = 1; tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT; @@ -5187,11 +5192,27 @@ static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) amdgpu_gfx_off_ctrl(adev, true); } else { preempt_disable(); - clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER); - clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER); - clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_UPPER); - if (clock_counter_hi_pre != clock_counter_hi_after) - clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, regGOLDEN_TSC_COUNT_LOWER); + if (amdgpu_ip_version(adev, SMUIO_HWIP, 0) < IP_VERSION(15, 0, 0)) { + clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_UPPER); + clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_LOWER); + clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_UPPER); + if (clock_counter_hi_pre != clock_counter_hi_after) + clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_LOWER); + } else { + clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); + clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); + clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); + if (clock_counter_hi_pre != clock_counter_hi_after) + clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, + regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); + } preempt_enable(); } clock = clock_counter_lo | (clock_counter_hi_after << 32ULL); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c index eb9725ae1607a6..557d15b90ad274 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c @@ -1405,7 +1405,7 @@ static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev, /* * Configure apertures: * LDS: 0x20000000'00000000 - 0x20000001'00000000 (4GB) - * Scratch: 0x10000000'00000000 - 0x10000001'00000000 (4GB) + * Scratch: 0x10000000'00000000 - 0x11ffffff'ffffffff (128PB 57-bit) */ sh_mem_bases = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, (adev->gmc.private_aperture_start >> 58)); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index 73223d97a87f59..ac90d8e9d86a83 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -1571,6 +1571,71 @@ static void gfx_v6_0_setup_spi(struct amdgpu_device *adev) mutex_unlock(&adev->grbm_idx_mutex); } +/** + * gfx_v6_0_setup_tcc() - setup which TCCs are used + * + * @adev: amdgpu_device pointer + * + * Verify whether the current GPU has any TCCs disabled, + * which can happen when the GPU is harvested and some + * memory channels are disabled, reducing the memory bus width. + * For example, on the Radeon HD 7870 XT (Tahiti LE). + * + * If some TCCs are disabled, we need to make sure that + * the disabled TCCs are not used, and the remaining TCCs + * are used optimally. + * + * TCP_CHAN_STEER_LO/HI control which TCC is used by TCP channels. + * TCP_ADDR_CONFIG.NUM_TCC_BANKS controls how many channels are used. + * + * For optimal performance: + * - Rely on the CHAN_STEER from the golden registers table, + * only skip disabled TCCs but keep the mapping order. + * - Limit NUM_TCC_BANKS to number of active TCCs to avoid thrashing, + * which performs better than using the same TCC twice. + */ +static void gfx_v6_0_setup_tcc(struct amdgpu_device *adev) +{ + u32 i, tcc, tcp_addr_config, num_active_tcc = 0; + u64 chan_steer, patched_chan_steer = 0; + const u32 num_max_tcc = adev->gfx.config.max_texture_channel_caches; + const u32 dis_tcc_mask = + amdgpu_gfx_create_bitmask(num_max_tcc) & + (REG_GET_FIELD(RREG32(mmCGTS_TCC_DISABLE), + CGTS_TCC_DISABLE, TCC_DISABLE) | + REG_GET_FIELD(RREG32(mmCGTS_USER_TCC_DISABLE), + CGTS_USER_TCC_DISABLE, TCC_DISABLE)); + + /* When no TCC is disabled, the golden registers table already has optimal TCC setup */ + if (!dis_tcc_mask) + return; + + /* Each 4-bit nibble contains the index of a TCC used by all TCPs */ + chan_steer = RREG32(mmTCP_CHAN_STEER_LO) | ((u64)RREG32(mmTCP_CHAN_STEER_HI) << 32ull); + + /* Patch the TCP to TCC mapping to skip disabled TCCs */ + for (i = 0; i < num_max_tcc; ++i) { + tcc = (chan_steer >> (u64)(4 * i)) & 0xf; + + if (!((1 << tcc) & dis_tcc_mask)) { + /* Copy enabled TCC indices to the patched register value. */ + patched_chan_steer |= (u64)tcc << (u64)(4 * num_active_tcc); + ++num_active_tcc; + } + } + + WARN_ON(num_active_tcc != num_max_tcc - hweight32(dis_tcc_mask)); + + /* Patch number of TCCs used by TCPs */ + tcp_addr_config = REG_SET_FIELD(RREG32(mmTCP_ADDR_CONFIG), + TCP_ADDR_CONFIG, NUM_TCC_BANKS, + num_active_tcc - 1); + + WREG32(mmTCP_ADDR_CONFIG, tcp_addr_config); + WREG32(mmTCP_CHAN_STEER_HI, upper_32_bits(patched_chan_steer)); + WREG32(mmTCP_CHAN_STEER_LO, lower_32_bits(patched_chan_steer)); +} + static void gfx_v6_0_config_init(struct amdgpu_device *adev) { adev->gfx.config.double_offchip_lds_buf = 0; @@ -1729,6 +1794,7 @@ static void gfx_v6_0_constants_init(struct amdgpu_device *adev) gfx_v6_0_tiling_mode_table_init(adev); gfx_v6_0_setup_rb(adev); + gfx_v6_0_setup_tcc(adev); gfx_v6_0_setup_spi(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 7e9d753f4a808d..8fe6d0db3dad97 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -5660,9 +5660,6 @@ static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c index b9671fc39e2a88..da4a0cf4aad0c3 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c @@ -654,9 +654,15 @@ static int gmc_v12_0_early_init(struct amdgpu_ip_block *ip_block) adev->gmc.shared_aperture_start = 0x2000000000000000ULL; adev->gmc.shared_aperture_end = adev->gmc.shared_aperture_start + (4ULL << 30) - 1; + adev->gmc.private_aperture_start = 0x1000000000000000ULL; - adev->gmc.private_aperture_end = - adev->gmc.private_aperture_start + (4ULL << 30) - 1; + if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) + adev->gmc.private_aperture_end = + adev->gmc.private_aperture_start + (1ULL << 57) - 1; + else + adev->gmc.private_aperture_end = + adev->gmc.private_aperture_start + (4ULL << 30) - 1; + adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF; return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c index 9fe8d10ab2705a..cffb1e6bab353f 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_0.c @@ -802,6 +802,7 @@ static const struct amd_ip_funcs jpeg_v2_0_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v2_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v2_0_dec_ring_get_rptr, .get_wptr = jpeg_v2_0_dec_ring_get_wptr, .set_wptr = jpeg_v2_0_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c index 20983f126b4907..13a6e24c624a24 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c @@ -693,6 +693,7 @@ static const struct amd_ip_funcs jpeg_v2_6_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v2_5_dec_ring_get_rptr, .get_wptr = jpeg_v2_5_dec_ring_get_wptr, .set_wptr = jpeg_v2_5_dec_ring_set_wptr, @@ -724,6 +725,7 @@ static const struct amdgpu_ring_funcs jpeg_v2_5_dec_ring_vm_funcs = { static const struct amdgpu_ring_funcs jpeg_v2_6_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v2_5_dec_ring_get_rptr, .get_wptr = jpeg_v2_5_dec_ring_get_wptr, .set_wptr = jpeg_v2_5_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c index 98f5e0622bc58a..d0445df39d2c04 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v3_0.c @@ -594,6 +594,7 @@ static const struct amd_ip_funcs jpeg_v3_0_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v3_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v3_0_dec_ring_get_rptr, .get_wptr = jpeg_v3_0_dec_ring_get_wptr, .set_wptr = jpeg_v3_0_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c index 0bd83820dd20ca..6fd4238a8471a9 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0.c @@ -759,6 +759,7 @@ static const struct amd_ip_funcs jpeg_v4_0_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v4_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v4_0_dec_ring_get_rptr, .get_wptr = jpeg_v4_0_dec_ring_get_wptr, .set_wptr = jpeg_v4_0_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c index 4b4aa955362423..0c746580de1130 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c @@ -736,15 +736,35 @@ static void jpeg_v4_0_3_dec_ring_set_wptr(struct amdgpu_ring *ring) */ void jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring *ring) { - if (!amdgpu_sriov_vf(ring->adev)) { + struct amdgpu_device *adev = ring->adev; + + if (!amdgpu_sriov_vf(adev)) { + int jpeg_inst = GET_INST(JPEG, ring->me); + uint32_t value = 0x80004000; /* default DS14 */ + amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET, 0, 0, PACKETJ_TYPE0)); - amdgpu_ring_write(ring, 0x62a04); /* PCTL0_MMHUB_DEEPSLEEP_IB */ + + /* PCTL0__MMHUB_DEEPSLEEP_IB could be different on different mmhub version */ + switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { + case IP_VERSION(4, 1, 0): + amdgpu_ring_write(ring, 0x69004); + value = 0x80010000; + break; + case IP_VERSION(4, 2, 0): + amdgpu_ring_write(ring, 0x60804); + if (jpeg_inst & 1) + value = 0x80010000; + break; + default: + amdgpu_ring_write(ring, 0x62a04); + break; + } amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR, 0, 0, PACKETJ_TYPE0)); - amdgpu_ring_write(ring, 0x80004000); + amdgpu_ring_write(ring, value); } } @@ -757,15 +777,35 @@ void jpeg_v4_0_3_dec_ring_insert_start(struct amdgpu_ring *ring) */ void jpeg_v4_0_3_dec_ring_insert_end(struct amdgpu_ring *ring) { - if (!amdgpu_sriov_vf(ring->adev)) { + struct amdgpu_device *adev = ring->adev; + + if (!amdgpu_sriov_vf(adev)) { + int jpeg_inst = GET_INST(JPEG, ring->me); + uint32_t value = 0x00004000; /* default DS14 */ + amdgpu_ring_write(ring, PACKETJ(regUVD_JRBC_EXTERNAL_REG_INTERNAL_OFFSET, 0, 0, PACKETJ_TYPE0)); - amdgpu_ring_write(ring, 0x62a04); + + /* PCTL0__MMHUB_DEEPSLEEP_IB could be different on different mmhub version */ + switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { + case IP_VERSION(4, 1, 0): + amdgpu_ring_write(ring, 0x69004); + value = 0x00010000; + break; + case IP_VERSION(4, 2, 0): + amdgpu_ring_write(ring, 0x60804); + if (jpeg_inst & 1) + value = 0x00010000; + break; + default: + amdgpu_ring_write(ring, 0x62a04); + break; + } amdgpu_ring_write(ring, PACKETJ(JRBC_DEC_EXTERNAL_REG_WRITE_ADDR, 0, 0, PACKETJ_TYPE0)); - amdgpu_ring_write(ring, 0x00004000); + amdgpu_ring_write(ring, value); } } @@ -1179,6 +1219,7 @@ static const struct amd_ip_funcs jpeg_v4_0_3_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v4_0_3_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v4_0_3_dec_ring_get_rptr, .get_wptr = jpeg_v4_0_3_dec_ring_get_wptr, .set_wptr = jpeg_v4_0_3_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c index 54fd9c800c40af..a43582b9c876c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c @@ -804,6 +804,7 @@ static const struct amd_ip_funcs jpeg_v4_0_5_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v4_0_5_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v4_0_5_dec_ring_get_rptr, .get_wptr = jpeg_v4_0_5_dec_ring_get_wptr, .set_wptr = jpeg_v4_0_5_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c index 46bf15dce2bd04..72a4b2d0676fa4 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c @@ -680,6 +680,7 @@ static const struct amd_ip_funcs jpeg_v5_0_0_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v5_0_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v5_0_0_dec_ring_get_rptr, .get_wptr = jpeg_v5_0_0_dec_ring_get_wptr, .set_wptr = jpeg_v5_0_0_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c index edecbfe66c79a1..250316704dfac4 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c @@ -884,6 +884,7 @@ static const struct amd_ip_funcs jpeg_v5_0_1_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v5_0_1_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v5_0_1_dec_ring_get_rptr, .get_wptr = jpeg_v5_0_1_dec_ring_get_wptr, .set_wptr = jpeg_v5_0_1_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c index 1821dced936fb9..e7546816baba34 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_3_0.c @@ -661,6 +661,7 @@ static const struct amd_ip_funcs jpeg_v5_3_0_ip_funcs = { static const struct amdgpu_ring_funcs jpeg_v5_3_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_JPEG, .align_mask = 0xf, + .no_user_fence = true, .get_rptr = jpeg_v5_3_0_dec_ring_get_rptr, .get_wptr = jpeg_v5_3_0_dec_ring_get_wptr, .set_wptr = jpeg_v5_3_0_dec_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index faac21ee5739c0..312604847e068e 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -30,34 +30,6 @@ #define AMDGPU_USERQ_PROC_CTX_SZ PAGE_SIZE #define AMDGPU_USERQ_GANG_CTX_SZ PAGE_SIZE -static int -mes_userq_map_gtt_bo_to_gart(struct amdgpu_bo *bo) -{ - int ret; - - ret = amdgpu_bo_reserve(bo, true); - if (ret) { - DRM_ERROR("Failed to reserve bo. ret %d\n", ret); - goto err_reserve_bo_failed; - } - - ret = amdgpu_ttm_alloc_gart(&bo->tbo); - if (ret) { - DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret); - goto err_map_bo_gart_failed; - } - - amdgpu_bo_unreserve(bo); - bo = amdgpu_bo_ref(bo); - - return 0; - -err_map_bo_gart_failed: - amdgpu_bo_unreserve(bo); -err_reserve_bo_failed: - return ret; -} - static int mes_userq_create_wptr_mapping(struct amdgpu_device *adev, struct amdgpu_userq_mgr *uq_mgr, @@ -65,55 +37,62 @@ mes_userq_create_wptr_mapping(struct amdgpu_device *adev, uint64_t wptr) { struct amdgpu_bo_va_mapping *wptr_mapping; - struct amdgpu_vm *wptr_vm; struct amdgpu_userq_obj *wptr_obj = &queue->wptr_obj; + struct amdgpu_bo *obj; + struct amdgpu_vm *vm = queue->vm; + struct drm_exec exec; int ret; - wptr_vm = queue->vm; - ret = amdgpu_bo_reserve(wptr_vm->root.bo, false); - if (ret) - return ret; - wptr &= AMDGPU_GMC_HOLE_MASK; - wptr_mapping = amdgpu_vm_bo_lookup_mapping(wptr_vm, wptr >> PAGE_SHIFT); - amdgpu_bo_unreserve(wptr_vm->root.bo); - if (!wptr_mapping) { - DRM_ERROR("Failed to lookup wptr bo\n"); - return -EINVAL; - } - wptr_obj->obj = wptr_mapping->bo_va->base.bo; - if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { - DRM_ERROR("Requested GART mapping for wptr bo larger than one page\n"); - return -EINVAL; - } + drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2); + drm_exec_until_all_locked(&exec) { + ret = amdgpu_vm_lock_pd(vm, &exec, 1); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto fail_lock; + + wptr_mapping = amdgpu_vm_bo_lookup_mapping(vm, wptr >> PAGE_SHIFT); + if (!wptr_mapping) { + ret = -EINVAL; + goto fail_lock; + } - ret = mes_userq_map_gtt_bo_to_gart(wptr_obj->obj); - if (ret) { - DRM_ERROR("Failed to map wptr bo to GART\n"); - return ret; + obj = wptr_mapping->bo_va->base.bo; + ret = drm_exec_lock_obj(&exec, &obj->tbo.base); + drm_exec_retry_on_contention(&exec); + if (unlikely(ret)) + goto fail_lock; } - ret = amdgpu_bo_reserve(wptr_obj->obj, true); - if (ret) { - DRM_ERROR("Failed to reserve wptr bo\n"); - return ret; + wptr_obj->obj = amdgpu_bo_ref(wptr_mapping->bo_va->base.bo); + if (wptr_obj->obj->tbo.base.size > PAGE_SIZE) { + ret = -EINVAL; + goto fail_map; } /* TODO use eviction fence instead of pinning. */ ret = amdgpu_bo_pin(wptr_obj->obj, AMDGPU_GEM_DOMAIN_GTT); if (ret) { - drm_file_err(uq_mgr->file, "[Usermode queues] Failed to pin wptr bo\n"); - goto unresv_bo; + DRM_ERROR("Failed to pin wptr bo. ret %d\n", ret); + goto fail_map; + } + + ret = amdgpu_ttm_alloc_gart(&wptr_obj->obj->tbo); + if (ret) { + DRM_ERROR("Failed to bind bo to GART. ret %d\n", ret); + goto fail_map; } queue->wptr_obj.gpu_addr = amdgpu_bo_gpu_offset(wptr_obj->obj); - amdgpu_bo_unreserve(wptr_obj->obj); + drm_exec_fini(&exec); return 0; -unresv_bo: - amdgpu_bo_unreserve(wptr_obj->obj); +fail_map: + amdgpu_bo_unref(&wptr_obj->obj); +fail_lock: + drm_exec_fini(&exec); return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c index db14a1a326d20a..b6f832c538603c 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c @@ -54,6 +54,8 @@ #define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10_BASE_IDX 3 #define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1_nbif_4_10 0x4f0af6 #define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1_nbif_4_10_BASE_IDX 3 +#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10 0x0021 +#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10_BASE_IDX 2 static void nbif_v6_3_1_remap_hdp_registers(struct amdgpu_device *adev) { @@ -65,7 +67,12 @@ static void nbif_v6_3_1_remap_hdp_registers(struct amdgpu_device *adev) static u32 nbif_v6_3_1_get_rev_id(struct amdgpu_device *adev) { - u32 tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0); + u32 tmp; + + if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) + tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_nbif_4_10); + else + tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0); tmp &= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK; tmp >>= RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c index 73a709773e85ba..2a8582e87f2b99 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0.c @@ -32,6 +32,7 @@ #include "mp/mp_15_0_0_sh_mask.h" MODULE_FIRMWARE("amdgpu/psp_15_0_0_toc.bin"); +MODULE_FIRMWARE("amdgpu/psp_15_0_0_ta.bin"); static int psp_v15_0_0_init_microcode(struct psp_context *psp) { diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index f38004e6064e5c..4d0dc58c904584 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -890,7 +890,7 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se /* write the fence */ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -900,7 +900,7 @@ static void sdma_v4_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se addr += 4; amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c index fea576a7f397f0..efb3fde919ee3b 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v3_1.c @@ -242,6 +242,10 @@ static void uvd_v3_1_mc_resume(struct amdgpu_device *adev) uint64_t addr; uint32_t size; + /* When the keyselect is already set, don't perturb it. */ + if (RREG32(mmUVD_FW_START)) + return; + /* program the VCPU memory controller bits 0-27 */ addr = (adev->uvd.inst->gpu_addr + AMDGPU_UVD_FIRMWARE_OFFSET) >> 3; size = AMDGPU_UVD_FIRMWARE_SIZE(adev) >> 3; @@ -284,6 +288,12 @@ static int uvd_v3_1_fw_validate(struct amdgpu_device *adev) int i; uint32_t keysel = adev->uvd.keyselect; + if (RREG32(mmUVD_FW_START) & UVD_FW_STATUS__PASS_MASK) { + dev_dbg(adev->dev, "UVD keyselect already set: 0x%x (on CPU: 0x%x)\n", + RREG32(mmUVD_FW_START), adev->uvd.keyselect); + return 0; + } + WREG32(mmUVD_FW_START, keysel); for (i = 0; i < 10; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c index 73ce3d211ed64c..8a9ba2276275cf 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c @@ -93,6 +93,11 @@ static void uvd_v4_2_ring_set_wptr(struct amdgpu_ring *ring) static int uvd_v4_2_early_init(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + + /* UVD doesn't work without DPM, it needs DPM to ungate it. */ + if (!amdgpu_dpm) + return -ENOENT; + adev->uvd.num_uvd_inst = 1; uvd_v4_2_set_ring_funcs(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c index e35fae9cdaf66e..0442bfcfd384d3 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c @@ -2113,6 +2113,7 @@ static const struct amd_ip_funcs vcn_v2_0_ip_funcs = { static const struct amdgpu_ring_funcs vcn_v2_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_DEC, .align_mask = 0xf, + .no_user_fence = true, .secure_submission_supported = true, .get_rptr = vcn_v2_0_dec_ring_get_rptr, .get_wptr = vcn_v2_0_dec_ring_get_wptr, @@ -2145,6 +2146,7 @@ static const struct amdgpu_ring_funcs vcn_v2_0_enc_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v2_0_enc_ring_get_rptr, .get_wptr = vcn_v2_0_enc_ring_get_wptr, .set_wptr = vcn_v2_0_enc_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c index 006a1545119717..8b8184fe6764b1 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c @@ -1778,6 +1778,7 @@ static void vcn_v2_5_dec_ring_set_wptr(struct amdgpu_ring *ring) static const struct amdgpu_ring_funcs vcn_v2_5_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_DEC, .align_mask = 0xf, + .no_user_fence = true, .secure_submission_supported = true, .get_rptr = vcn_v2_5_dec_ring_get_rptr, .get_wptr = vcn_v2_5_dec_ring_get_wptr, @@ -1879,6 +1880,7 @@ static const struct amdgpu_ring_funcs vcn_v2_5_enc_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v2_5_enc_ring_get_rptr, .get_wptr = vcn_v2_5_enc_ring_get_wptr, .set_wptr = vcn_v2_5_enc_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index 02d5c5af65f232..81bba3ec2a9378 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -1856,6 +1856,7 @@ static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_DEC, .align_mask = 0x3f, .nop = VCN_DEC_SW_CMD_NO_OP, + .no_user_fence = true, .secure_submission_supported = true, .get_rptr = vcn_v3_0_dec_ring_get_rptr, .get_wptr = vcn_v3_0_dec_ring_get_wptr, @@ -1909,7 +1910,7 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, struct ttm_operation_ctx ctx = { false, false }; struct amdgpu_device *adev = p->adev; struct amdgpu_bo_va_mapping *map; - uint32_t *msg, num_buffers; + uint32_t *msg, num_buffers, len_dw; struct amdgpu_bo *bo; uint64_t start, end; unsigned int i; @@ -1930,6 +1931,11 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, return -EINVAL; } + if (end - addr < 16) { + DRM_ERROR("VCN messages must be at least 4 DWORDs!\n"); + return -EINVAL; + } + bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); @@ -1946,8 +1952,8 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, msg = ptr + addr - start; - /* Check length */ if (msg[1] > end - addr) { + DRM_ERROR("VCN message header does not fit in BO!\n"); r = -EINVAL; goto out; } @@ -1955,9 +1961,19 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, if (msg[3] != RDECODE_MSG_CREATE) goto out; + len_dw = msg[1] / 4; num_buffers = msg[2]; + + /* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */ + if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) { + DRM_ERROR("VCN message has too many buffers!\n"); + r = -EINVAL; + goto out; + } + for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) { uint32_t offset, size, *create; + uint64_t buf_end; if (msg[0] != RDECODE_MESSAGE_CREATE) continue; @@ -1965,14 +1981,16 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, offset = msg[1]; size = msg[2]; - if (offset + size > end) { + if (size < 4 || check_add_overflow(offset, size, &buf_end) || + buf_end > end - addr) { + DRM_ERROR("VCN message buffer exceeds BO bounds!\n"); r = -EINVAL; goto out; } create = ptr + addr + offset - start; - /* H246, HEVC and VP9 can run on any instance */ + /* H264, HEVC and VP9 can run on any instance */ if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11) continue; @@ -2021,6 +2039,7 @@ static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, static const struct amdgpu_ring_funcs vcn_v3_0_dec_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_DEC, .align_mask = 0xf, + .no_user_fence = true, .secure_submission_supported = true, .get_rptr = vcn_v3_0_dec_ring_get_rptr, .get_wptr = vcn_v3_0_dec_ring_get_wptr, @@ -2123,6 +2142,7 @@ static const struct amdgpu_ring_funcs vcn_v3_0_enc_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v3_0_enc_ring_get_rptr, .get_wptr = vcn_v3_0_enc_ring_get_wptr, .set_wptr = vcn_v3_0_enc_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index d17219be50f393..ff7269bafae8ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -1826,7 +1826,7 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, struct ttm_operation_ctx ctx = { false, false }; struct amdgpu_device *adev = p->adev; struct amdgpu_bo_va_mapping *map; - uint32_t *msg, num_buffers; + uint32_t *msg, num_buffers, len_dw; struct amdgpu_bo *bo; uint64_t start, end; unsigned int i; @@ -1847,6 +1847,11 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, return -EINVAL; } + if (end - addr < 16) { + DRM_ERROR("VCN messages must be at least 4 DWORDs!\n"); + return -EINVAL; + } + bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); @@ -1863,8 +1868,8 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, msg = ptr + addr - start; - /* Check length */ if (msg[1] > end - addr) { + DRM_ERROR("VCN message header does not fit in BO!\n"); r = -EINVAL; goto out; } @@ -1872,9 +1877,19 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, if (msg[3] != RDECODE_MSG_CREATE) goto out; + len_dw = msg[1] / 4; num_buffers = msg[2]; + + /* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */ + if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) { + DRM_ERROR("VCN message has too many buffers!\n"); + r = -EINVAL; + goto out; + } + for (i = 0, msg = &msg[6]; i < num_buffers; ++i, msg += 4) { uint32_t offset, size, *create; + uint64_t buf_end; if (msg[0] != RDECODE_MESSAGE_CREATE) continue; @@ -1882,7 +1897,9 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, offset = msg[1]; size = msg[2]; - if (offset + size > end) { + if (size < 4 || check_add_overflow(offset, size, &buf_end) || + buf_end > end - addr) { + DRM_ERROR("VCN message buffer exceeds BO bounds!\n"); r = -EINVAL; goto out; } @@ -1913,9 +1930,10 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start) { int i; + uint32_t len; - for (i = start; i < ib->length_dw && ib->ptr[i] >= 8; i += ib->ptr[i] / 4) { - if (ib->ptr[i + 1] == id) + for (i = start; (len = amdgpu_ib_get_value(ib, i)) >= 8; i += len / 4) { + if (amdgpu_ib_get_value(ib, i + 1) == id) return i; } return -1; @@ -1926,8 +1944,6 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, struct amdgpu_ib *ib) { struct amdgpu_ring *ring = amdgpu_job_ring(job); - struct amdgpu_vcn_decode_buffer *decode_buffer; - uint64_t addr; uint32_t val; int idx = 0, sidx; @@ -1938,20 +1954,22 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx)) >= 0) { val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */ if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { - decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[idx + 6]; + uint32_t valid_buf_flag = amdgpu_ib_get_value(ib, idx + 6); + uint64_t msg_buffer_addr; - if (!(decode_buffer->valid_buf_flag & 0x1)) + if (!(valid_buf_flag & 0x1)) return 0; - addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 | - decode_buffer->msg_buffer_address_lo; - return vcn_v4_0_dec_msg(p, job, addr); + msg_buffer_addr = ((u64)amdgpu_ib_get_value(ib, idx + 7)) << 32 | + amdgpu_ib_get_value(ib, idx + 8); + return vcn_v4_0_dec_msg(p, job, msg_buffer_addr); } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) { sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx); - if (sidx >= 0 && ib->ptr[sidx + 2] == RENCODE_ENCODE_STANDARD_AV1) + if (sidx >= 0 && + amdgpu_ib_get_value(ib, sidx + 2) == RENCODE_ENCODE_STANDARD_AV1) return vcn_v4_0_limit_sched(p, job); } - idx += ib->ptr[idx] / 4; + idx += amdgpu_ib_get_value(ib, idx) / 4; } return 0; } @@ -1978,6 +1996,7 @@ static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .extra_bytes = sizeof(struct amdgpu_vcn_rb_metadata), .get_rptr = vcn_v4_0_unified_ring_get_rptr, .get_wptr = vcn_v4_0_unified_ring_get_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c index e78526a4e521e2..210e41cf293746 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c @@ -1758,6 +1758,7 @@ static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c index 1f6a22983c0dd5..1571cc5a148c87 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c @@ -1483,6 +1483,7 @@ static struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v4_0_5_unified_ring_get_rptr, .get_wptr = vcn_v4_0_5_unified_ring_get_wptr, .set_wptr = vcn_v4_0_5_unified_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c index 6109124f852e52..d5f49fa33bee4b 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c @@ -1207,6 +1207,7 @@ static const struct amdgpu_ring_funcs vcn_v5_0_0_unified_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v5_0_0_unified_ring_get_rptr, .get_wptr = vcn_v5_0_0_unified_ring_get_wptr, .set_wptr = vcn_v5_0_0_unified_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c index c28c6aff17aaa0..54fbf8d73ca679 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c @@ -1419,6 +1419,7 @@ static const struct amdgpu_ring_funcs vcn_v5_0_1_unified_ring_vm_funcs = { .type = AMDGPU_RING_TYPE_VCN_ENC, .align_mask = 0x3f, .nop = VCN_ENC_CMD_NO_OP, + .no_user_fence = true, .get_rptr = vcn_v5_0_1_unified_ring_get_rptr, .get_wptr = vcn_v5_0_1_unified_ring_get_wptr, .set_wptr = vcn_v5_0_1_unified_ring_set_wptr, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 462a32abf720ab..f95bf6d9553463 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -776,6 +777,9 @@ static int kfd_ioctl_get_process_apertures_new(struct file *filp, goto out_unlock; } + if (args->num_of_nodes > kfd_topology_get_num_devices()) + return -EINVAL; + /* Fill in process-aperture information for all available * nodes, but not more than args->num_of_nodes as that is * the amount of memory allocated by user @@ -1356,7 +1360,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep, peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]); if (WARN_ON_ONCE(!peer_pdd)) continue; - kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY); + kfd_flush_tlb(peer_pdd); } kfree(devices_arr); @@ -1451,7 +1455,7 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep, if (WARN_ON_ONCE(!peer_pdd)) continue; if (flush_tlb) - kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT); + kfd_flush_tlb(peer_pdd); /* Remove dma mapping after tlb flush to avoid IO_PAGE_FAULT */ err = amdgpu_amdkfd_gpuvm_dmaunmap_mem(mem, peer_pdd->drm_priv); @@ -1692,6 +1696,16 @@ static int kfd_ioctl_smi_events(struct file *filep, return kfd_smi_event_open(pdd->dev, &args->anon_fd); } +static int kfd_ioctl_svm_validate(void *kdata, unsigned int usize) +{ + struct kfd_ioctl_svm_args *args = kdata; + size_t expected = struct_size(args, attrs, args->nattr); + + if (expected == SIZE_MAX || usize < expected) + return -EINVAL; + return 0; +} + #if IS_ENABLED(CONFIG_HSA_AMD_SVM) static int kfd_ioctl_set_xnack_mode(struct file *filep, @@ -3206,7 +3220,11 @@ static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, v #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ - .cmd_drv = 0, .name = #ioctl} + .validate = NULL, .cmd_drv = 0, .name = #ioctl} + +#define AMDKFD_IOCTL_DEF_V(ioctl, _func, _validate, _flags) \ + [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ + .validate = _validate, .cmd_drv = 0, .name = #ioctl} /** Ioctl table */ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { @@ -3303,7 +3321,8 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS, kfd_ioctl_smi_events, 0), - AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0), + AMDKFD_IOCTL_DEF_V(AMDKFD_IOC_SVM, kfd_ioctl_svm, + kfd_ioctl_svm_validate, 0), AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE, kfd_ioctl_set_xnack_mode, 0), @@ -3428,6 +3447,12 @@ static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) memset(kdata, 0, usize); } + if (ioctl->validate) { + retcode = ioctl->validate(kdata, usize); + if (retcode) + goto err_i1; + } + retcode = func(filep, process, kdata); if (cmd & IOC_OUT) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 3ddf06c755b527..c4cf595abca6c5 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -572,7 +572,7 @@ static int allocate_vmid(struct device_queue_manager *dqm, qpd->vmid, qpd->page_table_base); /* invalidate the VM context after pasid and vmid mapping is set up */ - kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); + kfd_flush_tlb(qpd_to_pdd(qpd)); if (dqm->dev->kfd2kgd->set_scratch_backing_va) dqm->dev->kfd2kgd->set_scratch_backing_va(dqm->dev->adev, @@ -610,7 +610,7 @@ static void deallocate_vmid(struct device_queue_manager *dqm, if (flush_texture_cache_nocpsch(q->device, qpd)) dev_err(dev, "Failed to flush TC\n"); - kfd_flush_tlb(qpd_to_pdd(qpd), TLB_FLUSH_LEGACY); + kfd_flush_tlb(qpd_to_pdd(qpd)); /* Release the vmid mapping */ set_pasid_vmid_mapping(dqm, 0, qpd->vmid); @@ -1284,7 +1284,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, dqm->dev->adev, qpd->vmid, qpd->page_table_base); - kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); + kfd_flush_tlb(pdd); } /* Take a safe reference to the mm_struct, which may otherwise diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c index e8da0b4527dc5f..04c5e26f01ed91 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c @@ -342,20 +342,14 @@ static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id) static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id) { - if (KFD_GC_VERSION(pdd->dev) >= IP_VERSION(12, 1, 0)) - pdd->lds_base = pdd->dev->adev->gmc.shared_aperture_start; - else - pdd->lds_base = MAKE_LDS_APP_BASE_V9(); + pdd->lds_base = MAKE_LDS_APP_BASE_V9(); pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base); pdd->gpuvm_base = AMDGPU_VA_RESERVED_BOTTOM; pdd->gpuvm_limit = pdd->dev->kfd->shared_resources.gpuvm_size - 1; - if (KFD_GC_VERSION(pdd->dev) >= IP_VERSION(12, 1, 0)) - pdd->scratch_base = pdd->dev->adev->gmc.private_aperture_start; - else - pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9(); + pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9(); pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base); /* @@ -365,6 +359,25 @@ static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id) pdd->qpd.cwsr_base = AMDGPU_VA_RESERVED_TRAP_START(pdd->dev->adev); } +static void kfd_init_apertures_v12(struct kfd_process_device *pdd, uint8_t id) +{ + pdd->lds_base = pdd->dev->adev->gmc.shared_aperture_start; + pdd->lds_limit = pdd->dev->adev->gmc.shared_aperture_end; + + pdd->gpuvm_base = AMDGPU_VA_RESERVED_BOTTOM; + pdd->gpuvm_limit = + pdd->dev->kfd->shared_resources.gpuvm_size - 1; + + pdd->scratch_base = pdd->dev->adev->gmc.private_aperture_start; + pdd->scratch_limit = pdd->dev->adev->gmc.private_aperture_end; + + /* + * Place TBA/TMA on opposite side of VM hole to prevent + * stray faults from triggering SVM on these pages. + */ + pdd->qpd.cwsr_base = AMDGPU_VA_RESERVED_TRAP_START(pdd->dev->adev); +} + int kfd_init_apertures(struct kfd_process *process) { uint8_t id = 0; @@ -412,9 +425,11 @@ int kfd_init_apertures(struct kfd_process *process) kfd_init_apertures_vi(pdd, id); break; default: - if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) + if (KFD_GC_VERSION(dev) >= IP_VERSION(12, 1, 0)) { + kfd_init_apertures_v12(pdd, id); + } else if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1)) { kfd_init_apertures_v9(pdd, id); - else { + } else { WARN(1, "Unexpected ASIC family %u", dev->adev->asic_type); return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c index 562d475cf4c999..bb70e57ae4d52f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c @@ -70,7 +70,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct cik_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c index d6067316d7f494..77fb41e2486a46 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c @@ -70,7 +70,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct v10_compute_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c index e3a7acb0ccbc8e..a1e3cf2384dd3b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c @@ -96,7 +96,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct v11_compute_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c index 0b97376fc6f9f1..b3e122d7876e08 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c @@ -77,7 +77,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct v12_compute_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c index eef6bdce4be39d..c90c0d99b1e3f3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c @@ -131,7 +131,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct v12_1_compute_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index a535f151cb5fdd..e856bee6280585 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -113,7 +113,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct v9_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static bool mqd_on_vram(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c index 69c1b8a690b868..f02ef2d44a07f0 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c @@ -73,7 +73,6 @@ static void update_cu_mask(struct mqd_manager *mm, void *mqd, static void set_priority(struct vi_mqd *m, struct queue_properties *q) { m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; - /* m->cp_hqd_queue_priority = q->priority; */ } static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 035687a17d896f..c8a2a3eb46ff4f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -1047,10 +1047,13 @@ extern struct srcu_struct kfd_processes_srcu; typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p, void *data); +typedef int amdkfd_ioctl_validate_t(void *kdata, unsigned int usize); + struct amdkfd_ioctl_desc { unsigned int cmd; int flags; amdkfd_ioctl_t *func; + amdkfd_ioctl_validate_t *validate; unsigned int cmd_drv; const char *name; }; @@ -1191,6 +1194,7 @@ static inline struct kfd_node *kfd_node_by_irq_ids(struct amdgpu_device *adev, return NULL; } int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_node **kdev); +uint32_t kfd_topology_get_num_devices(void); int kfd_numa_node_to_apic_id(int numa_node_id); uint32_t kfd_gpu_node_num(void); @@ -1550,13 +1554,13 @@ void kfd_signal_reset_event(struct kfd_node *dev); void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid); void kfd_signal_process_terminate_event(struct kfd_process *p); -static inline void kfd_flush_tlb(struct kfd_process_device *pdd, - enum TLB_FLUSH_TYPE type) +static inline void kfd_flush_tlb(struct kfd_process_device *pdd) { struct amdgpu_device *adev = pdd->dev->adev; struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv); - amdgpu_vm_flush_compute_tlb(adev, vm, type, pdd->dev->xcc_mask); + amdgpu_vm_flush_compute_tlb(adev, vm, TLB_FLUSH_HEAVYWEIGHT, + pdd->dev->xcc_mask); } static inline bool kfd_flush_tlb_after_unmap(struct kfd_dev *dev) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index f5d2847e1cbb44..3d172e35e57ce6 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -590,7 +590,8 @@ int pqm_update_queue_properties(struct process_queue_manager *pqm, return err; if (kfd_queue_buffer_get(vm, (void *)p->queue_address, &p->ring_bo, - p->queue_size)) { + p->queue_size + + pqn->q->properties.metadata_queue_size)) { pr_debug("ring buf 0x%llx size 0x%llx not mapped on GPU\n", p->queue_address, p->queue_size); amdgpu_bo_unreserve(vm->root.bo); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 080242f9981b0f..3d2c603f20858d 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1415,7 +1415,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start, if (r) break; } - kfd_flush_tlb(pdd, TLB_FLUSH_HEAVYWEIGHT); + kfd_flush_tlb(pdd); } return r; @@ -1557,7 +1557,7 @@ svm_range_map_to_gpus(struct svm_range *prange, unsigned long offset, } } - kfd_flush_tlb(pdd, TLB_FLUSH_LEGACY); + kfd_flush_tlb(pdd); } return r; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 995f2c2528a988..29dee26261ab1f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -2297,6 +2297,17 @@ int kfd_topology_remove_device(struct kfd_node *gpu) return res; } +uint32_t kfd_topology_get_num_devices(void) +{ + uint32_t num_devices; + + down_read(&topology_lock); + num_devices = sys_props.num_devices; + up_read(&topology_lock); + + return num_devices; +} + /* kfd_topology_enum_kfd_devices - Enumerate through all devices in KFD * topology. If GPU device is found @idx, then valid kfd_dev pointer is * returned through @kdev diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 2328c1aa0ead13..0aee65503642d7 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1891,7 +1891,11 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) goto error; } - init_data.asic_id.chip_family = adev->family; + /* special handling for early revisions of GC 11.5.4 */ + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4)) + init_data.asic_id.chip_family = AMDGPU_FAMILY_GC_11_5_4; + else + init_data.asic_id.chip_family = adev->family; init_data.asic_id.pci_revision_id = adev->pdev->revision; init_data.asic_id.hw_internal_rev = adev->external_rev_id; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 304437c2284d8b..527d0ad69348e8 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -101,23 +101,22 @@ bool amdgpu_dm_crtc_vrr_active(const struct dm_crtc_state *dm_state) /** * amdgpu_dm_crtc_set_panel_sr_feature() - Manage panel self-refresh features. - * - * @vblank_work: is a pointer to a struct vblank_control_work object. - * @vblank_enabled: indicates whether the DRM vblank counter is currently - * enabled (true) or disabled (false). - * @allow_sr_entry: represents whether entry into the self-refresh mode is - * allowed (true) or not allowed (false). + * @dm: amdgpu display manager instance. + * @acrtc: CRTC whose panel self-refresh state is being updated. + * @stream: DC stream associated with @acrtc. + * @vblank_enabled: Whether the DRM vblank counter is currently enabled. + * @allow_sr_entry: Whether entry into self-refresh mode is allowed. * * The DRM vblank counter enable/disable action is used as the trigger to enable * or disable various panel self-refresh features: * * Panel Replay and PSR SU * - Enable when: - * - VRR is disabled - * - vblank counter is disabled - * - entry is allowed: usermode demonstrates an adequate number of fast - * commits) - * - CRC capture window isn't active + * - VRR is disabled + * - vblank counter is disabled + * - entry is allowed: usermode demonstrates an adequate number of fast + * commits + * - CRC capture window isn't active * - Keep enabled even when vblank counter gets enabled * * PSR1 diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index a09761f9882d12..5b0245eb5fdb24 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -993,6 +993,45 @@ dm_helpers_read_acpi_edid(struct amdgpu_dm_connector *aconnector) return drm_edid_read_custom(connector, dm_helpers_probe_acpi_edid, connector); } +static const struct drm_edid * +dm_helpers_read_vbios_hardcoded_edid(struct dc_link *link, struct amdgpu_dm_connector *aconnector) +{ + struct dc_bios *bios = link->ctx->dc_bios; + struct embedded_panel_info info; + const struct drm_edid *edid; + enum bp_result r; + + if (!dc_is_embedded_signal(link->connector_signal) || + !bios->funcs->get_embedded_panel_info) + return NULL; + + memset(&info, 0, sizeof(info)); + r = bios->funcs->get_embedded_panel_info(bios, &info); + + if (r != BP_RESULT_OK) { + dm_error("Error when reading embedded panel info: %u\n", r); + return NULL; + } + + if (!info.fake_edid || !info.fake_edid_size) { + dm_error("Embedded panel info doesn't contain an EDID\n"); + return NULL; + } + + edid = drm_edid_alloc(info.fake_edid, info.fake_edid_size); + + if (!drm_edid_valid(edid)) { + dm_error("EDID from embedded panel info is invalid\n"); + drm_edid_free(edid); + return NULL; + } + + aconnector->base.display_info.width_mm = info.panel_width_mm; + aconnector->base.display_info.height_mm = info.panel_height_mm; + + return edid; +} + void populate_hdmi_info_from_connector(struct drm_hdmi_info *hdmi, struct dc_edid_caps *edid_caps) { edid_caps->scdc_present = hdmi->scdc.supported; @@ -1013,6 +1052,9 @@ enum dc_edid_status dm_helpers_read_local_edid( if (link->aux_mode) ddc = &aconnector->dm_dp_aux.aux.ddc; + else if (link->ddc_hw_inst == GPIO_DDC_LINE_UNKNOWN && + dc_is_embedded_signal(link->connector_signal)) + ddc = NULL; else ddc = &aconnector->i2c->base; @@ -1023,6 +1065,8 @@ enum dc_edid_status dm_helpers_read_local_edid( drm_edid = dm_helpers_read_acpi_edid(aconnector); if (drm_edid) drm_info(connector->dev, "Using ACPI provided EDID for %s\n", connector->name); + else if (!ddc) + drm_edid = dm_helpers_read_vbios_hardcoded_edid(link, aconnector); else drm_edid = drm_edid_read_ddc(connector, ddc); drm_edid_connector_update(connector, drm_edid); diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c index 73e3c45eeeba6b..bbd8d52330b556 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c @@ -1295,6 +1295,60 @@ static enum bp_result bios_parser_get_embedded_panel_info( return BP_RESULT_FAILURE; } +static enum bp_result get_embedded_panel_extra_info( + struct bios_parser *bp, + struct embedded_panel_info *info, + const uint32_t table_offset) +{ + uint8_t *record = bios_get_image(&bp->base, table_offset, 1); + ATOM_PANEL_RESOLUTION_PATCH_RECORD *panel_res_record; + ATOM_FAKE_EDID_PATCH_RECORD *fake_edid_record; + + while (*record != ATOM_RECORD_END_TYPE) { + switch (*record) { + case LCD_MODE_PATCH_RECORD_MODE_TYPE: + record += sizeof(ATOM_PATCH_RECORD_MODE); + break; + case LCD_RTS_RECORD_TYPE: + record += sizeof(ATOM_LCD_RTS_RECORD); + break; + case LCD_CAP_RECORD_TYPE: + record += sizeof(ATOM_LCD_MODE_CONTROL_CAP); + break; + case LCD_FAKE_EDID_PATCH_RECORD_TYPE: + fake_edid_record = (ATOM_FAKE_EDID_PATCH_RECORD *)record; + if (fake_edid_record->ucFakeEDIDLength) { + if (fake_edid_record->ucFakeEDIDLength == 128) + info->fake_edid_size = + fake_edid_record->ucFakeEDIDLength; + else + info->fake_edid_size = + fake_edid_record->ucFakeEDIDLength * 128; + + info->fake_edid = fake_edid_record->ucFakeEDIDString; + + record += struct_size(fake_edid_record, + ucFakeEDIDString, + info->fake_edid_size); + } else { + /* empty fake edid record must be 3 bytes long */ + record += sizeof(ATOM_FAKE_EDID_PATCH_RECORD) + 1; + } + break; + case LCD_PANEL_RESOLUTION_RECORD_TYPE: + panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record; + info->panel_width_mm = panel_res_record->usHSize; + info->panel_height_mm = panel_res_record->usVSize; + record += sizeof(ATOM_PANEL_RESOLUTION_PATCH_RECORD); + break; + default: + return BP_RESULT_BADBIOSTABLE; + } + } + + return BP_RESULT_OK; +} + static enum bp_result get_embedded_panel_info_v1_2( struct bios_parser *bp, struct embedded_panel_info *info) @@ -1411,6 +1465,10 @@ static enum bp_result get_embedded_panel_info_v1_2( if (ATOM_PANEL_MISC_API_ENABLED & lvds->ucLVDS_Misc) info->lcd_timing.misc_info.API_ENABLED = true; + if (lvds->usExtInfoTableOffset) + return get_embedded_panel_extra_info(bp, info, + le16_to_cpu(lvds->usExtInfoTableOffset) + DATA_TABLES(LCD_Info)); + return BP_RESULT_OK; } @@ -1536,6 +1594,10 @@ static enum bp_result get_embedded_panel_info_v1_3( (uint32_t) (ATOM_PANEL_MISC_V13_GREY_LEVEL & lvds->ucLCD_Misc) >> ATOM_PANEL_MISC_V13_GREY_LEVEL_SHIFT; + if (lvds->usExtInfoTableOffset) + return get_embedded_panel_extra_info(bp, info, + le16_to_cpu(lvds->usExtInfoTableOffset) + DATA_TABLES(LCD_Info)); + return BP_RESULT_OK; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 03d125f794b058..896f4719cf7b64 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -5049,7 +5049,7 @@ void resource_build_bit_depth_reduction_params(struct dc_stream_state *stream, option = DITHER_OPTION_SPATIAL8; break; case COLOR_DEPTH_101010: - option = DITHER_OPTION_TRUN10; + option = DITHER_OPTION_SPATIAL10; break; default: option = DITHER_OPTION_DISABLE; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 4c4239cac863da..8044c80971f042 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1638,7 +1638,7 @@ struct dc_scratch_space { struct dc_link_training_overrides preferred_training_settings; struct dp_audio_test_data audio_test_data; - uint8_t ddc_hw_inst; + enum gpio_ddc_line ddc_hw_inst; uint8_t hpd_src; diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index b15360bcdacf76..b3cedaa596c027 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -958,7 +958,10 @@ void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv) { uint32_t i; - if (!dc_dmub_srv || !dc_dmub_srv->dmub) { + if (!dc_dmub_srv) + return; + + if (!dc_dmub_srv->dmub) { DC_LOG_ERROR("%s: invalid parameters.", __func__); return; } @@ -1163,7 +1166,10 @@ void dc_dmub_srv_enable_dpia_trace(const struct dc *dc) { struct dc_dmub_srv *dc_dmub_srv = dc->ctx->dmub_srv; - if (!dc_dmub_srv || !dc_dmub_srv->dmub) { + if (!dc_dmub_srv) + return; + + if (!dc_dmub_srv->dmub) { DC_LOG_ERROR("%s: invalid parameters.", __func__); return; } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 2ba3d3a3aac581..d2b36cfb28c3d4 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -1071,7 +1071,9 @@ void dce110_link_encoder_hw_init( ASSERT(result == BP_RESULT_OK); } - aux_initialize(enc110); + + if (enc110->aux_regs) + aux_initialize(enc110); /* reinitialize HPD. * hpd_initialize() will pass DIG_FE id to HW context. diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/lib_float_math.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/lib_float_math.c index e17b5ceba4471e..dc5bc649f3ac97 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/lib_float_math.c +++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_standalone_libraries/lib_float_math.c @@ -23,7 +23,7 @@ double math_mod(const double arg1, const double arg2) return arg2; if (isNaN(arg2)) return arg1; - return arg1 - arg1 * ((int)(arg1 / arg2)); + return arg1 - arg2 * ((int)(arg1 / arg2)); } double math_min2(const double arg1, const double arg2) diff --git a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c index a2c46350e44e8a..95f8b7c7d657a1 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/gpio_service.c @@ -646,6 +646,9 @@ enum gpio_result dal_ddc_change_mode( enum gpio_ddc_line dal_ddc_get_line( const struct ddc *ddc) { + if (!ddc) + return GPIO_DDC_LINE_UNKNOWN; + return (enum gpio_ddc_line)dal_gpio_get_enum(ddc->pin_data); } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c index 21815ad01a2951..409cc6e6cd846e 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c @@ -549,7 +549,9 @@ static bool construct_phy(struct dc_link *link, goto ddc_create_fail; } - if (!link->ddc->ddc_pin) { + /* Embedded display connectors such as LVDS may not have DDC. */ + if (!link->ddc->ddc_pin && + !dc_is_embedded_signal(link->connector_signal)) { DC_ERROR("Failed to get I2C info for connector!\n"); goto ddc_create_fail; } diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c index 3d52973dd7f20d..15f220671fbed8 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c @@ -753,7 +753,8 @@ static struct link_encoder *dce60_link_encoder_create( enc_init_data, &link_enc_feature, &link_enc_regs[link_regs_id], - &link_enc_aux_regs[enc_init_data->channel - 1], + enc_init_data->channel == CHANNEL_ID_UNKNOWN ? + NULL : &link_enc_aux_regs[enc_init_data->channel - 1], enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c index 89927727a0d9e8..42d0bd656f7939 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c @@ -759,7 +759,8 @@ static struct link_encoder *dce80_link_encoder_create( enc_init_data, &link_enc_feature, &link_enc_regs[link_regs_id], - &link_enc_aux_regs[enc_init_data->channel - 1], + enc_init_data->channel == CHANNEL_ID_UNKNOWN ? + NULL : &link_enc_aux_regs[enc_init_data->channel - 1], enc_init_data->hpd_source >= ARRAY_SIZE(link_enc_hpd_regs) ? NULL : &link_enc_hpd_regs[enc_init_data->hpd_source]); return &enc110->base; diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c index c3a6ae14de18bf..43b72b9113be1a 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c @@ -92,9 +92,14 @@ #include "dml/dcn32/dcn32_fpu.h" #include "dc_state_priv.h" +#include "dc_fpu.h" #include "dml2_0/dml2_wrapper.h" +#if !defined(DC_RUN_WITH_PREEMPTION_ENABLED) +#define DC_RUN_WITH_PREEMPTION_ENABLED(code) code +#endif + #define DC_LOGGER_INIT(logger) enum dcn32_clk_src_array_id { @@ -1684,7 +1689,8 @@ static void dcn32_enable_phantom_plane(struct dc *dc, if (curr_pipe->top_pipe && curr_pipe->top_pipe->plane_state == curr_pipe->plane_state) phantom_plane = prev_phantom_plane; else - phantom_plane = dc_state_create_phantom_plane(dc, context, curr_pipe->plane_state); + DC_RUN_WITH_PREEMPTION_ENABLED(phantom_plane = + dc_state_create_phantom_plane(dc, context, curr_pipe->plane_state)); if (!phantom_plane) continue; diff --git a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h index 38a77fa9b4afde..a0f03fb67605eb 100644 --- a/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h +++ b/drivers/gpu/drm/amd/display/include/grph_object_ctrl_defs.h @@ -153,6 +153,10 @@ struct embedded_panel_info { uint32_t drr_enabled; uint32_t min_drr_refresh_rate; bool realtek_eDPToLVDS; + uint16_t panel_width_mm; + uint16_t panel_height_mm; + uint16_t fake_edid_size; + const uint8_t *fake_edid; }; struct dc_firmware_info { diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c index 2b5ac21fee3997..1d6e30269d5679 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c @@ -104,6 +104,21 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr) PP_GFXOFF_MASK); hwmgr->pp_table_version = PP_TABLE_V0; hwmgr->od_enabled = false; + switch (hwmgr->chip_id) { + case CHIP_BONAIRE: + /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM + * R7 260X cards with old MC ucode: MCLK DPM is unstable + */ + if (adev->pdev->subsystem_vendor == 0x106B || + adev->pdev->device == 0x6658) { + dev_info(adev->dev, "disabling MCLK DPM on quirky ASIC"); + adev->pm.pp_feature &= ~PP_MCLK_DPM_MASK; + hwmgr->feature_mask &= ~PP_MCLK_DPM_MASK; + } + break; + default: + break; + } smu7_init_function_pointers(hwmgr); break; case AMDGPU_FAMILY_CZ: diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index e38222877f7ef2..bc6acdb52c2692 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -787,7 +787,7 @@ static int smu7_setup_dpm_tables_v0(struct pp_hwmgr *hwmgr) hwmgr->dyn_state.vddc_dependency_on_mclk; struct phm_cac_leakage_table *std_voltage_table = hwmgr->dyn_state.cac_leakage_table; - uint32_t i; + uint32_t i, clk; PP_ASSERT_WITH_CODE(allowed_vdd_sclk_table != NULL, "SCLK dependency table is missing. This table is mandatory", return -EINVAL); @@ -804,10 +804,12 @@ static int smu7_setup_dpm_tables_v0(struct pp_hwmgr *hwmgr) data->dpm_table.sclk_table.count = 0; for (i = 0; i < allowed_vdd_sclk_table->count; i++) { + clk = min(allowed_vdd_sclk_table->entries[i].clk, data->sclk_cap); + if (i == 0 || data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count-1].value != - allowed_vdd_sclk_table->entries[i].clk) { + clk) { data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value = - allowed_vdd_sclk_table->entries[i].clk; + clk; data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = (i == 0) ? 1 : 0; data->dpm_table.sclk_table.count++; } @@ -2802,6 +2804,10 @@ static int smu7_patch_dependency_tables_with_leakage(struct pp_hwmgr *hwmgr) if (tmp) return -EINVAL; + tmp = smu7_patch_vddc(hwmgr, hwmgr->dyn_state.vddc_dependency_on_display_clock); + if (tmp) + return -EINVAL; + tmp = smu7_patch_vce_vddc(hwmgr, hwmgr->dyn_state.vce_clock_voltage_dependency_table); if (tmp) return -EINVAL; @@ -2885,6 +2891,8 @@ static int smu7_hwmgr_backend_fini(struct pp_hwmgr *hwmgr) { kfree(hwmgr->dyn_state.vddc_dep_on_dal_pwrl); hwmgr->dyn_state.vddc_dep_on_dal_pwrl = NULL; + kfree(hwmgr->dyn_state.vddc_dependency_on_display_clock); + hwmgr->dyn_state.vddc_dependency_on_display_clock = NULL; kfree(hwmgr->backend); hwmgr->backend = NULL; @@ -2955,6 +2963,70 @@ static int smu7_update_edc_leakage_table(struct pp_hwmgr *hwmgr) return ret; } +static int smu7_init_voltage_dependency_on_display_clock_table(struct pp_hwmgr *hwmgr) +{ + struct phm_clock_voltage_dependency_table *table; + + if (!amdgpu_device_ip_get_ip_block(hwmgr->adev, AMD_IP_BLOCK_TYPE_DCE)) + return 0; + + table = kzalloc(struct_size(table, entries, 4), GFP_KERNEL); + if (!table) + return -ENOMEM; + + if (hwmgr->chip_id >= CHIP_POLARIS10) { + table->entries[0].clk = 38918; + table->entries[1].clk = 45900; + table->entries[2].clk = 66700; + table->entries[3].clk = 113200; + + table->entries[0].v = 700; + table->entries[1].v = 740; + table->entries[2].v = 800; + table->entries[3].v = 900; + } else { + if (hwmgr->chip_family == AMDGPU_FAMILY_CZ) { + table->entries[0].clk = 35200; + table->entries[1].clk = 35200; + table->entries[2].clk = 46700; + table->entries[3].clk = 64300; + } else { + table->entries[0].clk = 0; + table->entries[1].clk = 35200; + table->entries[2].clk = 54000; + table->entries[3].clk = 62500; + } + + table->entries[0].v = 0; + table->entries[1].v = 720; + table->entries[2].v = 810; + table->entries[3].v = 900; + } + + table->count = 4; + hwmgr->dyn_state.vddc_dependency_on_display_clock = table; + return 0; +} + +static void smu7_set_sclk_cap(struct pp_hwmgr *hwmgr) +{ + struct amdgpu_device *adev = hwmgr->adev; + struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend); + + data->sclk_cap = 0xffffffff; + + if (hwmgr->od_enabled) + return; + + /* R9 390X board: last sclk dpm level is unstable, use lower sclk */ + if (adev->pdev->device == 0x67B0 && + adev->pdev->subsystem_vendor == 0x1043) + data->sclk_cap = 104000; /* 1040 MHz */ + + if (data->sclk_cap != 0xffffffff) + dev_info(adev->dev, "sclk cap: %u kHz on quirky ASIC\n", data->sclk_cap * 10); +} + static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr) { struct amdgpu_device *adev = hwmgr->adev; @@ -2966,6 +3038,7 @@ static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr) return -ENOMEM; hwmgr->backend = data; + smu7_set_sclk_cap(hwmgr); smu7_patch_voltage_workaround(hwmgr); smu7_init_dpm_defaults(hwmgr); @@ -2983,6 +3056,10 @@ static int smu7_hwmgr_backend_init(struct pp_hwmgr *hwmgr) smu7_get_elb_voltages(hwmgr); } + result = smu7_init_voltage_dependency_on_display_clock_table(hwmgr); + if (result) + goto fail; + if (hwmgr->pp_table_version == PP_TABLE_V1) { smu7_complete_dependency_tables(hwmgr); smu7_set_private_data_based_on_pptable_v1(hwmgr); @@ -3079,13 +3156,40 @@ static int smu7_force_dpm_highest(struct pp_hwmgr *hwmgr) return 0; } +static uint32_t smu7_lookup_vddc_from_dispclk(struct pp_hwmgr *hwmgr) +{ + const struct amd_pp_display_configuration *cfg = hwmgr->display_config; + const struct phm_clock_voltage_dependency_table *vddc_dep_on_dispclk = + hwmgr->dyn_state.vddc_dependency_on_display_clock; + uint32_t i; + + if (!vddc_dep_on_dispclk || !vddc_dep_on_dispclk->count || + !cfg || !cfg->num_display || !cfg->display_clk) + return 0; + + /* Start from 1 because ClocksStateUltraLow should not be used according to DC. */ + for (i = 1; i < vddc_dep_on_dispclk->count; ++i) + if (vddc_dep_on_dispclk->entries[i].clk >= cfg->display_clk) + return vddc_dep_on_dispclk->entries[i].v; + + return vddc_dep_on_dispclk->entries[vddc_dep_on_dispclk->count - 1].v; +} + +static void smu7_apply_minimum_dce_voltage_request(struct pp_hwmgr *hwmgr) +{ + uint32_t req_vddc = smu7_lookup_vddc_from_dispclk(hwmgr); + + smum_send_msg_to_smc_with_parameter(hwmgr, + PPSMC_MSG_VddC_Request, + req_vddc * VOLTAGE_SCALE, + NULL); +} + static int smu7_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr) { struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend); - if (hwmgr->pp_table_version == PP_TABLE_V1) - phm_apply_dal_min_voltage_request(hwmgr); -/* TO DO for v0 iceland and Ci*/ + smu7_apply_minimum_dce_voltage_request(hwmgr); if (!data->sclk_dpm_key_disabled) { if (data->dpm_level_enable_mask.sclk_dpm_enable_mask) @@ -3821,7 +3925,7 @@ static int smu7_get_pp_table_entry_callback_func_v0(struct pp_hwmgr *hwmgr, /* Performance levels are arranged from low to high. */ performance_level->memory_clock = memory_clock; - performance_level->engine_clock = engine_clock; + performance_level->engine_clock = min(engine_clock, data->sclk_cap); pcie_gen_from_bios = visland_clk_info->ucPCIEGen; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.h index d9e8b386bd4d38..66adabeab6a3aa 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.h @@ -234,6 +234,7 @@ struct smu7_hwmgr { uint32_t pcie_gen_cap; uint32_t pcie_lane_cap; uint32_t pcie_spc_cap; + uint32_t sclk_cap; struct smu7_leakage_voltage vddc_leakage; struct smu7_leakage_voltage vddci_leakage; struct smu7_leakage_voltage vddcgfx_leakage; diff --git a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h index c661185753b428..2f49c95342a142 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h +++ b/drivers/gpu/drm/amd/pm/powerplay/inc/hwmgr.h @@ -631,6 +631,7 @@ struct phm_dynamic_state_info { struct phm_clock_voltage_dependency_table *vddci_dependency_on_mclk; struct phm_clock_voltage_dependency_table *vddc_dependency_on_mclk; struct phm_clock_voltage_dependency_table *mvdd_dependency_on_mclk; + struct phm_clock_voltage_dependency_table *vddc_dependency_on_display_clock; struct phm_clock_voltage_dependency_table *vddc_dep_on_dal_pwrl; struct phm_clock_array *valid_sclk_values; struct phm_clock_array *valid_mclk_values; diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c index 62ebec1c6fe3c3..3650e7beeb6712 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/ci_smumgr.c @@ -245,7 +245,7 @@ static void ci_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr) smu_data->power_tune_defaults = &defaults_hawaii_pro; break; case 0x67B8: - case 0x66B0: + case 0x67B0: smu_data->power_tune_defaults = &defaults_hawaii_xt; break; case 0x6640: @@ -543,12 +543,11 @@ static int ci_populate_dw8(struct pp_hwmgr *hwmgr, uint32_t fuse_table_offset) { struct ci_smumgr *smu_data = (struct ci_smumgr *)(hwmgr->smu_backend); const struct ci_pt_defaults *defaults = smu_data->power_tune_defaults; - uint32_t temp; if (ci_read_smc_sram_dword(hwmgr, fuse_table_offset + offsetof(SMU7_Discrete_PmFuses, TdcWaterfallCtl), - (uint32_t *)&temp, SMC_RAM_END)) + (uint32_t *)&smu_data->power_tune_table.TdcWaterfallCtl, SMC_RAM_END)) PP_ASSERT_WITH_CODE(false, "Attempt to read PmFuses.DW6 (SviLoadLineEn) from SMC Failed!", return -EINVAL); @@ -1217,7 +1216,7 @@ static int ci_populate_single_memory_level( } memory_level->EnabledForThrottle = 1; - memory_level->EnabledForActivity = 1; + memory_level->EnabledForActivity = 0; memory_level->UpH = data->current_profile_setting.mclk_up_hyst; memory_level->DownH = data->current_profile_setting.mclk_down_hyst; memory_level->VoltageDownH = 0; @@ -1322,16 +1321,25 @@ static int ci_populate_all_memory_levels(struct pp_hwmgr *hwmgr) return result; } + if (data->mclk_dpm_key_disabled && dpm_table->mclk_table.count) { + /* Populate the table with the highest MCLK level when MCLK DPM is disabled */ + for (i = 0; i < dpm_table->mclk_table.count - 1; i++) { + levels[i] = levels[dpm_table->mclk_table.count - 1]; + levels[i].DisplayWatermark = PPSMC_DISPLAY_WATERMARK_HIGH; + } + } + smu_data->smc_state_table.MemoryLevel[0].EnabledForActivity = 1; dev_id = adev->pdev->device; - if ((dpm_table->mclk_table.count >= 2) - && ((dev_id == 0x67B0) || (dev_id == 0x67B1))) { - smu_data->smc_state_table.MemoryLevel[1].MinVddci = - smu_data->smc_state_table.MemoryLevel[0].MinVddci; - smu_data->smc_state_table.MemoryLevel[1].MinMvdd = - smu_data->smc_state_table.MemoryLevel[0].MinMvdd; + if ((dpm_table->mclk_table.count >= 2) && + ((dev_id == 0x67B0) || (dev_id == 0x67B1)) && + (adev->pdev->revision == 0)) { + smu_data->smc_state_table.MemoryLevel[1].MinVddc = + smu_data->smc_state_table.MemoryLevel[0].MinVddc; + smu_data->smc_state_table.MemoryLevel[1].MinVddcPhases = + smu_data->smc_state_table.MemoryLevel[0].MinVddcPhases; } smu_data->smc_state_table.MemoryLevel[0].ActivityLevel = 0x1F; CONVERT_FROM_HOST_TO_SMC_US(smu_data->smc_state_table.MemoryLevel[0].ActivityLevel); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c index 2b4faab376930b..23c9f14bb20860 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c @@ -425,6 +425,7 @@ static int aldebaran_set_default_dpm_table(struct smu_context *smu) dpm_table->dpm_levels[0].enabled = true; dpm_table->dpm_levels[1].value = pptable->GfxclkFmax; dpm_table->dpm_levels[1].enabled = true; + dpm_table->flags |= SMU_DPM_TABLE_FINE_GRAINED; } else { dpm_table->count = 1; dpm_table->dpm_levels[0].value = smu->smu_table.boot_values.gfxclk / 100; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c index 32d5e2170d80a6..aa9deb7cd9da6d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_12_ppt.c @@ -262,8 +262,9 @@ static void smu_v13_0_12_init_xgmi_data(struct smu_context *smu, int ret; if (smu_table->tables[SMU_TABLE_SMU_METRICS].version >= 0x13) { - max_width = (uint8_t)static_metrics->MaxXgmiWidth; - max_speed = (uint16_t)static_metrics->MaxXgmiBitrate; + max_width = (uint8_t)SMUQ10_ROUND(static_metrics->MaxXgmiWidth); + max_speed = + (uint16_t)SMUQ10_ROUND(static_metrics->MaxXgmiBitrate); ret = 0; } else { MetricsTable_t *metrics = (MetricsTable_t *)smu_table->metrics_table; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c index 870bcc86fd7948..c62b12d672d48d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c @@ -1122,6 +1122,7 @@ static int smu_v13_0_6_set_default_dpm_table(struct smu_context *smu) /* gfxclk dpm table setup */ dpm_table = &dpm_context->dpm_tables.gfx_table; dpm_table->clk_type = SMU_GFXCLK; + dpm_table->flags = SMU_DPM_TABLE_FINE_GRAINED; if (smu_cmn_feature_is_enabled(smu, SMU_FEATURE_DPM_GFXCLK_BIT)) { /* In the case of gfxclk, only fine-grained dpm is honored. * Get min/max values from FW. diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c index c3ebfac062a7a5..58d12073f7b7e1 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c @@ -2372,6 +2372,7 @@ static int smu_v14_0_2_od_restore_table_single(struct smu_context *smu, long inp } od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + od_table->OverDriveTable.FeatureCtrlMask &= ~BIT(PP_OD_FEATURE_FAN_LEGACY_BIT); break; case PP_OD_EDIT_FAN_ZERO_RPM_ENABLE: od_table->OverDriveTable.FanZeroRpmEnable = @@ -2400,7 +2401,8 @@ static int smu_v14_0_2_od_restore_table_single(struct smu_context *smu, long inp od_table->OverDriveTable.FanMinimumPwm = boot_overdrive_table->OverDriveTable.FanMinimumPwm; od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_LEGACY_BIT); + od_table->OverDriveTable.FeatureCtrlMask &= ~BIT(PP_OD_FEATURE_FAN_CURVE_BIT); break; default: dev_info(adev->dev, "Invalid table index: %ld\n", input); @@ -2570,6 +2572,7 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, od_table->OverDriveTable.FanLinearPwmPoints[input[0]] = input[2]; od_table->OverDriveTable.FanMode = FAN_MODE_MANUAL_LINEAR; od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + od_table->OverDriveTable.FeatureCtrlMask &= ~BIT(PP_OD_FEATURE_FAN_LEGACY_BIT); break; case PP_OD_EDIT_ACOUSTIC_LIMIT: @@ -2639,7 +2642,7 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, break; case PP_OD_EDIT_FAN_MINIMUM_PWM: - if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_CURVE_BIT)) { + if (!smu_v14_0_2_is_od_feature_supported(smu, PP_OD_FEATURE_FAN_LEGACY_BIT)) { dev_warn(adev->dev, "Fan curve setting not supported!\n"); return -ENOTSUPP; } @@ -2657,7 +2660,8 @@ static int smu_v14_0_2_od_edit_dpm_table(struct smu_context *smu, od_table->OverDriveTable.FanMinimumPwm = input[0]; od_table->OverDriveTable.FanMode = FAN_MODE_AUTO; - od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_CURVE_BIT); + od_table->OverDriveTable.FeatureCtrlMask |= BIT(PP_OD_FEATURE_FAN_LEGACY_BIT); + od_table->OverDriveTable.FeatureCtrlMask &= ~BIT(PP_OD_FEATURE_FAN_CURVE_BIT); break; case PP_OD_EDIT_FAN_ZERO_RPM_ENABLE: diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_core.c b/drivers/gpu/drm/amd/ras/rascore/ras_core.c index 3f56f26abd6da6..1ad555eff59276 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_core.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_core.c @@ -62,14 +62,16 @@ int ras_core_convert_timestamp_to_time(struct ras_core_context *ras_core, uint64_t timestamp, struct ras_time *tm) { int days_in_month[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - uint64_t month = 0, day = 0, hour = 0, minute = 0, second = 0; + uint64_t month = 0, day = 0, hour = 0, minute = 0, second = 0, remainder; uint32_t year = 0; int seconds_per_day = 24 * 60 * 60; int seconds_per_hour = 60 * 60; int seconds_per_minute = 60; int days, remaining_seconds; - days = div64_u64_rem(timestamp, seconds_per_day, (uint64_t *)&remaining_seconds); + days = div64_u64_rem(timestamp, seconds_per_day, &remainder); + /* remainder will always be less than seconds_per_day. */ + remaining_seconds = remainder; /* utc_timestamp follows the Unix epoch */ year = 1970; @@ -505,8 +507,11 @@ bool ras_core_is_enabled(struct ras_core_context *ras_core) uint64_t ras_core_get_utc_second_timestamp(struct ras_core_context *ras_core) { - if (ras_core && ras_core->sys_fn && - ras_core->sys_fn->get_utc_second_timestamp) + if (!ras_core) + return 0; + + if (ras_core->sys_fn && + ras_core->sys_fn->get_utc_second_timestamp) return ras_core->sys_fn->get_utc_second_timestamp(ras_core); RAS_DEV_ERR(ras_core->dev, "Failed to get system timestamp!\n"); @@ -528,7 +533,9 @@ bool ras_core_ras_interrupt_detected(struct ras_core_context *ras_core) ras_core->sys_fn->detect_ras_interrupt) return ras_core->sys_fn->detect_ras_interrupt(ras_core); - RAS_DEV_ERR(ras_core->dev, "Failed to detect ras interrupt!\n"); + if (ras_core && ras_core->dev) + RAS_DEV_ERR(ras_core->dev, "Failed to detect ras interrupt!\n"); + return false; } diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c index 2abe8553e479e3..4fff0b3af75c1a 100644 --- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c +++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c @@ -222,7 +222,7 @@ int ras_umc_log_pending_bad_bank(struct ras_core_context *ras_core) mutex_lock(&ras_umc->pending_ecc_lock); list_for_each_entry_safe(ecc_node, tmp, &ras_umc->pending_ecc_list, node){ - if (ecc_node && !ras_umc_log_bad_bank(ras_core, &ecc_node->ecc)) { + if (!ras_umc_log_bad_bank(ras_core, &ecc_node->ecc)) { list_del(&ecc_node->node); kfree(ecc_node); } diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig new file mode 100644 index 00000000000000..a1d4498c5d788e --- /dev/null +++ b/drivers/gpu/drm/apple/Kconfig @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +config DRM_APPLE + tristate "DRM Support for Apple display controllers" + depends on DRM && OF && ARM64 + depends on ARCH_APPLE || COMPILE_TEST + depends on APPLE_RTKIT + depends on OF_ADDRESS + select APPLE_PMP_REPORT + select DRM_CLIENT_SELECTION + select DRM_KMS_HELPER + select DRM_KMS_DMA_HELPER + select DRM_GEM_DMA_HELPER + select VIDEOMODE_HELPERS + select BACKLIGHT_CLASS_DEVICE + select MULTIPLEXER + help + Say Y if you have an Apple Silicon chipset. + +config DRM_APPLE_AUDIO + bool "DisplayPort/HDMI Audio support" + default y + depends on DRM_APPLE + depends on SND + select SND_PCM + select SND_DMAENGINE_PCM + +config DRM_APPLE_DEBUG + bool "Enable additional driver debugging" + depends on DRM_APPLE + depends on EXPERT # only for developers diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile new file mode 100644 index 00000000000000..b22839bdd611a7 --- /dev/null +++ b/drivers/gpu/drm/apple/Makefile @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +CFLAGS_trace.o = -I$(src) + +appledrm-y := apple_drv.o +appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o +appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o +appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o +appledrm-y += connector.o +appledrm-y += ibootep.o +appledrm-y += iomfb_v12_3.o +appledrm-y += iomfb_v13_3.o +appledrm-y += epic/dpavservep.o +appledrm-y += plane.o + + +appledrm-$(CONFIG_TRACING) += trace.o + +obj-$(CONFIG_DRM_APPLE) += appledrm.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c new file mode 100644 index 00000000000000..b7f16298826925 --- /dev/null +++ b/drivers/gpu/drm/apple/afk.c @@ -0,0 +1,1181 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Sven Peter */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "afk.h" +#include "trace.h" + +struct afk_receive_message_work { + struct apple_dcp_afkep *ep; + u64 message; + struct work_struct work; +}; + +#define RBEP_TYPE GENMASK(63, 48) + +enum rbep_msg_type { + RBEP_INIT = 0x80, + RBEP_INIT_ACK = 0xa0, + RBEP_GETBUF = 0x89, + RBEP_GETBUF_ACK = 0xa1, + RBEP_INIT_TX = 0x8a, + RBEP_INIT_RX = 0x8b, + RBEP_START = 0xa3, + RBEP_START_ACK = 0x86, + RBEP_SEND = 0xa2, + RBEP_RECV = 0x85, + RBEP_SHUTDOWN = 0xc0, + RBEP_SHUTDOWN_ACK = 0xc1, +}; + +#define BLOCK_SHIFT 6 + +#define GETBUF_SIZE GENMASK(31, 16) +#define GETBUF_TAG GENMASK(15, 0) +#define GETBUF_ACK_DVA GENMASK(47, 0) + +#define INITRB_OFFSET GENMASK(47, 32) +#define INITRB_SIZE GENMASK(31, 16) +#define INITRB_TAG GENMASK(15, 0) + +#define SEND_WPTR GENMASK(31, 0) + +static void afk_send(struct apple_dcp_afkep *ep, u64 message) +{ + dcp_send_message(ep->dcp, ep->endpoint, message); +} + +struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, + const struct apple_epic_service_ops *ops) +{ + struct apple_dcp_afkep *afkep; + int ret; + + afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); + if (!afkep) + return ERR_PTR(-ENOMEM); + + afkep->ops = ops; + afkep->dcp = dcp; + afkep->endpoint = endpoint; + afkep->wq = alloc_ordered_workqueue("apple-dcp-afkep%02x", + WQ_MEM_RECLAIM, endpoint); + if (!afkep->wq) { + ret = -ENOMEM; + goto out_free_afkep; + } + + // TODO: devm_ for wq + + init_completion(&afkep->started); + init_completion(&afkep->stopped); + spin_lock_init(&afkep->lock); + + return afkep; + +out_free_afkep: + devm_kfree(dcp->dev, afkep); + return ERR_PTR(ret); +} + +void afk_shutdown(struct apple_dcp_afkep *afkep) +{ + afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); + int ret; + + ret = wait_for_completion_timeout(&afkep->stopped, msecs_to_jiffies(1000)); + if (ret <= 0) { + dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); + } + + destroy_workqueue(afkep->wq); +} + +int afk_start(struct apple_dcp_afkep *ep) +{ + int ret; + + reinit_completion(&ep->started); + apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); + + ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); + if (ret <= 0) + return -ETIMEDOUT; + else + return 0; +} + +static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) +{ + u16 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; + u16 tag = FIELD_GET(GETBUF_TAG, message); + u64 reply; + + trace_afk_getbuf(ep, size, tag); + + if (ep->bfr) { + dev_err(ep->dcp->dev, + "Got GETBUF message but buffer already exists\n"); + return; + } + + ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, + GFP_KERNEL); + if (!ep->bfr) { + dev_err(ep->dcp->dev, "Failed to allocate %d bytes buffer\n", + size); + return; + } + + ep->bfr_size = size; + ep->bfr_tag = tag; + + reply = FIELD_PREP(RBEP_TYPE, RBEP_GETBUF_ACK); + reply |= FIELD_PREP(GETBUF_ACK_DVA, ep->bfr_dma); + afk_send(ep, reply); +} + +static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, + struct afk_ringbuffer *bfr) +{ + u16 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; + u16 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; + u16 tag = FIELD_GET(INITRB_TAG, message); + u32 bufsz, end; + + if (tag != ep->bfr_tag) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", + ep->endpoint, ep->bfr_tag, tag); + return; + } + + if (bfr->ready) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: buffer is already initialized\n", + ep->endpoint); + return; + } + + if (base >= ep->bfr_size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", + ep->endpoint, base, ep->bfr_size); + return; + } + + end = base + size; + if (end > ep->bfr_size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: requested end 0x%x > max size 0x%lx\n", + ep->endpoint, end, ep->bfr_size); + return; + } + + bfr->hdr = ep->bfr + base; + bufsz = le32_to_cpu(bfr->hdr->bufsz); + if (bufsz + sizeof(*bfr->hdr) != size) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", + ep->endpoint, bufsz, sizeof(*bfr->hdr)); + return; + } + + bfr->buf = bfr->hdr + 1; + bfr->bufsz = bufsz; + bfr->ready = true; + + if (ep->rxbfr.ready && ep->txbfr.ready) + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_START)); +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) +static void afk_populate_service_debugfs(struct apple_epic_service *srv); +static void afk_remove_service_debugfs(struct apple_epic_service *srv); +#else +static void afk_populate_service_debugfs(struct apple_epic_service *srv) +{ +} +static void afk_remove_service_debugfs(struct apple_epic_service *srv) +{ +} +#endif + +static const struct apple_epic_service_ops * +afk_match_service(struct apple_dcp_afkep *ep, const char *name) +{ + const struct apple_epic_service_ops *ops; + + if (!name[0]) + return NULL; + if (!ep->ops) + return NULL; + + for (ops = ep->ops; ops->name[0]; ops++) { + if (strcmp(ops->name, name)) + continue; + + return ops; + } + + return NULL; +} + +static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep, + u32 channel) +{ + for (u32 i = 0; i < ep->num_channels; i++) + if (ep->services[i].enabled && ep->services[i].channel == channel) + return &ep->services[i]; + + return NULL; +} + +static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, + u8 *payload, size_t payload_size) +{ + char name[32]; + s64 epic_unit = -1; + u32 ch_idx; + const char *service_name = name; + const char *epic_name = NULL, *epic_class = NULL; + const struct apple_epic_service_ops *ops; + struct dcp_parse_ctx ctx; + u8 *props = payload + sizeof(name); + size_t props_size = payload_size - sizeof(name); + + WARN_ON(afk_epic_find_service(ep, channel)); + + if (payload_size < sizeof(name)) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, payload_size); + return; + } + + if (ep->num_channels >= AFK_MAX_CHANNEL) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: too many enabled services!\n", + ep->endpoint); + return; + } + + strscpy(name, payload, sizeof(name)); + + /* + * in DCP firmware 13.2 DCP reports interface-name as name which starts + * with "dispext%d" using -1 s ID for "dcp". In the 12.3 firmware + * EPICProviderClass was used. If the init call has props parse them and + * use EPICProviderClass to match the service. + */ + if (props_size > 36) { + int ret = parse(props, props_size, &ctx); + if (ret) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: Failed to parse service init props for %s\n", + ep->endpoint, name); + return; + } + ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit); + if (ret) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: failed to extract init props: %d\n", + ep->endpoint, ret); + return; + } + service_name = epic_class; + } else { + service_name = name; + } + + if (ep->match_epic_name) + ops = afk_match_service(ep, epic_name); + else + ops = afk_match_service(ep, service_name); + + if (!ops) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: unable to match service %s on channel %d\n", + ep->endpoint, service_name, channel); + goto free; + } + + ch_idx = ep->num_channels++; + spin_lock_init(&ep->services[ch_idx].lock); + ep->services[ch_idx].enabled = true; + ep->services[ch_idx].torndown = false; + ep->services[ch_idx].ops = ops; + ep->services[ch_idx].ep = ep; + ep->services[ch_idx].channel = channel; + ep->services[ch_idx].cmd_tag = 0; + ops->init(&ep->services[ch_idx], epic_name, epic_class, epic_unit); + dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", + ep->endpoint, service_name, channel); + + afk_populate_service_debugfs(&ep->services[ch_idx]); + +free: + kfree(epic_name); + kfree(epic_class); +} + +static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) +{ + struct apple_epic_service *service; + const struct apple_epic_service_ops *ops; + unsigned long flags; + + service = afk_epic_find_service(ep, channel); + if (!service) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", + ep->endpoint, channel); + return; + } + + afk_remove_service_debugfs(service); + + // TODO: think through what locking is necessary + spin_lock_irqsave(&service->lock, flags); + /* + * teardown must not disable the service since since it may be sent as + * side effect of a COMMAND which for which a reply is expected. + * Seen with DCP's "av" endpoint during the close afk_service_call. + */ + service->torndown = true; + ops = service->ops; + spin_unlock_irqrestore(&service->lock, flags); + + if (ops->teardown) + ops->teardown(service); +} + +static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, + u16 tag, void *payload, size_t payload_size) +{ + struct epic_cmd *cmd = payload; + struct apple_epic_service *service; + unsigned long flags; + u8 idx = tag & 0xff; + void *rxbuf, *txbuf; + dma_addr_t rxbuf_dma, txbuf_dma; + size_t rxlen, txlen; + + service = afk_epic_find_service(ep, channel); + if (!service) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: command reply on disabled channel %u\n", + ep->endpoint, channel); + return; + } + + if (payload_size < sizeof(*cmd)) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d too small: %ld\n", + ep->endpoint, channel, payload_size); + return; + } + + if (idx >= MAX_PENDING_CMDS) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d out of range: %d\n", + ep->endpoint, channel, idx); + return; + } + + spin_lock_irqsave(&service->lock, flags); + if (service->cmds[idx].done) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d already handled\n", + ep->endpoint, channel); + spin_unlock_irqrestore(&service->lock, flags); + return; + } + + if (tag != service->cmds[idx].tag) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: command reply on channel %d has invalid tag: expected 0x%04x != 0x%04x\n", + ep->endpoint, channel, tag, service->cmds[idx].tag); + spin_unlock_irqrestore(&service->lock, flags); + return; + } + + service->cmds[idx].done = true; + service->cmds[idx].retcode = le32_to_cpu(cmd->retcode); + if (service->cmds[idx].free_on_ack) { + /* defer freeing until we're no longer in atomic context */ + rxbuf = service->cmds[idx].rxbuf; + txbuf = service->cmds[idx].txbuf; + rxlen = service->cmds[idx].rxlen; + txlen = service->cmds[idx].txlen; + rxbuf_dma = service->cmds[idx].rxbuf_dma; + txbuf_dma = service->cmds[idx].txbuf_dma; + bitmap_release_region(service->cmd_map, idx, 0); + } else { + rxbuf = txbuf = NULL; + rxlen = txlen = 0; + } + if (service->cmds[idx].completion) + complete(service->cmds[idx].completion); + + spin_unlock_irqrestore(&service->lock, flags); + + if (rxbuf && rxlen) + dma_free_coherent(ep->dcp->dev, rxlen, rxbuf, rxbuf_dma); + if (txbuf && txlen) + dma_free_coherent(ep->dcp->dev, txlen, txbuf, txbuf_dma); +} + +struct epic_std_service_ap_call { + __le32 unk0; + __le32 unk1; + __le32 type; + __le32 len; + __le32 magic; + u8 _unk[48]; +} __attribute__((packed)); + +static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, + u32 type, struct epic_hdr *ehdr, + struct epic_sub_hdr *eshdr, + void *payload, size_t payload_size) +{ + struct apple_epic_service *service = afk_epic_find_service(ep, channel); + + if (!service) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: std service notify on disabled channel %u\n", + ep->endpoint, channel); + return; + } + if (service->torndown) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: std service notify on torn down service " + "(chan:%u)\n", ep->endpoint, channel); + return; + } + + if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_NOTIFY) { + struct epic_std_service_ap_call *call = payload; + size_t call_size; + void *reply; + int ret; + + if (payload_size < sizeof(*call)) + return; + + call_size = le32_to_cpu(call->len); + if (payload_size < sizeof(*call) + call_size) + return; + + if (!service->ops->call) + return; + reply = kzalloc(payload_size, GFP_KERNEL); + if (!reply) + return; + + ret = service->ops->call(service, le32_to_cpu(call->type), + payload + sizeof(*call), call_size, + reply + sizeof(*call), call_size); + if (ret) { + kfree(reply); + return; + } + + memcpy(reply, call, sizeof(*call)); + afk_send_epic(ep, channel, le16_to_cpu(eshdr->tag), + EPIC_TYPE_NOTIFY_ACK, EPIC_CAT_REPLY, + EPIC_SUBTYPE_STD_SERVICE, reply, payload_size); + kfree(reply); + + return; + } + + if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_REPORT) { + if (service->ops->report) + service->ops->report(service, le16_to_cpu(eshdr->type), + payload, payload_size); + return; + } + + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", + ep->endpoint, channel, type, eshdr->category); + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, + payload_size, true); +} + +static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, + u8 *data, size_t data_size) +{ + struct apple_epic_service *service; + struct epic_hdr *ehdr = (struct epic_hdr *)data; + struct epic_sub_hdr *eshdr = + (struct epic_sub_hdr *)(data + sizeof(*ehdr)); + u16 subtype = le16_to_cpu(eshdr->type); + u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); + size_t payload_size; + + if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, data_size); + return; + } + payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); + + trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); + + service = afk_epic_find_service(ep, channel); + + if (!service) { + if (type != EPIC_TYPE_NOTIFY && type != EPIC_TYPE_REPLY) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: expected notify but got 0x%x on channel %d\n", + ep->endpoint, type, channel); + return; + } + if (eshdr->category != EPIC_CAT_REPORT) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: expected report but got 0x%x on channel %d\n", + ep->endpoint, eshdr->category, channel); + return; + } + if (subtype == EPIC_SUBTYPE_TEARDOWN) { + dev_dbg(ep->dcp->dev, + "AFK[ep:%02x]: teardown without service on channel %d\n", + ep->endpoint, channel); + return; + } + if (subtype != EPIC_SUBTYPE_ANNOUNCE) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: expected announce but got 0x%x on channel %d\n", + ep->endpoint, subtype, channel); + return; + } + + return afk_recv_handle_init(ep, channel, payload, payload_size); + } + + if (!service) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d has no service\n", + ep->endpoint, channel); + return; + } + + if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_REPORT && + subtype == EPIC_SUBTYPE_TEARDOWN) + return afk_recv_handle_teardown(ep, channel); + + if (type == EPIC_TYPE_REPLY && eshdr->category == EPIC_CAT_REPLY) + return afk_recv_handle_reply(ep, channel, + le16_to_cpu(eshdr->tag), payload, + payload_size); + + if (subtype == EPIC_SUBTYPE_STD_SERVICE) + return afk_recv_handle_std_service( + ep, channel, type, ehdr, eshdr, payload, payload_size); + + dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " + "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, + payload_size, true); +} + +static bool afk_recv(struct apple_dcp_afkep *ep) +{ + struct afk_qe *hdr; + u32 rptr, wptr; + u32 magic, size, channel, type; + + if (!ep->rxbfr.ready) { + dev_err(ep->dcp->dev, "AFK[ep:%02x]: got RECV but not ready\n", + ep->endpoint); + return false; + } + + rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); + wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + trace_afk_recv_rwptr_pre(ep, rptr, wptr); + + if (rptr == wptr) + return false; + + if (rptr > (ep->rxbfr.bufsz - sizeof(*hdr))) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: rptr out of bounds: 0x%x > 0x%lx\n", + ep->endpoint, rptr, ep->rxbfr.bufsz - sizeof(*hdr)); + return false; + } + + dma_rmb(); + + hdr = ep->rxbfr.buf + rptr; + magic = le32_to_cpu(hdr->magic); + size = le32_to_cpu(hdr->size); + trace_afk_recv_qe(ep, rptr, magic, size); + + if (magic != QE_MAGIC) { + dev_warn(ep->dcp->dev, "AFK[ep:%02x]: invalid queue entry magic: 0x%x\n", + ep->endpoint, magic); + return false; + } + + /* + * If there's not enough space for the payload the co-processor inserted + * the current dummy queue entry and we have to advance to the next one + * which will contain the real data. + */ + if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + rptr = 0; + hdr = ep->rxbfr.buf + rptr; + magic = le32_to_cpu(hdr->magic); + size = le32_to_cpu(hdr->size); + trace_afk_recv_qe(ep, rptr, magic, size); + + if (magic != QE_MAGIC) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: invalid next queue entry magic: 0x%x\n", + ep->endpoint, magic); + return false; + } + + ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + } + + if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { + dev_warn(ep->dcp->dev, + "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", + ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); + return false; + } + + channel = le32_to_cpu(hdr->channel); + type = le32_to_cpu(hdr->type); + + rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + if (WARN_ON(rptr > ep->rxbfr.bufsz)) + rptr = 0; + if (rptr == ep->rxbfr.bufsz) + rptr = 0; + + dma_mb(); + + ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + trace_afk_recv_rwptr_post(ep, rptr, wptr); + + /* + * TODO: this is theoretically unsafe since DCP could overwrite data + * after the read pointer was updated above. Do it anyway since + * it avoids 2 problems in the DCP tracer: + * 1. the tracer sees replies before the notifies from dcp + * 2. the tracer tries to read buffers after they are unmapped. + */ + afk_recv_handle(ep, channel, type, hdr->data, size); + + return true; +} + +static void afk_receive_message_worker(struct work_struct *work_) +{ + struct afk_receive_message_work *work; + u16 type; + + work = container_of(work_, struct afk_receive_message_work, work); + + type = FIELD_GET(RBEP_TYPE, work->message); + switch (type) { + case RBEP_INIT_ACK: + break; + + case RBEP_START_ACK: + complete_all(&work->ep->started); + break; + + case RBEP_SHUTDOWN_ACK: + complete_all(&work->ep->stopped); + break; + + case RBEP_GETBUF: + afk_getbuf(work->ep, work->message); + break; + + case RBEP_INIT_TX: + afk_init_rxtx(work->ep, work->message, &work->ep->txbfr); + break; + + case RBEP_INIT_RX: + afk_init_rxtx(work->ep, work->message, &work->ep->rxbfr); + break; + + case RBEP_RECV: + while (afk_recv(work->ep)) + ; + break; + + default: + dev_err(work->ep->dcp->dev, + "Received unknown AFK message type: 0x%x\n", type); + } + + kfree(work); +} + +int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) +{ + struct afk_receive_message_work *work; + + // TODO: comment why decoupling from rtkit thread is required here + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return -ENOMEM; + + work->ep = ep; + work->message = message; + INIT_WORK(&work->work, afk_receive_message_worker); + queue_work(ep->wq, &work->work); + + return 0; +} + +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len) +{ + u32 rptr, wptr; + struct afk_qe *hdr, *hdr2; + struct epic_hdr *ehdr; + struct epic_sub_hdr *eshdr; + unsigned long flags; + size_t total_epic_size, total_size; + int ret; + + spin_lock_irqsave(&ep->lock, flags); + + dma_rmb(); + rptr = le32_to_cpu(ep->txbfr.hdr->rptr); + wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + trace_afk_send_rwptr_pre(ep, rptr, wptr); + total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; + total_size = sizeof(*hdr) + total_epic_size; + + hdr = hdr2 = NULL; + + /* + * We need to figure out how to place the entire headers and payload + * into the ring buffer: + * - If the write pointer is in front of the read pointer we just need + * enough space inbetween to store everything. + * - If the read pointer has already wrapper around the end of the + * buffer we can + * a) either store the entire payload at the writer pointer if + * there's enough space until the end, + * b) or just store the queue entry at the write pointer to indicate + * that we need to wrap to the start and then store the headers + * and the payload at the beginning of the buffer. The queue + * header has to be store twice in this case. + * In either case we have to ensure that there's always enough space + * so that we don't accidentally overwrite other buffers. + */ + if (wptr < rptr) { + /* + * If wptr < rptr we can't wrap around and only have to make + * sure that there's enough space for the entire payload. + */ + if (wptr + total_size > rptr) { + ret = -ENOMEM; + goto out; + } + + hdr = ep->txbfr.buf + wptr; + wptr += sizeof(*hdr); + } else { + /* We need enough space to place at least a queue entry */ + if (wptr + sizeof(*hdr) > ep->txbfr.bufsz) { + ret = -ENOMEM; + goto out; + } + + /* + * If we can place a single queue entry but not the full payload + * we need to place one queue entry at the end of the ring + * buffer and then another one together with the entire + * payload at the beginning. + */ + if (wptr + total_size > ep->txbfr.bufsz) { + /* + * Ensure there's space for the queue entry at the + * beginning + */ + if (sizeof(*hdr) > rptr) { + ret = -ENOMEM; + goto out; + } + + /* + * Place two queue entries to indicate we want to wrap + * around to the firmware. + */ + hdr = ep->txbfr.buf + wptr; + hdr2 = ep->txbfr.buf; + wptr = sizeof(*hdr); + + /* Ensure there's enough space for the entire payload */ + if (wptr + total_epic_size > rptr) { + ret = -ENOMEM; + goto out; + } + } else { + /* We have enough space to place the entire payload */ + hdr = ep->txbfr.buf + wptr; + wptr += sizeof(*hdr); + } + } + /* + * At this point we're guaranteed that hdr (and possibly hdr2) point + * to a buffer large enough to fit the queue entry and that we have + * enough space at wptr to store the payload. + */ + + hdr->magic = cpu_to_le32(QE_MAGIC); + hdr->size = cpu_to_le32(total_epic_size); + hdr->channel = cpu_to_le32(channel); + hdr->type = cpu_to_le32(etype); + if (hdr2) + memcpy(hdr2, hdr, sizeof(*hdr)); + + ehdr = ep->txbfr.buf + wptr; + memset(ehdr, 0, sizeof(*ehdr)); + ehdr->version = 2; + ehdr->seq = cpu_to_le16(ep->qe_seq++); + ehdr->timestamp = cpu_to_le64(0); + wptr += sizeof(*ehdr); + + eshdr = ep->txbfr.buf + wptr; + memset(eshdr, 0, sizeof(*eshdr)); + eshdr->length = cpu_to_le32(payload_len); + eshdr->version = 4; + eshdr->category = ecat; + eshdr->type = cpu_to_le16(stype); + eshdr->timestamp = cpu_to_le64(0); + eshdr->tag = cpu_to_le16(tag); + if (ecat == EPIC_CAT_REPLY) + eshdr->inline_len = cpu_to_le32(payload_len - 4); + else + eshdr->inline_len = cpu_to_le32(0); + wptr += sizeof(*eshdr); + + memcpy(ep->txbfr.buf + wptr, payload, payload_len); + wptr += payload_len; + wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + if (wptr == ep->txbfr.bufsz) + wptr = 0; + trace_afk_send_rwptr_post(ep, rptr, wptr); + + ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | + FIELD_PREP(SEND_WPTR, wptr)); + ret = 0; + +out: + spin_unlock_irqrestore(&ep->lock, flags); + return ret; +} + +int afk_send_command(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode) +{ + struct epic_cmd cmd; + void *rxbuf, *txbuf; + dma_addr_t rxbuf_dma, txbuf_dma; + unsigned long flags; + int ret, idx; + u16 tag; + struct apple_dcp_afkep *ep = service->ep; + DECLARE_COMPLETION_ONSTACK(completion); + + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, + GFP_KERNEL); + if (!rxbuf) + return -ENOMEM; + txbuf = dma_alloc_coherent(ep->dcp->dev, payload_len, &txbuf_dma, + GFP_KERNEL); + if (!txbuf) { + ret = -ENOMEM; + goto err_free_rxbuf; + } + + memcpy(txbuf, payload, payload_len); + + memset(&cmd, 0, sizeof(cmd)); + cmd.retcode = cpu_to_le32(0); + cmd.rxbuf = cpu_to_le64(rxbuf_dma); + cmd.rxlen = cpu_to_le32(output_len); + cmd.txbuf = cpu_to_le64(txbuf_dma); + cmd.txlen = cpu_to_le32(payload_len); + + spin_lock_irqsave(&service->lock, flags); + idx = bitmap_find_free_region(service->cmd_map, MAX_PENDING_CMDS, 0); + if (idx < 0) { + ret = -ENOSPC; + goto err_unlock; + } + + tag = (service->cmd_tag & 0xff) << 8; + tag |= idx & 0xff; + service->cmd_tag++; + + service->cmds[idx].tag = tag; + service->cmds[idx].rxbuf = rxbuf; + service->cmds[idx].txbuf = txbuf; + service->cmds[idx].rxbuf_dma = rxbuf_dma; + service->cmds[idx].txbuf_dma = txbuf_dma; + service->cmds[idx].rxlen = output_len; + service->cmds[idx].txlen = payload_len; + service->cmds[idx].free_on_ack = false; + service->cmds[idx].done = false; + service->cmds[idx].completion = &completion; + init_completion(&completion); + + spin_unlock_irqrestore(&service->lock, flags); + + ret = afk_send_epic(service->ep, service->channel, tag, + EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, + sizeof(cmd)); + if (ret) + goto err_free_cmd; + + ret = wait_for_completion_timeout(&completion, + msecs_to_jiffies(MSEC_PER_SEC)); + + if (ret <= 0) { + spin_lock_irqsave(&service->lock, flags); + /* + * Check again while we're inside the lock to make sure + * the command wasn't completed just after + * wait_for_completion_timeout returned. + */ + if (!service->cmds[idx].done) { + service->cmds[idx].completion = NULL; + service->cmds[idx].free_on_ack = true; + spin_unlock_irqrestore(&service->lock, flags); + return -ETIMEDOUT; + } + spin_unlock_irqrestore(&service->lock, flags); + } + + ret = 0; + if (retcode) + *retcode = service->cmds[idx].retcode; + if (output && output_len) + memcpy(output, rxbuf, output_len); + +err_free_cmd: + spin_lock_irqsave(&service->lock, flags); + bitmap_release_region(service->cmd_map, idx, 0); +err_unlock: + spin_unlock_irqrestore(&service->lock, flags); + dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); +err_free_rxbuf: + dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); + return ret; +} + +int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t data_len, size_t data_pad, + void *output, size_t output_len, size_t output_pad) +{ + struct epic_service_call *call; + void *bfr; + size_t bfr_len = max(data_len + data_pad, output_len + output_pad) + + sizeof(*call); + int ret; + u32 retcode; + u32 retlen; + + bfr = kzalloc(bfr_len, GFP_KERNEL); + if (!bfr) + return -ENOMEM; + + call = bfr; + + memset(call, 0, sizeof(*call)); + call->group = cpu_to_le16(group); + call->command = cpu_to_le32(command); + call->data_len = cpu_to_le32(data_len + data_pad); + call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); + + memcpy(bfr + sizeof(*call), data, data_len); + + ret = afk_send_command(service, EPIC_SUBTYPE_STD_SERVICE, bfr, bfr_len, + bfr, bfr_len, &retcode); + if (ret) + goto out; + if (retcode) { + ret = -EINVAL; + goto out; + } + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le16_to_cpu(call->group) != group || + le32_to_cpu(call->command) != command) { + ret = -EINVAL; + goto out; + } + + retlen = le32_to_cpu(call->data_len); + if (output_len < retlen) + retlen = output_len; + if (output && output_len) { + memset(output, 0, output_len); + memcpy(output, bfr + sizeof(*call), retlen); + } + +out: + kfree(bfr); + return ret; +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) + +#define AFK_DEBUGFS_MAX_REPLY 8192 + +static ssize_t service_call_write_file(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + void *buf; + int ret; + struct { + u32 group; + u32 command; + } call_info; + + if (count < sizeof(call_info)) + return -EINVAL; + if (!srv->debugfs.scratch) { + srv->debugfs.scratch = \ + devm_kzalloc(srv->ep->dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); + if (!srv->debugfs.scratch) + return -ENOMEM; + } + + ret = copy_from_user(&call_info, user_buf, sizeof(call_info)); + if (ret == sizeof(call_info)) + return -EFAULT; + user_buf += sizeof(call_info); + count -= sizeof(call_info); + + buf = kmalloc(count, GFP_KERNEL); + if (!buf) + return -ENOMEM; + ret = copy_from_user(buf, user_buf, count); + if (ret == count) { + kfree(buf); + return -EFAULT; + } + + memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); + dma_mb(); + + ret = afk_service_call(srv, call_info.group, call_info.command, buf, count, 0, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY, 0); + kfree(buf); + + if (ret < 0) + return ret; + + return count + sizeof(call_info); +} + +static ssize_t service_call_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + + if (!srv->debugfs.scratch) + return -EINVAL; + + return simple_read_from_buffer(user_buf, count, ppos, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY); +} + +static const struct file_operations service_call_fops = { + .open = simple_open, + .write = service_call_write_file, + .read = service_call_read_file, +}; + +static ssize_t service_raw_call_write_file(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + u32 retcode; + int ret; + + if (!srv->debugfs.scratch) { + srv->debugfs.scratch = \ + devm_kzalloc(srv->ep->dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); + if (!srv->debugfs.scratch) + return -ENOMEM; + } + + memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); + ret = copy_from_user(srv->debugfs.scratch, user_buf, count); + if (ret == count) + return -EFAULT; + + ret = afk_send_command(srv, EPIC_SUBTYPE_STD_SERVICE, srv->debugfs.scratch, count, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY, &retcode); + if (ret < 0) + return ret; + if (retcode) + return -EINVAL; + + return count; +} + +static ssize_t service_raw_call_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct apple_epic_service *srv = file->private_data; + + if (!srv->debugfs.scratch) + return -EINVAL; + + return simple_read_from_buffer(user_buf, count, ppos, + srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY); +} + +static const struct file_operations service_raw_call_fops = { + .open = simple_open, + .write = service_raw_call_write_file, + .read = service_raw_call_read_file, +}; + +static void afk_populate_service_debugfs(struct apple_epic_service *srv) +{ + if (!srv->ep->debugfs_entry || !srv->ops) + return; + + if (strcmp(srv->ops->name, "DCPAVAudioInterface") == 0) { + srv->debugfs.entry = debugfs_create_dir(srv->ops->name, + srv->ep->debugfs_entry); + debugfs_create_file("call", 0600, srv->debugfs.entry, srv, + &service_call_fops); + debugfs_create_file("raw_call", 0600, srv->debugfs.entry, srv, + &service_raw_call_fops); + } +} + +static void afk_remove_service_debugfs(struct apple_epic_service *srv) +{ + if (srv->debugfs.entry) { + debugfs_remove_recursive(srv->debugfs.entry); + srv->debugfs.entry = NULL; + } +} + +#endif diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h new file mode 100644 index 00000000000000..a339c00a2a0138 --- /dev/null +++ b/drivers/gpu/drm/apple/afk.h @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * AFK (Apple Firmware Kit) EPIC (EndPoint Interface Client) support + */ +/* Copyright 2022 Sven Peter */ + +#ifndef _DRM_APPLE_DCP_AFK_H +#define _DRM_APPLE_DCP_AFK_H + +#include +#include +#include + +#include "dcp.h" + +#define AFK_MAX_CHANNEL 16 +#define MAX_PENDING_CMDS 16 + +struct apple_epic_service_ops; +struct apple_dcp_afkep; + +struct epic_cmd_info { + u16 tag; + + void *rxbuf; + void *txbuf; + dma_addr_t rxbuf_dma; + dma_addr_t txbuf_dma; + size_t rxlen; + size_t txlen; + + u32 retcode; + bool done; + bool free_on_ack; + struct completion *completion; +}; + +struct apple_epic_service { + const struct apple_epic_service_ops *ops; + struct apple_dcp_afkep *ep; + + struct epic_cmd_info cmds[MAX_PENDING_CMDS]; + DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); + u8 cmd_tag; + spinlock_t lock; + + u32 channel; + bool enabled; + bool torndown; + + void *cookie; + + struct { + struct dentry *entry; + u8 *scratch; + } debugfs; +}; + +enum epic_subtype; + +struct apple_epic_service_ops { + const char name[32]; + + void (*init)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); + int (*call)(struct apple_epic_service *service, u32 idx, + const void *data, size_t data_size, void *reply, + size_t reply_size); + int (*report)(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size); + void (*teardown)(struct apple_epic_service *service); +}; + +struct afk_ringbuffer_header { + __le32 bufsz; + u32 unk; + u32 _pad1[14]; + __le32 rptr; + u32 _pad2[15]; + __le32 wptr; + u32 _pad3[15]; +}; + +struct afk_qe { +#define QE_MAGIC 0x20504f49 // ' POI' + __le32 magic; + __le32 size; + __le32 channel; + __le32 type; + u8 data[]; +}; + +struct epic_hdr { + u8 version; + __le16 seq; + u8 _pad; + __le32 unk; + __le64 timestamp; +} __attribute__((packed)); + +struct epic_sub_hdr { + __le32 length; + u8 version; + u8 category; + __le16 type; + __le64 timestamp; + __le16 tag; + __le16 unk; + __le32 inline_len; +} __attribute__((packed)); + +struct epic_cmd { + __le32 retcode; + __le64 rxbuf; + __le64 txbuf; + __le32 rxlen; + __le32 txlen; + u8 rxcookie; + u8 txcookie; +} __attribute__((packed)); + +struct epic_service_call { + u8 _pad0[2]; + __le16 group; + __le32 command; + __le32 data_len; +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 + __le32 magic; + u8 _pad1[48]; +} __attribute__((packed)); +static_assert(sizeof(struct epic_service_call) == 64); + +enum epic_type { + EPIC_TYPE_NOTIFY = 0, + EPIC_TYPE_COMMAND = 3, + EPIC_TYPE_REPLY = 4, + EPIC_TYPE_NOTIFY_ACK = 8, +}; + +enum epic_category { + EPIC_CAT_REPORT = 0x00, + EPIC_CAT_NOTIFY = 0x10, + EPIC_CAT_REPLY = 0x20, + EPIC_CAT_COMMAND = 0x30, +}; + +enum epic_subtype { + EPIC_SUBTYPE_ANNOUNCE = 0x30, + EPIC_SUBTYPE_TEARDOWN = 0x32, + EPIC_SUBTYPE_STD_SERVICE = 0xc0, +}; + +struct afk_ringbuffer { + bool ready; + struct afk_ringbuffer_header *hdr; + u32 rptr; + void *buf; + size_t bufsz; +}; + +struct apple_dcp_afkep { + struct apple_dcp *dcp; + + u32 endpoint; + struct workqueue_struct *wq; + + struct completion started; + struct completion stopped; + + void *bfr; + u16 bfr_tag; + size_t bfr_size; + dma_addr_t bfr_dma; + + struct afk_ringbuffer txbfr; + struct afk_ringbuffer rxbfr; + + spinlock_t lock; + u16 qe_seq; + + const struct apple_epic_service_ops *ops; + struct apple_epic_service services[AFK_MAX_CHANNEL]; + u32 num_channels; + + struct dentry *debugfs_entry; + + bool match_epic_name; +}; + +struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, + const struct apple_epic_service_ops *ops); +int afk_start(struct apple_dcp_afkep *ep); +void afk_shutdown(struct apple_dcp_afkep *ep); +int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len); +int afk_send_command(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode); +int afk_service_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t data_len, size_t data_pad, + void *output, size_t output_len, size_t output_pad); +#endif diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c new file mode 100644 index 00000000000000..31b86e909014ad --- /dev/null +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -0,0 +1,699 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ +/* Based on meson driver which is + * Copyright (C) 2016 BayLibre, SAS + * Author: Neil Armstrong + * Copyright (C) 2015 Amlogic, Inc. All rights reserved. + * Copyright (C) 2014 Endless Mobile + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "plane.h" + +#define DRIVER_NAME "apple" +#define DRIVER_DESC "Apple display controller DRM driver" + +#define MAX_COPROCESSORS 3 + +struct apple_drm_private { + struct drm_device drm; +}; + +DEFINE_DRM_GEM_DMA_FOPS(apple_fops); + +#define DART_PAGE_SIZE 16384 + +static int apple_drm_gem_dumb_create(struct drm_file *file_priv, + struct drm_device *drm, + struct drm_mode_create_dumb *args) +{ + args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); + args->size = round_up(args->pitch * args->height, DART_PAGE_SIZE); + + return drm_gem_dma_dumb_create_internal(file_priv, drm, args); +} + +static const struct drm_driver apple_drm_driver = { + DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), + DRM_FBDEV_DMA_DRIVER_OPS, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .major = 1, + .minor = 0, + .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC | DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, + .fops = &apple_fops, +}; + +static enum drm_connector_status +apple_connector_detect(struct drm_connector *connector, bool force) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + + return apple_connector->connected ? connector_status_connected : + connector_status_disconnected; +} + +static void apple_connector_oob_hotplug(struct drm_connector *connector, + enum drm_connector_status status) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + + printk("#### oob_hotplug status:0x%x ####\n", (u32)status); + + if (status == connector_status_connected) + dcp_dptx_connect_oob(apple_connector->dcp, 0); + else if (status == connector_status_disconnected) + dcp_dptx_disconnect_oob(apple_connector->dcp, 0); + else + dev_err(&apple_connector->dcp->dev, "unexpected connector status" + ":0x%x in oob_hotplug event\n", (u32)status); +} + +static void apple_crtc_atomic_enable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_crtc_state *crtc_state; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + if (crtc_state->active_changed && crtc_state->active) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dcp_poweron(apple_crtc->dcp); + /* Force the CTM to be set on first swap */ + crtc_state->color_mgmt_changed = true; + } + + if (crtc_state->active) + dcp_crtc_atomic_modeset(crtc, state); +} + +static void apple_crtc_atomic_disable(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct drm_crtc_state *crtc_state; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + if (crtc_state->active_changed && !crtc_state->active) { + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + dcp_poweroff(apple_crtc->dcp); + } + + if (crtc->state->event && !crtc->state->active) { + spin_lock_irq(&crtc->dev->event_lock); + drm_crtc_send_vblank_event(crtc, crtc->state->event); + spin_unlock_irq(&crtc->dev->event_lock); + + crtc->state->event = NULL; + } +} + +static void apple_crtc_atomic_begin(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + unsigned long flags; + + if (crtc->state->event) { + spin_lock_irqsave(&crtc->dev->event_lock, flags); + apple_crtc->event = crtc->state->event; + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + crtc->state->event = NULL; + } +} + +static void apple_crtc_cleanup(struct drm_crtc *crtc) +{ + drm_crtc_cleanup(crtc); + kfree(to_apple_crtc(crtc)); +} + +static int apple_crtc_parse_crc_source(const char *source, bool *enabled) +{ + int ret = 0; + + if (!source) { + *enabled = false; + } else if (strcmp(source, "auto") == 0) { + *enabled = true; + } else { + *enabled = false; + ret = -EINVAL; + } + + return ret; +} + +static int apple_crtc_set_crc_source(struct drm_crtc *crtc, const char *source) +{ + bool enabled = false; + + int ret = apple_crtc_parse_crc_source(source, &enabled); + + if (!ret) + dcp_set_crc(crtc, enabled); + + return ret; +} + +static int apple_crtc_verify_crc_source(struct drm_crtc *crtc, + const char *source, + size_t *values_cnt) +{ + bool enabled; + + if (apple_crtc_parse_crc_source(source, &enabled) < 0) { + pr_warn("dcp: Invalid CRC source name %s\n", source); + return -EINVAL; + } + + *values_cnt = 1; + + return 0; +} + +static const char * const apple_crtc_crc_sources[] = {"auto"}; + +static const char *const * apple_crtc_get_crc_sources(struct drm_crtc *crtc, + size_t *count) +{ + *count = ARRAY_SIZE(apple_crtc_crc_sources); + return apple_crtc_crc_sources; +} + +static const struct drm_crtc_funcs apple_crtc_funcs = { + .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, + .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, + .destroy = apple_crtc_cleanup, + .page_flip = drm_atomic_helper_page_flip, + .reset = drm_atomic_helper_crtc_reset, + .set_config = drm_atomic_helper_set_config, + .set_crc_source = apple_crtc_set_crc_source, + .verify_crc_source = apple_crtc_verify_crc_source, + .get_crc_sources = apple_crtc_get_crc_sources, + +}; + +static const struct drm_mode_config_funcs apple_mode_config_funcs = { + .atomic_check = drm_atomic_helper_check, + .atomic_commit = drm_atomic_helper_commit, + .fb_create = drm_gem_fb_create, +}; + +static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { + .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, +}; + +static void appledrm_connector_cleanup(struct drm_connector *connector) +{ + drm_connector_cleanup(connector); + kfree(to_apple_connector(connector)); +} + +static const struct drm_connector_funcs apple_connector_funcs = { + .fill_modes = drm_helper_probe_single_connector_modes, + .destroy = appledrm_connector_cleanup, + .reset = drm_atomic_helper_connector_reset, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .detect = apple_connector_detect, + .debugfs_init = apple_connector_debugfs_init, + .oob_hotplug_event = apple_connector_oob_hotplug, +}; + +static const struct drm_connector_helper_funcs apple_connector_helper_funcs = { + .get_modes = dcp_get_modes, + .mode_valid = dcp_mode_valid, +}; + +static const struct drm_crtc_helper_funcs apple_crtc_helper_funcs = { + .atomic_begin = apple_crtc_atomic_begin, + .atomic_check = dcp_crtc_atomic_check, + .atomic_flush = dcp_flush, + .atomic_enable = apple_crtc_atomic_enable, + .atomic_disable = apple_crtc_atomic_disable, + .mode_fixup = dcp_crtc_mode_fixup, +}; + +static int apple_probe_per_dcp(struct device *dev, + struct drm_device *drm, + struct platform_device *dcp, + int num, bool dcp_ext) +{ + struct apple_crtc *crtc; + struct apple_connector *connector; + struct apple_encoder *enc; + struct drm_plane *planes[DCP_MAX_PLANES]; + int ret, i; + int immutable_zpos = 0; + bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); + + planes[0] = apple_plane_init(drm, 1U << num, supports_l10r, + DRM_PLANE_TYPE_PRIMARY); + if (IS_ERR(planes[0])) + return PTR_ERR(planes[0]); + ret = drm_plane_create_zpos_immutable_property(planes[0], immutable_zpos); + if (ret) { + return ret; + } + + + /* Set up our other planes */ + for (i = 1; i < DCP_MAX_PLANES; i++) { + planes[i] = apple_plane_init(drm, 1U << num, supports_l10r, + DRM_PLANE_TYPE_OVERLAY); + if (IS_ERR(planes[i])) + return PTR_ERR(planes[i]); + immutable_zpos++; + ret = drm_plane_create_zpos_immutable_property(planes[i], immutable_zpos); + if (ret) { + return ret; + } + } + + /* + * Even though we have an overlay plane, we cannot expose it to legacy + * userspace for cursors as we cannot make the same guarantees as ye olde + * hardware cursor planes such userspace would expect us to. Modern userspace + * knows what to do with overlays. + */ + crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); + ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, + &apple_crtc_funcs, NULL); + if (ret) + return ret; + + drm_crtc_helper_add(&crtc->base, &apple_crtc_helper_funcs); + drm_crtc_enable_color_mgmt(&crtc->base, 0, true, 0); + + enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, + DRM_MODE_ENCODER_TMDS); + if (IS_ERR(enc)) + return PTR_ERR(enc); + enc->base.possible_crtcs = drm_crtc_mask(&crtc->base); + + connector = kzalloc(sizeof(*connector), GFP_KERNEL); + mutex_init(&connector->chunk_lock); + drm_connector_helper_add(&connector->base, + &apple_connector_helper_funcs); + + // HACK: + if (dcp_ext) + connector->base.fwnode = fwnode_handle_get(dcp->dev.fwnode); + + ret = drm_connector_init(drm, &connector->base, &apple_connector_funcs, + dcp_get_connector_type(dcp)); + if (ret) + return ret; + + ret = drm_connector_attach_vrr_capable_property(&connector->base); + if (ret) + return ret; + + connector->base.polled = DRM_CONNECTOR_POLL_HPD; + connector->connected = false; + connector->dcp = dcp; + + INIT_WORK(&connector->hotplug_wq, dcp_hotplug); + + crtc->dcp = dcp; + dcp_link(dcp, crtc, connector); + + return drm_connector_attach_encoder(&connector->base, &enc->base); +} + +static int apple_get_fb_resource(struct device *dev, const char *name, + struct resource *fb_r) +{ + int idx, ret = -ENODEV; + struct device_node *node; + + idx = of_property_match_string(dev->of_node, "memory-region-names", name); + + node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!node) { + dev_err(dev, "reserved-memory node '%s' not found\n", name); + return -ENODEV; + } + + if (!of_device_is_available(node)) { + dev_err(dev, "reserved-memory node '%s' is unavailable\n", name); + goto err; + } + + if (!of_device_is_compatible(node, "framebuffer")) { + dev_err(dev, "reserved-memory node '%s' is incompatible\n", + node->full_name); + goto err; + } + + ret = of_address_to_resource(node, 0, fb_r); + +err: + of_node_put(node); + return ret; +} + +static const struct of_device_id apple_dcp_id_tbl[] = { + { .compatible = "apple,dcp" }, + { .compatible = "apple,dcpext" }, + {}, +}; + +static int apple_drm_init_dcp(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + struct platform_device *dcp[MAX_COPROCESSORS]; + struct device_node *np; + u64 timeout; + int i, ret, num_dcp = 0; + + for_each_matching_node(np, apple_dcp_id_tbl) { + bool dcp_ext; + if (!of_device_is_available(np)) { + of_node_put(np); + continue; + } + dcp_ext = of_device_is_compatible(np, "apple,dcpext") || + of_property_present(np, "phys"); + + dcp[num_dcp] = of_find_device_by_node(np); + of_node_put(np); + if (!dcp[num_dcp]) + continue; + + device_link_add(dev, &dcp[num_dcp]->dev, DL_FLAG_AUTOREMOVE_SUPPLIER); + + ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], + num_dcp, dcp_ext); + if (ret) + continue; + + ret = dcp_start(dcp[num_dcp]); + if (ret) + continue; + + num_dcp++; + } + + if (num_dcp < 1) + return -ENODEV; + + /* + * Starting DPTX might take some time. + */ + timeout = get_jiffies_64() + msecs_to_jiffies(3000); + + for (i = 0; i < num_dcp; ++i) { + u64 jiffies = get_jiffies_64(); + u64 wait = time_after_eq64(jiffies, timeout) ? + 0 : + timeout - jiffies; + ret = dcp_wait_ready(dcp[i], wait); + /* There is nothing we can do if a dcp/dcpext does not boot + * (successfully). Ignoring it should not do any harm now. + * Needs to reevaluated when adding dcpext support. + */ + if (ret) + dev_warn(dev, "DCP[%d] not ready: %d\n", i, ret); + } + /* HACK: Wait for dcp* to settle before a modeset */ + msleep(100); + + return 0; +} + +static int apple_drm_init(struct device *dev) +{ + struct apple_drm_private *apple; + struct resource fb_r; + resource_size_t fb_size; + int ret; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + + ret = apple_get_fb_resource(dev, "framebuffer", &fb_r); + if (ret) + return ret; + + apple = devm_drm_dev_alloc(dev, &apple_drm_driver, + struct apple_drm_private, drm); + if (IS_ERR(apple)) + return PTR_ERR(apple); + + dev_set_drvdata(dev, apple); + + ret = component_bind_all(dev, apple); + if (ret) + return ret; + + ret = drmm_mode_config_init(&apple->drm); + if (ret) + goto err_unbind; + + /* + * IOMFB::UPPipeDCP_H13P::verify_surfaces produces the error "plane + * requires a minimum of 32x32 for the source buffer" if smaller + */ + apple->drm.mode_config.min_width = 32; + apple->drm.mode_config.min_height = 32; + + /* + * TODO: this is the max framebuffer size not the maximal supported + * output resolution. DCP reports the maximal framebuffer size take it + * from there. + * Hardcode it for now to the M1 Max DCP reported 'MaxSrcBufferWidth' + * and 'MaxSrcBufferHeight' of 16384. + */ + apple->drm.mode_config.max_width = 16384; + apple->drm.mode_config.max_height = 16384; + + apple->drm.mode_config.funcs = &apple_mode_config_funcs; + apple->drm.mode_config.helper_private = &apple_mode_config_helpers; + + ret = apple_drm_init_dcp(dev); + if (ret) + goto err_unbind; + + drm_mode_config_reset(&apple->drm); + + fb_size = fb_r.end - fb_r.start + 1; + ret = aperture_remove_conflicting_devices(fb_r.start, fb_size, + apple_drm_driver.name); + if (ret) { + dev_err(dev, "Failed remove fb: %d\n", ret); + goto err_unbind; + } + + ret = drm_dev_register(&apple->drm, 0); + if (ret) + goto err_unbind; + + drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); + + return 0; + +err_unbind: + component_unbind_all(dev, NULL); + return ret; +} + +static void apple_drm_uninit(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + drm_dev_unregister(&apple->drm); + drm_atomic_helper_shutdown(&apple->drm); + + component_unbind_all(dev, NULL); + + dev_set_drvdata(dev, NULL); +} + +static int apple_drm_bind(struct device *dev) +{ + return apple_drm_init(dev); +} + +static void apple_drm_unbind(struct device *dev) +{ + apple_drm_uninit(dev); +} + +const struct component_master_ops apple_drm_ops = { + .bind = apple_drm_bind, + .unbind = apple_drm_unbind, +}; + +static int add_dcp_components(struct device *dev, + struct component_match **matchptr) +{ + struct device_node *np, *endpoint, *port; + int num = 0; + + for_each_matching_node(np, apple_dcp_id_tbl) { + if (of_device_is_available(np)) { + drm_of_component_match_add(dev, matchptr, + component_compare_of, np); + num++; + for_each_endpoint_of_node(np, endpoint) { + port = of_graph_get_remote_port_parent(endpoint); + if (!port) + continue; + +#if !IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (of_device_is_compatible(port, "apple,dpaudio")) { + of_node_put(port); + continue; + } +#endif + + /* + * The ATC phy driver is not part of the component + * collection for the Apple display-subsystem so + * ignore it here. + */ + if (of_device_is_compatible(port, "apple,t8103-atcphy")) { + of_node_put(port); + continue; + } + + if (of_device_is_available(port)) + drm_of_component_match_add(dev, matchptr, + component_compare_of, + port); + of_node_put(port); + } + } + of_node_put(np); + } + + return num; +} + +static int apple_platform_probe(struct platform_device *pdev) +{ + struct device *mdev = &pdev->dev; + struct component_match *match = NULL; + int num_dcp; + + /* add DCP components, handle less than 1 as probe error */ + num_dcp = add_dcp_components(mdev, &match); + if (num_dcp < 1) + return -ENODEV; + + return component_master_add_with_match(mdev, &apple_drm_ops, match); +} + +static void apple_platform_remove(struct platform_device *pdev) +{ + component_master_del(&pdev->dev, &apple_drm_ops); +} + +static const struct of_device_id of_match[] = { + { .compatible = "apple,display-subsystem" }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +#ifdef CONFIG_PM_SLEEP +static int apple_platform_suspend(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple) + return drm_mode_config_helper_suspend(&apple->drm); + + return 0; +} + +static int apple_platform_resume(struct device *dev) +{ + struct apple_drm_private *apple = dev_get_drvdata(dev); + + if (apple) + drm_mode_config_helper_resume(&apple->drm); + + return 0; +} + +static const struct dev_pm_ops apple_platform_pm_ops = { + .suspend = apple_platform_suspend, + .resume = apple_platform_resume, +}; +#endif + +static struct platform_driver apple_platform_driver = { + .driver = { + .name = "apple-drm", + .of_match_table = of_match, +#ifdef CONFIG_PM_SLEEP + .pm = &apple_platform_pm_ops, +#endif + }, + .probe = apple_platform_probe, + .remove = apple_platform_remove, +}; + + + +static int __init appledrm_register(void) +{ + if (drm_firmware_drivers_only()) + return -ENODEV; + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + dcp_audio_register(); +#endif + dcp_register(); + platform_driver_register(&apple_platform_driver); + + return 0; +} + +static void __exit appledrm_unregister(void) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + dcp_audio_unregister(); +#endif + dcp_unregister(); + platform_driver_unregister(&apple_platform_driver); +} + +module_init(appledrm_register); +module_exit(appledrm_unregister); + +MODULE_AUTHOR("Asahi Linux contributors"); +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c new file mode 100644 index 00000000000000..38718e2f56117b --- /dev/null +++ b/drivers/gpu/drm/apple/audio.c @@ -0,0 +1,776 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * DCP Audio Bits + * + * Copyright (C) The Asahi Linux Contributors + * + * TODO: + * - figure some nice identification of the sound card (in case + * there's many DCP instances) + */ + +#define DEBUG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "av.h" +#include "dcp.h" +#include "audio.h" +#include "parser.h" + +#define DCPAUD_ELEMENTS_MAXSIZE 16384 +#define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 + +struct dcp_audio { + struct device *dev; + struct device *dcp_dev; + struct device *dma_dev; + struct device_link *dma_link; + struct dma_chan *chan; + struct snd_card *card; + struct snd_jack *jack; + struct snd_pcm_substream *substream; + unsigned int open_cookie; + + struct mutex data_lock; + bool dcp_connected; /// dcp status keep for delayed initialization + bool connected; + unsigned int connection_cookie; + + struct snd_pcm_chmap_elem selected_chmap; + struct dcp_sound_cookie selected_cookie; + void *elements; + void *productattrs; + + struct snd_pcm_chmap *chmap_info; +}; + +static const struct snd_pcm_hardware dcp_pcm_hw = { + .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE | + SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE, + .rates = SNDRV_PCM_RATE_CONTINUOUS, + .rate_min = 0, + .rate_max = UINT_MAX, + .channels_min = 1, + .channels_max = 16, + .buffer_bytes_max = SIZE_MAX, + .period_bytes_min = 4096, /* TODO */ + .period_bytes_max = SIZE_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, +}; + +static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) +{ + int ret; + + ret = dcp_audiosrv_get_elements(dcpaud->dcp_dev, dcpaud->elements, + DCPAUD_ELEMENTS_MAXSIZE); + if (ret < 0) + return ret; + + ret = dcp_audiosrv_get_product_attrs(dcpaud->dcp_dev, dcpaud->productattrs, + DCPAUD_PRODUCTATTRS_MAXSIZE); + if (ret < 0) + return ret; + + return 0; +} + +static int dcpaud_interval_bitmask(struct snd_interval *i, + unsigned int mask) +{ + struct snd_interval range; + if (!mask) + return -EINVAL; + + snd_interval_any(&range); + range.min = __ffs(mask); + range.max = __fls(mask); + return snd_interval_refine(i, &range); +} + +extern const struct snd_pcm_hw_constraint_list snd_pcm_known_rates; + +static void dcpaud_fill_fmt_sieve(struct snd_pcm_hw_params *params, + struct dcp_sound_format_mask *sieve) +{ + struct snd_interval *c = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_CHANNELS); + struct snd_interval *r = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + struct snd_mask *f = hw_param_mask(params, + SNDRV_PCM_HW_PARAM_FORMAT); + int i; + + sieve->nchans = GENMASK(c->max, c->min); + sieve->formats = f->bits[0] | ((u64) f->bits[1]) << 32; /* TODO: don't open-code */ + + for (i = 0; i < snd_pcm_known_rates.count; i++) { + unsigned int rate = snd_pcm_known_rates.list[i]; + + if (snd_interval_test(r, rate)) + sieve->rates |= 1u << i; + } +} + +static void dcpaud_consult_elements(struct dcp_audio *dcpaud, + struct snd_pcm_hw_params *params, + struct dcp_sound_format_mask *hits) +{ + struct dcp_sound_format_mask sieve; + struct dcp_parse_ctx elements = { + .dcp = dev_get_drvdata(dcpaud->dcp_dev), + .blob = dcpaud->elements + 4, + .len = DCPAUD_ELEMENTS_MAXSIZE - 4, + .pos = 0, + }; + + dcpaud_fill_fmt_sieve(params, &sieve); + dev_dbg(dcpaud->dev, "elements in: %llx %x %x\n", sieve.formats, sieve.nchans, sieve.rates); + parse_sound_constraints(&elements, &sieve, hits); + dev_dbg(dcpaud->dev, "elements out: %llx %x %x\n", hits->formats, hits->nchans, hits->rates); +} + +static int dcpaud_select_cookie(struct dcp_audio *dcpaud, + struct snd_pcm_hw_params *params) +{ + struct dcp_sound_format_mask sieve; + struct dcp_parse_ctx elements = { + .dcp = dev_get_drvdata(dcpaud->dcp_dev), + .blob = dcpaud->elements + 4, + .len = DCPAUD_ELEMENTS_MAXSIZE - 4, + .pos = 0, + }; + + dcpaud_fill_fmt_sieve(params, &sieve); + return parse_sound_mode(&elements, &sieve, &dcpaud->selected_chmap, + &dcpaud->selected_cookie); +} + +static int dcpaud_rule_channels(struct snd_pcm_hw_params *params, + struct snd_pcm_hw_rule *rule) +{ + struct dcp_audio *dcpaud = rule->private; + struct snd_interval *c = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_CHANNELS); + struct dcp_sound_format_mask hits = {0, 0, 0}; + + dcpaud_consult_elements(dcpaud, params, &hits); + + return dcpaud_interval_bitmask(c, hits.nchans); +} + +static int dcpaud_refine_fmt_mask(struct snd_mask *m, u64 mask) +{ + struct snd_mask mask_mask; + + if (!mask) + return -EINVAL; + mask_mask.bits[0] = mask; + mask_mask.bits[1] = mask >> 32; + + return snd_mask_refine(m, &mask_mask); +} + +static int dcpaud_rule_format(struct snd_pcm_hw_params *params, + struct snd_pcm_hw_rule *rule) +{ + struct dcp_audio *dcpaud = rule->private; + struct snd_mask *f = hw_param_mask(params, + SNDRV_PCM_HW_PARAM_FORMAT); + struct dcp_sound_format_mask hits; + + dcpaud_consult_elements(dcpaud, params, &hits); + + return dcpaud_refine_fmt_mask(f, hits.formats); +} + +static int dcpaud_rule_rate(struct snd_pcm_hw_params *params, + struct snd_pcm_hw_rule *rule) +{ + struct dcp_audio *dcpaud = rule->private; + struct snd_interval *r = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + struct dcp_sound_format_mask hits; + + dcpaud_consult_elements(dcpaud, params, &hits); + + return snd_interval_rate_bits(r, hits.rates); +} + +static int dcpaud_init_dma(struct dcp_audio *dcpaud) +{ + struct dma_chan *chan; + if (dcpaud->chan) + return 0; + + chan = of_dma_request_slave_channel(dcpaud->dev->of_node, "tx"); + /* squelch dma channel request errors, the driver will try again alter */ + if (!chan) { + dev_warn(dcpaud->dev, "audio TX DMA channel request failed\n"); + return -ENXIO; + } else if (chan == ERR_PTR(-EPROBE_DEFER)) { + dev_info(dcpaud->dev, "audio TX DMA channel is not ready yet\n"); + return -ENXIO; + } else if (IS_ERR(chan)) { + dev_warn(dcpaud->dev, "audio TX DMA channel request failed: %ld\n", PTR_ERR(chan)); + return PTR_ERR(chan); + } + dcpaud->chan = chan; + + snd_pcm_set_managed_buffer(dcpaud->substream, SNDRV_DMA_TYPE_DEV_IRAM, + dcpaud->chan->device->dev, 1024 * 1024, + SIZE_MAX); + + return 0; +} + +static int dcp_pcm_open(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct snd_dmaengine_dai_dma_data dma_data = { + .flags = SND_DMAENGINE_PCM_DAI_FLAG_PACK, + }; + struct snd_pcm_hardware hw; + int ret; + + mutex_lock(&dcpaud->data_lock); + ret = dcpaud_init_dma(dcpaud); + if (ret < 0) + return ret; + + if (!dcpaud->connected) { + mutex_unlock(&dcpaud->data_lock); + return -ENXIO; + } + dcpaud->open_cookie = dcpaud->connection_cookie; + mutex_unlock(&dcpaud->data_lock); + + ret = dcpaud_read_remote_info(dcpaud); + if (ret < 0) + return ret; + + snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_FORMAT, + dcpaud_rule_format, dcpaud, + SNDRV_PCM_HW_PARAM_CHANNELS, SNDRV_PCM_HW_PARAM_RATE, -1); + snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, + dcpaud_rule_channels, dcpaud, + SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_RATE, -1); + snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_RATE, + dcpaud_rule_rate, dcpaud, + SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, -1); + + hw = dcp_pcm_hw; + hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED; + hw.periods_min = 2; + hw.periods_max = UINT_MAX; + hw.period_bytes_min = 256; + hw.period_bytes_max = SIZE_MAX; // TODO dma_get_max_seg_size(dma_dev); + hw.buffer_bytes_max = SIZE_MAX; + hw.fifo_size = 16; + ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, &dma_data, + &hw, dcpaud->chan); + if (ret) + return ret; + substream->runtime->hw = hw; + + return snd_dmaengine_pcm_open(substream, dcpaud->chan); +} + +static int dcp_pcm_close(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + dcpaud->selected_chmap.channels = 0; + + return snd_dmaengine_pcm_close(substream); +} + +static int dcpaud_connection_up(struct dcp_audio *dcpaud) +{ + bool ret; + mutex_lock(&dcpaud->data_lock); + ret = dcpaud->connected && + dcpaud->open_cookie == dcpaud->connection_cookie; + mutex_unlock(&dcpaud->data_lock); + return ret; +} + +static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct dma_slave_config slave_config; + struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); + int ret; + + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + ret = dcpaud_select_cookie(dcpaud, params); + if (ret < 0) + return ret; + if (!ret) + return -EINVAL; + + memset(&slave_config, 0, sizeof(slave_config)); + ret = snd_hwparams_to_dma_slave_config(substream, params, &slave_config); + dev_info(dcpaud->dev, "snd_hwparams_to_dma_slave_config: %d\n", ret); + if (ret < 0) + return ret; + + slave_config.direction = DMA_MEM_TO_DEV; + /* + * The data entry from the DMA controller to the DPA peripheral + * is 32-bit wide no matter the actual sample size. + */ + slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + ret = dmaengine_slave_config(chan, &slave_config); + dev_info(dcpaud->dev, "dmaengine_slave_config: %d\n", ret); + return ret; +} + +static int dcp_pcm_hw_free(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + if (!dcpaud_connection_up(dcpaud)) + return 0; + + return dcp_audiosrv_unprepare(dcpaud->dcp_dev); +} + +static int dcp_pcm_prepare(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + return dcp_audiosrv_prepare(dcpaud->dcp_dev, + &dcpaud->selected_cookie); +} + +static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + WARN_ON(pm_runtime_get_sync(dcpaud->dev) < 0); + ret = dcp_audiosrv_startlink(dcpaud->dcp_dev, + &dcpaud->selected_cookie); + if (ret < 0) + return ret; + break; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + break; + + default: + return -EINVAL; + } + + ret = snd_dmaengine_pcm_trigger(substream, cmd); + if (ret < 0) + return ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + break; + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + ret = dcp_audiosrv_stoplink(dcpaud->dcp_dev); + pm_runtime_mark_last_busy(dcpaud->dev); + __pm_runtime_put_autosuspend(dcpaud->dev); + if (ret < 0) + return ret; + break; + } + + return 0; +} + +struct snd_pcm_ops dcp_playback_ops = { + .open = dcp_pcm_open, + .close = dcp_pcm_close, + .hw_params = dcp_pcm_hw_params, + .hw_free = dcp_pcm_hw_free, + .prepare = dcp_pcm_prepare, + .trigger = dcp_pcm_trigger, + .pointer = snd_dmaengine_pcm_pointer, +}; + +// Transitional workaround: for the chmap control TLV, advertise options +// copied from hdmi-codec.c +#include "hdmi-codec-chmap.h" + +static int dcpaud_chmap_ctl_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct snd_pcm_chmap *info = snd_kcontrol_chip(kcontrol); + struct dcp_audio *dcpaud = info->private_data; + unsigned int i; + + for (i = 0; i < info->max_channels; i++) + ucontrol->value.integer.value[i] = \ + (i < dcpaud->selected_chmap.channels) ? + dcpaud->selected_chmap.map[i] : SNDRV_CHMAP_UNKNOWN; + + return 0; +} + + +static int dcpaud_create_chmap_ctl(struct dcp_audio *dcpaud) +{ + struct snd_pcm *pcm = dcpaud->substream->pcm; + struct snd_pcm_chmap *chmap_info; + int ret; + + ret = snd_pcm_add_chmap_ctls(pcm, SNDRV_PCM_STREAM_PLAYBACK, NULL, + dcp_pcm_hw.channels_max, 0, &chmap_info); + if (ret < 0) + return ret; + + chmap_info->kctl->get = dcpaud_chmap_ctl_get; + chmap_info->chmap = hdmi_codec_8ch_chmaps; + chmap_info->private_data = dcpaud; + + return 0; +} + +static int dcpaud_create_pcm(struct dcp_audio *dcpaud) +{ + struct snd_card *card = dcpaud->card; + struct snd_pcm *pcm; + int ret; + +#define NUM_PLAYBACK 1 +#define NUM_CAPTURE 0 + + ret = snd_pcm_new(card, card->shortname, 0, NUM_PLAYBACK, NUM_CAPTURE, &pcm); + if (ret) + return ret; + + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &dcp_playback_ops); + dcpaud->substream = pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + pcm->nonatomic = true; + pcm->private_data = dcpaud; + strscpy(pcm->name, card->shortname, sizeof(pcm->name)); + + return 0; +} + +/* expects to be called with data_lock locked and unlocks it */ +static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) +{ + struct snd_pcm_substream *substream = dcpaud->substream; + + if (!dcpaud->card || dcpaud->connected == connected) { + mutex_unlock(&dcpaud->data_lock); + return; + } + + dcpaud->connected = connected; + if (connected) + dcpaud->connection_cookie++; + mutex_unlock(&dcpaud->data_lock); + + snd_jack_report(dcpaud->jack, connected ? SND_JACK_AVOUT : 0); + + if (!connected) { + snd_pcm_stream_lock(substream); + if (substream->runtime) + snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); + snd_pcm_stream_unlock(substream); + } +} + +static int dcpaud_create_jack(struct dcp_audio *dcpaud) +{ + struct snd_card *card = dcpaud->card; + + return snd_jack_new(card, "HDMI/DP", SND_JACK_AVOUT, + &dcpaud->jack, true, false); +} + +static void dcpaud_set_card_names(struct dcp_audio *dcpaud) +{ + struct snd_card *card = dcpaud->card; + + strscpy(card->driver, "apple_dcp", sizeof(card->driver)); + strscpy(card->longname, "Apple DisplayPort", sizeof(card->longname)); + strscpy(card->shortname, "Apple DisplayPort", sizeof(card->shortname)); +} + +#ifdef CONFIG_SND_DEBUG +static void dcpaud_expose_debugfs_blob(struct dcp_audio *dcpaud, const char *name, void *base, size_t size) +{ + struct debugfs_blob_wrapper *wrapper; + wrapper = devm_kzalloc(dcpaud->dev, sizeof(*wrapper), GFP_KERNEL); + if (!wrapper) + return; + wrapper->data = base; + wrapper->size = size; + debugfs_create_blob(name, 0600, dcpaud->card->debugfs_root, wrapper); +} +#else +static void dcpaud_expose_debugfs_blob(struct dcp_audio *dcpaud, const char *name, void *base, size_t size) {} +#endif + +extern bool hdmi_audio; + +static int dcpaud_init_snd_card(struct dcp_audio *dcpaud) +{ + int ret; + if (!hdmi_audio) + return -ENODEV; + + + ret = snd_card_new(dcpaud->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, + THIS_MODULE, 0, &dcpaud->card); + if (ret) + return ret; + + dcpaud_set_card_names(dcpaud); + + ret = dcpaud_create_pcm(dcpaud); + if (ret) + goto err_free_card; + + ret = dcpaud_create_chmap_ctl(dcpaud); + if (ret) + goto err_free_card; + + ret = dcpaud_create_jack(dcpaud); + if (ret) + goto err_free_card; + + ret = snd_card_register(dcpaud->card); + if (ret) + goto err_free_card; + + return 0; +err_free_card: + dev_warn(dcpaud->dev, "Failed to initialize sound card: %d\n", ret); + snd_card_free(dcpaud->card); + dcpaud->card = NULL; + return ret; +} + +void dcpaud_connect(struct platform_device *pdev, bool connected) +{ + struct dcp_audio *dcpaud = platform_get_drvdata(pdev); + + mutex_lock(&dcpaud->data_lock); + + dcpaud_report_hotplug(dcpaud, connected); +} + +void dcpaud_disconnect(struct platform_device *pdev) +{ + struct dcp_audio *dcpaud = platform_get_drvdata(pdev); + + mutex_lock(&dcpaud->data_lock); + + dcpaud_report_hotplug(dcpaud, false); +} + +static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + struct device_node *endpoint, *dcp_node = NULL; + struct platform_device *dcp_pdev, *dma_pdev; + struct of_phandle_args dma_spec; + int index; + int ret; + + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable runtime PM: %d\n", ret); + + /* find linked DCP instance */ + endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); + if (endpoint) { + dcp_node = of_graph_get_remote_port_parent(endpoint); + of_node_put(endpoint); + } + if (!dcp_node || !of_device_is_available(dcp_node)) { + of_node_put(dcp_node); + dev_info(dev, "No audio support\n"); + goto rpm_put; + } + + index = of_property_match_string(dev->of_node, "dma-names", "tx"); + if (index < 0) { + dev_err(dev, "No dma-names property\n"); + goto rpm_put; + } + + if (of_parse_phandle_with_args(dev->of_node, "dmas", "#dma-cells", index, + &dma_spec) || !dma_spec.np) { + dev_err(dev, "Failed to parse dmas property\n"); + goto rpm_put; + } + + dcp_pdev = of_find_device_by_node(dcp_node); + of_node_put(dcp_node); + if (!dcp_pdev) { + dev_info(dev, "No DP/HDMI audio device, dcp not ready\n"); + goto rpm_put; + } + dcpaud->dcp_dev = &dcp_pdev->dev; + + dma_pdev = of_find_device_by_node(dma_spec.np); + of_node_put(dma_spec.np); + if (!dma_pdev) { + dev_info(dev, "No DMA device\n"); + goto rpm_put; + } + dcpaud->dma_dev = &dma_pdev->dev; + + dcpaud->dma_link = device_link_add(dev, dcpaud->dma_dev, + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE | + DL_FLAG_STATELESS); + + /* ignore errors to prevent audio issues affecting the display side */ + ret = dcpaud_init_snd_card(dcpaud); + + if (!ret) { + dcpaud_expose_debugfs_blob(dcpaud, "selected_cookie", &dcpaud->selected_cookie, + sizeof(dcpaud->selected_cookie)); + dcpaud_expose_debugfs_blob(dcpaud, "elements", dcpaud->elements, + DCPAUD_ELEMENTS_MAXSIZE); + dcpaud_expose_debugfs_blob(dcpaud, "product_attrs", dcpaud->productattrs, + DCPAUD_PRODUCTATTRS_MAXSIZE); + } + +rpm_put: + pm_runtime_put(dev); + + return 0; +} + +static void dcpaud_comp_unbind(struct device *dev, struct device *main, + void *data) +{ + struct dcp_audio *dcpaud = dev_get_drvdata(dev); + + /* snd_card_free_when_closed() checks for NULL */ + snd_card_free_when_closed(dcpaud->card); + + if (dcpaud->dma_link) + device_link_del(dcpaud->dma_link); +} + +static const struct component_ops dcpaud_comp_ops = { + .bind = dcpaud_comp_bind, + .unbind = dcpaud_comp_unbind, +}; + +static int dcpaud_probe(struct platform_device *pdev) +{ + struct dcp_audio *dcpaud; + + dcpaud = devm_kzalloc(&pdev->dev, sizeof(*dcpaud), GFP_KERNEL); + if (!dcpaud) + return -ENOMEM; + + dcpaud->elements = devm_kzalloc(&pdev->dev, DCPAUD_ELEMENTS_MAXSIZE, + GFP_KERNEL); + if (!dcpaud->elements) + return -ENOMEM; + + dcpaud->productattrs = devm_kzalloc(&pdev->dev, DCPAUD_PRODUCTATTRS_MAXSIZE, + GFP_KERNEL); + if (!dcpaud->productattrs) + return -ENOMEM; + + dcpaud->dev = &pdev->dev; + mutex_init(&dcpaud->data_lock); + platform_set_drvdata(pdev, dcpaud); + + return component_add(&pdev->dev, &dcpaud_comp_ops); +} + +static void dcpaud_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcpaud_comp_ops); +} + +static void dcpaud_shutdown(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcpaud_comp_ops); +} + +static __maybe_unused int dcpaud_suspend(struct device *dev) +{ + /* + * Using snd_power_change_state() does not work since the sound card + * is what resumes runtime PM. + */ + + return 0; +} + +static __maybe_unused int dcpaud_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(dcpaud_pm_ops, dcpaud_suspend, dcpaud_resume, NULL); + +static const struct of_device_id dcpaud_of_match[] = { + { .compatible = "apple,dpaudio" }, + {} +}; + +static struct platform_driver dcpaud_driver = { + .driver = { + .name = "dcp-dp-audio", + .of_match_table = dcpaud_of_match, + .pm = pm_ptr(&dcpaud_pm_ops), + }, + .probe = dcpaud_probe, + .remove = dcpaud_remove, + .shutdown = dcpaud_shutdown, +}; + +void __init dcp_audio_register(void) +{ + platform_driver_register(&dcpaud_driver); +} + +void __exit dcp_audio_unregister(void) +{ + platform_driver_unregister(&dcpaud_driver); +} + diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h new file mode 100644 index 00000000000000..83b990dc6c343f --- /dev/null +++ b/drivers/gpu/drm/apple/audio.h @@ -0,0 +1,20 @@ +#ifndef __AUDIO_H__ +#define __AUDIO_H__ + +#include + +struct device; +struct platform_device; +struct dcp_sound_cookie; + +int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie); +int dcp_audiosrv_startlink(struct device *dev, struct dcp_sound_cookie *cookie); +int dcp_audiosrv_stoplink(struct device *dev); +int dcp_audiosrv_unprepare(struct device *dev); +int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize); +int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize); + +void dcpaud_connect(struct platform_device *pdev, bool connected); +void dcpaud_disconnect(struct platform_device *pdev); + +#endif /* __AUDIO_H__ */ diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c new file mode 100644 index 00000000000000..0d3c752f62d5f5 --- /dev/null +++ b/drivers/gpu/drm/apple/av.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2023 Martin Povišer */ + +// #define DEBUG + +#include +#include +#include +#include +#include +#include +#include + +#include "audio.h" +#include "afk.h" +#include "av.h" +#include "dcp.h" +#include "dcp-internal.h" + +struct dcp_av_audio_cmds { + /* commands in group 0*/ + u32 open; + u32 close; + u32 prepare; + u32 start_link; + u32 stop_link; + u32 unprepare; + /* commands in group 1*/ + u32 get_elements; + u32 get_product_attrs; +}; + +static const struct dcp_av_audio_cmds dcp_av_audio_cmds_v12_3 = { + .open = 6, + .close = 7, + .prepare = 8, + .start_link = 9, + .stop_link = 12, + .unprepare = 13, + .get_elements = 18, + .get_product_attrs = 20, +}; + +static const struct dcp_av_audio_cmds dcp_av_audio_cmds_v13_5 = { + .open = 4, + .close = 5, + .prepare = 6, + .start_link = 7, + .stop_link = 10, + .unprepare = 11, + .get_elements = 16, + .get_product_attrs = 18, +}; + +struct audiosrv_data { + struct platform_device *audio_dev; + bool plugged; + struct mutex plug_lock; + + struct apple_epic_service *srv; + struct rw_semaphore srv_rwsem; + /* Workqueue for starting the audio service */ + struct work_struct start_av_service_wq; + + struct dcp_av_audio_cmds cmds; + + bool warned_get_elements; + bool warned_get_product_attrs; + bool is_open; +}; + +static void av_interface_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static void av_interface_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct audiosrv_data *asrv = dcp->audiosrv; + + service->enabled = false; + + mutex_lock(&asrv->plug_lock); + + asrv->plugged = false; + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + + mutex_unlock(&asrv->plug_lock); +} + +static void av_audiosrv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct audiosrv_data *asrv = dcp->audiosrv; + + mutex_lock(&asrv->plug_lock); + + down_write(&asrv->srv_rwsem); + asrv->srv = service; + up_write(&asrv->srv_rwsem); + + asrv->plugged = true; + mutex_unlock(&asrv->plug_lock); + schedule_work(&asrv->start_av_service_wq); +} + +static void av_audiosrv_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct audiosrv_data *asrv = dcp->audiosrv; + + mutex_lock(&asrv->plug_lock); + + down_write(&asrv->srv_rwsem); + asrv->srv = NULL; + up_write(&asrv->srv_rwsem); + + asrv->plugged = false; + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + + mutex_unlock(&asrv->plug_lock); +} + +int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.prepare, cookie, + sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, + 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +int dcp_audiosrv_startlink(struct device *dev, struct dcp_sound_cookie *cookie) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.start_link, cookie, + sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, + 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +int dcp_audiosrv_stoplink(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.stop_link, NULL, 0, 64, + NULL, 0, 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +int dcp_audiosrv_unprepare(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + down_write(&asrv->srv_rwsem); + ret = afk_service_call(asrv->srv, 0, asrv->cmds.unprepare, NULL, 0, 64, + NULL, 0, 64); + up_write(&asrv->srv_rwsem); + + return ret; +} + +static int +dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, + u32 command, void *output, size_t output_maxsize, + size_t *output_size) +{ + struct { + __le64 max_size; + u8 _pad1[24]; + __le64 used_size; + u8 _pad2[8]; + } __attribute__((packed)) *hdr; + static_assert(sizeof(*hdr) == 48); + size_t bfr_len = output_maxsize + sizeof(*hdr); + void *bfr; + int ret; + + bfr = kzalloc(bfr_len, GFP_KERNEL); + if (!bfr) + return -ENOMEM; + + hdr = bfr; + hdr->max_size = cpu_to_le64(output_maxsize); + ret = afk_service_call(service, group, command, hdr, sizeof(*hdr), output_maxsize, + bfr, sizeof(*hdr) + output_maxsize, 0); + if (ret) + return ret; + + if (output) + memcpy(output, bfr + sizeof(*hdr), output_maxsize); + + if (output_size) + *output_size = le64_to_cpu(hdr->used_size); + + return 0; +} + +int dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + size_t size; + int ret; + + down_write(&asrv->srv_rwsem); + ret = dcp_audiosrv_osobject_call(asrv->srv, 1, asrv->cmds.get_elements, + elements, maxsize, &size); + up_write(&asrv->srv_rwsem); + + if (ret && asrv->warned_get_elements) { + dev_err(dev, "audiosrv: error getting elements: %d\n", ret); + asrv->warned_get_elements = true; + } else { + dev_dbg(dev, "audiosrv: got %zd bytes worth of elements\n", size); + } + + return ret; +} + +int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + size_t size; + int ret; + + down_write(&asrv->srv_rwsem); + ret = dcp_audiosrv_osobject_call(asrv->srv, 1, + asrv->cmds.get_product_attrs, attrs, + maxsize, &size); + up_write(&asrv->srv_rwsem); + + if (ret && asrv->warned_get_product_attrs) { + dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); + asrv->warned_get_product_attrs = true; + } else { + dev_dbg(dev, "audiosrv: got %zd bytes worth of product attributes\n", size); + } + + return ret; +} + +static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, + const void *data, size_t data_size) +{ + dev_dbg(service->ep->dcp->dev, "got audio report %d size %zx\n", idx, data_size); +#ifdef DEBUG + print_hex_dump(KERN_DEBUG, "audio report: ", DUMP_PREFIX_NONE, 16, 1, data, data_size, true); +#endif + + return 0; +} + +static const struct apple_epic_service_ops avep_ops[] = { + { + .name = "DCPAVSimpleVideoInterface", + .init = av_interface_init, + .teardown = av_interface_teardown, + }, + { + .name = "DCPAVAudioInterface", + .init = av_audiosrv_init, + .report = av_audiosrv_report, + .teardown = av_audiosrv_teardown, + }, + {} +}; + +void av_service_connect(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + scoped_guard(rwsem_write, &asrv->srv_rwsem) { + if (!asrv->srv) + return; + service = asrv->srv; + } + + /* open AV audio service */ + dev_info(dcp->dev, "%s: starting audio service, plugged:%d\n", __func__, asrv->plugged); + if (asrv->is_open) + return; + + ret = afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, + NULL, 0, 32); + if (ret) { + dev_err(dcp->dev, "error opening audio service: %d\n", ret); + return; + } + mutex_lock(&asrv->plug_lock); + asrv->is_open = true; + + if (asrv->audio_dev) + dcpaud_connect(asrv->audio_dev, asrv->plugged); + mutex_unlock(&asrv->plug_lock); +} + +void av_service_disconnect(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + struct audiosrv_data *asrv = dcp->audiosrv; + int ret; + + scoped_guard(rwsem_write, &asrv->srv_rwsem) { + if (!asrv->srv) + return; + service = asrv->srv; + } + + /* close AV audio service */ + dev_info(dcp->dev, "%s: stopping audio service\n", __func__); + if (!asrv->is_open) + return; + + mutex_lock(&asrv->plug_lock); + + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + + mutex_unlock(&asrv->plug_lock); + + ret = afk_service_call(service, 0, asrv->cmds.close, NULL, 0, 16, + NULL, 0, 16); + if (ret) { + dev_err(dcp->dev, "error closing audio service: %d\n", ret); + } + if (service->torndown) + service->enabled = false; + asrv->is_open = false; +} + +static void av_work_service_start(struct work_struct *work) +{ + struct audiosrv_data *audiosrv_data; + struct apple_dcp *dcp; + + audiosrv_data = container_of(work, struct audiosrv_data, start_av_service_wq); + + scoped_guard(rwsem_read, &audiosrv_data->srv_rwsem) { + if (!audiosrv_data->srv || + !audiosrv_data->srv->ep || + !audiosrv_data->srv->ep->dcp) { + pr_err("%s: dcp: av: NULL ptr during startup\n", __func__); + return; + } + dcp = audiosrv_data->srv->ep->dcp; + } + + av_service_connect(dcp); +} + +int avep_init(struct apple_dcp *dcp) +{ + struct audiosrv_data *audiosrv_data; + struct platform_device *audio_pdev; + struct device *dev = dcp->dev; + struct device_node *endpoint, *audio_node = NULL; + + audiosrv_data = devm_kzalloc(dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); + if (!audiosrv_data) + return -ENOMEM; + init_rwsem(&audiosrv_data->srv_rwsem); + mutex_init(&audiosrv_data->plug_lock); + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + audiosrv_data->cmds = dcp_av_audio_cmds_v12_3; + break; + case DCP_FIRMWARE_V_13_5: + audiosrv_data->cmds = dcp_av_audio_cmds_v13_5; + break; + default: + dev_err(dcp->dev, "Audio not supported for firmware\n"); + return -ENODEV; + } + + dcp->audiosrv = audiosrv_data; + INIT_WORK(&audiosrv_data->start_av_service_wq, av_work_service_start); + + endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); + if (endpoint) { + audio_node = of_graph_get_remote_port_parent(endpoint); + of_node_put(endpoint); + } + if (!audio_node || !of_device_is_available(audio_node)) { + of_node_put(audio_node); + dev_info(dev, "No audio support\n"); + return 0; + } + + audio_pdev = of_find_device_by_node(audio_node); + of_node_put(audio_node); + if (!audio_pdev) { + dev_info(dev, "No DP/HDMI audio device not ready\n"); + return 0; + } + dcp->audiosrv->audio_dev = audio_pdev; + + device_link_add(&audio_pdev->dev, dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); + + dcp->avep = afk_init(dcp, AV_ENDPOINT, avep_ops); + if (IS_ERR(dcp->avep)) + return PTR_ERR(dcp->avep); + dcp->avep->debugfs_entry = dcp->ep_debugfs[AV_ENDPOINT - 0x20]; + return afk_start(dcp->avep); +} diff --git a/drivers/gpu/drm/apple/av.h b/drivers/gpu/drm/apple/av.h new file mode 100644 index 00000000000000..c00cbef549fd2e --- /dev/null +++ b/drivers/gpu/drm/apple/av.h @@ -0,0 +1,17 @@ +#ifndef __AV_H__ +#define __AV_H__ + +#include "parser.h" + +//int avep_audiosrv_startlink(struct apple_dcp *dcp, struct dcp_sound_cookie *cookie); +//int avep_audiosrv_stoplink(struct apple_dcp *dcp); + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +void av_service_connect(struct apple_dcp *dcp); +void av_service_disconnect(struct apple_dcp *dcp); +#else +static inline void av_service_connect(struct apple_dcp *dcp) { } +static inline void av_service_disconnect(struct apple_dcp *dcp) { } +#endif + +#endif /* __AV_H__ */ diff --git a/drivers/gpu/drm/apple/connector.c b/drivers/gpu/drm/apple/connector.c new file mode 100644 index 00000000000000..15b3664d85631e --- /dev/null +++ b/drivers/gpu/drm/apple/connector.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + +#include "connector.h" + +#include "linux/err.h" +#include +#include +#include +#include +#include + +#include + +#include "dcp-internal.h" + +enum dcp_chunk_type { + DCP_CHUNK_COLOR_ELEMENTS, + DCP_CHUNK_TIMING_ELELMENTS, + DCP_CHUNK_DISPLAY_ATTRIBUTES, + DCP_CHUNK_TRANSPORT, + DCP_CHUNK_NUM_TYPES, +}; + +static int chunk_show(struct seq_file *m, + enum dcp_chunk_type chunk_type) +{ + struct apple_connector *apple_con = m->private; + struct dcp_chunks *chunk = NULL; + + mutex_lock(&apple_con->chunk_lock); + + switch (chunk_type) { + case DCP_CHUNK_COLOR_ELEMENTS: + chunk = &apple_con->color_elements; + break; + case DCP_CHUNK_TIMING_ELELMENTS: + chunk = &apple_con->timing_elements; + break; + case DCP_CHUNK_DISPLAY_ATTRIBUTES: + chunk = &apple_con->display_attributes; + break; + case DCP_CHUNK_TRANSPORT: + chunk = &apple_con->transport; + break; + default: + break; + } + + if (chunk) + seq_write(m, chunk->data, chunk->length); + + mutex_unlock(&apple_con->chunk_lock); + + return 0; +} + +#define CONNECTOR_DEBUGFS_ENTRY(name, type) \ +static int chunk_ ## name ## _show(struct seq_file *m, void *data) \ +{ \ + return chunk_show(m, type); \ +} \ +static int chunk_ ## name ## _open(struct inode *inode, struct file *file) \ +{ \ + return single_open(file, chunk_ ## name ## _show, inode->i_private); \ +} \ +static const struct file_operations chunk_ ## name ## _fops = { \ + .owner = THIS_MODULE, \ + .open = chunk_ ## name ## _open, \ + .read = seq_read, \ + .llseek = seq_lseek, \ + .release = single_release, \ +} + +CONNECTOR_DEBUGFS_ENTRY(color, DCP_CHUNK_COLOR_ELEMENTS); +CONNECTOR_DEBUGFS_ENTRY(timing, DCP_CHUNK_TIMING_ELELMENTS); +CONNECTOR_DEBUGFS_ENTRY(display_attribs, DCP_CHUNK_DISPLAY_ATTRIBUTES); +CONNECTOR_DEBUGFS_ENTRY(transport, DCP_CHUNK_TRANSPORT); + +static void dcp_afk_debugfs_root(struct platform_device *pdev, int ep, struct dentry *root) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) + struct dentry *entry = NULL; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + switch (ep) { + case AV_ENDPOINT: + entry = debugfs_create_dir("avep", root); + break; + default: + break; + } + + if (!IS_ERR_OR_NULL(entry)) + dcp->ep_debugfs[ep - 0x20] = entry; +#endif +} + +void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root) +{ + struct apple_connector *apple_con = to_apple_connector(connector); + + debugfs_create_file("ColorElements", 0444, root, apple_con, + &chunk_color_fops); + debugfs_create_file("TimingElements", 0444, root, apple_con, + &chunk_timing_fops); + debugfs_create_file("DisplayAttributes", 0444, root, apple_con, + &chunk_display_attribs_fops); + debugfs_create_file("Transport", 0444, root, apple_con, + &chunk_transport_fops); + + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_DisplayPort: + case DRM_MODE_CONNECTOR_HDMIA: + dcp_afk_debugfs_root(apple_con->dcp, AV_ENDPOINT, root); + break; + default: + break; + } +} + +static void dcp_connector_set_dict(struct apple_connector *connector, + struct dcp_chunks *dict, + struct dcp_chunks *chunks) +{ + if (dict->data) + devm_kfree(&connector->dcp->dev, dict->data); + + *dict = *chunks; +} + +void dcp_connector_update_dict(struct apple_connector *connector, const char *key, + struct dcp_chunks *chunks) +{ + mutex_lock(&connector->chunk_lock); + if (!strcmp(key, "ColorElements")) + dcp_connector_set_dict(connector, &connector->color_elements, chunks); + else if (!strcmp(key, "TimingElements")) + dcp_connector_set_dict(connector, &connector->timing_elements, chunks); + else if (!strcmp(key, "DisplayAttributes")) + dcp_connector_set_dict(connector, &connector->display_attributes, chunks); + else if (!strcmp(key, "Transport")) + dcp_connector_set_dict(connector, &connector->transport, chunks); + + chunks->data = NULL; + chunks->length = 0; + + mutex_unlock(&connector->chunk_lock); +} diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h new file mode 100644 index 00000000000000..ef2c23737aac64 --- /dev/null +++ b/drivers/gpu/drm/apple/connector.h @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* "Copyright" 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_CONNECTOR_H__ +#define __APPLE_CONNECTOR_H__ + +#include + +#include +#include "drm/drm_connector.h" +#include "drm/drm_edid.h" + +struct apple_connector; + +#include "dcp-internal.h" + +void dcp_hotplug(struct work_struct *work); + +struct apple_connector { + struct drm_connector base; + bool connected; + + struct platform_device *dcp; + + const struct drm_edid *drm_edid; + + /* Workqueue for sending hotplug events to the associated device */ + struct work_struct hotplug_wq; + + struct mutex chunk_lock; + + struct dcp_chunks color_elements; + struct dcp_chunks timing_elements; + struct dcp_chunks display_attributes; + struct dcp_chunks transport; +}; + +#define to_apple_connector(x) container_of(x, struct apple_connector, base) + +void apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root); + +void dcp_connector_update_dict(struct apple_connector *connector, const char *key, + struct dcp_chunks *chunks); +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h new file mode 100644 index 00000000000000..0a6859448e19eb --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_INTERNAL_H__ +#define __APPLE_DCP_INTERNAL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dptxep.h" +#include "iomfb.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "epic/dpavservep.h" + +#define DCP_MAX_PLANES 2 + +struct apple_dcp; +struct apple_dcp_afkep; + +struct dcpav_service_epic; + +enum dcp_firmware_version { + DCP_FIRMWARE_UNKNOWN, + DCP_FIRMWARE_V_12_3, + DCP_FIRMWARE_V_13_5, +}; + +enum { + SYSTEM_ENDPOINT = 0x20, + TEST_ENDPOINT = 0x21, + DCP_EXPERT_ENDPOINT = 0x22, + DISP0_ENDPOINT = 0x23, + DPAVSERV_ENDPOINT = 0x28, + AV_ENDPOINT = 0x29, + DPTX_ENDPOINT = 0x2a, + HDCP_ENDPOINT = 0x2b, + REMOTE_ALLOC_ENDPOINT = 0x2d, + IOMFB_ENDPOINT = 0x37, +}; + +/* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ +struct dcp_chunks { + size_t length; + void *data; +}; + +#define DCP_MAX_MAPPINGS (128) /* should be enough */ +#define MAX_DISP_REGISTERS (7) + +struct dcp_mem_descriptor { + size_t size; + void *buf; + dma_addr_t dva; + struct sg_table map; + u64 reg; +}; + +/* Limit on call stack depth (arbitrary). Some nesting is required */ +#define DCP_MAX_CALL_DEPTH 8 + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + +struct dcp_channel { + dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; + void *cookies[DCP_MAX_CALL_DEPTH]; + void *output[DCP_MAX_CALL_DEPTH]; + u16 end[DCP_MAX_CALL_DEPTH]; + + /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ + u8 depth; + /* Already warned about busy channel */ + bool warned_busy; +}; + +struct dcp_fb_reference { + struct list_head head; + struct drm_framebuffer *fb; + u32 swap_id; +}; + +#define MAX_NOTCH_HEIGHT 160 + +struct dcp_brightness { + struct backlight_device *bl_dev; + u32 maximum; + u32 dac; + int nits; + int scale; + bool update; +}; + +struct audiosrv_data; + +/** laptop/AiO integrated panel parameters from DT */ +struct dcp_panel { + /// panel width in millimeter + int width_mm; + /// panel height in millimeter + int height_mm; + /// panel has a mini-LED backlight + bool has_mini_led; +}; + +struct apple_dcp_hw_data { + u32 num_dptx_ports; +}; + +/* TODO: move IOMFB members to its own struct */ +struct apple_dcp { + struct device *dev; + struct platform_device *piodma; + struct iommu_domain *iommu_dom; + struct apple_rtkit *rtk; + struct apple_crtc *crtc; + struct apple_connector *connector; + + struct apple_dcp_hw_data hw; + + /* firmware version and compatible firmware version */ + enum dcp_firmware_version fw_compat; + + /* Coprocessor control register */ + void __iomem *coproc_reg; + + /* DCP has crashed */ + bool crashed; + + /************* IOMFB ************************************************** + * everything below is mostly used inside IOMFB but it could make * + * sense to keep some of the members in apple_dcp. * + **********************************************************************/ + + /* clock rate request by dcp in */ + struct clk *clk; + + /* DCP shared memory */ + void *shmem; + + /* Display registers mappable to the DCP */ + struct resource *disp_registers[MAX_DISP_REGISTERS]; + unsigned int nr_disp_registers; + + struct resource disp_bw_scratch_res; + struct resource disp_bw_doorbell_res; + u32 disp_bw_scratch_index; + u32 disp_bw_scratch_offset; + u32 disp_bw_doorbell_index; + u32 disp_bw_doorbell_offset; + + u32 index; + + /* Bitmap of memory descriptors used for mappings made by the DCP */ + DECLARE_BITMAP(memdesc_map, DCP_MAX_MAPPINGS); + + /* Indexed table of memory descriptors */ + struct dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; + + struct dcp_channel ch_cmd, ch_oobcmd; + struct dcp_channel ch_cb, ch_oobcb, ch_async, ch_oobasync; + + /* iomfb EP callback handlers */ + const iomfb_cb_handler *cb_handlers; + + /* Active chunked transfer. There can only be one at a time. */ + struct dcp_chunks chunks; + + /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ + union { + struct dcp_swap_submit_req_v12_3 v12_3; + struct dcp_swap_submit_req_v13_3 v13_3; + } swap; + + /* swap id of the last completed swap */ + u32 last_swap_id; + ktime_t swap_start; + + /* Current display mode */ + bool during_modeset; + bool valid_mode; + bool use_timestamps; + bool vrr_enabled; + struct dcp_set_digital_out_mode_req mode; + + /* completion for active turning true */ + struct completion start_done; + + /* Is the DCP booted? */ + bool active; + + /* eDP display without DP-HDMI conversion */ + bool main_display; + + /* clear all surfaces on init */ + bool surfaces_cleared; + + /* enable CRC calculation */ + bool crc_enabled; + + /* Modes valid for the connected display */ + struct dcp_display_mode *modes; + unsigned int nr_modes; + + /* Attributes of the connector */ + int connector_type; + + /* Attributes of the connected display */ + int width_mm, height_mm; + + unsigned notch_height; + + /* Workqueue for sending vblank events when a dcp swap is not possible */ + struct work_struct vblank_wq; + + /* List of referenced drm_framebuffers which can be unreferenced + * on the next successfully completed swap. + */ + struct list_head swapped_out_fbs; + + struct dcp_brightness brightness; + /* Workqueue for updating the initial brightness */ + struct work_struct bl_register_wq; + struct mutex bl_register_mutex; + /* Workqueue for updating the brightness */ + struct work_struct bl_update_wq; + + /* integrated panel if present */ + struct dcp_panel panel; + + struct apple_dcp_afkep *systemep; + struct completion systemep_done; + + struct apple_dcp_afkep *ibootep; + struct apple_dcp_afkep *dcpavservep; + struct dcpavserv dcpavserv; + + struct apple_dcp_afkep *avep; + struct audiosrv_data *audiosrv; + + struct apple_dcp_afkep *dptxep; + + struct dptx_port dptxport[2]; + + /* debugfs entries */ + struct dentry *ep_debugfs[0x20]; + + /* these fields are output port specific */ + struct phy *phy; + struct mux_control *xbar; + struct typec_mux *typec_mux; + + struct gpio_desc *hdmi_hpd; + struct gpio_desc *hdmi_pwren; + struct gpio_desc *dp2hdmi_pwren; + + struct mutex hpd_mutex; + + u32 dptx_phy; + u32 dptx_die; + int hdmi_hpd_irq; +}; + +void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); + +int dcp_backlight_register(struct apple_dcp *dcp); +int dcp_backlight_update(struct apple_dcp *dcp); +bool dcp_has_panel(struct apple_dcp *dcp); + +#define DCP_AUDIO_MAX_CHANS 15 + +#endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c new file mode 100644 index 00000000000000..83ba20f7f02568 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp.c @@ -0,0 +1,1378 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "afk.h" +#include "av.h" +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "parser.h" +#include "trace.h" + +#define APPLE_DCP_COPROC_CPU_CONTROL 0x44 +#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) + +#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) + +static bool show_notch; +module_param(show_notch, bool, 0644); +MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); + +bool hdmi_audio; +module_param(hdmi_audio, bool, 0644); +MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); + +static bool unstable_edid = true; +module_param(unstable_edid, bool, 0644); +MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); + +bool force_vrr; +module_param(force_vrr, bool, 0644); +MODULE_PARM_DESC(force_vrr, "Always enable Adaptive Sync/ProMotion on supported displays"); + +/* copied and simplified from drm_vblank.c */ +static void send_vblank_event(struct drm_device *dev, + struct drm_pending_vblank_event *e, + u64 seq, ktime_t now) +{ + struct timespec64 tv; + + if (e->event.base.type != DRM_EVENT_FLIP_COMPLETE) + return; + + tv = ktime_to_timespec64(now); + e->event.vbl.sequence = seq; + /* + * e->event is a user space structure, with hardcoded unsigned + * 32-bit seconds/microseconds. This is safe as we always use + * monotonic timestamps since linux-4.15 + */ + e->event.vbl.tv_sec = tv.tv_sec; + e->event.vbl.tv_usec = tv.tv_nsec / 1000; + + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); +} + +/** + * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip + * + * Compensate for unknown slack between page flip and arrival of the + * swap_complete callback. Minimal observed duration on DCP with HDMI output + * was around 2.3 ms. If the fb swap was submitted closer to the expected + * swap_complete it gets a penalty of one frame duration. This is on the border + * of unreasonable considering that Apple advertises support for 240 Hz (frame + * duration of 4.167 ms). + * It is unreasonable considering kwin's kms commit scheduling. Kwin commits + * 1.5 ms + the mode's vblank time before the expected next page flip + * completion. This results in presenting at half the display's rate for HDMI + * outputs. + * This might be a difference between dcp and dcpext. + */ +static void dcp_crtc_send_page_flip_event(struct apple_crtc *crtc, + struct drm_pending_vblank_event *e, + ktime_t now, ktime_t start) +{ + struct drm_device *dev = crtc->base.dev; + u64 seq; + unsigned int pipe = drm_crtc_index(&crtc->base); + ktime_t flip; + + seq = 0; + if (start != KTIME_MIN) { + s64 delta = ktime_us_delta(now, start); + if (delta <= 500) + flip = now; + else if (delta >= 2500) + flip = ktime_sub_us(now, 1000); + else + flip = ktime_sub_us(now, (delta - 500) / 2); + } else { + flip = now; + } + e->pipe = pipe; + send_vblank_event(dev, e, seq, flip); +} + +/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ +void dcp_drm_crtc_vblank(struct apple_crtc *crtc) +{ + unsigned long flags; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + crtc->event = NULL; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + +void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) +{ + unsigned long flags; + struct apple_crtc *crtc = dcp->crtc; + + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + if (crtc->event->event.base.type == DRM_EVENT_FLIP_COMPLETE) + dcp_crtc_send_page_flip_event(crtc, crtc->event, now, dcp->swap_start); + else + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + crtc->event = NULL; + dcp->swap_start = KTIME_MIN; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); +} + +void dcp_set_dimensions(struct apple_dcp *dcp) +{ + int i; + int width_mm = dcp->width_mm; + int height_mm = dcp->height_mm; + + if (width_mm == 0 || height_mm == 0) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + + /* Set the connector info */ + if (dcp->connector) { + struct drm_connector *connector = &dcp->connector->base; + + mutex_lock(&connector->dev->mode_config.mutex); + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; + mutex_unlock(&connector->dev->mode_config.mutex); + } + + /* + * Fix up any probed modes. Modes are created when parsing + * TimingElements, dimensions are calculated when parsing + * DisplayAttributes, and TimingElements may be sent first + */ + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = width_mm; + dcp->modes[i].mode.height_mm = height_mm; + } +} + +bool dcp_has_panel(struct apple_dcp *dcp) +{ + return dcp->panel.width_mm > 0; +} + +int dcp_set_crc(struct drm_crtc *crtc, bool enabled) +{ + struct apple_crtc *ac = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(ac->dcp); + + dcp->crc_enabled = enabled; + + return 0; +} + +/* + * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp + * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks + * send a vblank event via a workqueue. + */ +static void dcp_delayed_vblank(struct work_struct *work) +{ + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, vblank_wq); + mdelay(5); + dcp_drm_crtc_vblank(dcp->crtc); +} + +static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) +{ + struct apple_dcp *dcp = cookie; + + trace_dcp_recv_msg(dcp, endpoint, message); + + switch (endpoint) { + case IOMFB_ENDPOINT: + return iomfb_recv_msg(dcp, message); + case AV_ENDPOINT: + afk_receive_message(dcp->avep, message); + return; + case SYSTEM_ENDPOINT: + afk_receive_message(dcp->systemep, message); + return; + case DISP0_ENDPOINT: + afk_receive_message(dcp->ibootep, message); + return; + case DPAVSERV_ENDPOINT: + afk_receive_message(dcp->dcpavservep, message); + return; + case DPTX_ENDPOINT: + afk_receive_message(dcp->dptxep, message); + return; + default: + WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + } +} + +static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +{ + struct apple_dcp *dcp = cookie; + + dcp->crashed = true; + dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->connector) { + dcp->connector->connected = 0; + drm_edid_free(dcp->connector->drm_edid); + dcp->connector->drm_edid = NULL; + schedule_work(&dcp->connector->hotplug_wq); + } + complete(&dcp->start_done); +} + +static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp *dcp = cookie; + + if (bfr->iova) { + struct iommu_domain *domain = + iommu_get_domain_for_dev(dcp->dev); + phys_addr_t phy_addr; + + if (!domain) + return -ENOMEM; + + // TODO: get map from device-tree + phy_addr = iommu_iova_to_phys(domain, bfr->iova); + if (!phy_addr) + return -ENOMEM; + + // TODO: verify phy_addr, cache attribute + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; + + bfr->is_mapped = true; + dev_info(dcp->dev, + "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)phy_addr, + (uintptr_t)bfr->buffer); + } else { + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + + dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); + } + + return 0; +} + +static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_dcp *dcp = cookie; + + if (bfr->is_mapped) + memunmap(bfr->buffer); + else + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); +} + +static struct apple_rtkit_ops rtkit_ops = { + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) +{ + trace_dcp_send_msg(dcp, endpoint, message); + apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, + true); +} + +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_plane_state *new_state; + struct drm_plane *plane; + struct drm_crtc_state *crtc_state; + int plane_idx, plane_count = 0; + bool needs_modeset; + + if (dcp->crashed) + return -EINVAL; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + if (!needs_modeset && !dcp->connector->connected) { + dev_err(dcp->dev, "crtc_atomic_check: disconnected but no modeset\n"); + return -EINVAL; + } + + for_each_new_plane_in_state(state, plane, new_state, plane_idx) { + /* skip planes not for this crtc */ + if (new_state->crtc != crtc) + continue; + + plane_count += 1; + } + + if (plane_count > DCP_MAX_PLANES) { + dev_err(dcp->dev, "crtc_atomic_check: Blend supports only 2 layers!\n"); + return -EINVAL; + } + + // if (dcp->vrr_enabled != crtc_state->vrr_enabled) { + // crtc_state->mode_changed = true; + // } + + return 0; +} + +int dcp_get_connector_type(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return (dcp->connector_type); +} + +#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +{ + int ret = 0; + + if (!dcp->phy) { + dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); + return -ENODEV; + } + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + if (!dcp->dptxport[port].enabled) { + dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); + ret = -ENODEV; + goto out_unlock; + } + + if (dcp->dptxport[port].connected) + goto out_unlock; + + reinit_completion(&dcp->dptxport[port].linkcfg_completion); + dcp->dptxport[port].atcphy = dcp->phy; + dptxport_connect(dcp->dptxport[port].service, 0, dcp->dptx_phy, dcp->dptx_die); + dptxport_request_display(dcp->dptxport[port].service); + dcp->dptxport[port].connected = true; + + mutex_unlock(&dcp->hpd_mutex); + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, + DPTX_CONNECT_TIMEOUT); + if (ret < 0) + dev_warn(dcp->dev, "dcp_dptx_connect: port %d link complete failed:%d\n", + port, ret); + else + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(DPTX_CONNECT_TIMEOUT - ret)); + + usleep_range(5, 10); + + if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) + dptxport_set_hpd(dcp->dptxport[port].service, true); + + if (dcp->avep) + av_service_connect(dcp); + + return 0; + +out_unlock: + mutex_unlock(&dcp->hpd_mutex); + return ret; +} + +static void disconnected_hpd_event(struct apple_connector *con) +{ + if (con && con->connected) { + con->connected = 0; + drm_kms_helper_connector_hotplug_event(&con->base); + } +} + +static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) +{ + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + if (dcp->dptxport[port].enabled && dcp->dptxport[port].connected) { + dptxport_release_display(dcp->dptxport[port].service); + dcp->dptxport[port].connected = false; + } + mutex_unlock(&dcp->hpd_mutex); + + return 0; +} + +int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + return dcp_dptx_connect(dcp, port); +} + +int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + disconnected_hpd_event(dcp->connector); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->dptxport[port].enabled) + dptxport_set_hpd(dcp->dptxport[port].service, false); + + return dcp_dptx_disconnect(dcp, port); +} + +static irqreturn_t dcp_dp2hdmi_hpd(int irq, void *data) +{ + struct apple_dcp *dcp = data; + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + + /* do nothing on disconnect and trust that dcp detects it itself. + * Parallel disconnect HPDs result drm disabling the CRTC even when it + * should not. + * The interrupt should be changed to rising but for now the disconnect + * IRQs might be helpful for debugging. + */ + dev_info(dcp->dev, "DP2HDMI HPD irq, connected:%d\n", connected); + + if (connected) { + msleep(500); + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); + } + + if (connected) + dcp_dptx_connect(dcp, 0); + + return IRQ_HANDLED; +} + +void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + dcp->crtc = crtc; + dcp->connector = connector; +} + + +bool dcp_fw_compat_is_12_x(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->fw_compat == DCP_FIRMWARE_V_12_3; +} + +int dcp_start(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; + + init_completion(&dcp->start_done); + + /* start RTKit endpoints */ + ret = systemep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start system endpoint: %d\n", ret); + + if (unstable_edid && !dcp_has_panel(dcp)) { + ret = dpavservep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start DPAVSERV endpoint: %d", + ret); + } + + if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + ret = ibootep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", + ret); + + ret = dptxep_init(dcp); + if (ret) { + dev_warn(dcp->dev, "Failed to start DPTX endpoint: %d\n", + ret); +#ifdef DCP_DPTX_DISCONNECT_ON_INIT + /* + * This disconnect / connect cycle on init is only necessary + * when using dcp0 on j473, j474s and presumedly j475c. + * Since dcp0 is not used at the moment let's avoid this + * since it is possibly the cause for startup issues. + */ + } else if (dcp->dptxport[0].enabled) { + bool connected; + /* force disconnect on start - necessary if the display + * is already up from m1n1 + */ + dptxport_set_hpd(dcp->dptxport[0].service, false); + dptxport_release_display(dcp->dptxport[0].service); + usleep_range(10 * USEC_PER_MSEC, 25 * USEC_PER_MSEC); + + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + // necessary on j473/j474 but not on j314c + if (connected) + dcp_dptx_connect(dcp, 0); +#endif + } + } else if (dcp->phy) { + dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); + } + ret = iomfb_start_rtkit(dcp); + if (ret) + dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (hdmi_audio) { + ret = avep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start AV endpoint: %d", ret); + ret = 0; + } +#endif + + return ret; +} + +static void _dcp_poweroff(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweroff_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_poweroff_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) +{ + // check HPD state before enabling the edge triggered IRQ + if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + if (connected) + dcp_dptx_connect(dcp, 0); + else + _dcp_poweroff(dcp); + } + + if (dcp->hdmi_hpd_irq) + enable_irq(dcp->hdmi_hpd_irq); + + return 0; +} + +int dcp_wait_ready(struct platform_device *pdev, u64 timeout) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; + + if (dcp->crashed) + return -ENODEV; + if (dcp->active) + return dcp_enable_dp2hdmi_hpd(dcp); + if (timeout <= 0) + return -ETIMEDOUT; + + ret = wait_for_completion_timeout(&dcp->start_done, timeout); + if (ret < 0) + return ret; + + if (dcp->crashed) + return -ENODEV; + + if (dcp->active) + dcp_enable_dp2hdmi_hpd(dcp); + + return dcp->active ? 0 : -ETIMEDOUT; +} + +static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_sleep_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_sleep_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +void dcp_poweron(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + if (connected) + dcp_dptx_connect(dcp, 0); + } + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweron_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_poweron_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } + + if (dcp->avep) + av_service_connect(dcp); +} + +void dcp_poweroff(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp->avep) + av_service_disconnect(dcp); + + _dcp_poweroff(dcp); + + if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + if (!connected) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } +} + +static void dcp_work_register_backlight(struct work_struct *work) +{ + int ret; + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, bl_register_wq); + + mutex_lock(&dcp->bl_register_mutex); + if (dcp->brightness.bl_dev) + goto out_unlock; + + /* try to register backlight device, */ + ret = dcp_backlight_register(dcp); + if (ret) { + dev_err(dcp->dev, "Unable to register backlight device\n"); + dcp->brightness.maximum = 0; + } + +out_unlock: + mutex_unlock(&dcp->bl_register_mutex); +} + +static void dcp_work_update_backlight(struct work_struct *work) +{ + struct apple_dcp *dcp; + + dcp = container_of(work, struct apple_dcp, bl_update_wq); + + dcp_backlight_update(dcp); +} + +static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) +{ + int ret; + struct device_node *node __free(device_node) = of_get_child_by_name(dcp->dev->of_node, "piodma"); + + if (!node) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to get piodma child DT node\n"); + + dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); + + ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); + if (ret) + goto err_destroy_pdev; + + ret = of_dma_configure(&dcp->piodma->dev, node, true); + if (ret) { + ret = dev_err_probe(dcp->dev, ret, + "Failed to configure IOMMU child DMA\n"); + goto err_destroy_pdev; + } + + dcp->iommu_dom = iommu_get_domain_for_dev(&dcp->piodma->dev); + if (IS_ERR(dcp->iommu_dom)) { + ret = dev_err_probe(dcp->dev, PTR_ERR(dcp->iommu_dom), + "Failed to get default iommu domain for " + "piodma device\n"); + dcp->iommu_dom = NULL; + goto err_destroy_pdev; + } + + return 0; +err_destroy_pdev: + of_platform_device_destroy(&dcp->piodma->dev, NULL); + return ret; +} + +static int dcp_get_bw_scratch_reg(struct apple_dcp *dcp, u32 expected) +{ + struct of_phandle_args ph_args; + u32 addr_idx, disp_idx, offset; + int ret; + + ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-scratch", + "#apple,bw-scratch-cells", 0, &ph_args); + if (ret < 0) { + dev_err(dcp->dev, "Failed to read 'apple,bw-scratch': %d\n", ret); + return ret; + } + + if (ph_args.args_count != 3) { + dev_err(dcp->dev, "Unexpected 'apple,bw-scratch' arg count %d\n", + ph_args.args_count); + ret = -EINVAL; + goto err_of_node_put; + } + + addr_idx = ph_args.args[0]; + disp_idx = ph_args.args[1]; + offset = ph_args.args[2]; + + if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { + dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-scratch': %d\n", + disp_idx); + ret = -EINVAL; + goto err_of_node_put; + } + + ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_scratch_res); + if (ret < 0) { + dev_err(dcp->dev, "Failed to get 'apple,bw-scratch' resource %d from %pOF\n", + addr_idx, ph_args.np); + goto err_of_node_put; + } + if (offset > resource_size(&dcp->disp_bw_scratch_res) - 4) { + ret = -EINVAL; + goto err_of_node_put; + } + + dcp->disp_registers[disp_idx] = &dcp->disp_bw_scratch_res; + dcp->disp_bw_scratch_index = disp_idx; + dcp->disp_bw_scratch_offset = offset; + ret = 0; + +err_of_node_put: + of_node_put(ph_args.np); + return ret; +} + +static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) +{ + struct of_phandle_args ph_args; + u32 addr_idx, disp_idx; + int ret; + + ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-doorbell", + "#apple,bw-doorbell-cells", 0, &ph_args); + if (ret < 0) { + dev_err(dcp->dev, "Failed to read 'apple,bw-doorbell': %d\n", ret); + return ret; + } + + if (ph_args.args_count != 2) { + dev_err(dcp->dev, "Unexpected 'apple,bw-doorbell' arg count %d\n", + ph_args.args_count); + ret = -EINVAL; + goto err_of_node_put; + } + + addr_idx = ph_args.args[0]; + disp_idx = ph_args.args[1]; + + if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { + dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-doorbell': %d\n", + disp_idx); + ret = -EINVAL; + goto err_of_node_put; + } + + ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_doorbell_res); + if (ret < 0) { + dev_err(dcp->dev, "Failed to get 'apple,bw-doorbell' resource %d from %pOF\n", + addr_idx, ph_args.np); + goto err_of_node_put; + } + dcp->disp_bw_doorbell_index = disp_idx; + dcp->disp_registers[disp_idx] = &dcp->disp_bw_doorbell_res; + ret = 0; + +err_of_node_put: + of_node_put(ph_args.np); + return ret; +} + +static int dcp_get_disp_regs(struct apple_dcp *dcp) +{ + struct platform_device *pdev = to_platform_device(dcp->dev); + int count = pdev->num_resources - 1; + int i, ret; + + if (count <= 0 || count > MAX_DISP_REGISTERS) + return -EINVAL; + + for (i = 0; i < count; ++i) { + dcp->disp_registers[i] = + platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + } + + /* load pmgr bandwidth scratch resource and offset */ + ret = dcp_get_bw_scratch_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; + + /* load pmgr bandwidth doorbell resource if present (only on t8103) */ + if (of_property_present(dcp->dev->of_node, "apple,bw-doorbell")) { + ret = dcp_get_bw_doorbell_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; + } + + dcp->nr_disp_registers = count; + return 0; +} + +#define DCP_FW_VERSION_MIN_LEN 3 +#define DCP_FW_VERSION_MAX_LEN 5 +#define DCP_FW_VERSION_STR_LEN (DCP_FW_VERSION_MAX_LEN * 4) + +static int dcp_read_fw_version(struct device *dev, const char *name, + char *version_str) +{ + u32 ver[DCP_FW_VERSION_MAX_LEN]; + int len_str; + int len; + + len = of_property_read_variable_u32_array(dev->of_node, name, ver, + DCP_FW_VERSION_MIN_LEN, + DCP_FW_VERSION_MAX_LEN); + + switch (len) { + case 3: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d", ver[0], ver[1], ver[2]); + break; + case 4: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3]); + break; + case 5: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3], ver[4]); + break; + default: + len_str = strscpy(version_str, "UNKNOWN", + DCP_FW_VERSION_STR_LEN); + if (len >= 0) + len = -EOVERFLOW; + break; + } + + if (len_str >= DCP_FW_VERSION_STR_LEN) + dev_warn(dev, "'%s' truncated: '%s'\n", name, version_str); + + return len; +} + +static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) +{ + char compat_str[DCP_FW_VERSION_STR_LEN]; + char fw_str[DCP_FW_VERSION_STR_LEN]; + int ret; + + /* firmware version is just informative */ + dcp_read_fw_version(dev, "apple,firmware-version", fw_str); + + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0) { + dev_err(dev, "Could not read 'apple,firmware-compat': %d\n", ret); + return DCP_FIRMWARE_UNKNOWN; + } + + if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_12_3; + /* + * m1n1 reports firmware version 13.5 as compatible with 13.3. This is + * only true for the iomfb endpoint. The interface for the dptx-port + * endpoint changed between 13.3 and 13.5. The driver will only support + * firmware 13.5. Check the actual firmware version for compat version + * 13.3 until m1n1 reports 13.5 as "firmware-compat". + */ + else if ((strncmp(compat_str, "13.3.0", sizeof(compat_str)) == 0) && + (strncmp(fw_str, "13.5.0", sizeof(compat_str)) == 0)) + return DCP_FIRMWARE_V_13_5; + else if (strncmp(compat_str, "13.5.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_13_5; + + dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", + compat_str, fw_str); + + return DCP_FIRMWARE_UNKNOWN; +} + +static int dcp_comp_bind(struct device *dev, struct device *main, void *data) +{ + struct device_node *panel_np; + struct apple_dcp *dcp = dev_get_drvdata(dev); + u32 cpu_ctrl; + int ret; + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; + + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); + if (IS_ERR(dcp->coproc_reg)) + return PTR_ERR(dcp->coproc_reg); + + of_property_read_u32(dev->of_node, "apple,dcp-index", + &dcp->index); + of_property_read_u32(dev->of_node, "apple,dptx-phy", + &dcp->dptx_phy); + of_property_read_u32(dev->of_node, "apple,dptx-die", + &dcp->dptx_die); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) + dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", + dcp->index, dcp->dptx_phy, dcp->dptx_die); + mutex_init(&dcp->hpd_mutex); + + if (!show_notch) + ret = of_property_read_u32(dev->of_node, "apple,notch-height", + &dcp->notch_height); + + if (dcp->notch_height > MAX_NOTCH_HEIGHT) + dcp->notch_height = MAX_NOTCH_HEIGHT; + if (dcp->notch_height > 0) + dev_info(dev, "Detected display with notch of %u pixel\n", dcp->notch_height); + + /* initialize brightness scale to a sensible default to avoid divide by 0*/ + dcp->brightness.scale = 65536; + panel_np = of_get_compatible_child(dev->of_node, "apple,panel-mini-led"); + if (panel_np) + dcp->panel.has_mini_led = true; + else + panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + + if (panel_np) { + const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + + if (of_device_is_available(panel_np)) { + ret = of_property_read_u32(panel_np, "apple,max-brightness", + &dcp->brightness.maximum); + if (ret) + dev_err(dev, "Missing property 'apple,max-brightness'\n"); + } + + of_property_read_u32(panel_np, "width-mm", &dcp->panel.width_mm); + /* use adjusted height as long as the notch is hidden */ + of_property_read_u32(panel_np, height_prop[!dcp->notch_height], + &dcp->panel.height_mm); + + of_node_put(panel_np); + dcp->connector_type = DRM_MODE_CONNECTOR_eDP; + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + mutex_init(&dcp->bl_register_mutex); + INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); + } else if (of_property_match_string(dev->of_node, "apple,connector-type", "HDMI-A") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_HDMIA; + else if (of_property_match_string(dev->of_node, "apple,connector-type", "DP") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_DisplayPort; + else if (of_property_match_string(dev->of_node, "apple,connector-type", "USB-C") >= 0) + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + else + dcp->connector_type = DRM_MODE_CONNECTOR_Unknown; + + ret = dcp_create_piodma_iommu_dev(dcp); + if (ret || !dcp->iommu_dom) + return dev_err_probe(dev, ret, + "Failed to created PIODMA iommu child device"); + + ret = dcp_get_disp_regs(dcp); + if (ret) { + dev_err(dev, "failed to find display registers\n"); + return ret; + } + + dcp->clk = devm_clk_get(dev, NULL); + if (IS_ERR(dcp->clk)) + return dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); + + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); + // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry + set_bit(0, dcp->memdesc_map); + + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + + dcp->swapped_out_fbs = + (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + if (IS_ERR(dcp->rtk)) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + + ret = apple_rtkit_wake(dcp->rtk); + if (ret) + return dev_err_probe(dev, ret, + "Failed to boot RTKit: %d\n", ret); + return ret; +} + +/* + * We need to shutdown DCP before tearing down the display subsystem. Otherwise + * the DCP will crash and briefly flash a green screen of death. + */ +static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (!dcp) + return; + + if (dcp->hdmi_hpd_irq) + disable_irq(dcp->hdmi_hpd_irq); + + typec_mux_put(dcp->typec_mux); + + if (dcp->avep) { + av_service_disconnect(dcp); + afk_shutdown(dcp->avep); + dcp->avep = NULL; + } + + if (dcp->dptxep) { + afk_shutdown(dcp->dptxep); + dcp->dptxep = NULL; + } + + if (dcp->ibootep) { + afk_shutdown(dcp->ibootep); + dcp->ibootep = NULL; + } + + if (dcp->systemep) { + afk_shutdown(dcp->systemep); + dcp->systemep = NULL; + } + + if (dcp->dcpavservep) { + afk_shutdown(dcp->dcpavservep); + dcp->dcpavservep = NULL; + } + + if (dcp->shmem) + iomfb_shutdown(dcp); + + if (dcp->piodma) { + dcp->iommu_dom = NULL; + of_platform_device_destroy(&dcp->piodma->dev, NULL); + dcp->piodma = NULL; + } + + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + } + cancel_work_sync(&dcp->vblank_wq); + + devm_clk_put(dev, dcp->clk); + dcp->clk = NULL; +} + +static const struct component_ops dcp_comp_ops = { + .bind = dcp_comp_bind, + .unbind = dcp_comp_unbind, +}; + +static int dcp_platform_probe(struct platform_device *pdev) +{ + enum dcp_firmware_version fw_compat; + struct device *dev = &pdev->dev; + struct apple_dcp *dcp; + u32 mux_index; + + fw_compat = dcp_check_firmware_version(dev); + if (fw_compat == DCP_FIRMWARE_UNKNOWN) + return -ENODEV; + + /* Check for "apple,bw-scratch" to avoid probing appledrm with outdated + * device trees. This prevents replacing simpledrm and ending up without + * display. + */ + if (!of_property_present(dev->of_node, "apple,bw-scratch")) + return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " + "Use devicetree matching this kernel.\n"); + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + + dcp->fw_compat = fw_compat; + dcp->dev = dev; + dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + + platform_set_drvdata(pdev, dcp); + + dcp->phy = devm_phy_optional_get(dev, "dp-phy"); + if (IS_ERR(dcp->phy)) { + dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); + return PTR_ERR(dcp->phy); + } + if (dcp->phy) { + int ret; + /* + * Request DP2HDMI related GPIOs as optional for DP-altmode + * compatibility. J180D misses a dp2hdmi-pwren GPIO in the + * template ADT. TODO: check device ADT + */ + dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); + if (IS_ERR(dcp->hdmi_hpd)) + return PTR_ERR(dcp->hdmi_hpd); + if (dcp->hdmi_hpd) { + int irq = gpiod_to_irq(dcp->hdmi_hpd); + if (irq < 0) { + dev_err(dev, "failed to translate HDMI hpd GPIO to IRQ\n"); + return irq; + } + dcp->hdmi_hpd_irq = irq; + + ret = devm_request_threaded_irq(dev, dcp->hdmi_hpd_irq, + NULL, dcp_dp2hdmi_hpd, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, + "dp2hdmi-hpd-irq", dcp); + if (ret < 0) { + dev_err(dev, "failed to request HDMI hpd irq %d: %d\n", + irq, ret); + return ret; + } + } + + /* + * Power DP2HDMI on as it is required for the HPD irq. + * TODO: check if one is sufficient for the hpd to save power + * on battery powered Macbooks. + */ + dcp->hdmi_pwren = devm_gpiod_get_optional(dev, "hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->hdmi_pwren)) + return PTR_ERR(dcp->hdmi_pwren); + + dcp->dp2hdmi_pwren = devm_gpiod_get_optional(dev, "dp2hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->dp2hdmi_pwren)) + return PTR_ERR(dcp->dp2hdmi_pwren); + + ret = of_property_read_u32(dev->of_node, "mux-index", &mux_index); + if (!ret) { + dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); + if (IS_ERR(dcp->xbar)) { + dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); + return PTR_ERR(dcp->xbar); + } + ret = mux_control_select(dcp->xbar, mux_index); + if (ret) + dev_warn(dev, "mux_control_select failed: %d\n", ret); + + /* + * Switch atcphy to DP-only. should move to a Macbook Pro + * 14-/16-inch specific DP-to-HDMI drm_bridge. + */ + dcp->typec_mux = fwnode_typec_mux_get(dev_fwnode(dcp->dev)); + if (!IS_ERR_OR_NULL(dcp->typec_mux)) { + struct typec_altmode alt = { + .svid = USB_TYPEC_DP_SID, + }; + struct typec_mux_state state = { + .alt = &alt, + .mode = TYPEC_DP_STATE_C, + }; + int ret = typec_mux_set(dcp->typec_mux, &state); + dev_info(dev, "typec_mux_set() returned: %d\n", ret); + } else { + dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", + IS_ERR(dcp->typec_mux) ? PTR_ERR(dcp->typec_mux) : 0); + dcp->typec_mux = NULL; + } + } + } + + return component_add(&pdev->dev, &dcp_comp_ops); +} + +static void dcp_platform_remove(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_comp_ops); +} + +static void dcp_platform_shutdown(struct platform_device *pdev) +{ + component_del(&pdev->dev, &dcp_comp_ops); +} + +static int dcp_platform_suspend(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dcp->hdmi_hpd_irq) { + disable_irq(dcp->hdmi_hpd_irq); + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); + + return 0; +} + +static int dcp_platform_resume(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->hdmi_hpd_irq) + enable_irq(dcp->hdmi_hpd_irq); + + if (dcp->avep) + av_service_connect(dcp); + + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(dcp_platform_pm_ops, + dcp_platform_suspend, dcp_platform_resume); + + +static const struct apple_dcp_hw_data apple_dcp_hw_t6020 = { + .num_dptx_ports = 1, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { + .num_dptx_ports = 2, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { + .num_dptx_ports = 0, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { + .num_dptx_ports = 2, +}; + +static const struct of_device_id of_match[] = { + { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, + { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, + { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, + { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, + {} +}; +MODULE_DEVICE_TABLE(of, of_match); + +static struct platform_driver apple_platform_driver = { + .probe = dcp_platform_probe, + .remove = dcp_platform_remove, + .shutdown = dcp_platform_shutdown, + .driver = { + .name = "apple-dcp", + .of_match_table = of_match, + .pm = pm_sleep_ptr(&dcp_platform_pm_ops), + }, +}; + +void __init dcp_register(void) +{ + platform_driver_register(&apple_platform_driver); +} + +void __exit dcp_unregister(void) +{ + platform_driver_unregister(&apple_platform_driver); +} diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h new file mode 100644 index 00000000000000..bd20876847e0c3 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp.h @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_H__ +#define __APPLE_DCP_H__ + +#include +#include +#include + +#include "connector.h" +#include "dcp-internal.h" +#include "parser.h" + +struct apple_crtc { + struct drm_crtc base; + struct drm_pending_vblank_event *event; + bool vsync_disabled; + + /* Reference to the DCP device owning this CRTC */ + struct platform_device *dcp; +}; + +#define to_apple_crtc(x) container_of(x, struct apple_crtc, base) + +struct apple_encoder { + struct drm_encoder base; +}; + +#define to_apple_encoder(x) container_of(x, struct apple_encoder, base) + +void dcp_poweroff(struct platform_device *pdev); +void dcp_poweron(struct platform_device *pdev); +int dcp_set_crc(struct drm_crtc *crtc, bool enabled); +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); +int dcp_get_connector_type(struct platform_device *pdev); +bool dcp_fw_compat_is_12_x(struct platform_device *pdev); +void dcp_link(struct platform_device *pdev, struct apple_crtc *apple, + struct apple_connector *connector); +int dcp_start(struct platform_device *pdev); +int dcp_wait_ready(struct platform_device *pdev, u64 timeout); +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); +bool dcp_is_initialized(struct platform_device *pdev); +void apple_crtc_vblank(struct apple_crtc *apple); +void dcp_drm_crtc_vblank(struct apple_crtc *crtc); +int dcp_get_modes(struct drm_connector *connector); +enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode); +int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, + struct drm_atomic_state *state); +bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode); +void dcp_set_dimensions(struct apple_dcp *dcp); +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); + +int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); +int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); + +int iomfb_start_rtkit(struct apple_dcp *dcp); +void iomfb_shutdown(struct apple_dcp *dcp); +/* rtkit message handler for IOMFB messages */ +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); + +int systemep_init(struct apple_dcp *dcp); +int dptxep_init(struct apple_dcp *dcp); +int ibootep_init(struct apple_dcp *dcp); +int dpavservep_init(struct apple_dcp *dcp); +int avep_init(struct apple_dcp *dcp); + + +void __init dcp_register(void); +void __exit dcp_unregister(void); + +void __init dcp_audio_register(void); +void __exit dcp_audio_unregister(void); + +#endif diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c new file mode 100644 index 00000000000000..9eb0c7d4eb5345 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright (C) The Asahi Linux Contributors */ + +#include +#include +#include +#include + +#include +#include +#include +#include "linux/jiffies.h" + +#include "dcp.h" +#include "dcp-internal.h" + +#define MIN_BRIGHTNESS_PART1 2U +#define MAX_BRIGHTNESS_PART1 99U +#define MIN_BRIGHTNESS_PART2 103U +#define MAX_BRIGHTNESS_PART2 510U + +/* + * lookup for display brightness 2 to 99 nits + * */ +static u32 brightness_part1[] = { + 0x0000000, 0x0810038, 0x0f000bd, 0x143011c, + 0x1850165, 0x1bc01a1, 0x1eb01d4, 0x2140200, + 0x2380227, 0x2590249, 0x2770269, 0x2930285, + 0x2ac02a0, 0x2c402b8, 0x2d902cf, 0x2ee02e4, + 0x30102f8, 0x314030b, 0x325031c, 0x335032d, + 0x345033d, 0x354034d, 0x362035b, 0x3700369, + 0x37d0377, 0x38a0384, 0x3960390, 0x3a2039c, + 0x3ad03a7, 0x3b803b3, 0x3c303bd, 0x3cd03c8, + 0x3d703d2, 0x3e103dc, 0x3ea03e5, 0x3f303ef, + 0x3fc03f8, 0x4050400, 0x40d0409, 0x4150411, + 0x41d0419, 0x4250421, 0x42d0429, 0x4340431, + 0x43c0438, 0x443043f, 0x44a0446, 0x451044d, + 0x4570454, 0x45e045b, 0x4640461, 0x46b0468, + 0x471046e, 0x4770474, 0x47d047a, 0x4830480, + 0x4890486, 0x48e048b, 0x4940491, 0x4990497, + 0x49f049c, 0x4a404a1, 0x4a904a7, 0x4ae04ac, + 0x4b304b1, 0x4b804b6, 0x4bd04bb, 0x4c204c0, + 0x4c704c5, 0x4cc04c9, 0x4d004ce, 0x4d504d3, + 0x4d904d7, 0x4de04dc, 0x4e204e0, 0x4e704e4, + 0x4eb04e9, 0x4ef04ed, 0x4f304f1, 0x4f704f5, + 0x4fb04f9, 0x4ff04fd, 0x5030501, 0x5070505, + 0x50b0509, 0x50f050d, 0x5130511, 0x5160515, + 0x51a0518, 0x51e051c, 0x5210520, 0x5250523, + 0x5290527, 0x52c052a, 0x52f052e, 0x5330531, + 0x5360535, 0x53a0538, 0x53d053b, 0x540053f, + 0x5440542, 0x5470545, 0x54a0548, 0x54d054c, + 0x550054f, 0x5530552, 0x5560555, 0x5590558, + 0x55c055b, 0x55f055e, 0x5620561, 0x5650564, + 0x5680567, 0x56b056a, 0x56e056d, 0x571056f, + 0x5740572, 0x5760575, 0x5790578, 0x57c057b, + 0x57f057d, 0x5810580, 0x5840583, 0x5870585, + 0x5890588, 0x58c058b, 0x58f058d +}; + +static u32 brightness_part12[] = { 0x58f058d, 0x59d058f }; + +/* + * lookup table for display brightness 103.3 to 510 nits + * */ +static u32 brightness_part2[] = { + 0x59d058f, 0x5b805ab, 0x5d105c5, 0x5e805dd, + 0x5fe05f3, 0x6120608, 0x625061c, 0x637062e, + 0x6480640, 0x6580650, 0x6680660, 0x677066f, + 0x685067e, 0x693068c, 0x6a00699, 0x6ac06a6, + 0x6b806b2, 0x6c406be, 0x6cf06ca, 0x6da06d5, + 0x6e506df, 0x6ef06ea, 0x6f906f4, 0x70206fe, + 0x70c0707, 0x7150710, 0x71e0719, 0x7260722, + 0x72f072a, 0x7370733, 0x73f073b, 0x7470743, + 0x74e074a, 0x7560752, 0x75d0759, 0x7640760, + 0x76b0768, 0x772076e, 0x7780775, 0x77f077c, + 0x7850782, 0x78c0789, 0x792078f, 0x7980795, + 0x79e079b, 0x7a407a1, 0x7aa07a7, 0x7af07ac, + 0x7b507b2, 0x7ba07b8, 0x7c007bd, 0x7c507c2, + 0x7ca07c8, 0x7cf07cd, 0x7d407d2, 0x7d907d7, + 0x7de07dc, 0x7e307e1, 0x7e807e5, 0x7ec07ea, + 0x7f107ef, 0x7f607f3, 0x7fa07f8, 0x7fe07fc +}; + + +static int dcp_get_brightness(struct backlight_device *bd) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + return dcp->brightness.nits; +} + +#define SCALE_FACTOR (1 << 10) + +static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) +{ + u32 frac; + u64 low, high; + u32 interpolated = (tbl_size - 1) * ((val - min) * SCALE_FACTOR) / (max - min); + + size_t index = interpolated / SCALE_FACTOR; + + if (WARN(index + 1 >= tbl_size, "invalid index %zu for brightness %u\n", index, val)) + return tbl[tbl_size / 2]; + + frac = interpolated & (SCALE_FACTOR - 1); + low = tbl[index]; + high = tbl[index + 1]; + + return ((frac * high) + ((SCALE_FACTOR - frac) * low)) / SCALE_FACTOR; +} + +static u32 calculate_dac(struct apple_dcp *dcp, int val) +{ + u32 dac; + + if (val <= MIN_BRIGHTNESS_PART1) + return 16 * brightness_part1[0]; + else if (val == MAX_BRIGHTNESS_PART1) + return 16 * brightness_part1[ARRAY_SIZE(brightness_part1) - 1]; + else if (val == MIN_BRIGHTNESS_PART2) + return 16 * brightness_part2[0]; + else if (val >= MAX_BRIGHTNESS_PART2) + return brightness_part2[ARRAY_SIZE(brightness_part2) - 1]; + + if (val < MAX_BRIGHTNESS_PART1) { + dac = interpolate(val, MIN_BRIGHTNESS_PART1, MAX_BRIGHTNESS_PART1, + brightness_part1, ARRAY_SIZE(brightness_part1)); + } else if (val > MIN_BRIGHTNESS_PART2) { + dac = interpolate(val, MIN_BRIGHTNESS_PART2, MAX_BRIGHTNESS_PART2, + brightness_part2, ARRAY_SIZE(brightness_part2)); + } else { + dac = interpolate(val, MAX_BRIGHTNESS_PART1, MIN_BRIGHTNESS_PART2, + brightness_part12, ARRAY_SIZE(brightness_part12)); + } + + return 16 * dac; +} + +static int drm_crtc_set_brightness(struct apple_dcp *dcp) +{ + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc *crtc = &dcp->crtc->base; + int ret = 0; + + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + ret = drm_modeset_lock(&crtc->mutex, &ctx); + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + return -EDEADLK; + } else if (ret == -ERESTARTSYS) { + return -ERESTARTSYS; + } + + if (!dcp->brightness.update) + goto done; + + state = drm_atomic_state_alloc(crtc->dev); + if (!state) { + ret = -ENOMEM; + goto done; + } + + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + + crtc_state->color_mgmt_changed |= true; + + ret = drm_atomic_commit(state); + +fail: + drm_atomic_state_put(state); +done: + drm_modeset_drop_locks(&ctx); + + return ret; +} + +int dcp_backlight_update(struct apple_dcp *dcp) +{ + /* + * Do not actively try to change brightness if no mode is set. + * TODO: should this be reflected the in backlight's power property? + * defer this hopefully until it becomes irrelevant due to proper + * drm integrated backlight handling + */ + if (!dcp->valid_mode) + return 0; + + /* Wait 1 vblank cycle in the hope an atomic swap has already updated + * the brightness */ + msleep((1001 + 23) / 24); // 42ms for 23.976 fps + + return drm_crtc_set_brightness(dcp); +} + +static int dcp_set_brightness(struct backlight_device *bd) +{ + int ret = 0; + struct apple_dcp *dcp = bl_get_data(bd); + struct drm_modeset_acquire_ctx ctx; + int brightness = backlight_get_brightness(bd); + + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); + ret = drm_modeset_lock(&dcp->crtc->base.mutex, &ctx); + if (ret == -EDEADLK) { + drm_modeset_backoff(&ctx); + return -EDEADLK; + } else if (ret == -ERESTARTSYS) { + return -ERESTARTSYS; + } + + dcp->brightness.dac = calculate_dac(dcp, brightness); + dcp->brightness.update = true; + + drm_modeset_drop_locks(&ctx); + + return dcp_backlight_update(dcp); +} + +static const struct backlight_ops dcp_backlight_ops = { + .options = BL_CORE_SUSPENDRESUME, + .get_brightness = dcp_get_brightness, + .update_status = dcp_set_brightness, +}; + +int dcp_backlight_register(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct backlight_device *bl_dev; + struct backlight_properties props = { + .type = BACKLIGHT_PLATFORM, + .brightness = dcp->brightness.nits, + .scale = BACKLIGHT_SCALE_LINEAR, + }; + props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + + bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, + &dcp_backlight_ops, &props); + if (IS_ERR(bl_dev)) + return PTR_ERR(bl_dev); + + dcp->brightness.bl_dev = bl_dev; + dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); + + return 0; +} diff --git a/drivers/gpu/drm/apple/dcp_trace.c b/drivers/gpu/drm/apple/dcp_trace.c new file mode 100644 index 00000000000000..d18e71af73a74d --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_trace.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +#define CREATE_TRACE_POINTS +#include "dcp_trace.h" \ No newline at end of file diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c new file mode 100644 index 00000000000000..384e3906494edc --- /dev/null +++ b/drivers/gpu/drm/apple/dptxep.c @@ -0,0 +1,640 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Sven Peter */ + +#include +#include +#include +#include + +#include "afk.h" +#include "dcp.h" +#include "dptxep.h" +#include "parser.h" +#include "trace.h" + +struct dcpdptx_connection_cmd { + __le32 unk; + __le32 target; +} __attribute__((packed)); + +struct dcpdptx_hotplug_cmd { + u8 _pad0[16]; + __le32 unk; +} __attribute__((packed)); + +struct dptxport_apcall_link_rate { + __le32 retcode; + u8 _unk0[12]; + __le32 link_rate; + u8 _unk1[12]; +} __attribute__((packed)); + +struct dptxport_apcall_lane_count { + __le32 retcode; + u8 _unk0[12]; + __le64 lane_count; + u8 _unk1[8]; +} __attribute__((packed)); + +struct dptxport_apcall_set_active_lane_count { + __le32 retcode; + u8 _unk0[12]; + __le64 lane_count; + u8 _unk1[8]; +} __packed; + +struct dptxport_apcall_get_support { + __le32 retcode; + u8 _unk0[12]; + __le32 supported; + u8 _unk1[12]; +} __attribute__((packed)); + +struct dptxport_apcall_max_drive_settings { + __le32 retcode; + u8 _unk0[12]; + __le32 max_drive_settings[2]; + u8 _unk1[8]; +}; + +struct dptxport_apcall_drive_settings { + __le32 retcode; + u8 _unk0[12]; + __le32 unk1; + __le32 unk2; + __le32 unk3; + __le32 unk4; + __le32 unk5; + __le32 unk6; + __le32 unk7; +}; + +int dptxport_validate_connection(struct apple_epic_service *service, u8 core, + u8 atc, u8 die) +{ + struct dptx_port *dptx = service->cookie; + struct dcpdptx_connection_cmd cmd, resp; + int ret; + u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | + DCPDPTX_REMOTE_PORT_CONNECTED; + + trace_dptxport_validate_connection(dptx, core, atc, die); + + cmd.target = cpu_to_le32(target); + cmd.unk = cpu_to_le32(0x100); + ret = afk_service_call(service, 0, 12, &cmd, sizeof(cmd), 40, &resp, + sizeof(resp), 40); + if (ret) + return ret; + + if (le32_to_cpu(resp.target) != target) + return -EINVAL; + if (le32_to_cpu(resp.unk) != 0x100) + return -EINVAL; + + return 0; +} + +int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, + u8 die) +{ + struct dptx_port *dptx = service->cookie; + struct dcpdptx_connection_cmd cmd, resp; + u32 unk_field = 0x0; // seen as 0x100 under some conditions + int ret; + u32 target = FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | + FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | + DCPDPTX_REMOTE_PORT_CONNECTED; + + trace_dptxport_connect(dptx, core, atc, die); + + cmd.target = cpu_to_le32(target); + cmd.unk = cpu_to_le32(unk_field); + ret = afk_service_call(service, 0, 11, &cmd, sizeof(cmd), 24, &resp, + sizeof(resp), 24); + if (ret) + return ret; + + if (le32_to_cpu(resp.target) != target) + return -EINVAL; + if (le32_to_cpu(resp.unk) != unk_field) + dev_notice(service->ep->dcp->dev, "unexpected unk field in reply: 0x%x (0x%x)\n", + le32_to_cpu(resp.unk), unk_field); + + return 0; +} + +int dptxport_request_display(struct apple_epic_service *service) +{ + return afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); +} + +int dptxport_release_display(struct apple_epic_service *service) +{ + return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); +} + +int dptxport_set_hpd(struct apple_epic_service *service, bool hpd) +{ + struct dcpdptx_hotplug_cmd cmd, resp; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + + if (hpd) + cmd.unk = cpu_to_le32(1); + + ret = afk_service_call(service, 8, 8, &cmd, sizeof(cmd), 12, &resp, + sizeof(resp), 12); + if (ret) + return ret; + if (le32_to_cpu(resp.unk) != 1) + return -EINVAL; + return 0; +} + +static int +dptxport_call_get_max_drive_settings(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_max_drive_settings *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->max_drive_settings[0] = cpu_to_le32(0x3); + reply->max_drive_settings[1] = cpu_to_le32(0x3); + + return 0; +} + +static int +dptxport_call_get_drive_settings(struct apple_epic_service *service, + const void *request_, size_t request_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct dptxport_apcall_drive_settings *request = request_; + struct dptxport_apcall_drive_settings *reply = reply_; + + if (reply_size < sizeof(*reply) || request_size < sizeof(*request)) + return -EINVAL; + + *reply = *request; + + /* Clear the rest of the buffer */ + memset(reply_ + sizeof(*reply), 0, reply_size - sizeof(*reply)); + + /* + * retcode appears to be lane count, seeing 2 for USB-C dp alt mode + * with lanes splitted for DP/USB3. + */ + if (cpu_to_le32(reply->retcode) != dptx->lane_count) + dev_err(service->ep->dcp->dev, + "get_drive_settings: unexpected retcode %d\n", + reply->retcode); + + reply->retcode = cpu_to_le32(dptx->lane_count); + reply->unk5 = cpu_to_le32(dptx->drive_settings[0]); + reply->unk6 = cpu_to_le32(0); + reply->unk7 = cpu_to_le32(dptx->drive_settings[1]); + + return 0; +} + +static int +dptxport_call_set_drive_settings(struct apple_epic_service *service, + const void *request_, size_t request_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct dptxport_apcall_drive_settings *request = request_; + struct dptxport_apcall_drive_settings *reply = reply_; + + if (reply_size < sizeof(*reply) || request_size < sizeof(*request)) + return -EINVAL; + + *reply = *request; + reply->retcode = cpu_to_le32(0); + + dev_info(service->ep->dcp->dev, "set_drive_settings: %d:%d:%d:%d:%d:%d:%d\n", + request->unk1, request->unk2, request->unk3, request->unk4, + request->unk5, request->unk6, request->unk7); + + dptx->drive_settings[0] = cpu_to_le32(reply->unk5); + dptx->drive_settings[1] = cpu_to_le32(reply->unk7); + + return 0; +} + +static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_link_rate *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + + return 0; +} + +static int dptxport_call_get_max_lane_count(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_lane_count *reply = reply_; + struct dptx_port *dptx = service->cookie; + struct apple_dcp *dcp = service->ep->dcp; + union phy_configure_opts phy_ops; + int ret; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + ret = phy_validate(dptx->atcphy, PHY_MODE_DP, 0, &phy_ops); + if (ret < 0) { + dev_err(dcp->dev, "phy_validate failed: %d\n", ret); + reply->retcode = cpu_to_le32(1); + reply->lane_count = cpu_to_le64(0); + } else { + if (phy_ops.dp.lanes < 2) { + // phy_validate might return 0 lanes if atc phy is not + // yet switched to DP mode + dev_dbg(dcp->dev, "get_max_lane_count: phy lanes: %d\n", + phy_ops.dp.lanes); + // default to 4 lanes + dptx->lane_count = 4; + } else { + dptx->lane_count = phy_ops.dp.lanes; + } + reply->retcode = cpu_to_le32(0); + reply->lane_count = cpu_to_le64(dptx->lane_count); + } + + return 0; +} + +static int dptxport_call_set_active_lane_count(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + struct apple_dcp *dcp = service->ep->dcp; + const struct dptxport_apcall_set_active_lane_count *request = data; + struct dptxport_apcall_set_active_lane_count *reply = reply_; + int ret = 0; + int retcode = 0; + + if (reply_size < sizeof(*reply)) + return -1; + if (data_size < sizeof(*request)) + return -1; + + u64 lane_count = le64_to_cpu(request->lane_count); + + if (dptx->lane_count < lane_count) + dev_err(dcp->dev, "set_active_lane_count: unexpected lane " + "count:%llu phy: %d\n", lane_count, dptx->lane_count); + + switch (lane_count) { + case 0 ... 2: + case 4: + dptx->phy_ops.dp.lanes = lane_count; + // Use dptx phy index > 3 as indication for dptx-phy or + // lpdptx-phy and configure the number of lanes for those + dptx->phy_ops.dp.set_lanes = (dcp->dptx_phy > 3); + break; + default: + dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); + retcode = 1; + lane_count = 0; + break; + } + + if (dptx->phy_ops.dp.set_lanes) { + if (dptx->atcphy) { + ret = phy_configure(dptx->atcphy, &dptx->phy_ops); + if (ret) + return ret; + } + dptx->phy_ops.dp.set_lanes = 0; + dptx->lane_count = lane_count; + } + + reply->retcode = cpu_to_le32(retcode); + reply->lane_count = cpu_to_le64(lane_count); + + if (lane_count > 0) + complete(&dptx->linkcfg_completion); + + return ret; +} + +static int dptxport_call_get_link_rate(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + struct dptxport_apcall_link_rate *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->link_rate = cpu_to_le32(dptx->link_rate); + + return 0; +} + +static int +dptxport_call_will_change_link_config(struct apple_epic_service *service) +{ + struct dptx_port *dptx = service->cookie; + + dptx->phy_ops.dp.set_lanes = 0; + dptx->phy_ops.dp.set_rate = 0; + dptx->phy_ops.dp.set_voltages = 0; + + return 0; +} + +static int +dptxport_call_did_change_link_config(struct apple_epic_service *service) +{ + /* assume the link config did change and wait a little bit */ + mdelay(10); + + return 0; +} + +static int dptxport_call_set_link_rate(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct dptxport_apcall_link_rate *request = data; + struct dptxport_apcall_link_rate *reply = reply_; + u32 link_rate, phy_link_rate; + bool phy_set_rate = false; + int ret; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + if (data_size < sizeof(*request)) + return -EINVAL; + + link_rate = le32_to_cpu(request->link_rate); + trace_dptxport_call_set_link_rate(dptx, link_rate); + + switch (link_rate) { + case LINK_RATE_RBR: + phy_link_rate = 1620; + phy_set_rate = true; + break; + case LINK_RATE_HBR: + phy_link_rate = 2700; + phy_set_rate = true; + break; + case LINK_RATE_HBR2: + phy_link_rate = 5400; + phy_set_rate = true; + break; + case LINK_RATE_HBR3: + phy_link_rate = 8100; + phy_set_rate = true; + break; + case 0: + phy_link_rate = 0; + phy_set_rate = true; + break; + default: + dev_err(service->ep->dcp->dev, + "DPTXPort: Unsupported link rate 0x%x requested\n", + link_rate); + link_rate = 0; + phy_set_rate = false; + break; + } + + if (phy_set_rate) { + dptx->phy_ops.dp.link_rate = phy_link_rate; + dptx->phy_ops.dp.set_rate = 1; + + if (dptx->atcphy) { + ret = phy_configure(dptx->atcphy, &dptx->phy_ops); + if (ret) + return ret; + } + + //if (dptx->phy_ops.dp.set_rate) + dptx->link_rate = dptx->pending_link_rate = link_rate; + + } + + //dptx->pending_link_rate = link_rate; + reply->retcode = cpu_to_le32(0); + reply->link_rate = cpu_to_le32(link_rate); + + return 0; +} + +static int dptxport_call_get_supports_hpd(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_get_support *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->supported = cpu_to_le32(0); + return 0; +} + +static int +dptxport_call_get_supports_downspread(struct apple_epic_service *service, + void *reply_, size_t reply_size) +{ + struct dptxport_apcall_get_support *reply = reply_; + + if (reply_size < sizeof(*reply)) + return -EINVAL; + + reply->retcode = cpu_to_le32(0); + reply->supported = cpu_to_le32(0); + return 0; +} + +static int +dptxport_call_activate(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + const struct apple_dcp *dcp = service->ep->dcp; + + // TODO: hack, use phy_set_mode to select the correct DCP(EXT) input + // for standalone phy (i.e. not atc phy). + if (!dcp->typec_mux) + phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + + return 0; +} + +static int +dptxport_call_deactivate(struct apple_epic_service *service, + const void *data, size_t data_size, + void *reply, size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + + /* deactivate phy */ + phy_set_mode_ext(dptx->atcphy, PHY_MODE_INVALID, 0); + + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + + return 0; +} + +static int dptxport_call(struct apple_epic_service *service, u32 idx, + const void *data, size_t data_size, void *reply, + size_t reply_size) +{ + struct dptx_port *dptx = service->cookie; + trace_dptxport_apcall(dptx, idx, data_size); + + switch (idx) { + case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: + return dptxport_call_will_change_link_config(service); + case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: + return dptxport_call_did_change_link_config(service); + case DPTX_APCALL_GET_MAX_LINK_RATE: + return dptxport_call_get_max_link_rate(service, reply, + reply_size); + case DPTX_APCALL_GET_LINK_RATE: + return dptxport_call_get_link_rate(service, reply, reply_size); + case DPTX_APCALL_SET_LINK_RATE: + return dptxport_call_set_link_rate(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_GET_MAX_LANE_COUNT: + return dptxport_call_get_max_lane_count(service, reply, reply_size); + case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: + return dptxport_call_set_active_lane_count(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_GET_SUPPORTS_HPD: + return dptxport_call_get_supports_hpd(service, reply, + reply_size); + case DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD: + return dptxport_call_get_supports_downspread(service, reply, + reply_size); + case DPTX_APCALL_GET_MAX_DRIVE_SETTINGS: + return dptxport_call_get_max_drive_settings(service, reply, + reply_size); + case DPTX_APCALL_GET_DRIVE_SETTINGS: + return dptxport_call_get_drive_settings(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_SET_DRIVE_SETTINGS: + return dptxport_call_set_drive_settings(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_ACTIVATE: + return dptxport_call_activate(service, data, data_size, + reply, reply_size); + case DPTX_APCALL_DEACTIVATE: + return dptxport_call_deactivate(service, data, data_size, + reply, reply_size); + default: + /* just try to ACK and hope for the best... */ + dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", + idx); + fallthrough; + case DPTX_APCALL_GET_DOWN_SPREAD: + case DPTX_APCALL_SET_DOWN_SPREAD: + memcpy(reply, data, min(reply_size, data_size)); + if (reply_size >= 4) + memset(reply, 0, 4); + return 0; + } +} + +static void dptxport_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + + if (strcmp(name, "dcpdptx-port-epic")) + return; + if (strcmp(class, "AppleDCPDPTXRemotePort")) + return; + + trace_dptxport_init(service->ep->dcp, unit); + + switch (unit) { + case 0: + case 1: + if (service->ep->dcp->dptxport[unit].enabled) { + dev_err(service->ep->dcp->dev, + "DPTXPort: unit %lld already exists\n", unit); + return; + } + service->ep->dcp->dptxport[unit].unit = unit; + service->ep->dcp->dptxport[unit].service = service; + service->ep->dcp->dptxport[unit].enabled = true; + service->cookie = (void *)&service->ep->dcp->dptxport[unit]; + complete(&service->ep->dcp->dptxport[unit].enable_completion); + break; + default: + dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", + unit); + } +} + +static const struct apple_epic_service_ops dptxep_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptxport_init, + .call = dptxport_call, + }, + {} +}; + +int dptxep_init(struct apple_dcp *dcp) +{ + int ret; + u32 port; + unsigned long timeout = msecs_to_jiffies(1000); + + init_completion(&dcp->dptxport[0].enable_completion); + init_completion(&dcp->dptxport[1].enable_completion); + init_completion(&dcp->dptxport[0].linkcfg_completion); + init_completion(&dcp->dptxport[1].linkcfg_completion); + + dcp->dptxep = afk_init(dcp, DPTX_ENDPOINT, dptxep_ops); + if (IS_ERR(dcp->dptxep)) + return PTR_ERR(dcp->dptxep); + + ret = afk_start(dcp->dptxep); + if (ret) + return ret; + + for (port = 0; port < dcp->hw.num_dptx_ports; port++) { + ret = wait_for_completion_timeout(&dcp->dptxport[port].enable_completion, + timeout); + if (!ret) + return -ETIMEDOUT; + else if (ret < 0) + return ret; + timeout = ret; + } + + return 0; +} diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h new file mode 100644 index 00000000000000..0bf2534054fd7b --- /dev/null +++ b/drivers/gpu/drm/apple/dptxep.h @@ -0,0 +1,70 @@ +#ifndef __APPLE_DCP_DPTXEP_H__ +#define __APPLE_DCP_DPTXEP_H__ + +#include +#include + +enum dptx_apcall { + DPTX_APCALL_ACTIVATE = 0, + DPTX_APCALL_DEACTIVATE = 1, + DPTX_APCALL_GET_MAX_DRIVE_SETTINGS = 2, + DPTX_APCALL_SET_DRIVE_SETTINGS = 3, + DPTX_APCALL_GET_DRIVE_SETTINGS = 4, + DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG = 5, + DPTX_APCALL_DID_CHANGE_LINK_CONFIG = 6, + DPTX_APCALL_GET_MAX_LINK_RATE = 7, + DPTX_APCALL_GET_LINK_RATE = 8, + DPTX_APCALL_SET_LINK_RATE = 9, + DPTX_APCALL_GET_MAX_LANE_COUNT = 10, + DPTX_APCALL_GET_ACTIVE_LANE_COUNT = 11, + DPTX_APCALL_SET_ACTIVE_LANE_COUNT = 12, + DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD = 13, + DPTX_APCALL_GET_DOWN_SPREAD = 14, + DPTX_APCALL_SET_DOWN_SPREAD = 15, + DPTX_APCALL_GET_SUPPORTS_LANE_MAPPING = 16, + DPTX_APCALL_SET_LANE_MAP = 17, + DPTX_APCALL_GET_SUPPORTS_HPD = 18, + DPTX_APCALL_FORCE_HOTPLUG_DETECT = 19, + DPTX_APCALL_INACTIVE_SINK_DETECTED = 20, + DPTX_APCALL_SET_TILED_DISPLAY_HINTS = 21, + DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, + DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, + DPTX_APCALL_DEVICE_NOT_STARTED = 24, +}; + +#define DCPDPTX_REMOTE_PORT_CORE GENMASK(3, 0) +#define DCPDPTX_REMOTE_PORT_ATC GENMASK(7, 4) +#define DCPDPTX_REMOTE_PORT_DIE GENMASK(11, 8) +#define DCPDPTX_REMOTE_PORT_CONNECTED BIT(15) + +enum dptx_link_rate { + LINK_RATE_RBR = 0x06, + LINK_RATE_HBR = 0x0a, + LINK_RATE_HBR2 = 0x14, + LINK_RATE_HBR3 = 0x1e, +}; + +struct apple_epic_service; + +struct dptx_port { + bool enabled, connected; + struct completion enable_completion; + struct completion linkcfg_completion; + u32 unit; + struct apple_epic_service *service; + union phy_configure_opts phy_ops; + struct phy *atcphy; + struct mux_control *mux; + u32 lane_count; + u32 link_rate, pending_link_rate; + u32 drive_settings[2]; +}; + +int dptxport_validate_connection(struct apple_epic_service *service, u8 core, + u8 atc, u8 die); +int dptxport_connect(struct apple_epic_service *service, u8 core, u8 atc, + u8 die); +int dptxport_request_display(struct apple_epic_service *service); +int dptxport_release_display(struct apple_epic_service *service); +int dptxport_set_hpd(struct apple_epic_service *service, bool hpd); +#endif diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c new file mode 100644 index 00000000000000..2de9d2fe4c24a3 --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dpavservep.h" + +#include + +#include +#include +#include + +#include "../afk.h" +#include "../dcp.h" +#include "../dcp-internal.h" +#include "../trace.h" + +static void dcpavserv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + trace_dcpavserv_init(dcp, unit); + + if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + if (dcp->dcpavserv.enabled) { + dev_err(dcp->dev, + "DCPAVSERV: unit %lld already exists\n", unit); + return; + } + dcp->dcpavserv.service = service; + dcp->dcpavserv.enabled = true; + service->cookie = &dcp->dcpavserv; + complete(&dcp->dcpavserv.enable_completion); + } +} + +static void dcpavserv_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + service->enabled = false; + + if (dcp->dcpavserv.enabled) { + dcp->dcpavserv.enabled = false; + dcp->dcpavserv.service = NULL; + service->cookie = NULL; + reinit_completion(&dcp->dcpavserv.enable_completion); + } +} + +static void dcpdpserv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static void dcpdpserv_teardown(struct apple_epic_service *service) +{ + service->enabled = false; +} + +struct dcpavserv_status_report { + u32 unk00[4]; + u8 flag0; + u8 flag1; + u8 flag2; + u8 flag3; + u32 unk14[3]; + u32 status; + u32 unk24[3]; +} __packed; + +struct dpavserv_copy_edid_cmd { + __le64 max_size; + u8 _pad1[24]; + __le64 used_size; + u8 _pad2[8]; +} __packed; + +#define EDID_LEADING_DATA_SIZE 8 +#define EDID_BLOCK_SIZE 128 +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E +#define EDID_MAX_SIZE SZ_32K +#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) + +struct dpavserv_copy_edid_resp { + __le64 max_size; + u8 _pad1[24]; + __le64 used_size; + u8 _pad2[8]; + u8 data[]; +} __packed; + +static int parse_report(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size) +{ +#if defined(DEBUG) + struct apple_dcp *dcp = service->ep->dcp; + const struct epic_service_call *call; + const void *payload; + size_t payload_size; + + dev_dbg(dcp->dev, "dcpavserv[ch:%u]: report type:%02x len:%zu\n", + service->channel, type, data_size); + + if (type != EPIC_SUBTYPE_STD_SERVICE) + return 0; + + if (data_size < sizeof(*call)) + return 0; + + call = data; + + if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC) { + dev_warn(dcp->dev, "dcpavserv[ch:%u]: report magic 0x%08x != 0x%08x\n", + service->channel, le32_to_cpu(call->magic), EPIC_SERVICE_CALL_MAGIC); + return 0; + } + + payload_size = data_size - sizeof(*call); + if (payload_size < le32_to_cpu(call->data_len)) { + dev_warn(dcp->dev, "dcpavserv[ch:%u]: report payload size %zu call len %u\n", + service->channel, payload_size, le32_to_cpu(call->data_len)); + return 0; + } + payload_size = le32_to_cpu(call->data_len); + payload = data + sizeof(*call); + + if (le16_to_cpu(call->group) == 2 && le16_to_cpu(call->command) == 0) { + if (payload_size == sizeof(struct dcpavserv_status_report)) { + const struct dcpavserv_status_report *stat = payload; + dev_info(dcp->dev, "dcpavserv[ch:%u]: flags: 0x%02x,0x%02x,0x%02x,0x%02x status:%u\n", + service->channel, stat->flag0, stat->flag1, + stat->flag2, stat->flag3, stat->status); + } else { + dev_dbg(dcp->dev, "dcpavserv[ch:%u]: report payload size %zu\n", service->channel, payload_size); + } + } else { + print_hex_dump(KERN_DEBUG, "dcpavserv report: ", DUMP_PREFIX_NONE, + 16, 1, payload, payload_size, true); + } +#endif + + return 0; +} + +static int dcpavserv_report(struct apple_epic_service *service, + enum epic_subtype type, const void *data, + size_t data_size) +{ + return parse_report(service, type, data, data_size); +} + +static int dcpdpserv_report(struct apple_epic_service *service, + enum epic_subtype type, const void *data, + size_t data_size) +{ + return parse_report(service, type, data, data_size); +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service) +{ + struct dpavserv_copy_edid_cmd cmd; + struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + int num_blocks; + u64 data_size; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); + resp = kzalloc(sizeof(*resp) + EDID_BUF_SIZE, GFP_KERNEL); + if (!resp) + return ERR_PTR(-ENOMEM); + + ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, + sizeof(resp) + EDID_BUF_SIZE, 0); + if (ret < 0) + return ERR_PTR(ret); + + if (le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) + return ERR_PTR(-EIO); + + // print_hex_dump(KERN_DEBUG, "dpavserv EDID cmd: ", DUMP_PREFIX_NONE, + // 16, 1, resp, 192, true); + + data_size = le64_to_cpu(resp->used_size); + if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE) + return ERR_PTR(-EIO); + + num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; + if ((1 + num_blocks) * EDID_BLOCK_SIZE != data_size - EDID_LEADING_DATA_SIZE) + return ERR_PTR(-EIO); + + return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, + data_size - EDID_LEADING_DATA_SIZE); +} + +static const struct apple_epic_service_ops dpavservep_ops[] = { + { + .name = "dcpav-service-epic", + .init = dcpavserv_init, + .teardown = dcpavserv_teardown, + .report = dcpavserv_report, + }, + { + .name = "dcpdp-service-epic", + .init = dcpdpserv_init, + .teardown = dcpdpserv_teardown, + .report = dcpdpserv_report, + }, + {}, +}; + +int dpavservep_init(struct apple_dcp *dcp) +{ + int ret; + + init_completion(&dcp->dcpavserv.enable_completion); + + dcp->dcpavservep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + if (IS_ERR(dcp->dcpavservep)) + return PTR_ERR(dcp->dcpavservep); + + dcp->dcpavservep->match_epic_name = true; + + ret = afk_start(dcp->dcpavservep); + if (ret) + return ret; + + ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, + msecs_to_jiffies(1000)); + if (ret >= 0) + return 0; + + return ret; +} diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h new file mode 100644 index 00000000000000..858ff14b0bd7be --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef _DRM_APPLE_EPIC_DPAVSERV_H +#define _DRM_APPLE_EPIC_DPAVSERV_H + +#include +#include + +struct drm_edid; +struct apple_epic_service; + +struct dcpavserv { + bool enabled; + struct completion enable_completion; + u32 unit; + struct apple_epic_service *service; +}; + +const struct drm_edid *dcpavserv_copy_edid(struct apple_epic_service *service); + +#endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/hdmi-codec-chmap.h b/drivers/gpu/drm/apple/hdmi-codec-chmap.h new file mode 100644 index 00000000000000..f98e1e86b89602 --- /dev/null +++ b/drivers/gpu/drm/apple/hdmi-codec-chmap.h @@ -0,0 +1,123 @@ +// copied from sound/soc/codecs/hdmi-codec.c + +#include + +/* Channel maps for multi-channel playbacks, up to 8 n_ch */ +static const struct snd_pcm_chmap_elem hdmi_codec_8ch_chmaps[] = { + { .channels = 2, /* CA_ID 0x00 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR } }, + { .channels = 4, /* CA_ID 0x01 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA } }, + { .channels = 4, /* CA_ID 0x02 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC } }, + { .channels = 4, /* CA_ID 0x03 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC } }, + { .channels = 6, /* CA_ID 0x04 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x05 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x06 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x07 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 6, /* CA_ID 0x08 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 6, /* CA_ID 0x09 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 6, /* CA_ID 0x0A */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 6, /* CA_ID 0x0B */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, + { .channels = 8, /* CA_ID 0x0C */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x0D */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x0E */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x0F */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, + { .channels = 8, /* CA_ID 0x10 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x11 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x12 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x13 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, + SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, + { .channels = 8, /* CA_ID 0x14 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x15 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x16 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x17 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x18 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x19 */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1A */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1B */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1C */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1D */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1E */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { .channels = 8, /* CA_ID 0x1F */ + .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, + SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, + SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, + { } +}; diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c new file mode 100644 index 00000000000000..ae4bc8a69f2a8d --- /dev/null +++ b/drivers/gpu/drm/apple/ibootep.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2023 */ + +#include + +#include "afk.h" +#include "dcp.h" + +static void disp_service_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + + +static const struct apple_epic_service_ops ibootep_ops[] = { + { + .name = "disp0-service", + .init = disp_service_init, + }, + {} +}; + +int ibootep_init(struct apple_dcp *dcp) +{ + dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ibootep_ops); + afk_start(dcp->ibootep); + + return 0; +} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c new file mode 100644 index 00000000000000..1d90e4a2597303 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb.c @@ -0,0 +1,559 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "iomfb_internal.h" +#include "parser.h" +#include "trace.h" + +static int dcp_tx_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_CMD: + return 0x00000; + case DCP_CONTEXT_OOBCB: + case DCP_CONTEXT_OOBCMD: + return 0x08000; + default: + return -EINVAL; + } +} + +static int dcp_channel_offset(enum dcp_context_id id) +{ + switch (id) { + case DCP_CONTEXT_ASYNC: + return 0x40000; + case DCP_CONTEXT_OOBASYNC: + return 0x48000; + case DCP_CONTEXT_CB: + return 0x60000; + case DCP_CONTEXT_OOBCB: + return 0x68000; + default: + return dcp_tx_offset(id); + } +} + +static inline u64 dcpep_set_shmem(u64 dart_va) +{ + return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_SET_SHMEM) | + FIELD_PREP(IOMFB_SHMEM_FLAG, IOMFB_SHMEM_FLAG_VALUE) | + FIELD_PREP(IOMFB_SHMEM_DVA, dart_va); +} + +static inline u64 dcpep_msg(enum dcp_context_id id, u32 length, u16 offset) +{ + return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_MSG) | + FIELD_PREP(IOMFB_MSG_CONTEXT, id) | + FIELD_PREP(IOMFB_MSG_OFFSET, offset) | + FIELD_PREP(IOMFB_MSG_LENGTH, length); +} + +static inline u64 dcpep_ack(enum dcp_context_id id) +{ + return dcpep_msg(id, 0, 0) | IOMFB_MSG_ACK; +} + +/* + * A channel is busy if we have sent a message that has yet to be + * acked. The driver must not sent a message to a busy channel. + */ +static bool dcp_channel_busy(struct dcp_channel *ch) +{ + return (ch->depth != 0); +} + +/* + * Get the context ID passed to the DCP for a command we push. The rule is + * simple: callback contexts are used when replying to the DCP, command + * contexts are used otherwise. That corresponds to a non/zero call stack + * depth. This rule frees the caller from tracking the call context manually. + */ +static enum dcp_context_id dcp_call_context(struct apple_dcp *dcp, bool oob) +{ + u8 depth = oob ? dcp->ch_oobcmd.depth : dcp->ch_cmd.depth; + + if (depth) + return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; + else + return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; +} + +/* Get a channel for a context */ +static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, + enum dcp_context_id context) +{ + switch (context) { + case DCP_CONTEXT_CB: + return &dcp->ch_cb; + case DCP_CONTEXT_CMD: + return &dcp->ch_cmd; + case DCP_CONTEXT_OOBCB: + return &dcp->ch_oobcb; + case DCP_CONTEXT_OOBCMD: + return &dcp->ch_oobcmd; + case DCP_CONTEXT_ASYNC: + return &dcp->ch_async; + case DCP_CONTEXT_OOBASYNC: + return &dcp->ch_oobasync; + default: + return NULL; + } +} + +/* Get the start of a packet: after the end of the previous packet */ +static u16 dcp_packet_start(struct dcp_channel *ch, u8 depth) +{ + if (depth > 0) + return ch->end[depth - 1]; + else + return 0; +} + +/* Pushes and pops the depth of the call stack with safety checks */ +static u8 dcp_push_depth(u8 *depth) +{ + u8 ret = (*depth)++; + + WARN_ON(ret >= DCP_MAX_CALL_DEPTH); + return ret; +} + +static u8 dcp_pop_depth(u8 *depth) +{ + WARN_ON((*depth) == 0); + + return --(*depth); +} + +/* Call a DCP function given by a tag */ +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie) +{ + enum dcp_context_id context = dcp_call_context(dcp, oob); + struct dcp_channel *ch = dcp_get_channel(dcp, context); + + struct dcp_packet_header header = { + .in_len = in_len, + .out_len = out_len, + + /* Tag is reversed due to endianness of the fourcc */ + .tag[0] = call->tag[3], + .tag[1] = call->tag[2], + .tag[2] = call->tag[1], + .tag[3] = call->tag[0], + }; + + u8 depth = dcp_push_depth(&ch->depth); + u16 offset = dcp_packet_start(ch, depth); + + void *out = dcp->shmem + dcp_tx_offset(context) + offset; + void *out_data = out + sizeof(header); + size_t data_len = sizeof(header) + in_len + out_len; + + memcpy(out, &header, sizeof(header)); + + if (in_len > 0) + memcpy(out_data, data, in_len); + + trace_iomfb_push(dcp, call, context, offset, depth); + + ch->callbacks[depth] = cb; + ch->cookies[depth] = cookie; + ch->output[depth] = out + sizeof(header) + in_len; + ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); + + dcp_send_message(dcp, IOMFB_ENDPOINT, + dcpep_msg(context, data_len, offset)); +} + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +int dcp_parse_tag(char tag[4]) +{ + u32 d[3]; + int i; + + if (tag[3] != 'D') + return -EINVAL; + + for (i = 0; i < 3; ++i) { + d[i] = (u32)(tag[i] - '0'); + + if (d[i] > 9) + return -EINVAL; + } + + return d[0] + (d[1] * 10) + (d[2] * 100); +} + +/* Ack a callback from the DCP */ +void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context) +{ + struct dcp_channel *ch = dcp_get_channel(dcp, context); + + dcp_pop_depth(&ch->depth); + dcp_send_message(dcp, IOMFB_ENDPOINT, + dcpep_ack(context)); +} + +/* + * Helper to send a DRM hotplug event. The DCP is accessed from a single + * (RTKit) thread. To handle hotplug callbacks, we need to call + * drm_kms_helper_hotplug_event, which does an atomic commit (via DCP) and + * waits for vblank (a DCP callback). That means we deadlock if we call from + * the RTKit thread! Instead, move the call to another thread via a workqueue. + */ +void dcp_hotplug(struct work_struct *work) +{ + struct apple_connector *connector; + struct apple_dcp *dcp; + + connector = container_of(work, struct apple_connector, hotplug_wq); + + dcp = platform_get_drvdata(connector->dcp); + dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, + connector->connected, dcp->valid_mode, dcp->nr_modes); + + if (!connector->connected) { + drm_edid_free(connector->drm_edid); + drm_connector_set_vrr_capable_property(&connector->base, false); + connector->drm_edid = NULL; + } + + /* + * DCP defers link training until we set a display mode. But we set + * display modes from atomic_flush, so userspace needs to trigger a + * flush, or the CRTC gets no signal. + */ + if (connector->base.state && !dcp->valid_mode && connector->connected) + drm_connector_set_link_status_property(&connector->base, + DRM_MODE_LINK_STATUS_BAD); + + drm_kms_helper_connector_hotplug_event(&connector->base); +} + +static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length, u16 offset) +{ + struct device *dev = dcp->dev; + struct dcp_packet_header *hdr = data; + void *in, *out; + int tag = dcp_parse_tag(hdr->tag); + struct dcp_channel *ch = dcp_get_channel(dcp, context); + u8 depth; + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + return; + } + + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + // TODO: verify that in_len and out_len match our prototypes + // for now just clear the out data to have at least consistent results + if (hdr->out_len) + memset(out, 0, hdr->out_len); + + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); + + if (dcp->cb_handlers[tag](dcp, tag, out, in)) + dcp_ack(dcp, context); +} + +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct dcp_packet_header *header = data; + struct dcp_channel *ch = dcp_get_channel(dcp, context); + void *cookie; + dcp_callback_t cb; + + if (!ch) { + dev_warn(dcp->dev, "ignoring ack on context %X\n", context); + return; + } + + dcp_pop_depth(&ch->depth); + + cb = ch->callbacks[ch->depth]; + cookie = ch->cookies[ch->depth]; + + ch->callbacks[ch->depth] = NULL; + ch->cookies[ch->depth] = NULL; + + if (cb) + cb(dcp, data + sizeof(*header) + header->in_len, cookie); +} + +static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcp_context_id ctx_id; + u16 offset; + u32 length; + int channel_offset; + void *data; + + ctx_id = FIELD_GET(IOMFB_MSG_CONTEXT, message); + offset = FIELD_GET(IOMFB_MSG_OFFSET, message); + length = FIELD_GET(IOMFB_MSG_LENGTH, message); + + channel_offset = dcp_channel_offset(ctx_id); + + if (channel_offset < 0) { + dev_warn(dcp->dev, "invalid context received %u\n", ctx_id); + return; + } + + data = dcp->shmem + channel_offset + offset; + + if (FIELD_GET(IOMFB_MSG_ACK, message)) + dcpep_handle_ack(dcp, ctx_id, data, length); + else + dcpep_handle_cb(dcp, ctx_id, data, length, offset); +} + +int dcp_get_modes(struct drm_connector *connector) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode; + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); + + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + return 0; + } + + drm_mode_probed_add(connector, mode); + } + + if (dcp->nr_modes && dcp->dcpavserv.enabled && + !apple_connector->drm_edid) { + const struct drm_edid *edid; + edid = dcpavserv_copy_edid(dcp->dcpavserv.service); + if (IS_ERR_OR_NULL(edid)) { + dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + } else { + drm_edid_free(apple_connector->drm_edid); + apple_connector->drm_edid = edid; + } + } + if (dcp->nr_modes && apple_connector->drm_edid) + drm_edid_connector_update(connector, apple_connector->drm_edid); + + return dcp->nr_modes; +} + +/* The user may own drm_display_mode, so we need to search for our copy */ +struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + const struct drm_display_mode *mode) +{ + int i; + + for (i = 0; i < dcp->nr_modes; ++i) { + if (drm_mode_match(mode, &dcp->modes[i].mode, + DRM_MODE_MATCH_TIMINGS | + DRM_MODE_MATCH_CLOCK)) + return &dcp->modes[i]; + } + + return NULL; +} + +enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, + const struct drm_display_mode *mode) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = apple_connector->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return lookup_mode(dcp, mode) ? MODE_OK : MODE_BAD; +} + +int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); + struct drm_crtc_state *crtc_state; + int ret = -EIO; + bool modeset; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + if (!crtc_state) + return 0; + + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || !dcp->valid_mode; + + if (!modeset) + return 0; + + /* ignore no mode, poweroff is handled elsewhere */ + if (crtc_state->mode.hdisplay == 0 && crtc_state->mode.vdisplay == 0) + return 0; + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + ret = iomfb_modeset_v12_3(dcp, crtc_state); + break; + case DCP_FIRMWARE_V_13_5: + ret = iomfb_modeset_v13_3(dcp, crtc_state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", + dcp->fw_compat); + break; + } + + return ret; +} + +bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct platform_device *pdev = apple_crtc->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + /* TODO: support synthesized modes through scaling */ + return lookup_mode(dcp, mode) != NULL; +} + + +void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp_channel_busy(&dcp->ch_cmd)) + { + if (!dcp->ch_cmd.warned_busy) { + dev_err(dcp->dev, "unexpected busy command channel\n"); + dcp->ch_cmd.warned_busy = true; + } + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). + */ + schedule_work(&dcp->vblank_wq); + return; + } else if (dcp->ch_cmd.warned_busy) { + dcp->ch_cmd.warned_busy = false; + } + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_flush_v12_3(dcp, crtc, state); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_flush_v13_3(dcp, crtc, state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +static void iomfb_start(struct apple_dcp *dcp) +{ + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_start_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_start_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + +bool dcp_is_initialized(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return dcp->active; +} + +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +{ + enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); + + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) + iomfb_start(dcp); + else if (type == IOMFB_MESSAGE_TYPE_MSG) + dcpep_got_msg(dcp, message); + else + dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); +} + +int iomfb_start_rtkit(struct apple_dcp *dcp) +{ + dma_addr_t shmem_iova; + apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); + + dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, + GFP_KERNEL); + + dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); + + return 0; +} + +void iomfb_shutdown(struct apple_dcp *dcp) +{ + /* We're going down */ + dcp->active = false; + dcp->valid_mode = false; + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_shutdown_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_shutdown_v13_3(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h new file mode 100644 index 00000000000000..8a871db0b94a70 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb.h @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCPEP_H__ +#define __APPLE_DCPEP_H__ + +#include + +#include "version_utils.h" + +/* Fixed size of shared memory between DCP and AP */ +#define DCP_SHMEM_SIZE 0x100000 + +/* DCP message contexts */ +enum dcp_context_id { + /* Callback */ + DCP_CONTEXT_CB = 0, + + /* Command */ + DCP_CONTEXT_CMD = 2, + + /* Asynchronous */ + DCP_CONTEXT_ASYNC = 3, + + /* Out-of-band callback */ + DCP_CONTEXT_OOBCB = 4, + + /* Out-of-band command */ + DCP_CONTEXT_OOBCMD = 6, + + /* Out-of-band Asynchronous */ + DCP_CONTEXT_OOBASYNC = 7, + + DCP_NUM_CONTEXTS +}; + +/* RTKit endpoint message types */ +enum dcpep_type { + /* Set shared memory */ + IOMFB_MESSAGE_TYPE_SET_SHMEM = 0, + + /* DCP is initialized */ + IOMFB_MESSAGE_TYPE_INITIALIZED = 1, + + /* Remote procedure call */ + IOMFB_MESSAGE_TYPE_MSG = 2, +}; + +/* + * IOMFB supports the setting of a number of parameters + * that alter various aspects of the connected sink's + * behaviour at runtime. + */ +enum iomfb_parameter { + IOMFBPARAM_ADAPTIVE_SYNC = 14, +}; + +#define IOMFB_MESSAGE_TYPE GENMASK_ULL( 3, 0) + +/* Message */ +#define IOMFB_MSG_LENGTH GENMASK_ULL(63, 32) +#define IOMFB_MSG_OFFSET GENMASK_ULL(31, 16) +#define IOMFB_MSG_CONTEXT GENMASK_ULL(11, 8) +#define IOMFB_MSG_ACK BIT_ULL(6) + +/* Set shmem */ +#define IOMFB_SHMEM_DVA GENMASK_ULL(63, 16) +#define IOMFB_SHMEM_FLAG GENMASK_ULL( 7, 4) +#define IOMFB_SHMEM_FLAG_VALUE 4 + +struct dcp_packet_header { + char tag[4]; + u32 in_len; + u32 out_len; +} __packed; + +#define DCP_IS_NULL(ptr) ((ptr) ? 1 : 0) +#define DCP_PACKET_ALIGNMENT (0x40) + +enum iomfb_property_id { + IOMFB_PROPERTY_NITS = 15, // divide by Brightness_Scale +}; + +#define IOMFB_BRIGHTNESS_MIN 0x10000000 + +/* Structures used in v12.0 firmware */ + +#define SWAP_SURFACES 4 +/* We have 4 surfaces, but we can only ever blend two */ +#define MAX_BLEND_SURFACES 2 + +struct dcp_iouserclient { + /* Handle for the IOUserClient. macOS sets this to a kernel VA. */ + u64 handle; + u32 unk; + u8 flag1; + u8 flag2; + u8 padding[2]; +} __packed; + +/* + * Update background color to struct dcp_swap.bg_color + */ +#define IOMFB_SET_BACKGROUND BIT(31) + +struct dcp_allocate_bandwidth_req { + u64 unk1; + u64 unk2; + u64 unk3; + u8 unk1_null; + u8 unk2_null; + u8 padding[8]; +} __packed; + +struct dcp_allocate_bandwidth_resp { + u64 unk1; + u64 unk2; + u32 ret; +} __packed; + +struct dcp_rt_bandwidth { + u64 unk1; + u64 reg_scratch; + u64 reg_doorbell; + u32 unk2; + u32 doorbell_bit; + u32 padding[7]; +} __packed; + +struct frame_sync_props { + u8 unk[28]; +}; + +struct dcp_set_frame_sync_props_req { + struct frame_sync_props props; + u8 frame_sync_props_null; + u8 padding[3]; +} __packed; + +struct dcp_set_frame_sync_props_resp { + struct frame_sync_props props; +} __packed; + +/* Method calls */ + +enum dcpep_method { + dcpep_late_init_signal, + dcpep_setup_video_limits, + dcpep_set_create_dfb, + dcpep_start_signal, + dcpep_swap_start, + dcpep_swap_submit, + dcpep_set_display_device, + dcpep_set_digital_out_mode, + dcpep_create_default_fb, + dcpep_set_display_refresh_properties, + dcpep_flush_supports_power, + dcpep_set_power_state, + dcpep_first_client_open, + dcpep_set_parameter_dcp, + dcpep_enable_disable_video_power_savings, + dcpep_is_main_display, + iomfbep_a131_pmu_service_matched, + iomfbep_a132_backlight_service_matched, + iomfbep_a358_vi_set_temperature_hint, + iomfbep_get_color_remap_mode, + iomfbep_last_client_close, + iomfbep_abort_swaps_dcp, + iomfbep_set_matrix, + dcpep_num_methods +}; + +#define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } + +struct dcp_method_entry { + const char *name; + char tag[4]; +}; + +#define IOMFB_MAX_CB (1000) +struct apple_dcp; + +typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); + +/* Prototypes */ + +struct dcp_set_digital_out_mode_req { + u32 color_mode_id; + u32 timing_mode_id; +} __packed; + +struct dcp_map_buf_req { + u64 buffer; + u8 unk; + u8 buf_null; + u8 vaddr_null; + u8 dva_null; +} __packed; + +struct dcp_map_buf_resp { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; + +struct dcp_unmap_buf_resp { + u64 buffer; + u64 vaddr; + u64 dva; + u8 unk; + u8 buf_null; +} __packed; + +struct dcp_allocate_buffer_req { + u32 unk0; + u64 size; + u32 unk2; + u8 paddr_null; + u8 dva_null; + u8 dva_size_null; + u8 padding; +} __packed; + +struct dcp_allocate_buffer_resp { + u64 paddr; + u64 dva; + u64 dva_size; + u32 mem_desc_id; +} __packed; + +struct dcp_map_physical_req { + u64 paddr; + u64 size; + u32 flags; + u8 dva_null; + u8 dva_size_null; + u8 padding[2]; +} __packed; + +struct dcp_map_physical_resp { + u64 dva; + u64 dva_size; + u32 mem_desc_id; +} __packed; + +struct dcp_swap_start_req { + u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct dcp_swap_start_resp { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct dcp_get_uint_prop_req { + char obj[4]; + char key[0x40]; + u64 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_get_uint_prop_resp { + u64 value; + u8 ret; + u8 padding[3]; +} __packed; + +struct iomfb_sr_set_property_int_req { + char obj[4]; + char key[0x40]; + u64 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct iomfb_set_fx_prop_req { + char obj[4]; + char key[0x40]; + u32 value; +} __packed; + +struct dcp_set_power_state_req { + u64 unklong; + u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; + +struct dcp_set_power_state_resp { + u32 unkint; + u32 ret; +} __packed; + +struct dcp_set_dcpav_prop_chunk_req { + char data[0x1000]; + u32 offset; + u32 length; +} __packed; + +struct dcp_set_dcpav_prop_end_req { + char key[0x40]; +} __packed; + +struct dcp_set_parameter_dcp { + u32 param; + u32 value[8]; + u32 count; +} __packed; + +struct dcp_swap_complete_intent_gated { + u32 swap_id; + u8 unkBool; + u32 unkInt; + u32 width; + u32 height; +} __packed; + +struct dcp_read_edt_data_req { + char key[0x40]; + u32 count; + u32 value[8]; +} __packed; + +struct dcp_read_edt_data_resp { + u32 value[8]; + u8 ret; +} __packed; + +struct iomfb_property { + u32 id; + u32 value; +} __packed; + +struct iomfb_get_color_remap_mode_req { + u32 mode; + u8 mode_null; + u8 padding[3]; +} __packed; + +struct iomfb_get_color_remap_mode_resp { + u32 mode; + u32 ret; +} __packed; + +struct iomfb_last_client_close_req { + u8 unkint_null; + u8 padding[3]; +} __packed; + +struct iomfb_last_client_close_resp { + u32 unkint; +} __packed; + +struct io_user_client { + u64 addr; + u32 unk; + u8 flag1; + u8 flag2; + u8 pad[2]; +} __packed; + +struct iomfb_abort_swaps_dcp_req { + struct io_user_client client; + u8 client_null; + u8 pad[3]; +} __packed; + +struct iomfb_abort_swaps_dcp_resp { + struct io_user_client client; + u32 ret; +} __packed; + +struct iomfb_set_matrix_req { + u32 location; + u64 matrix[9]; + u8 matrix_null; + u8 padding[3]; +} __packed; + +struct iomfb_set_matrix_resp { + u32 ret; +} __packed; + +struct dcpep_get_tiling_state_req { + u32 event; + u32 param; + u32 value; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcpep_get_tiling_state_resp { + u32 value; + u32 ret; +} __packed; + +#endif diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h new file mode 100644 index 00000000000000..75e9d7b0e8cc84 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include +#include + +#include "dcp-internal.h" + +struct apple_dcp; + +typedef void (*dcp_callback_t)(struct apple_dcp *, void *, void *); + + +#define DCP_THUNK_VOID(func, handle) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], 0, 0, NULL, cb, cookie); \ + } + +#define DCP_THUNK_OUT(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, dcp_callback_t cb, \ + void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], 0, sizeof(T), NULL, cb, cookie); \ + } + +#define DCP_THUNK_IN(func, handle, T) \ + static void func(struct apple_dcp *dcp, bool oob, T *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], sizeof(T), 0, data, cb, cookie); \ + } + +#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ + static void func(struct apple_dcp *dcp, bool oob, T_in *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[handle], sizeof(T_in), sizeof(T_out), data, \ + cb, cookie); \ + } + +#define IOMFB_THUNK_INOUT(name) \ + static void iomfb_ ## name(struct apple_dcp *dcp, bool oob, \ + struct iomfb_ ## name ## _req *data, \ + dcp_callback_t cb, void *cookie) \ + { \ + dcp_push(dcp, oob, &dcp_methods[iomfbep_ ## name], \ + sizeof(struct iomfb_ ## name ## _req), \ + sizeof(struct iomfb_ ## name ## _resp), \ + data, cb, cookie); \ + } + +/* + * Define type-safe trampolines. Define typedefs to enforce type-safety on the + * input data (so if the types don't match, gcc errors out). + */ + +#define TRAMPOLINE_VOID(func, handler) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + trace_iomfb_callback(dcp, tag, #handler); \ + handler(dcp); \ + return true; \ + } + +#define TRAMPOLINE_IN(func, handler, T_in) \ + typedef void (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + callback_##handler cb = handler; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ + typedef T_out (*callback_##handler)(struct apple_dcp *, T_in *); \ + \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + callback_##handler cb = handler; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + *typed_out = cb(dcp, in); \ + return true; \ + } + +#define TRAMPOLINE_OUT(func, handler, T_out) \ + static bool __maybe_unused func(struct apple_dcp *dcp, int tag, void *out, void *in) \ + { \ + T_out *typed_out = out; \ + \ + trace_iomfb_callback(dcp, tag, #handler); \ + *typed_out = handler(dcp); \ + return true; \ + } + +/* Call a DCP function given by a tag */ +void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, + u32 in_len, u32 out_len, void *data, dcp_callback_t cb, + void *cookie); + +/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ +int dcp_parse_tag(char tag[4]); + +void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context); + +/* The user may own drm_display_mode, so we need to search for our copy */ +struct dcp_display_mode *lookup_mode(struct apple_dcp *dcp, + const struct drm_display_mode *mode); diff --git a/drivers/gpu/drm/apple/iomfb_plane.h b/drivers/gpu/drm/apple/iomfb_plane.h new file mode 100644 index 00000000000000..0701978200311a --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_plane.h @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + + +#ifndef __APPLE_IOMFB_PLANE_H__ +#define __APPLE_IOMFB_PLANE_H__ + +#include + +#include + +#define DCP_SURF_MAX_PLANES 3 + +#define DCP_FORMAT_BGRA fourcc_code('A', 'R', 'G', 'B') +#define DCP_FORMAT_RGBA fourcc_code('A', 'B', 'G', 'R') + +#define DCP_FORMAT_W30R fourcc_code('r', '0', '3', 'w') // wide gamut packed 10-bit RGB without alpha +#define DCP_FORMAT_L10R fourcc_code('r', '0', '1', 'l') // full range packed 10-bit RGB with alpha + +#define DCP_FORMAT_420V fourcc_code('v', '0', '2', '4') // NV12 video range 2 plane 8-bit YCbCr +#define DCP_FORMAT_420F fourcc_code('f', '0', '2', '4') // NV12 full range 2 plane 8-bit YCbCr +#define DCP_FORMAT_422V fourcc_code('v', '2', '2', '4') // NV16 video range 2 plane 8-bit YCbCr +#define DCP_FORMAT_422F fourcc_code('f', '2', '2', '4') // NV16 full range 2 plane 8-bit YCbCr +#define DCP_FORMAT_444V fourcc_code('v', '4', '4', '4') // NV24 video range 2 plane 8-bit YCbCr +#define DCP_FORMAT_444F fourcc_code('f', '4', '4', '4') // NV24 full range 2 plane 8-bit YCbCr + +#define DCP_FORMAT_X420 fourcc_code('0', '2', '4', 'x') // P010 video range 2 plane 10-bit YCbCR +#define DCP_FORMAT_X422 fourcc_code('2', '2', '4', 'x') // P210 video range 2 plane 10-bit YCbCR +#define DCP_FORMAT_X444 fourcc_code('4', '4', '4', 'x') // P410 video range 2 plane 10-bit YCbCR + +#define DCP_FORMAT_XF20 fourcc_code('0', '2', 'f', 'x') // P010 full range 2 plane 10-bit YCbCR +#define DCP_FORMAT_XF22 fourcc_code('2', '2', 'f', 'x') // P210 full range 2 plane 10-bit YCbCR +#define DCP_FORMAT_XF44 fourcc_code('4', '4', 'f', 'x') // P410 full range 2 plane 10-bit YCbCR + +enum dcp_colorspace { + DCP_COLORSPACE_BG_SRGB = 0, + DCP_COLORSPACE_BT601 = 1, + DCP_COLORSPACE_BT709 = 2, + DCP_COLORSPACE_BG_BT2020 = 9, + DCP_COLORSPACE_NATIVE = 12, +}; + +enum dcp_xfer_func { + DCP_XFER_FUNC_BT601 = 1, + DCP_XFER_FUNC_BT1886 = 2, + DCP_XFER_FUNC_SDR = 13, + DCP_XFER_FUNC_HDR = 16, +}; + +struct dcp_rect { + u32 x; + u32 y; + u32 w; + u32 h; +} __packed; + +/* Information describing a plane of a planar compressed surface */ +struct dcp_plane_info { + u32 width; + u32 height; + u32 base; + u32 offset; + u32 stride; + u32 size; + u16 tile_size; + u8 tile_w; + u8 tile_h; + u32 unk[13]; +} __packed; + +struct dcp_component_types { + u8 count; + u8 types[7]; +} __packed; + +/* Information describing a surface */ +struct dcp_surface { + u8 is_tiled; + u8 is_tearing_allowed; + u8 is_premultiplied; + u32 plane_cnt; + u32 plane_cnt2; + u32 format; /* DCP fourcc */ + u32 ycbcr_matrix; + u8 xfer_func; + u8 colorspace; + u32 stride; + u16 pix_size; + u8 pel_w; + u8 pel_h; + u32 offset; + u32 width; + u32 height; + u32 buf_size; + u64 protection_opts; + u32 surface_id; + struct dcp_component_types comp_types[DCP_SURF_MAX_PLANES]; + u64 has_comp; + struct dcp_plane_info planes[DCP_SURF_MAX_PLANES]; + u64 has_planes; + u32 compression_info[DCP_SURF_MAX_PLANES][13]; + u64 has_compr_info; + u32 unk_num; + u32 unk_denom; +} __packed; + +#endif /* __APPLE_IOMFB_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c new file mode 100644 index 00000000000000..0e5d5908a3c9b6 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -0,0 +1,1521 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright 2021 Alyssa Rosenzweig + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "iomfb_internal.h" +#include "parser.h" +#include "trace.h" +#include "version_utils.h" + +/* Register defines used in bandwidth setup structure */ +#define REG_DOORBELL_BIT(idx) (2 + (idx)) + +extern bool force_vrr; + +struct dcp_wait_cookie { + struct kref refcount; + struct completion done; +}; + +static void release_wait_cookie(struct kref *ref) +{ + struct dcp_wait_cookie *cookie; + cookie = container_of(ref, struct dcp_wait_cookie, refcount); + + kfree(cookie); +} + +DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); +DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); +DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); + +IOMFB_THUNK_INOUT(set_matrix); +IOMFB_THUNK_INOUT(get_color_remap_mode); +IOMFB_THUNK_INOUT(last_client_close); +IOMFB_THUNK_INOUT(abort_swaps_dcp); + +DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, + struct DCP_FW_NAME(dcp_swap_submit_req), + struct DCP_FW_NAME(dcp_swap_submit_resp)); + +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, + struct dcp_swap_start_resp); + +DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, + struct dcp_set_power_state_req, + struct dcp_set_power_state_resp); + +DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, + struct dcp_set_digital_out_mode_req, u32); + +DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); + +DCP_THUNK_OUT(dcp_set_display_refresh_properties, + dcpep_set_display_refresh_properties, u32); + +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +DCP_THUNK_INOUT(dcp_late_init_signal, dcpep_late_init_signal, u32, u32); +#else +DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); +#endif +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); +DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); +DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); +DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); +DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); + +DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, + struct dcp_set_parameter_dcp, u32); + +DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, + dcpep_enable_disable_video_power_savings, u32, int); + +DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); + +/* DCP callback handlers */ +static void dcpep_cb_nop(struct apple_dcp *dcp) +{ + /* No operation */ +} + +static u8 dcpep_cb_true(struct apple_dcp *dcp) +{ + return true; +} + +static u8 dcpep_cb_false(struct apple_dcp *dcp) +{ + return false; +} + +static u32 dcpep_cb_zero(struct apple_dcp *dcp) +{ + return 0; +} + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + ktime_t now = ktime_get(); + trace_iomfb_swap_complete(dcp, resp->swap_id); + dcp->last_swap_id = resp->swap_id; + + dcp_drm_crtc_page_flip(dcp, now); + if (dcp->crc_enabled) { + u32 crc32 = 0; + drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); + } +} + +/* special */ +static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) +{ + // ack D100 cb_match_pmu_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + iomfb_a358_vi_set_temperature_hint(dcp, false, + complete_vi_set_temperature_hint, + NULL); + + // return false for deferred ACK + return false; +} + +static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_pmu_service_2 + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service_2(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a131_pmu_service_matched(dcp, false, complete_pmu_service_matched, + out); + + // return false for deferred ACK + return false; +} + +static void complete_backlight_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_backlight_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + if (!dcp_has_panel(dcp)) { + u8 *succ = out; + *succ = true; + return true; + } + + iomfb_a132_backlight_service_matched(dcp, false, complete_backlight_service_matched, out); + + // return false for deferred ACK + return false; +} + +static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) +{ + switch (prop->id) { + case IOMFB_PROPERTY_NITS: + { + if (dcp_has_panel(dcp)) { + dcp->brightness.nits = prop->value / dcp->brightness.scale; + /* notify backlight device of the initial brightness */ + if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); + trace_iomfb_brightness(dcp, prop->value); + } + break; + } + default: + dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); + } +} + +static struct dcp_get_uint_prop_resp +dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) +{ + struct dcp_get_uint_prop_resp resp = (struct dcp_get_uint_prop_resp){ + .value = 0 + }; + + if (dcp->panel.has_mini_led && + memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ + if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { + /* + * TODO: value from j314c, find out if it is temperature in + * centigrade C and which temperature sensor reports it + */ + resp.value = 3029; + resp.ret = true; + } + } + + return resp; +} + +static u8 iomfbep_cb_sr_set_property_int(struct apple_dcp *dcp, + struct iomfb_sr_set_property_int_req *req) +{ + if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ + if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { + if (!req->value_null) + dcp->brightness.scale = req->value; + } + } + + return 1; +} + +static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) +{ + // TODO: trace this, see if there properties which needs to used later +} + +/* + * Callback to map a buffer allocated with allocate_buf for PIODMA usage. + * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated + * stream of the display DART, rather than the expected DCP DART. + */ +static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct dcp_mem_descriptor *memdesc; + struct sg_table *map; + ssize_t ret; + + if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) + goto reject; + + memdesc = &dcp->memdesc[req->buffer]; + map = &memdesc->map; + + if (!map->sgl) + goto reject; + + /* use the piodma iommu domain to map against the right IOMMU */ + ret = iommu_map_sgtable(dcp->iommu_dom, memdesc->dva, map, + IOMMU_READ | IOMMU_WRITE); + + /* HACK: expect size to be 16K aligned since the iommu API only maps + * full pages + */ + if (ret < 0 || ret != ALIGN(memdesc->size, SZ_16K)) { + dev_err(dcp->dev, "iommu_map_sgtable() returned %zd instead of expected buffer size of %zu\n", ret, memdesc->size); + goto reject; + } + + return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; + +reject: + dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", + req->buffer); + return (struct dcp_map_buf_resp){ .ret = EINVAL }; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *resp) +{ + struct dcp_mem_descriptor *memdesc; + + if (resp->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", + resp->buffer); + return; + } + + memdesc = &dcp->memdesc[resp->buffer]; + + if (!memdesc->buf) { + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx\n", + resp->buffer, resp->dva); + return; + } + + if (memdesc->dva != resp->dva) { + dev_warn(dcp->dev, "unmap buffer %llu address mismatch " + "memdesc.dva:%llx dva:%llx\n", resp->buffer, + memdesc->dva, resp->dva); + return; + } + + /* use the piodma iommu domain to unmap from the right IOMMU */ + /* HACK: expect size to be 16K aligned since the iommu API only maps + * full pages + */ + iommu_unmap(dcp->iommu_dom, memdesc->dva, ALIGN(memdesc->size, SZ_16K)); +} + +/* + * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be + * physically contiguous, however we should save the sgtable in case the + * buffer needs to be later mapped for PIODMA. + */ +static struct dcp_allocate_buffer_resp +dcpep_cb_allocate_buffer(struct apple_dcp *dcp, + struct dcp_allocate_buffer_req *req) +{ + struct dcp_allocate_buffer_resp resp = { 0 }; + struct dcp_mem_descriptor *memdesc; + size_t size; + u32 id; + + resp.dva_size = ALIGN(req->size, 4096); + resp.mem_desc_id = + find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + + if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + resp.dva_size = 0; + resp.mem_desc_id = 0; + return resp; + } + id = resp.mem_desc_id; + set_bit(id, dcp->memdesc_map); + + memdesc = &dcp->memdesc[id]; + + memdesc->size = resp.dva_size; + /* HACK: align size to 16K since the iommu API only maps full pages */ + size = ALIGN(resp.dva_size, SZ_16K); + memdesc->buf = dma_alloc_coherent(dcp->dev, size, + &memdesc->dva, GFP_KERNEL); + + dma_get_sgtable(dcp->dev, &memdesc->map, memdesc->buf, memdesc->dva, + size); + resp.dva = memdesc->dva; + + return resp; +} + +static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) +{ + struct dcp_mem_descriptor *memdesc; + size_t size; + u32 id = *mem_desc_id; + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, + "unmap request for out of range mem_desc_id %u", id); + return 0; + } + + if (!test_and_clear_bit(id, dcp->memdesc_map)) { + dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u\n", + id); + return 0; + } + + memdesc = &dcp->memdesc[id]; + size = ALIGN(memdesc->size, SZ_16K); + if (memdesc->buf) { + dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); + memdesc->buf = NULL; + memset(&memdesc->map, 0, sizeof(memdesc->map)); + } else { + memdesc->reg = 0; + } + + memdesc->size = 0; + + return 1; +} + +/* Validate that the specified region is a display register */ +static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +{ + int i; + + for (i = 0; i < dcp->nr_disp_registers; ++i) { + struct resource *r = dcp->disp_registers[i]; + + if ((start >= r->start) && (end <= r->end)) + return true; + } + + return false; +} + +/* + * Map contiguous physical memory into the DCP's address space. The firmware + * uses this to map the display registers we advertise in + * sr_map_device_memory_with_index, so we bounds check against that to guard + * safe against malicious coprocessors. + */ +static struct dcp_map_physical_resp +dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) +{ + int size = ALIGN(req->size, 4096); + dma_addr_t dva; + u32 id; + + if (!is_disp_register(dcp, req->paddr, req->paddr + size - 1)) { + dev_err(dcp->dev, "refusing to map phys address %llx size %llx\n", + req->paddr, req->size); + return (struct dcp_map_physical_resp){}; + } + + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + set_bit(id, dcp->memdesc_map); + dcp->memdesc[id].size = size; + dcp->memdesc[id].reg = req->paddr; + + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); + + return (struct dcp_map_physical_resp){ + .dva_size = size, + .mem_desc_id = id, + .dva = dva, + }; +} + +static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) +{ + return clk_get_rate(dcp->clk); +} + +static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *dcp, + struct DCP_FW_NAME(dcp_map_reg_req) *req) +{ + if (req->index >= dcp->nr_disp_registers) { + dev_warn(dcp->dev, "attempted to read invalid reg index %u\n", + req->index); + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; + } else { + struct resource *rsrc = dcp->disp_registers[req->index]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), + DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); +#endif + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ + .addr = rsrc->start, + .length = resource_size(rsrc), +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .dva = dva, +#endif + }; + } +} + +static struct dcp_read_edt_data_resp +dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) +{ + return (struct dcp_read_edt_data_resp){ + .value[0] = req->value[0], + .ret = 0, + }; +} + +static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, + u8 *enabled) +{ + /* + * update backlight brightness on next swap, on non mini-LED displays + * DCP seems to set an invalid iDAC value after coming out of DPMS. + * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" + */ + dcp->brightness.update = true; + schedule_work(&dcp->bl_update_wq); +} + +/* Chunked data transfer for property dictionaries */ +static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) +{ + if (dcp->chunks.data != NULL) { + dev_warn(dcp->dev, "ignoring spurious transfer start\n"); + return false; + } + + dcp->chunks.length = *length; + dcp->chunks.data = devm_kzalloc(dcp->dev, *length, GFP_KERNEL); + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "failed to allocate chunks\n"); + return false; + } + + return true; +} + +static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_chunk_req *req) +{ + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious chunk\n"); + return false; + } + + if (req->offset + req->length > dcp->chunks.length) { + dev_warn(dcp->dev, "ignoring overflowing chunk\n"); + return false; + } + + memcpy(dcp->chunks.data + req->offset, req->data, req->length); + return true; +} + +static bool dcpep_process_chunks(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + // struct apple_connector *connector = dcp->connector; + struct dcp_parse_ctx ctx; + int ret; //, i; + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious end\n"); + return false; + } + + /* used just as opaque pointer for tracing */ + ctx.dcp = dcp; + + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); + + if (ret) { + dev_warn(dcp->dev, "bad header on dcpav props\n"); + return false; + } + + if (!strcmp(req->key, "TimingElements")) { + dcp->modes = enumerate_modes(&ctx, &dcp->nr_modes, + dcp->width_mm, dcp->height_mm, + dcp->notch_height); + + if (IS_ERR(dcp->modes)) { + dev_warn(dcp->dev, "failed to parse modes\n"); + dcp->modes = NULL; + dcp->nr_modes = 0; + return false; + } + if (dcp->nr_modes == 0) + dev_warn(dcp->dev, "TimingElements without valid modes!\n"); + } else if (!strcmp(req->key, "DisplayAttributes")) { + ret = parse_display_attributes(&ctx, &dcp->width_mm, + &dcp->height_mm); + + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } + + dcp_set_dimensions(dcp); + } + + // if (connector) { + // for (i = 0; i < dcp->nr_modes; i++) { + // if (dcp->modes[i].vrr) { + // drm_connector_set_vrr_capable_property(&connector->base, true); + // break; + // } + // } + // } + + return true; +} + +static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + u8 resp = dcpep_process_chunks(dcp, req); + + /* move chunked data to connector to provide it via debugfs */ + dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); + dcp->chunks.data = NULL; + dcp->chunks.length = 0; + + return resp; +} + +/* Boot sequence */ +static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + dev_dbg(dcp->dev, "boot done\n"); + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); +} + +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 v_true = 1; + dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); +#else + dcp_late_init_signal(dcp, false, boot_5, NULL); +#endif +} + +static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 v_true = true; + + dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); +} + +static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_setup_video_limits(dcp, false, boot_3, NULL); +} + +static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_create_default_fb(dcp, false, boot_2, NULL); +} + +/* Use special function signature to defer the ACK */ +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); + return false; +} + +static struct dcp_allocate_bandwidth_resp dcpep_cb_allocate_bandwidth(struct apple_dcp *dcp, + struct dcp_allocate_bandwidth_req *req) +{ + return (struct dcp_allocate_bandwidth_resp){ + .unk1 = req->unk1, + .unk2 = req->unk2, + .ret = 1, + }; +} + +static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) +{ + struct dcp_rt_bandwidth rt_bw = (struct dcp_rt_bandwidth){ + .reg_scratch = 0, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; + + if (dcp->disp_bw_scratch_index) { + u32 offset = dcp->disp_bw_scratch_offset; + u32 index = dcp->disp_bw_scratch_index; + rt_bw.reg_scratch = dcp->disp_registers[index]->start + offset; + } + + if (dcp->disp_bw_doorbell_index) { + u32 index = dcp->disp_bw_doorbell_index; + rt_bw.reg_doorbell = dcp->disp_registers[index]->start; + rt_bw.doorbell_bit = REG_DOORBELL_BIT(dcp->index); + /* + * This is most certainly not padding. t8103-dcp crashes without + * setting this immediately during modeset on 12.3 and 13.5 + * firmware. + */ + rt_bw.padding[3] = 0x4; + } + + return rt_bw; +} + +static struct dcp_set_frame_sync_props_resp +dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req *req) +{ + return (struct dcp_set_frame_sync_props_resp){}; +} + +/* Callback to get the current time as milliseconds since the UNIX epoch */ +static u64 dcpep_cb_get_time(struct apple_dcp *dcp) +{ + return ktime_to_ms(ktime_get_real()); +} + +struct dcp_swap_cookie { + struct kref refcount; + struct completion done; + u32 swap_id; +}; + +static void release_swap_cookie(struct kref *ref) +{ + struct dcp_swap_cookie *cookie; + cookie = container_of(ref, struct dcp_swap_cookie, refcount); + + kfree(cookie); +} + +static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + + if (resp->ret) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + + if (cookie) { + struct dcp_swap_cookie *info = cookie; + info->swap_id = resp->swap_id; + } + + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); +} + +static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req req = { + .unklong = 1, + }; + + dcp_set_power_state(dcp, false, &req, dcp_on_final, cookie); +} + +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + int ret; + u32 handle; + dev_info(dcp->dev, "dcp_poweron() starting\n"); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + handle = dcp->main_display ? 0 : 2; + + dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, + cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(10000)); + + if (ret == 0) { + dev_warn(dcp->dev, "wait for power timed out, connector will be broken\n"); + } else if (ret > 0) { + int msecs = jiffies_to_msecs(ret); + if (msecs > 6000) + dev_info(dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); + else + dev_warn(dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); + } else { + drm_connector_set_link_status_property(&dcp->connector->base, + DRM_MODE_LINK_STATUS_BAD); + dev_warn(dcp->dev, "wait for completion error: %d\n", ret); + } + + kref_put(&cookie->refcount, release_wait_cookie);; + + /* Force a brightness update after poweron, to restore the brightness */ + dcp->brightness.update = true; +} + +static void complete_set_powerstate(struct apple_dcp *dcp, void *out, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, + cookie); +} + +static void aborted_swaps_dcp_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_last_client_close_req last_client_req = {}; + iomfb_last_client_close(dcp, false, &last_client_req, + last_client_closed_poff, cookie); +} + +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) +{ + int ret, swap_id; + struct iomfb_abort_swaps_dcp_req abort_req = { + .client = { + .flag2 = 1, + }, + }; + struct dcp_swap_cookie *cookie; + struct dcp_wait_cookie *poff_cookie; + struct dcp_swap_start_req swap_req = { 0 }; + struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + // clear surfaces + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; + swap->unkU32Ptr_null = true; + swap->unkU32out_null = true; +#endif + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(50)); + swap_id = cookie->swap_id; + kref_put(&cookie->refcount, release_swap_cookie); + if (ret <= 0) { + dcp->crashed = true; + return; + } + + dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, swap_id); + + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); + if (!poff_cookie) + return; + init_completion(&poff_cookie->done); + kref_init(&poff_cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&poff_cookie->refcount); + + iomfb_abort_swaps_dcp(dcp, false, &abort_req, + aborted_swaps_dcp_poff, poff_cookie); + ret = wait_for_completion_timeout(&poff_cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setPowerState(0) timeout %u ms\n", 1000); + else if (ret > 0) + dev_dbg(dcp->dev, + "setPowerState(0) finished with %d ms to spare", + jiffies_to_msecs(ret)); + + kref_put(&poff_cookie->refcount, release_wait_cookie); + + dev_info(dcp->dev, "dcp_poweroff() done\n"); +} + +static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_power_state_req power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, false, &power_req, complete_set_powerstate, cookie); +} + +static void aborted_swaps_dcp_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_last_client_close_req req = { 0 }; + iomfb_last_client_close(dcp, false, &req, last_client_closed_sleep, cookie); +} + +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) +{ + int ret; + struct iomfb_abort_swaps_dcp_req req = { + .client = { + .flag2 = 1, + }, + }; + + struct dcp_wait_cookie *cookie; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, + cookie); + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setDCPPower(0) timeout %u ms\n", 1000); + + kref_put(&cookie->refcount, release_wait_cookie); + dev_info(dcp->dev, "dcp_sleep() done\n"); +} + +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + struct apple_connector *connector = dcp->connector; + + /* DCP issues hotplug_gated callbacks after SetPowerState() calls on + * devices with display (macbooks, imacs). This must not result in + * connector state changes on DRM side. Some applications won't enable + * a CRTC with a connector in disconnected state. Weston after DPMS off + * is one example. dcp_is_main_display() returns true on devices with + * integrated display. Ignore the hotplug_gated() callbacks there. + */ + if (dcp->main_display) + return; + + if (dcp->during_modeset) { + dev_info(dcp->dev, + "cb_hotplug() ignored during modeset connected:%llu\n", + *connected); + return; + } + + dev_info(dcp->dev, "cb_hotplug() connected:%llu, valid_mode:%d\n", + *connected, dcp->valid_mode); + + /* Hotplug invalidates mode. DRM doesn't always handle this. */ + if (!(*connected)) { + dcp->valid_mode = false; + /* after unplug swap will not complete until the next + * set_digital_out_mode */ + schedule_work(&dcp->vblank_wq); + } + + if (connector && connector->connected != !!(*connected)) { + connector->connected = !!(*connected); + dcp->valid_mode = false; + schedule_work(&connector->hotplug_wq); + } +} + +static void +dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, + struct dcp_swap_complete_intent_gated *info) +{ + trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, + info->width, info->height); +} + +static void +dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) +{ + trace_iomfb_abort_swap_ap_gated(dcp, *swap_id); +} + +static struct dcpep_get_tiling_state_resp +dcpep_cb_get_tiling_state(struct apple_dcp *dcp, + struct dcpep_get_tiling_state_req *req) +{ + return (struct dcpep_get_tiling_state_resp){ + .value = 0, + .ret = 1, + }; +} + +static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) +{ + return dcp_has_panel(dcp); +} + +TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); +TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); +TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); +TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); +TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, + struct DCP_FW_NAME(dc_swap_complete_resp)); +TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, + struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, + struct iomfb_set_fx_prop_req) +TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, + struct dcp_map_buf_req, struct dcp_map_buf_resp); +TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, + struct dcp_unmap_buf_resp); +TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, + struct iomfb_sr_set_property_int_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, + struct dcp_allocate_buffer_req, + struct dcp_allocate_buffer_resp); +TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, + struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, + u8); +TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, + struct DCP_FW_NAME(dcp_map_reg_req), + struct DCP_FW_NAME(dcp_map_reg_resp)); +TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, + struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); +TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); +TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, + struct dcp_set_dcpav_prop_chunk_req, u8); +TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, + struct dcp_set_dcpav_prop_end_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_bandwidth, dcpep_cb_allocate_bandwidth, + struct dcp_allocate_bandwidth_req, struct dcp_allocate_bandwidth_resp); +TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, + struct dcp_rt_bandwidth); +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, + struct dcp_set_frame_sync_props_req, + struct dcp_set_frame_sync_props_resp); +TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); +TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, + dcpep_cb_swap_complete_intent_gated, + struct dcp_swap_complete_intent_gated); +TRAMPOLINE_IN(trampoline_abort_swap_ap_gated, dcpep_cb_abort_swap_ap_gated, u32); +TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, + iomfbep_cb_enable_backlight_message_ap_gated, u8); +TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, + struct iomfb_property); +TRAMPOLINE_INOUT(trampoline_get_tiling_state, dcpep_cb_get_tiling_state, + struct dcpep_get_tiling_state_req, struct dcpep_get_tiling_state_resp); +TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_service, u8); + +/* + * Callback for swap requests. If a swap failed, we'll never get a swap + * complete event so we need to fake a vblank event early to avoid a hang. + */ + +static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + + if (resp->ret) { + dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + dcp->swap_start = ktime_get(); + + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} + +static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_resp *resp = data; + + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + + trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); +} + +/* Helpers to modeset and swap, used to flush */ +static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct dcp_swap_start_req start_req = { 0 }; + + if (dcp->connector && dcp->connector->connected) + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + else + dcp_drm_crtc_vblank(dcp->crtc); +} + +static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +/* Changes to Adaptive Sync require a trip through set_digital_out_mode */ +static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); +} + +static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 rate, void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .value = { + rate, /* minRR */ + 0, /* mediaTargetRate */ + 0, /* Fractional Rate (?) */ + 0, /* unused */ + }, +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .count = 3, +#else + .count = 1, +#endif + }; + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_adaptive_sync, cookie); +} + +int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, + struct drm_crtc_state *crtc_state) +{ + struct dcp_display_mode *mode; + struct dcp_wait_cookie *cookie; + struct dcp_color_mode *cmode = NULL; + int ret; + + mode = lookup_mode(dcp, &crtc_state->mode); + if (!mode) { + dev_err(dcp->dev, "no match for " DRM_MODE_FMT "\n", + DRM_MODE_ARG(&crtc_state->mode)); + return -EIO; + } + + dev_info(dcp->dev, + "set_digital_out_mode(color:%d timing:%d) " DRM_MODE_FMT "\n", + mode->color_mode_id, mode->timing_mode_id, + DRM_MODE_ARG(&crtc_state->mode)); + if (mode->color_mode_id == mode->sdr_rgb.id) + cmode = &mode->sdr_rgb; + else if (mode->color_mode_id == mode->sdr_444.id) + cmode = &mode->sdr_444; + else if (mode->color_mode_id == mode->sdr.id) + cmode = &mode->sdr; + else if (mode->color_mode_id == mode->best.id) + cmode = &mode->best; + if (cmode) + dev_info(dcp->dev, + "set_digital_out_mode() color mode depth:%hhu format:%u " + "colorimetry:%u eotf:%u range:%u vrr:%u\n", cmode->depth, + cmode->format, cmode->colorimetry, cmode->eotf, + cmode->range, mode->vrr); + + dcp->mode = (struct dcp_set_digital_out_mode_req){ + .color_mode_id = mode->color_mode_id, + .timing_mode_id = mode->timing_mode_id + }; + + /* Use DCP swap timestamps on MacBook Pros with VRR */ + dcp->use_timestamps = mode->vrr && dcp->main_display; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) { + return -ENOMEM; + } + + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->during_modeset = true; + + if (mode->vrr) { + dcp_set_adaptive_sync(dcp, force_vrr ? mode->min_vrr : 0, cookie); + } else { + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); + } + + /* + * The DCP firmware has an internal timeout of ~8 seconds for + * modesets. Add an extra 500ms to safe side that the modeset + * call has returned. + */ + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(8500)); + + kref_put(&cookie->refcount, release_wait_cookie); + dcp->during_modeset = false; + dev_info(dcp->dev, "set_digital_out_mode finished:%d\n", ret); + + if (ret == 0) { + dev_info(dcp->dev, "set_digital_out_mode timed out\n"); + return -EIO; + } else if (ret < 0) { + dev_info(dcp->dev, + "waiting on set_digital_out_mode failed:%d\n", ret); + return -EIO; + + } else if (ret > 0) { + dev_dbg(dcp->dev, + "set_digital_out_mode finished with %d to spare\n", + jiffies_to_msecs(ret)); + } + dcp->valid_mode = true; + dcp->vrr_enabled = mode->vrr && force_vrr; + + return 0; +} + +/* + * DCP timestamps are expressed in system timer ticks. Approximate + * this by converting from ktime nanoseconds to 24 MHz ticks. + */ +static u64 ns_to_mach(u64 ns) +{ + return ns * 3 / 125; +} + +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct drm_plane *plane; + struct drm_plane_state *new_state, *old_state; + struct drm_crtc_state *crtc_state; + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + int plane_idx, l; + int has_surface = 0; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + /* Reset all surfaces to defaults */ + memset(req, 0, sizeof(*req)); + for (l = 0; l < SWAP_SURFACES; l++) + req->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (l = 0; l < 5; l++) + req->surf2_null[l] = true; + req->unkU32Ptr_null = true; + req->unkU32out_null = true; +#endif + + /* + * Clear all surfaces on startup. The boot framebuffer in surface 0 + * sticks around. + */ + if (!dcp->surfaces_cleared) { + req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0x7; + req->swap.bg_color = 0xFF000000; + dcp->surfaces_cleared = true; + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { + struct apple_plane_state *apple_state = to_apple_plane_state(new_state); + + /* skip planes not for this crtc */ + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + + /* + * Plane order is nondeterministic for this iterator. DCP will + * almost always crash at some point if the z order of planes + * flip-flops around. Make sure we are always blending them + * in the correct order. + * + * Despite having 4 surfaces, we can only blend two. Surface 0 is + * also unusable on some machines, so ignore it. + */ + + l = MAX_BLEND_SURFACES - new_state->normalized_zpos; + + WARN_ON(l > MAX_BLEND_SURFACES); + + req->swap.swap_enabled |= BIT(l); + + if (old_state->fb && new_state->fb != old_state->fb) { + /* + * Race condition between a framebuffer unbind getting + * swapped out and GEM unreferencing a framebuffer. If + * we lose the race, the display gets IOVA faults and + * the DCP crashes. We need to extend the lifetime of + * the drm_framebuffer (and hence the GEM object) until + * after we get a swap complete for the swap unbinding + * it. + */ + struct dcp_fb_reference *entry = + kzalloc(sizeof(*entry), GFP_KERNEL); + if (entry) { + entry->fb = old_state->fb; + entry->swap_id = dcp->last_swap_id; + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + } + drm_framebuffer_get(old_state->fb); + } + + if (!new_state->fb || !new_state->visible) { + continue; + } + req->surf_null[l] = false; + has_surface = 1; + + req->swap.src_rect[l] = apple_state->src_rect; + req->swap.dst_rect[l] = apple_state->dst_rect; + + if (dcp->notch_height > 0) + req->swap.dst_rect[l].y += dcp->notch_height; + + req->surf_iova[l] = apple_state->iova; + req->surf[l].base = apple_state->surf; + + /* Use sRGB colorspace only for internal panels. External + * displays are expected to have EDID and user space can use + * the contained colorimetry information to provide native + * colors. + */ + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && + req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) + req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; + } + + if (!has_surface && !crtc_state->color_mgmt_changed) { + if (crtc_state->enable && crtc_state->active && + !crtc_state->planes_changed) { + schedule_work(&dcp->vblank_wq); + return; + } + + /* Set black background */ + req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; + req->swap.bg_color = 0xFF000000; + req->clear = 1; + } + + if (has_surface && (dcp->use_timestamps || crtc_state->vrr_enabled || force_vrr)) { + /* + * TODO: ascertain with certainty what these timestamps + * are. They are something to do with presentation timing, + * but that is all we know for sure. These values seem to + * work well with VRR. + */ + req->swap.unk_pres_ts1 = ns_to_mach(ktime_get_ns()); + req->swap.unk_pres_ts2 = ns_to_mach(ktime_to_ns(dcp->swap_start)); + req->swap.unk_pres_ts3 = req->swap.unk_pres_ts1; + } + + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + + /* update brightness if changed */ + if (dcp_has_panel(dcp) && dcp->brightness.update) { + req->swap.bl_unk = 1; + req->swap.bl_value = dcp->brightness.dac; + req->swap.bl_power = 0x40; + dcp->brightness.update = false; + } + + if (crtc_state->color_mgmt_changed) { + struct iomfb_set_matrix_req mat = { + .location = 9, + }; + + if (crtc_state->ctm) { + struct drm_color_ctm *ctm = (struct drm_color_ctm *)crtc_state->ctm->data; + memcpy(mat.matrix, ctm->matrix, sizeof(mat.matrix)); + } else { + mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; + } + + iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); + } else + do_swap(dcp, NULL, NULL); +} + +static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct apple_connector *connector; + int result = *(int *)out; + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); + + dcp->main_display = result != 0; + + connector = dcp->connector; + if (connector) { + connector->connected = dcp->nr_modes > 0; + schedule_work(&connector->hotplug_wq); + } + + dcp->active = true; + complete(&dcp->start_done); +} + +static void init_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_is_main_display(dcp, false, res_is_main_display, NULL); +} + +static void init_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_first_client_open(dcp, false, init_3, NULL); +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 val = 0; + dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); +} + +static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct iomfb_get_color_remap_mode_req color_remap = + (struct iomfb_get_color_remap_mode_req){ + .mode = 6, + }; + + dev_info(dcp->dev, "DCP booted\n"); + + iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); +} + +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) +{ + struct dcp_set_power_state_req req = { + /* defaults are ok */ + }; + + dcp_set_power_state(dcp, false, &req, NULL, NULL); +} diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h new file mode 100644 index 00000000000000..e74672da712efa --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +/* + * This file is intended to be included multiple times with IOMFB_VER + * defined to declare DCP firmware version dependent structs. + */ + +#ifdef DCP_FW_VER + +#include + +#include + +#include "iomfb.h" +#include "iomfb_plane.h" +#include "plane.h" +#include "version_utils.h" + +struct DCP_FW_NAME(dcp_swap) { + u64 unk_pres_ts1; + u64 unk_pres_ts2; + + u64 unk_10; + u64 unk_18; + u64 ts64_unk; + u64 unk_28; + u64 unk_pres_ts3; + u64 unk_38; + + u64 flags1; + u64 flags2; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 bg_color; + u8 unk_110[0x1b8]; + u32 unk_2c8; + u8 unk_2cc[0x14]; + u32 unk_2e0; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u16 unk_2e2; +#else + u8 unk_2e2[3]; +#endif + u64 bl_unk; + u32 bl_value; // min value is 0x10000000 + u8 bl_power; // constant 0x40 for on + u8 unk_2f3[0x2d]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unk_320[0x13f]; + u64 unk_1; +#endif +} __packed; + +/* Information describing a surface */ +struct DCP_FW_NAME(dcp_surface) { + struct dcp_surface base; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u8 padding[7]; +#else + u8 padding[47]; +#endif +} __packed; + +/* Prototypes */ + +struct DCP_FW_NAME(dcp_swap_submit_req) { + struct DCP_FW_NAME(dcp_swap) swap; + struct DCP_FW_NAME(dcp_surface) surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 unk_u64_a[SWAP_SURFACES]; + struct DCP_FW_NAME(dcp_surface) surf2[5]; + u64 surf2_iova[5]; +#endif + u8 unkbool; + u64 unkdouble; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 unkU64; + u8 unkbool2; +#endif + u32 clear; // or maybe switch to default fb? +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 unkU32Ptr; +#endif + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 surf2_null[5]; +#endif + u8 unkoutbool_null; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unkU32Ptr_null; + u8 unkU32out_null; +#endif + u8 padding[1]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_submit_resp) { + u8 unkoutbool; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u32 unkU32out; +#endif + u32 ret; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; + u8 unkbool; + u64 swap_data; +#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) + u8 swap_info[0x6c4]; +#else + u8 swap_info[0x6c5]; +#endif + u32 unkint; + u8 swap_info_null; +} __packed; + +struct DCP_FW_NAME(dcp_map_reg_req) { + char obj[4]; + u32 index; + u32 flags; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unk_u64_null; +#endif + u8 addr_null; + u8 length_null; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 padding[1]; +#else + u8 padding[2]; +#endif +} __packed; + +struct DCP_FW_NAME(dcp_map_reg_resp) { +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u64 dva; +#endif + u64 addr; + u64 length; + u32 ret; +} __packed; + + +struct apple_dcp; + +int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, + struct drm_crtc_state *crtc_state); +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state); +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp); +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c new file mode 100644 index 00000000000000..0fe08c42d64659 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A357", dcpep_set_create_dfb), + IOMFB_METHOD("A358", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A407", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A410", dcpep_set_display_device), + IOMFB_METHOD("A411", dcpep_is_main_display), + IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A422", iomfbep_set_matrix), + IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A439", dcpep_set_parameter_dcp), + IOMFB_METHOD("A443", dcpep_create_default_fb), + IOMFB_METHOD("A447", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A454", dcpep_first_client_open), + IOMFB_METHOD("A455", iomfbep_last_client_close), + IOMFB_METHOD("A460", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A463", dcpep_flush_supports_power), + IOMFB_METHOD("A464", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A468", dcpep_set_power_state), +}; + +#define DCP_FW v12_3 +#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* set_boolean_property */ + [106] = trampoline_nop, /* remove_property */ + [107] = trampoline_true, /* create_provider_service */ + [108] = trampoline_true, /* create_product_service */ + [109] = trampoline_true, /* create_pmu_service */ + [110] = trampoline_true, /* create_iomfb_service */ + [111] = trampoline_create_backlight_service, + [116] = dcpep_cb_boot_1, + [117] = trampoline_false, /* is_dark_boot */ + [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [120] = trampoline_read_edt_data, + [122] = trampoline_prop_start, + [123] = trampoline_prop_chunk, + [124] = trampoline_prop_end, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_get_time, + [211] = trampoline_nop, /* update_backlight_factor_prop */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [456] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [592] = trampoline_abort_swap_ap_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; + +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.h b/drivers/gpu/drm/apple/iomfb_v12_3.h new file mode 100644 index 00000000000000..7359685d981fe5 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v12_3.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V12_3_H__ +#define __APPLE_IOMFB_V12_3_H__ + +#include "version_utils.h" + +#define DCP_FW v12_3 +#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V12_3_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c new file mode 100644 index 00000000000000..0ac869d24eb01b --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A029", dcpep_setup_video_limits), + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A373", dcpep_set_create_dfb), + IOMFB_METHOD("A374", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A407", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A410", dcpep_set_display_device), + IOMFB_METHOD("A411", dcpep_is_main_display), + IOMFB_METHOD("A412", dcpep_set_digital_out_mode), + IOMFB_METHOD("A422", iomfbep_set_matrix), + IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A441", dcpep_set_parameter_dcp), + IOMFB_METHOD("A445", dcpep_create_default_fb), + IOMFB_METHOD("A449", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A456", dcpep_first_client_open), + IOMFB_METHOD("A457", iomfbep_last_client_close), + IOMFB_METHOD("A463", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A466", dcpep_flush_supports_power), + IOMFB_METHOD("A467", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A472", dcpep_set_power_state), +}; + +#define DCP_FW v13_3 +#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) + +#include "iomfb_template.c" + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = trampoline_true, /* did_boot_signal */ + [1] = trampoline_true, /* did_power_on_signal */ + [2] = trampoline_nop, /* will_power_off_signal */ + [3] = trampoline_rt_bandwidth, + [6] = trampoline_set_frame_sync_props, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, /* get_display_default_stride */ + [102] = trampoline_nop, /* set_number_property */ + [103] = trampoline_nop, /* trigger_user_cal_loader */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, /* remove_property */ + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_pmu_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, + [113] = trampoline_true, /* create_nvram_service? */ + [114] = trampoline_get_tiling_state, + [115] = trampoline_false, /* set_tiling_state */ + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, /* is_dark_boot */ + [122] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, + [129] = trampoline_allocate_bandwidth, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, /* sr_set_uint_prop */ + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency, + [411] = trampoline_map_reg, + [413] = trampoline_true, /* sr_set_property_dict */ + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, /* sr_set_property_bool */ + [451] = trampoline_allocate_buffer, + [452] = trampoline_map_physical, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, /* set_property_dict_0 */ + [561] = trampoline_true, /* set_property_dict */ + [563] = trampoline_true, /* set_property_int */ + [565] = trampoline_true, /* set_property_bool */ + [567] = trampoline_true, /* set_property_str */ + [574] = trampoline_zero, /* power_up_dart */ + [576] = trampoline_hotplug, + [577] = trampoline_nop, /* powerstate_notify */ + [582] = trampoline_true, /* create_default_fb_surface */ + [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ + [588] = trampoline_nop, /* resize_default_fb_surface_gated */ + [589] = trampoline_swap_complete, + [591] = trampoline_swap_complete_intent_gated, + [592] = trampoline_abort_swap_ap_gated, + [593] = trampoline_enable_backlight_message_ap_gated, + [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ + [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ + [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ + [598] = trampoline_nop, /* find_swap_function_gated */ +}; +void DCP_FW_NAME(iomfb_start)(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.h b/drivers/gpu/drm/apple/iomfb_v13_3.h new file mode 100644 index 00000000000000..bbb3156b40f893 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v13_3.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_V13_3_H__ +#define __APPLE_IOMFB_V13_3_H__ + +#include "version_utils.h" + +#define DCP_FW v13_3 +#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) + +#include "iomfb_template.h" + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_V13_3_H__ */ diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c new file mode 100644 index 00000000000000..cc559c8bd7196b --- /dev/null +++ b/drivers/gpu/drm/apple/parser.c @@ -0,0 +1,1062 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#include +#include +#include +#include +#include +#include + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +#include // for sound format masks +#endif + +#include "parser.h" +#include "trace.h" + +#define DCP_PARSE_HEADER 0xd3 + +enum dcp_parse_type { + DCP_TYPE_DICTIONARY = 1, + DCP_TYPE_ARRAY = 2, + DCP_TYPE_INT64 = 4, + DCP_TYPE_STRING = 9, + DCP_TYPE_BLOB = 10, + DCP_TYPE_BOOL = 11 +}; + +struct dcp_parse_tag { + unsigned int size : 24; + enum dcp_parse_type type : 5; + unsigned int padding : 2; + bool last : 1; +} __packed; + +static const void *parse_bytes(struct dcp_parse_ctx *ctx, size_t count) +{ + const void *ptr = ctx->blob + ctx->pos; + + if (ctx->pos + count > ctx->len) + return ERR_PTR(-EINVAL); + + ctx->pos += count; + return ptr; +} + +static const u32 *parse_u32(struct dcp_parse_ctx *ctx) +{ + return parse_bytes(ctx, sizeof(u32)); +} + +static const struct dcp_parse_tag *parse_tag(struct dcp_parse_ctx *ctx) +{ + const struct dcp_parse_tag *tag; + + /* Align to 32-bits */ + ctx->pos = round_up(ctx->pos, 4); + + tag = parse_bytes(ctx, sizeof(struct dcp_parse_tag)); + + if (IS_ERR(tag)) + return tag; + + if (tag->padding) + return ERR_PTR(-EINVAL); + + return tag; +} + +static const struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, + enum dcp_parse_type type) +{ + const struct dcp_parse_tag *tag = parse_tag(ctx); + + if (IS_ERR(tag)) + return tag; + + if (tag->type != type) + return ERR_PTR(-EINVAL); + + return tag; +} + +static int skip(struct dcp_parse_ctx *handle) +{ + const struct dcp_parse_tag *tag = parse_tag(handle); + int ret = 0; + int i; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + switch (tag->type) { + case DCP_TYPE_DICTIONARY: + for (i = 0; i < tag->size; ++i) { + ret |= skip(handle); /* key */ + ret |= skip(handle); /* value */ + } + + return ret; + + case DCP_TYPE_ARRAY: + for (i = 0; i < tag->size; ++i) + ret |= skip(handle); + + return ret; + + case DCP_TYPE_INT64: + handle->pos += sizeof(s64); + return 0; + + case DCP_TYPE_STRING: + case DCP_TYPE_BLOB: + handle->pos += tag->size; + return 0; + + case DCP_TYPE_BOOL: + return 0; + + default: + return -EINVAL; + } +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +static int skip_pair(struct dcp_parse_ctx *handle) +{ + int ret; + + ret = skip(handle); + if (ret) + return ret; + + return skip(handle); +} + +static bool consume_string(struct dcp_parse_ctx *ctx, const char *specimen) +{ + const struct dcp_parse_tag *tag; + const char *key; + ctx->pos = round_up(ctx->pos, 4); + + if (ctx->pos + sizeof(*tag) + strlen(specimen) - 1 > ctx->len) + return false; + tag = ctx->blob + ctx->pos; + key = ctx->blob + ctx->pos + sizeof(*tag); + if (tag->padding) + return false; + + if (tag->type != DCP_TYPE_STRING || + tag->size != strlen(specimen) || + strncmp(key, specimen, tag->size)) + return false; + + skip(ctx); + return true; +} +#endif + +/* Caller must free the result */ +static char *parse_string(struct dcp_parse_ctx *handle) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_STRING); + const char *in; + char *out; + + if (IS_ERR(tag)) + return (void *)tag; + + in = parse_bytes(handle, tag->size); + if (IS_ERR(in)) + return (void *)in; + + out = kmalloc(tag->size + 1, GFP_KERNEL); + + memcpy(out, in, tag->size); + out[tag->size] = '\0'; + return out; +} + +static int parse_int_bound(struct dcp_parse_ctx *handle, s64 *value, s64 min, s64 max) +{ + const void *tag = parse_tag_of_type(handle, DCP_TYPE_INT64); + const s64 *in; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + in = parse_bytes(handle, sizeof(s64)); + + if (IS_ERR(in)) + return PTR_ERR(in); + + memcpy(value, in, sizeof(*value)); + + if (*value < min || *value > max) + return -EINVAL; + + return 0; +} + +static int parse_int(struct dcp_parse_ctx *handle, s64 *value) +{ + return parse_int_bound(handle, value, S64_MIN, S64_MAX); +} + +static int parse_bool(struct dcp_parse_ctx *handle, bool *b) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_BOOL); + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + *b = !!tag->size; + return 0; +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +static int parse_blob(struct dcp_parse_ctx *handle, size_t size, u8 const **blob) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_BLOB); + const u8 *out; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + + if (tag->size < size) + return -EINVAL; + + out = parse_bytes(handle, tag->size); + + if (IS_ERR(out)) + return PTR_ERR(out); + + *blob = out; + return 0; +} +#endif + +struct iterator { + struct dcp_parse_ctx *handle; + u32 idx, len; +}; + +static int iterator_begin(struct dcp_parse_ctx *handle, struct iterator *it, + bool dict) +{ + const struct dcp_parse_tag *tag; + enum dcp_parse_type type = dict ? DCP_TYPE_DICTIONARY : DCP_TYPE_ARRAY; + + *it = (struct iterator) { + .handle = handle, + .idx = 0 + }; + + tag = parse_tag_of_type(it->handle, type); + if (IS_ERR(tag)) + return PTR_ERR(tag); + + it->len = tag->size; + return 0; +} + +#define dcp_parse_foreach_in_array(handle, it) \ + for (iterator_begin(handle, &it, false); it.idx < it.len; ++it.idx) +#define dcp_parse_foreach_in_dict(handle, it) \ + for (iterator_begin(handle, &it, true); it.idx < it.len; ++it.idx) + +int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) +{ + const u32 *header; + + *ctx = (struct dcp_parse_ctx) { + .blob = blob, + .len = size, + .pos = 0, + }; + + header = parse_u32(ctx); + if (IS_ERR(header)) + return PTR_ERR(header); + + if (*header != DCP_PARSE_HEADER) + return -EINVAL; + + return 0; +} + +static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim) +{ + struct iterator it; + int ret = 0; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "Active")) + ret = parse_int(it.handle, &dim->active); + else if (!strcmp(key, "Total")) + ret = parse_int(it.handle, &dim->total); + else if (!strcmp(key, "FrontPorch")) + ret = parse_int(it.handle, &dim->front_porch); + else if (!strcmp(key, "SyncWidth")) + ret = parse_int(it.handle, &dim->sync_width); + else if (!strcmp(key, "PreciseSyncRate")) + ret = parse_int(it.handle, &dim->precise_sync_rate); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + return 0; +} + +struct color_mode { + s64 colorimetry; + s64 depth; + s64 dynamic_range; + s64 eotf; + s64 id; + s64 pixel_encoding; + s64 score; +}; + +static int fill_color_mode(struct dcp_color_mode *color, + struct color_mode *cmode) +{ + if (color->score >= cmode->score) + return 0; + + if (cmode->colorimetry < 0 || cmode->colorimetry >= DCP_COLORIMETRY_COUNT) + return -EINVAL; + if (cmode->depth < 8 || cmode->depth > 12) + return -EINVAL; + if (cmode->dynamic_range < 0 || cmode->dynamic_range >= DCP_COLOR_YCBCR_RANGE_COUNT) + return -EINVAL; + if (cmode->eotf < 0 || cmode->eotf >= DCP_EOTF_COUNT) + return -EINVAL; + if (cmode->pixel_encoding < 0 || cmode->pixel_encoding >= DCP_COLOR_FORMAT_COUNT) + return -EINVAL; + + color->score = cmode->score; + color->id = cmode->id; + color->eotf = cmode->eotf; + color->format = cmode->pixel_encoding; + color->colorimetry = cmode->colorimetry; + color->range = cmode->dynamic_range; + color->depth = cmode->depth; + + return 0; +} + +static int parse_color_modes(struct dcp_parse_ctx *handle, + struct dcp_display_mode *out) +{ + struct iterator outer_it; + int ret = 0; + out->sdr_444.score = -1; + out->sdr_rgb.score = -1; + out->sdr.score = -1; + out->best.score = -1; + + dcp_parse_foreach_in_array(handle, outer_it) { + struct iterator it; + bool is_virtual = true; + struct color_mode cmode; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "Colorimetry")) + ret = parse_int(it.handle, &cmode.colorimetry); + else if (!strcmp(key, "Depth")) + ret = parse_int(it.handle, &cmode.depth); + else if (!strcmp(key, "DynamicRange")) + ret = parse_int(it.handle, &cmode.dynamic_range); + else if (!strcmp(key, "EOTF")) + ret = parse_int(it.handle, &cmode.eotf); + else if (!strcmp(key, "ID")) + ret = parse_int(it.handle, &cmode.id); + else if (!strcmp(key, "IsVirtual")) + ret = parse_bool(it.handle, &is_virtual); + else if (!strcmp(key, "PixelEncoding")) + ret = parse_int(it.handle, &cmode.pixel_encoding); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, &cmode.score); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + /* Skip virtual or partial entries */ + if (is_virtual || cmode.score < 0 || cmode.id < 0) + continue; + + trace_iomfb_color_mode(handle->dcp, cmode.id, cmode.score, + cmode.depth, cmode.colorimetry, + cmode.eotf, cmode.dynamic_range, + cmode.pixel_encoding); + + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { + if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && + cmode.depth <= 10) + fill_color_mode(&out->sdr_rgb, &cmode); + else if (cmode.pixel_encoding == DCP_COLOR_FORMAT_YCBCR444 && + cmode.depth <= 10) + fill_color_mode(&out->sdr_444, &cmode); + fill_color_mode(&out->sdr, &cmode); + } + fill_color_mode(&out->best, &cmode); + } + + return 0; +} + +/* + * Calculate the pixel clock for a mode given the 16:16 fixed-point refresh + * rate. The pixel clock is the refresh rate times the pixel count. DRM + * specifies the clock in kHz. The intermediate result may overflow a u32, so + * use a u64 where required. + */ +static u32 calculate_clock(struct dimension *horiz, struct dimension *vert) +{ + u32 pixels = horiz->total * vert->total; + u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); + + return DIV_ROUND_CLOSEST_ULL(clock >> 16, 1000); +} + +static int parse_mode(struct dcp_parse_ctx *handle, + struct dcp_display_mode *out, s64 *score, int width_mm, + int height_mm, unsigned notch_height) +{ + int ret = 0; + struct iterator it; + struct dimension horiz, vert; + s64 min_vrr = 0, max_vrr = 0; + s64 id = -1; + s64 best_color_mode = -1; + bool is_virtual = false; + struct drm_display_mode *mode = &out->mode; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (is_virtual) + skip(it.handle); + else if (!strcmp(key, "HorizontalAttributes")) + ret = parse_dimension(it.handle, &horiz); + else if (!strcmp(key, "VerticalAttributes")) + ret = parse_dimension(it.handle, &vert); + else if (!strcmp(key, "MinimumVariableRefreshRate")) + ret = parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); + else if (!strcmp(key, "MaximumVariableRefreshRate")) + ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); + else if (!strcmp(key, "ColorModes")) + ret = parse_color_modes(it.handle, out); + else if (!strcmp(key, "ID")) + ret = parse_int(it.handle, &id); + else if (!strcmp(key, "IsVirtual")) + ret = parse_bool(it.handle, &is_virtual); + else if (!strcmp(key, "Score")) + ret = parse_int(it.handle, score); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) { + trace_iomfb_parse_mode_fail(id, &horiz, &vert, best_color_mode, is_virtual, *score); + return ret; + } + } + if (out->sdr_rgb.score >= 0) + best_color_mode = out->sdr_rgb.id; + else if (out->sdr_444.score >= 0) + best_color_mode = out->sdr_444.id; + else if (out->sdr.score >= 0) + best_color_mode = out->sdr.id; + else if (out->best.score >= 0) + best_color_mode = out->best.id; + + trace_iomfb_parse_mode_success(id, &horiz, &vert, best_color_mode, + is_virtual, *score); + + /* + * Reject modes without valid color mode. + */ + if (best_color_mode < 0) + return -EINVAL; + + /* + * We need to skip virtual modes. In some cases, virtual modes are "too + * big" for the monitor and can cause breakage. It is unclear why the + * DCP reports these modes at all. Treat as a recoverable error. + */ + if (is_virtual) + return -EINVAL; + + /* + * HACK: + * Mark the 120 Hz mode on j314/j316 (identified by resolution) as vrr. + * Setting timestamps in the the swap_surface message to non-zero + * values drives the display at 120 fps. + */ + if (vert.precise_sync_rate >> 16 == 120 && + ((horiz.active == 3024 && vert.active == 1964) || + (horiz.active == 3456 && vert.active == 2234))) { + out->min_vrr = 24 << 16; + out->max_vrr = 120 << 16; + out->vrr = true; + } + + if (min_vrr && max_vrr) { + out->min_vrr = min_vrr; + out->max_vrr = max_vrr; + out->vrr = true; + } + + vert.active -= notch_height; + vert.sync_width += notch_height; + + /* From here we must succeed. Start filling out the mode. */ + *mode = (struct drm_display_mode) { + .type = DRM_MODE_TYPE_DRIVER, + .clock = calculate_clock(&horiz, &vert), + + .vdisplay = vert.active, + .vsync_start = vert.active + vert.front_porch, + .vsync_end = vert.active + vert.front_porch + vert.sync_width, + .vtotal = vert.total, + + .hdisplay = horiz.active, + .hsync_start = horiz.active + horiz.front_porch, + .hsync_end = horiz.active + horiz.front_porch + + horiz.sync_width, + .htotal = horiz.total, + + .width_mm = width_mm, + .height_mm = height_mm, + }; + + drm_mode_set_name(mode); + + out->timing_mode_id = id; + out->color_mode_id = best_color_mode; + + trace_iomfb_timing_mode(handle->dcp, id, *score, horiz.active, + vert.active, vert.precise_sync_rate, + best_color_mode); + + return 0; +} + +struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, + unsigned int *count, int width_mm, + int height_mm, unsigned notch_height) +{ + struct iterator it; + int ret; + struct dcp_display_mode *mode, *modes; + struct dcp_display_mode *best_mode = NULL; + s64 score, best_score = -1; + + ret = iterator_begin(handle, &it, false); + + if (ret) + return ERR_PTR(ret); + + /* Start with a worst case allocation */ + modes = kmalloc_array(it.len, sizeof(*modes), GFP_KERNEL); + *count = 0; + + if (!modes) + return ERR_PTR(-ENOMEM); + + for (; it.idx < it.len; ++it.idx) { + mode = &modes[*count]; + ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height); + + /* Errors for a single mode are recoverable -- just skip it. */ + if (ret) + continue; + + /* Process a successful mode */ + (*count)++; + + if (score > best_score) { + best_score = score; + best_mode = mode; + } + } + + if (best_mode != NULL) + best_mode->mode.type |= DRM_MODE_TYPE_PREFERRED; + + return modes; +} + +int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm) +{ + int ret = 0; + struct iterator it; + s64 width_cm = 0, height_cm = 0; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "MaxHorizontalImageSize")) + ret = parse_int(it.handle, &width_cm); + else if (!strcmp(key, "MaxVerticalImageSize")) + ret = parse_int(it.handle, &height_cm); + else + skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + /* 1cm = 10mm */ + *width_mm = 10 * width_cm; + *height_mm = 10 * height_cm; + + return 0; +} + +int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, + const char **class, s64 *unit) +{ + int ret = 0; + struct iterator it; + bool parsed_unit = false; + bool parsed_name = false; + bool parsed_class = false; + + *name = ERR_PTR(-ENOENT); + *class = ERR_PTR(-ENOENT); + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) { + ret = PTR_ERR(key); + break; + } + + if (!strcmp(key, "EPICName")) { + *name = parse_string(it.handle); + if (IS_ERR(*name)) + ret = PTR_ERR(*name); + else + parsed_name = true; + } else if (!strcmp(key, "EPICProviderClass")) { + *class = parse_string(it.handle); + if (IS_ERR(*class)) + ret = PTR_ERR(*class); + else + parsed_class = true; + } else if (!strcmp(key, "EPICUnit")) { + ret = parse_int(it.handle, unit); + if (!ret) + parsed_unit = true; + } else { + skip(it.handle); + } + + kfree(key); + if (ret) + break; + } + + if (!parsed_unit || !parsed_name || !parsed_class) + ret = -ENOENT; + + if (ret) { + if (!IS_ERR(*name)) { + kfree(*name); + *name = ERR_PTR(ret); + } + if (!IS_ERR(*class)) { + kfree(*class); + *class = ERR_PTR(ret); + } + } + + return ret; +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) +static int parse_sample_rate_bit(struct dcp_parse_ctx *handle, unsigned int *ratebit) +{ + s64 rate; + int ret = parse_int(handle, &rate); + + if (ret) + return ret; + + *ratebit = snd_pcm_rate_to_rate_bit(rate); + if (*ratebit == SNDRV_PCM_RATE_KNOT) { + /* + * The rate wasn't recognized, and unless we supply + * a supplementary constraint, the SNDRV_PCM_RATE_KNOT bit + * will allow any rate. So clear it. + */ + *ratebit = 0; + } + + return 0; +} + +static int parse_sample_fmtbit(struct dcp_parse_ctx *handle, u64 *fmtbit) +{ + s64 sample_size; + int ret = parse_int(handle, &sample_size); + + if (ret) + return ret; + + switch (sample_size) { + case 16: + *fmtbit = SNDRV_PCM_FMTBIT_S16; + break; + case 20: + *fmtbit = SNDRV_PCM_FMTBIT_S20; + break; + case 24: + *fmtbit = SNDRV_PCM_FMTBIT_S24; + break; + case 32: + *fmtbit = SNDRV_PCM_FMTBIT_S32; + break; + default: + *fmtbit = 0; + break; + } + + return 0; +} + +static struct { + const char *label; + u8 type; +} chan_position_names[] = { + { "Front Left", SNDRV_CHMAP_FL }, + { "Front Right", SNDRV_CHMAP_FR }, + { "Rear Left", SNDRV_CHMAP_RL }, + { "Rear Right", SNDRV_CHMAP_RR }, + { "Front Center", SNDRV_CHMAP_FC }, + { "Low Frequency Effects", SNDRV_CHMAP_LFE }, + { "Rear Center", SNDRV_CHMAP_RC }, + { "Front Left Center", SNDRV_CHMAP_FLC }, + { "Front Right Center", SNDRV_CHMAP_FRC }, + { "Rear Left Center", SNDRV_CHMAP_RLC }, + { "Rear Right Center", SNDRV_CHMAP_RRC }, + { "Front Left Wide", SNDRV_CHMAP_FLW }, + { "Front Right Wide", SNDRV_CHMAP_FRW }, + { "Front Left High", SNDRV_CHMAP_FLH }, + { "Front Center High", SNDRV_CHMAP_FCH }, + { "Front Right High", SNDRV_CHMAP_FRH }, + { "Top Center", SNDRV_CHMAP_TC }, +}; + +static void append_chmap(struct snd_pcm_chmap_elem *chmap, u8 type) +{ + if (!chmap || chmap->channels >= ARRAY_SIZE(chmap->map)) + return; + + chmap->map[chmap->channels] = type; + chmap->channels++; +} + +static int parse_chmap(struct dcp_parse_ctx *handle, struct snd_pcm_chmap_elem *chmap) +{ + struct iterator it; + int i, ret; + + if (!chmap) { + skip(handle); + return 0; + } + + chmap->channels = 0; + + dcp_parse_foreach_in_array(handle, it) { + for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) + if (consume_string(it.handle, chan_position_names[i].label)) + break; + + if (i == ARRAY_SIZE(chan_position_names)) { + ret = skip(it.handle); + if (ret) + return ret; + + append_chmap(chmap, SNDRV_CHMAP_UNKNOWN); + continue; + } + + append_chmap(chmap, chan_position_names[i].type); + } + + return 0; +} + +static int parse_chan_layout_element(struct dcp_parse_ctx *handle, + unsigned int *nchans_out, + struct snd_pcm_chmap_elem *chmap) +{ + struct iterator it; + int ret; + s64 nchans = 0; + + dcp_parse_foreach_in_dict(handle, it) { + if (consume_string(it.handle, "ActiveChannelCount")) + ret = parse_int(it.handle, &nchans); + else if (consume_string(it.handle, "ChannelLayout")) + ret = parse_chmap(it.handle, chmap); + else + ret = skip_pair(it.handle); + + if (ret) + return ret; + } + + if (nchans_out) + *nchans_out = nchans; + + return 0; +} + +static int parse_nchans_mask(struct dcp_parse_ctx *handle, unsigned int *mask) +{ + struct iterator it; + int ret; + + *mask = 0; + + dcp_parse_foreach_in_array(handle, it) { + int nchans; + + ret = parse_chan_layout_element(it.handle, &nchans, NULL); + if (ret) + return ret; + *mask |= 1 << nchans; + } + + return 0; +} + +static int parse_avep_element(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct dcp_sound_format_mask *hits) +{ + struct dcp_sound_format_mask mask = {0, 0, 0}; + struct iterator it; + int ret; + + dcp_parse_foreach_in_dict(handle, it) { + if (consume_string(handle, "StreamSampleRate")) + ret = parse_sample_rate_bit(it.handle, &mask.rates); + else if (consume_string(handle, "SampleSize")) + ret = parse_sample_fmtbit(it.handle, &mask.formats); + else if (consume_string(handle, "AudioChannelLayoutElements")) + ret = parse_nchans_mask(it.handle, &mask.nchans); + else + ret = skip_pair(it.handle); + + if (ret) + return ret; + } + + trace_avep_sound_mode(handle->dcp, mask.rates, mask.formats, mask.nchans); + + if (!(mask.rates & sieve->rates) || !(mask.formats & sieve->formats) || + !(mask.nchans & sieve->nchans)) + return 0; + + if (hits) { + hits->rates |= mask.rates; + hits->formats |= mask.formats; + hits->nchans |= mask.nchans; + } + + return 1; +} + +static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, + unsigned int selected_nchans, + struct snd_pcm_chmap_elem *chmap, + struct dcp_sound_cookie *cookie) +{ + struct iterator it; + struct dcp_parse_ctx save_handle; + int ret; + + dcp_parse_foreach_in_dict(handle, it) { + if (consume_string(it.handle, "AudioChannelLayoutElements")) { + struct iterator inner_it; + int nchans; + + dcp_parse_foreach_in_array(it.handle, inner_it) { + save_handle = *it.handle; + ret = parse_chan_layout_element(inner_it.handle, + &nchans, NULL); + if (ret) + return ret; + + if (nchans != selected_nchans) + continue; + + /* + * Now that we know this layout matches the + * selected channel number, reread the element + * and fill in the channel map. + */ + *inner_it.handle = save_handle; + ret = parse_chan_layout_element(inner_it.handle, + NULL, chmap); + if (ret) + return ret; + } + } else if (consume_string(it.handle, "ElementData")) { + const u8 *blob; + + ret = parse_blob(it.handle, sizeof(*cookie), &blob); + if (ret) + return ret; + + if (cookie) + memcpy(cookie, blob, sizeof(*cookie)); + } else { + ret = skip_pair(it.handle); + if (ret) + return ret; + } + } + + return 0; +} + +int parse_sound_constraints(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct dcp_sound_format_mask *hits) +{ + int ret; + struct iterator it; + + if (hits) { + hits->rates = 0; + hits->formats = 0; + hits->nchans = 0; + } + + dcp_parse_foreach_in_array(handle, it) { + ret = parse_avep_element(it.handle, sieve, hits); + + if (ret < 0) + return ret; + } + + return 0; +} + +int parse_sound_mode(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct snd_pcm_chmap_elem *chmap, + struct dcp_sound_cookie *cookie) +{ + struct dcp_parse_ctx save_handle; + struct iterator it; + int ret; + + dcp_parse_foreach_in_array(handle, it) { + save_handle = *it.handle; + ret = parse_avep_element(it.handle, sieve, NULL); + + if (!ret) + continue; + + if (ret < 0) + return ret; + + ret = parse_mode_in_avep_element(&save_handle, __ffs(sieve->nchans), + chmap, cookie); + if (ret < 0) + return ret; + return 1; + } + + return 0; +} +#endif + +int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mnits *entry) +{ + struct iterator it; + int ret; + s64 mnits = -1; + s64 idac = -1; + s64 timestamp = -1; + bool type_match = false; + + dcp_parse_foreach_in_dict(handle, it) { + char *key = parse_string(it.handle); + if (IS_ERR(key)) { + ret = PTR_ERR(key); + } else if (!strcmp(key, "mNits")) { + ret = parse_int(it.handle, &mnits); + } else if (!strcmp(key, "iDAC")) { + ret = parse_int(it.handle, &idac); + } else if (!strcmp(key, "logEvent")) { + const char * value = parse_string(it.handle); + if (!IS_ERR_OR_NULL(value)) { + type_match = strcmp(value, "Display (Event Forward)") == 0; + kfree(value); + } + } else if (!strcmp(key, "timestamp")) { + ret = parse_int(it.handle, ×tamp); + } else { + skip(it.handle); + } + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) { + pr_err("dcp parser: failed to parse mNits sys event\n"); + return ret; + } + } + + if (!type_match || mnits < 0 || idac < 0 || timestamp < 0) + return -EINVAL; + + entry->millinits = mnits; + entry->idac = idac; + entry->timestamp = timestamp; + + return 0; +} diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h new file mode 100644 index 00000000000000..52c4de46d2f79f --- /dev/null +++ b/drivers/gpu/drm/apple/parser.h @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2021 Alyssa Rosenzweig */ + +#ifndef __APPLE_DCP_PARSER_H__ +#define __APPLE_DCP_PARSER_H__ + +/* For mode parsing */ +#include + +struct apple_dcp; + +struct dcp_parse_ctx { + struct apple_dcp *dcp; + const void *blob; + u32 pos, len; +}; + +enum dcp_color_eotf { + DCP_EOTF_SDR_GAMMA = 0, // "SDR gamma" + DCP_EOTF_HDR_GAMMA = 1, // "HDR gamma" + DCP_EOTF_ST_2084 = 2, // "ST 2084 (PQ)" + DCP_EOTF_BT_2100 = 3, // "BT.2100 (HLG)" + DCP_EOTF_COUNT +}; + +enum dcp_color_format { + DCP_COLOR_FORMAT_RGB = 0, // "RGB" + DCP_COLOR_FORMAT_YCBCR420 = 1, // "YUV 4:2:0" + DCP_COLOR_FORMAT_YCBCR422 = 3, // "YUV 4:2:2" + DCP_COLOR_FORMAT_YCBCR444 = 2, // "YUV 4:4:4" + DCP_COLOR_FORMAT_DV_NATIVE = 4, // "DolbyVision (native)" + DCP_COLOR_FORMAT_DV_HDMI = 5, // "DolbyVision (HDMI)" + DCP_COLOR_FORMAT_YCBCR422_DP = 6, // "YCbCr 4:2:2 (DP tunnel)" + DCP_COLOR_FORMAT_YCBCR422_HDMI = 7, // "YCbCr 4:2:2 (HDMI tunnel)" + DCP_COLOR_FORMAT_DV_LL_YCBCR422 = 8, // "DolbyVision LL YCbCr 4:2:2" + DCP_COLOR_FORMAT_DV_LL_YCBCR422_DP = 9, // "DolbyVision LL YCbCr 4:2:2 (DP)" + DCP_COLOR_FORMAT_DV_LL_YCBCR422_HDMI = 10, // "DolbyVision LL YCbCr 4:2:2 (HDMI)" + DCP_COLOR_FORMAT_DV_LL_YCBCR444 = 11, // "DolbyVision LL YCbCr 4:4:4" + DCP_COLOR_FORMAT_DV_LL_RGB422 = 12, // "DolbyVision LL RGB 4:2:2" + DCP_COLOR_FORMAT_GRGB_BLUE_422 = 13, // "GRGB as YCbCr422 (Even line blue)" + DCP_COLOR_FORMAT_GRGB_RED_422 = 14, // "GRGB as YCbCr422 (Even line red)" + DCP_COLOR_FORMAT_COUNT +}; + +enum dcp_colorimetry { + DCP_COLORIMETRY_BT601 = 0, // "SMPTE 170M/BT.601" + DCP_COLORIMETRY_BT709 = 1, // "BT.701" + DCP_COLORIMETRY_XVYCC_601 = 2, // "xvYCC601" + DCP_COLORIMETRY_XVYCC_709 = 3, // "xvYCC709" + DCP_COLORIMETRY_SYCC_601 = 4, // "sYCC601" + DCP_COLORIMETRY_ADOBE_YCC_601 = 5, // "AdobeYCC601" + DCP_COLORIMETRY_BT2020_CYCC = 6, // "BT.2020 (c)" + DCP_COLORIMETRY_BT2020_YCC = 7, // "BT.2020 (nc)" + DCP_COLORIMETRY_VSVDB = 8, // "DolbyVision VSVDB" + DCP_COLORIMETRY_BT2020_RGB = 9, // "BT.2020 (RGB)" + DCP_COLORIMETRY_SRGB = 10, // "sRGB" + DCP_COLORIMETRY_SCRGB = 11, // "scRGB" + DCP_COLORIMETRY_SCRGB_FIXED = 12, // "scRGBfixed" + DCP_COLORIMETRY_ADOBE_RGB = 13, // "AdobeRGB" + DCP_COLORIMETRY_DCI_P3_RGB_D65 = 14, // "DCI-P3 (D65)" + DCP_COLORIMETRY_DCI_P3_RGB_THEATER = 15, // "DCI-P3 (Theater)" + DCP_COLORIMETRY_RGB = 16, // "Default RGB" + DCP_COLORIMETRY_COUNT +}; + +enum dcp_color_range { + DCP_COLOR_YCBCR_RANGE_FULL = 0, + DCP_COLOR_YCBCR_RANGE_LIMITED = 1, + DCP_COLOR_YCBCR_RANGE_COUNT +}; + +struct dcp_color_mode { + s64 score; + u32 id; + enum dcp_color_eotf eotf; + enum dcp_color_format format; + enum dcp_colorimetry colorimetry; + enum dcp_color_range range; + u8 depth; +}; + +/* + * Represents a single display mode. These mode objects are populated at + * runtime based on the TimingElements dictionary sent by the DCP. + */ +struct dcp_display_mode { + struct drm_display_mode mode; + u32 color_mode_id; + u32 timing_mode_id; + u32 min_vrr; + u32 max_vrr; + struct dcp_color_mode sdr_rgb; + struct dcp_color_mode sdr_444; + struct dcp_color_mode sdr; + struct dcp_color_mode best; + bool vrr; +}; + +struct dimension { + s64 total, front_porch, sync_width, active; + s64 precise_sync_rate; +}; + +int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); +struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, + unsigned int *count, int width_mm, + int height_mm, unsigned notch_height); +int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm); +int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, + const char **class, s64 *unit); + +struct dcp_sound_format_mask { + u64 formats; /* SNDRV_PCM_FMTBIT_* */ + unsigned int rates; /* SNDRV_PCM_RATE_* */ + unsigned int nchans; +}; + +struct dcp_sound_cookie { + u8 data[24]; +}; + +struct snd_pcm_chmap_elem; +int parse_sound_constraints(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct dcp_sound_format_mask *hits); +int parse_sound_mode(struct dcp_parse_ctx *handle, + struct dcp_sound_format_mask *sieve, + struct snd_pcm_chmap_elem *chmap, + struct dcp_sound_cookie *cookie); + +struct dcp_system_ev_mnits { + u32 timestamp; + u32 millinits; + u32 idac; +}; + +int parse_system_log_mnits(struct dcp_parse_ctx *handle, + struct dcp_system_ev_mnits *entry); + +#endif diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c new file mode 100644 index 00000000000000..2f0b76ad84ad65 --- /dev/null +++ b/drivers/gpu/drm/apple/plane.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + +#include "plane.h" + +#include "iomfb_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define FRAC_16_16(mult, div) (((mult) << 16) / (div)) + +static int apple_plane_atomic_check(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *new_plane_state; + struct drm_crtc_state *crtc_state; + struct drm_rect *dst; + int ret; + + new_plane_state = drm_atomic_get_new_plane_state(state, plane); + + if (!new_plane_state->crtc) + return 0; + + crtc_state = drm_atomic_get_crtc_state(state, new_plane_state->crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + /* + * DCP limits downscaling to 2x and upscaling to 4x. Attempting to + * scale outside these bounds errors out when swapping. + * + * This function also takes care of clipping the src/dest rectangles, + * which is required for correct operation. Partially off-screen + * surfaces may appear corrupted. + * + * DCP does not distinguish plane types in the hardware, so we set + * can_position. If the primary plane does not fill the screen, the + * hardware will fill in zeroes (black). + */ + ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state, + FRAC_16_16(1, 2), + FRAC_16_16(4, 1), + true, true); + if (ret < 0) + return ret; + + if (!new_plane_state->visible) + return 0; + + /* + * DCP does not allow a surface to clip off the screen, and will crash + * if any blended surface is smaller than 32x32. Reject the atomic op + * if the plane will crash DCP. + * + * This is most pertinent to cursors. Userspace should fall back to + * software cursors if the plane check is rejected. + */ + dst = &new_plane_state->dst; + if (drm_rect_width(dst) < 32 || drm_rect_height(dst) < 32) { + dev_err_once(state->dev->dev, + "Plane operation would have crashed DCP! Rejected!\n\ + DCP requires 32x32 of every plane to be within screen space.\n\ + Your compositor asked to overlay [%dx%d, %dx%d] on %dx%d.\n\ + This is not supported, and your compositor should have\n\ + switched to software compositing when this operation failed.\n\ + You should not have noticed this at all. If your screen\n\ + froze/hitched, or your compositor crashed, please report\n\ + this to the your compositor's developers. We will not\n\ + throw this error again until you next reboot.\n", + dst->x1, dst->y1, dst->x2, dst->y2, + crtc_state->mode.hdisplay, crtc_state->mode.vdisplay); + return -EINVAL; + } + + /* + * Pitches have to be 64-byte aligned. + */ + for (u32 i = 0; i < new_plane_state->fb->format->num_planes; i++) + if (new_plane_state->fb->pitches[i] & 63) + return -EINVAL; + + /* + * FIXME: dcp can currently only use multi-planar buffers using the same + * object for all planes. It has a mandatory iommu so it should + * be no problem to map multiple objects "linearly" into DCP + * virtual address space and calculate the offsets accordingly. + * Or maybe it can accept multiple BOs via the per plane field + * `base`. + */ + if (new_plane_state->fb->format->num_planes > 1) { + const struct drm_gem_object *first = new_plane_state->fb->obj[0]; + for (u32 i = 1; i < new_plane_state->fb->format->num_planes; i++) + if (new_plane_state->fb->obj[i] != NULL && + new_plane_state->fb->obj[i] != first) + return -EINVAL; + } + + return 0; +} + +/* + * DRM specifies rectangles as start and end coordinates. DCP specifies + * rectangles as a start coordinate and a width/height. Convert a DRM rectangle + * to a DCP rectangle. + */ +static struct dcp_rect drm_to_dcp_rect(const struct drm_rect *rect) +{ + return (struct dcp_rect){ .x = rect->x1, + .y = rect->y1, + .w = drm_rect_width(rect), + .h = drm_rect_height(rect), + }; +} + +static struct dcp_rect drm_to_dcp_rect_fp(const struct drm_rect *fp_rect) +{ + struct drm_rect rect; + drm_rect_fp_to_int(&rect, fp_rect); + return drm_to_dcp_rect(&rect); +} + +static u32 drm_format_to_dcp(u32 drm, enum drm_color_range range) +{ + bool fr = range == DRM_COLOR_YCBCR_FULL_RANGE; + switch (drm) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + return DCP_FORMAT_BGRA; + + case DRM_FORMAT_XBGR8888: + case DRM_FORMAT_ABGR8888: + return DCP_FORMAT_RGBA; + + case DRM_FORMAT_XRGB2101010: + case DRM_FORMAT_ARGB2101010: + return DCP_FORMAT_L10R; + + /* semi planar YCbCr formats, limited and full range */ + case DRM_FORMAT_NV12: + return fr ? DCP_FORMAT_420F : DCP_FORMAT_420V; + case DRM_FORMAT_NV16: + return fr ? DCP_FORMAT_422F : DCP_FORMAT_422V; + case DRM_FORMAT_NV24: + return fr ? DCP_FORMAT_444F : DCP_FORMAT_444V; + + /* semi planar 10-bit YCbCr formats, limited and full range */ + case DRM_FORMAT_P010: + return fr ? DCP_FORMAT_XF20 : DCP_FORMAT_X420; + case DRM_FORMAT_P210: + return fr ? DCP_FORMAT_XF22 : DCP_FORMAT_X422; + /* + * TODO: missing DRM fourcc for P410 + */ +#if defined(DRM_FORMAT_P410) + case DRM_FORMAT_P410: + return fr ? DCP_FORMAT_XF44 : DCP_FORMAT_X444; +#endif + } + + pr_warn("DRM format %X not supported in DCP\n", drm); + return 0; +} + +static enum dcp_xfer_func get_xfer_func(bool is_yuv, enum drm_color_encoding enc) +{ + if (!is_yuv) + return DCP_XFER_FUNC_SDR; + + switch (enc) { + case DRM_COLOR_YCBCR_BT601: + return DCP_XFER_FUNC_BT601; + case DRM_COLOR_YCBCR_BT709: + case DRM_COLOR_YCBCR_BT2020: + return DCP_XFER_FUNC_BT1886; + default: + return DCP_XFER_FUNC_SDR; + } +} + +static enum dcp_colorspace get_colorspace(bool is_yuv, + enum drm_color_encoding enc) +{ + if (!is_yuv) + return DCP_COLORSPACE_NATIVE; + + switch (enc) { + case DRM_COLOR_YCBCR_BT601: + return DCP_COLORSPACE_BT601; + case DRM_COLOR_YCBCR_BT709: + return DCP_COLORSPACE_BT709; + case DRM_COLOR_YCBCR_BT2020: + return DCP_COLORSPACE_BG_BT2020; + default: + return DCP_COLORSPACE_NATIVE; + } +} + +static void apple_plane_atomic_update(struct drm_plane *plane, + struct drm_atomic_state *state) +{ + struct drm_plane_state *base = drm_atomic_get_new_plane_state(state, plane); + struct apple_plane_state *new_state; + struct drm_gem_dma_object *obj; + bool is_premultiplied = false; + + if (!base) + return; + + new_state = to_apple_plane_state(base); + + if (!base->fb) { + memset(&new_state->surf, 0, sizeof(new_state->surf)); + return; + } + + struct drm_framebuffer *fb = base->fb; + const struct drm_format_info *fmt = fb->format; + /* + * DCP doesn't support XBGR8 / XRGB8 / XBGR2101010 natively. Blending as + * pre-multiplied alpha with a black background can be used as + * workaround for the bottommost plane. + */ + if (fmt->format == DRM_FORMAT_XRGB8888 || + fmt->format == DRM_FORMAT_XBGR8888 || + fmt->format == DRM_FORMAT_XBGR2101010) + is_premultiplied = true; + + new_state->src_rect = drm_to_dcp_rect_fp(&base->src); + new_state->dst_rect = drm_to_dcp_rect(&base->dst); + + new_state->surf = (struct dcp_surface){ + .is_premultiplied = is_premultiplied, + .plane_cnt = fb->format->num_planes, + .plane_cnt2 = fb->format->num_planes, + .format = drm_format_to_dcp(fmt->format, base->color_range), + .xfer_func = get_xfer_func(fmt->is_yuv, base->color_encoding), + .colorspace = get_colorspace(fmt->is_yuv, base->color_encoding), + .stride = fb->pitches[0], + .width = fb->width, + .height = fb->height, + .buf_size = fb->height * fb->pitches[0], + // .surface_id = req->swap.surf_ids[l], + + /* Only used for compressed or multiplanar surfaces */ + .pix_size = 1, + .pel_w = 1, + .pel_h = 1, + .has_comp = 1, + .has_planes = 1, + }; + + /* Populate plane information for planar formats */ + struct dcp_surface *surf = &new_state->surf; + for (int i = 0; fb->format->num_planes && i < fb->format->num_planes; i++) { + u32 width = drm_format_info_plane_width(fb->format, fb->width, i); + u32 height = drm_format_info_plane_height(fb->format, fb->height, i); + u32 bh = drm_format_info_block_height(fb->format, i); + u32 bw = drm_format_info_block_width(fb->format, i); + + surf->planes[i] = (struct dcp_plane_info){ + .width = width, + .height = height, + .base = fb->offsets[i] - fb->offsets[0], + .offset = fb->offsets[i] - fb->offsets[0], + .stride = fb->pitches[i], + .size = height * fb->pitches[i], + .tile_size = bw * bh, + .tile_w = bw, + .tile_h = bh, + }; + + if (i > 0) + surf->buf_size += surf->planes[i].size; + } + + /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts + * the address for source x/y offsets. Since IOMFB has a direct + * support source position prefer that. + */ + obj = drm_fb_dma_get_gem_obj(base->fb, 0); + if (obj) + new_state->iova = obj->dma_addr + base->fb->offsets[0]; +} + +static const struct drm_plane_helper_funcs apple_primary_plane_helper_funcs = { + .atomic_check = apple_plane_atomic_check, + .atomic_update = apple_plane_atomic_update, + .get_scanout_buffer = drm_fb_dma_get_scanout_buffer, +}; + +static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { + .atomic_check = apple_plane_atomic_check, + .atomic_update = apple_plane_atomic_update, +}; + +// Duplicate drm_atomic_helper_plane_reset but allocate struct apple_plane_state +static void apple_plane_reset(struct drm_plane *plane) +{ + struct apple_plane_state *state = to_apple_plane_state(plane->state); + if (state) + __drm_atomic_helper_plane_destroy_state(&state->base); + + kfree(state); + plane->state = NULL; + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (state) + __drm_atomic_helper_plane_reset(plane, &state->base); +} + +static struct drm_plane_state * +apple_plane_duplicate_state(struct drm_plane *plane) +{ + struct apple_plane_state *apple_plane_state, *old_apple_plane_state; + + if (!plane->state) + return NULL; + + old_apple_plane_state = to_apple_plane_state(plane->state); + apple_plane_state = kzalloc(sizeof(*apple_plane_state), GFP_KERNEL); + if (!apple_plane_state) + return NULL; + + __drm_atomic_helper_plane_duplicate_state(plane, &apple_plane_state->base); + + apple_plane_state->surf = old_apple_plane_state->surf; + + return &apple_plane_state->base; +} + +// void apple_plane_destroy_state(struct drm_plane *plane, +// struct drm_plane_state *state) +// { +// drm_atomic_helper_plane_destroy_state(plane, state); +// } + +static const struct drm_plane_funcs apple_plane_funcs = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .reset = apple_plane_reset, + .atomic_duplicate_state = apple_plane_duplicate_state, + // .atomic_destroy_state = apple_plane_destroy_state, + .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, +}; + +/* + * Table of supported formats, mapping from DRM fourccs to DCP fourccs. + * + * For future work, DCP supports more formats not listed, including YUV + * formats, an extra RGBA format, and a biplanar RGB10_A8 format (fourcc b3a8) + * used for HDR. + * + * Note: we don't have non-alpha formats but userspace breaks without XRGB. It + * doesn't matter for the primary plane, but cursors/overlays must not + * advertise formats without alpha. + */ +static const u32 dcp_primary_formats[] = { + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +static const u32 dcp_overlay_formats[] = { + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +/* + * Formats for the 12.x firmware which does not support "l10r" / ARGB2101010 + */ +static const u32 dcp_primary_formats_12_x[] = { + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +static const u32 dcp_overlay_formats_12_x[] = { + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + +u64 apple_format_modifiers[] = { + DRM_FORMAT_MOD_LINEAR, + DRM_FORMAT_MOD_INVALID +}; + +struct apple_plane { + struct drm_plane base; +}; + +struct drm_plane *apple_plane_init(struct drm_device *dev, + unsigned long possible_crtcs, + bool supports_l10r, + enum drm_plane_type type) +{ + struct apple_plane *plane; + const u32 *fmts; + u32 num_fmts; + + switch (type) { + case DRM_PLANE_TYPE_PRIMARY: + if (supports_l10r) { + fmts = dcp_primary_formats; + num_fmts = ARRAY_SIZE(dcp_primary_formats); + } else { + fmts = dcp_primary_formats_12_x; + num_fmts = ARRAY_SIZE(dcp_primary_formats_12_x); + } + plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, + &apple_plane_funcs, fmts, num_fmts, + apple_format_modifiers, type, NULL); + break; + case DRM_PLANE_TYPE_OVERLAY: + case DRM_PLANE_TYPE_CURSOR: + if (supports_l10r) { + fmts = dcp_overlay_formats; + num_fmts = ARRAY_SIZE(dcp_overlay_formats); + } else { + fmts = dcp_overlay_formats_12_x; + num_fmts = ARRAY_SIZE(dcp_overlay_formats_12_x); + } + plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, + &apple_plane_funcs, fmts, num_fmts, + apple_format_modifiers, type, NULL); + break; + default: + return ERR_PTR(-EINVAL); + } + + if (IS_ERR(plane)) + return ERR_PTR(PTR_ERR(plane)); + + drm_plane_create_color_properties(&plane->base, + (1 << DRM_COLOR_ENCODING_MAX) - 1, + (1 << DRM_COLOR_RANGE_MAX) - 1, + DRM_COLOR_YCBCR_BT709, + DRM_COLOR_YCBCR_LIMITED_RANGE); + + if (type == DRM_PLANE_TYPE_PRIMARY) + drm_plane_helper_add(&plane->base, &apple_primary_plane_helper_funcs); + else + drm_plane_helper_add(&plane->base, &apple_plane_helper_funcs); + + return &plane->base; +} diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h new file mode 100644 index 00000000000000..67d15938cf0dcb --- /dev/null +++ b/drivers/gpu/drm/apple/plane.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright (C) The Asahi Linux Contributors + */ + +#ifndef __APPLE_PLANE_H__ +#define __APPLE_PLANE_H__ + +#include + +#include + +#include "iomfb_plane.h" + +struct apple_plane_state { + struct drm_plane_state base; + struct dcp_surface surf; + struct dcp_rect src_rect; + struct dcp_rect dst_rect; + u64 iova; +}; + +#define to_apple_plane_state(x) container_of(x, struct apple_plane_state, base) + +struct drm_plane *apple_plane_init(struct drm_device *dev, + unsigned long possible_crtcs, + bool supports_l10r, + enum drm_plane_type type); + +#endif /* __APPLE_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c new file mode 100644 index 00000000000000..9fe7a0ce495aab --- /dev/null +++ b/drivers/gpu/drm/apple/systemep.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2022 Sven Peter */ + +#include + +#include "afk.h" +#include "dcp.h" +#include "parser.h" + +static bool enable_verbose_logging; +module_param(enable_verbose_logging, bool, 0644); +MODULE_PARM_DESC(enable_verbose_logging, "Enable DCP firmware verbose logging"); + +/* + * Serialized setProperty("gAFKConfigLogMask", 0xffff) IPC call which + * will set the DCP firmware log level to the most verbose setting + */ +#define SYSTEM_SET_PROPERTY 0x43 +static const u8 setprop_gAFKConfigLogMask_ffff[] = { + 0x14, 0x00, 0x00, 0x00, 0x67, 0x41, 0x46, 0x4b, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x4c, 0x6f, 0x67, 0x4d, 0x61, 0x73, + 0x6b, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x00, 0x84, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +struct systemep_work { + struct apple_epic_service *service; + struct work_struct work; +}; + +static void system_log_work(struct work_struct *work_) +{ + struct systemep_work *work = + container_of(work_, struct systemep_work, work); + + afk_send_command(work->service, SYSTEM_SET_PROPERTY, + setprop_gAFKConfigLogMask_ffff, + sizeof(setprop_gAFKConfigLogMask_ffff), NULL, + sizeof(setprop_gAFKConfigLogMask_ffff), NULL); + complete(&work->service->ep->dcp->systemep_done); + kfree(work); +} + +static void system_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct systemep_work *work; + + if (!enable_verbose_logging) + return; + + /* + * We're called from the service message handler thread and can't + * dispatch blocking message from there. + */ + work = kzalloc(sizeof(*work), GFP_KERNEL); + if (!work) + return; + + work->service = service; + INIT_WORK(&work->work, system_log_work); + schedule_work(&work->work); +} + +static void powerlog_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static int powerlog_report(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size) +{ + struct dcp_system_ev_mnits mnits; + struct dcp_parse_ctx parse_ctx; + struct apple_dcp *dcp = service->ep->dcp; + int ret; + + dev_dbg(dcp->dev, "systemep[ch:%u]: report type:%02x len:%zu\n", + service->channel, type, data_size); + + if (type != EPIC_SUBTYPE_STD_SERVICE) + return 0; + + ret = parse(data, data_size, &parse_ctx); + if (ret) { + dev_warn(service->ep->dcp->dev, "systemep: failed to parse report: %d\n", ret); + return ret; + } + + ret = parse_system_log_mnits(&parse_ctx, &mnits); + if (ret) { + /* ignore parse errors in the case dcp sends unknown log events */ + dev_dbg(dcp->dev, "systemep: failed to parse mNits event: %d\n", ret); + return 0; + } + + dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", + mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); + + dcp->brightness.nits = mnits.millinits / 1000; + + return 0; +} + +static const struct apple_epic_service_ops systemep_ops[] = { + { + .name = "system", + .init = system_init, + }, + { + .name = "powerlog-service", + .init = powerlog_init, + .report = powerlog_report, + }, + {} +}; + +int systemep_init(struct apple_dcp *dcp) +{ + init_completion(&dcp->systemep_done); + + dcp->systemep = afk_init(dcp, SYSTEM_ENDPOINT, systemep_ops); + afk_start(dcp->systemep); + + if (!enable_verbose_logging) + return 0; + + /* + * Timeouts aren't really fatal here: in the worst case we just weren't + * able to enable additional debug prints inside DCP + */ + if (!wait_for_completion_timeout(&dcp->systemep_done, + msecs_to_jiffies(MSEC_PER_SEC))) + dev_err(dcp->dev, "systemep: couldn't enable verbose logs\n"); + + return 0; +} diff --git a/drivers/gpu/drm/apple/trace.c b/drivers/gpu/drm/apple/trace.c new file mode 100644 index 00000000000000..6f40d5a583df01 --- /dev/null +++ b/drivers/gpu/drm/apple/trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Tracepoints for Apple DCP driver + * + * Copyright (C) The Asahi Linux Contributors + */ + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h new file mode 100644 index 00000000000000..a13dd34fb7aab1 --- /dev/null +++ b/drivers/gpu/drm/apple/trace.h @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright (C) The Asahi Linux Contributors */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM dcp + +#if !defined(_TRACE_DCP_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_DCP_H + +#include "afk.h" +#include "dptxep.h" +#include "dcp-internal.h" +#include "parser.h" + +#include +#include +#include + +#define show_dcp_endpoint(ep) \ + __print_symbolic(ep, { SYSTEM_ENDPOINT, "system" }, \ + { TEST_ENDPOINT, "test" }, \ + { DCP_EXPERT_ENDPOINT, "dcpexpert" }, \ + { DISP0_ENDPOINT, "disp0" }, \ + { DPTX_ENDPOINT, "dptxport" }, \ + { HDCP_ENDPOINT, "hdcp" }, \ + { REMOTE_ALLOC_ENDPOINT, "remotealloc" }, \ + { IOMFB_ENDPOINT, "iomfb" }) +#define print_epic_type(etype) \ + __print_symbolic(etype, { EPIC_TYPE_NOTIFY, "notify" }, \ + { EPIC_TYPE_COMMAND, "command" }, \ + { EPIC_TYPE_REPLY, "reply" }, \ + { EPIC_TYPE_NOTIFY_ACK, "notify-ack" }) + +#define print_epic_category(ecat) \ + __print_symbolic(ecat, { EPIC_CAT_REPORT, "report" }, \ + { EPIC_CAT_NOTIFY, "notify" }, \ + { EPIC_CAT_REPLY, "reply" }, \ + { EPIC_CAT_COMMAND, "command" }) + +#define show_dptxport_apcall(idx) \ + __print_symbolic( \ + idx, { DPTX_APCALL_ACTIVATE, "activate" }, \ + { DPTX_APCALL_DEACTIVATE, "deactivate" }, \ + { DPTX_APCALL_GET_MAX_DRIVE_SETTINGS, \ + "get_max_drive_settings" }, \ + { DPTX_APCALL_SET_DRIVE_SETTINGS, "set_drive_settings" }, \ + { DPTX_APCALL_GET_DRIVE_SETTINGS, "get_drive_settings" }, \ + { DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG, \ + "will_change_link_config" }, \ + { DPTX_APCALL_DID_CHANGE_LINK_CONFIG, \ + "did_change_link_config" }, \ + { DPTX_APCALL_GET_MAX_LINK_RATE, "get_max_link_rate" }, \ + { DPTX_APCALL_GET_LINK_RATE, "get_link_rate" }, \ + { DPTX_APCALL_SET_LINK_RATE, "set_link_rate" }, \ + { DPTX_APCALL_GET_MAX_LANE_COUNT, \ + "get_max_lane_count" }, \ + { DPTX_APCALL_GET_ACTIVE_LANE_COUNT, \ + "get_active_lane_count" }, \ + { DPTX_APCALL_SET_ACTIVE_LANE_COUNT, \ + "set_active_lane_count" }, \ + { DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD, \ + "get_supports_downspread" }, \ + { DPTX_APCALL_GET_DOWN_SPREAD, "get_downspread" }, \ + { DPTX_APCALL_SET_DOWN_SPREAD, "set_downspread" }, \ + { DPTX_APCALL_GET_SUPPORTS_LANE_MAPPING, \ + "get_supports_lane_mapping" }, \ + { DPTX_APCALL_SET_LANE_MAP, "set_lane_map" }, \ + { DPTX_APCALL_GET_SUPPORTS_HPD, "get_supports_hpd" }, \ + { DPTX_APCALL_FORCE_HOTPLUG_DETECT, "force_hotplug_detect" }, \ + { DPTX_APCALL_INACTIVE_SINK_DETECTED, \ + "inactive_sink_detected" }, \ + { DPTX_APCALL_SET_TILED_DISPLAY_HINTS, \ + "set_tiled_display_hints" }, \ + { DPTX_APCALL_DEVICE_NOT_RESPONDING, \ + "device_not_responding" }, \ + { DPTX_APCALL_DEVICE_BUSY_TIMEOUT, "device_busy_timeout" }, \ + { DPTX_APCALL_DEVICE_NOT_STARTED, "device_not_started" }) + +TRACE_EVENT(dcp_recv_msg, + TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), + TP_ARGS(dcp, endpoint, message), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u8, endpoint) + __field(u64, message)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = endpoint; + __entry->message = message;), + + TP_printk("%s: endpoint 0x%x (%s): received message 0x%016llx", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->message)); + +TRACE_EVENT(dcp_send_msg, + TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), + TP_ARGS(dcp, endpoint, message), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u8, endpoint) + __field(u64, message)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = endpoint; + __entry->message = message;), + + TP_printk("%s: endpoint 0x%x (%s): will send message 0x%016llx", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->message)); + +TRACE_EVENT( + afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u16 size, u16 tag), + TP_ARGS(ep, size, tag), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) __field(u16, size) + __field(u16, tag)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; __entry->size = size; + __entry->tag = tag;), + + TP_printk( + "%s: endpoint 0x%x (%s): get buffer with size 0x%x and tag 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->size, + __entry->tag)); + +DECLARE_EVENT_CLASS(afk_rwptr_template, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) __field(u32, rptr) + __field(u32, wptr)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->rptr = rptr; __entry->wptr = wptr;), + + TP_printk("%s: endpoint 0x%x (%s): rptr 0x%x, wptr 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->rptr, + __entry->wptr)); + +DEFINE_EVENT(afk_rwptr_template, afk_recv_rwptr_pre, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); +DEFINE_EVENT(afk_rwptr_template, afk_recv_rwptr_post, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); +DEFINE_EVENT(afk_rwptr_template, afk_send_rwptr_pre, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); +DEFINE_EVENT(afk_rwptr_template, afk_send_rwptr_post, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 wptr), + TP_ARGS(ep, rptr, wptr)); + +TRACE_EVENT( + afk_recv_qe, + TP_PROTO(struct apple_dcp_afkep *ep, u32 rptr, u32 magic, u32 size), + TP_ARGS(ep, rptr, magic, size), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) __field(u32, rptr) + __field(u32, magic) + __field(u32, size)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; __entry->rptr = rptr; + __entry->magic = magic; __entry->size = size;), + + TP_printk("%s: endpoint 0x%x (%s): QE rptr 0x%x, magic 0x%x, size 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->rptr, + __entry->magic, __entry->size)); + +TRACE_EVENT( + afk_recv_handle, + TP_PROTO(struct apple_dcp_afkep *ep, u32 channel, u32 type, + u32 data_size, struct epic_hdr *ehdr, + struct epic_sub_hdr *eshdr), + TP_ARGS(ep, channel, type, data_size, ehdr, eshdr), + + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) __field( + u8, endpoint) __field(u32, channel) __field(u32, type) + __field(u32, data_size) __field(u8, category) + __field(u16, subtype) + __field(u16, tag)), + + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->channel = channel; __entry->type = type; + __entry->data_size = data_size; + __entry->category = eshdr->category, + __entry->subtype = le16_to_cpu(eshdr->type), + __entry->tag = le16_to_cpu(eshdr->tag)), + + TP_printk( + "%s: endpoint 0x%x (%s): channel 0x%x, type 0x%x (%s), data_size 0x%x, category: 0x%x (%s), subtype: 0x%x, seq: 0x%x", + __get_str(devname), __entry->endpoint, + show_dcp_endpoint(__entry->endpoint), __entry->channel, + __entry->type, print_epic_type(__entry->type), + __entry->data_size, __entry->category, + print_epic_category(__entry->category), __entry->subtype, + __entry->tag)); + +TRACE_EVENT(iomfb_callback, + TP_PROTO(struct apple_dcp *dcp, int tag, const char *name), + TP_ARGS(dcp, tag, name), + + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __field(int, tag) + __field(const char *, name) + ), + + TP_fast_assign( + __assign_str(devname); + __entry->tag = tag; __entry->name = name; + ), + + TP_printk("%s: Callback D%03d %s", __get_str(devname), __entry->tag, + __entry->name)); + +TRACE_EVENT(iomfb_push, + TP_PROTO(struct apple_dcp *dcp, + const struct dcp_method_entry *method, int context, + int offset, int depth), + TP_ARGS(dcp, method, context, offset, depth), + + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __string(name, method->name) + __field(int, context) + __field(int, offset) + __field(int, depth)), + + TP_fast_assign( + __assign_str(devname); + __assign_str(name); + __entry->context = context; __entry->offset = offset; + __entry->depth = depth; + ), + + TP_printk("%s: Method %s: context %u, offset %u, depth %u", + __get_str(devname), __get_str(name), __entry->context, + __entry->offset, __entry->depth)); + +TRACE_EVENT(iomfb_swap_submit, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%d", + __entry->dcp, + __entry->swap_id) +); + +TRACE_EVENT(iomfb_swap_complete, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%d", + __entry->dcp, + __entry->swap_id + ) +); + +TRACE_EVENT(iomfb_swap_complete_intent_gated, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id, u32 width, u32 height), + TP_ARGS(dcp, swap_id, width, height), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + __field(u32, width) + __field(u32, height) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + __entry->height = height; + __entry->width = width; + ), + TP_printk("dcp=%llx, swap_id=%u %ux%u", + __entry->dcp, + __entry->swap_id, + __entry->width, + __entry->height + ) +); + +TRACE_EVENT(iomfb_abort_swap_ap_gated, + TP_PROTO(struct apple_dcp *dcp, u32 swap_id), + TP_ARGS(dcp, swap_id), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, swap_id) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->swap_id = swap_id; + ), + TP_printk("dcp=%llx, swap_id=%u", + __entry->dcp, + __entry->swap_id + ) +); + +DECLARE_EVENT_CLASS(iomfb_parse_mode_template, + TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), + TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score), + + TP_STRUCT__entry(__field(s64, id) + __field_struct(struct dimension, horiz) + __field_struct(struct dimension, vert) + __field(s64, best_color_mode) + __field(bool, is_virtual) + __field(s64, score)), + + TP_fast_assign(__entry->id = id; + __entry->horiz = *horiz; + __entry->vert = *vert; + __entry->best_color_mode = best_color_mode; + __entry->is_virtual = is_virtual; + __entry->score = score;), + + TP_printk("id: %lld, best_color_mode: %lld, resolution:%lldx%lld virtual: %d, score: %lld", + __entry->id, __entry->best_color_mode, + __entry->horiz.active, __entry->vert.active, + __entry->is_virtual, __entry->score)); + +DEFINE_EVENT(iomfb_parse_mode_template, iomfb_parse_mode_success, + TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), + TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score)); + +DEFINE_EVENT(iomfb_parse_mode_template, iomfb_parse_mode_fail, + TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), + TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score)); + +TRACE_EVENT(dcpavserv_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), + TP_ARGS(dcp, unit), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u64, unit)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = unit;), + + TP_printk("%s: dcpav-service unit %lld initialized", __get_str(devname), + __entry->unit)); + +TRACE_EVENT(dptxport_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), + TP_ARGS(dcp, unit), + + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u64, unit)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = unit;), + + TP_printk("%s: dptxport unit %lld initialized", __get_str(devname), + __entry->unit)); + +TRACE_EVENT( + dptxport_apcall, + TP_PROTO(struct dptx_port *dptx, int idx, size_t len), + TP_ARGS(dptx, idx, len), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) __field(int, idx) __field(size_t, len)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; __entry->idx = idx; __entry->len = len;), + + TP_printk("%s: dptx%d: AP Call %d (%s) with len %lu", __get_str(devname), + __entry->unit, + __entry->idx, show_dptxport_apcall(__entry->idx), __entry->len)); + +TRACE_EVENT( + dptxport_validate_connection, + TP_PROTO(struct dptx_port *dptx, u8 core, u8 atc, u8 die), + TP_ARGS(dptx, core, atc, die), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) __field(u8, core) __field(u8, atc) __field(u8, die)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; __entry->core = core; __entry->atc = atc; __entry->die = die;), + + TP_printk("%s: dptx%d: core %d, atc %d, die %d", __get_str(devname), + __entry->unit, __entry->core, __entry->atc, __entry->die)); + +TRACE_EVENT( + dptxport_connect, + TP_PROTO(struct dptx_port *dptx, u8 core, u8 atc, u8 die), + TP_ARGS(dptx, core, atc, die), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) __field(u8, core) __field(u8, atc) __field(u8, die)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; __entry->core = core; __entry->atc = atc; __entry->die = die;), + + TP_printk("%s: dptx%d: core %d, atc %d, die %d", __get_str(devname), + __entry->unit, __entry->core, __entry->atc, __entry->die)); + +TRACE_EVENT( + dptxport_call_set_link_rate, + TP_PROTO(struct dptx_port *dptx, u32 link_rate), + TP_ARGS(dptx, link_rate), + + TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) + __field(u32, unit) + __field(u32, link_rate)), + + TP_fast_assign(__assign_str(devname); + __entry->unit = dptx->unit; + __entry->link_rate = link_rate;), + + TP_printk("%s: dptx%d: link rate 0x%x", __get_str(devname), __entry->unit, + __entry->link_rate)); + +TRACE_EVENT(iomfb_brightness, + TP_PROTO(struct apple_dcp *dcp, u32 nits), + TP_ARGS(dcp, nits), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, nits) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->nits = nits; + ), + TP_printk("dcp=%llx, nits=%u (raw=0x%05x)", + __entry->dcp, + __entry->nits >> 16, + __entry->nits + ) +); + +#define show_eotf(eotf) \ + __print_symbolic(eotf, { 0, "SDR gamma"}, \ + { 1, "HDR gamma"}, \ + { 2, "ST 2084 (PQ)"}, \ + { 3, "BT.2100 (HLG)"}, \ + { 4, "unexpected"}) + +#define show_encoding(enc) \ + __print_symbolic(enc, { 0, "RGB"}, \ + { 1, "YUV 4:2:0"}, \ + { 3, "YUV 4:2:2"}, \ + { 2, "YUV 4:4:4"}, \ + { 4, "DolbyVision (native)"}, \ + { 5, "DolbyVision (HDMI)"}, \ + { 6, "YCbCr 4:2:2 (DP tunnel)"}, \ + { 7, "YCbCr 4:2:2 (HDMI tunnel)"}, \ + { 8, "DolbyVision LL YCbCr 4:2:2"}, \ + { 9, "DolbyVision LL YCbCr 4:2:2 (DP)"}, \ + {10, "DolbyVision LL YCbCr 4:2:2 (HDMI)"}, \ + {11, "DolbyVision LL YCbCr 4:4:4"}, \ + {12, "DolbyVision LL RGB 4:2:2"}, \ + {13, "GRGB as YCbCr422 (Even line blue)"}, \ + {14, "GRGB as YCbCr422 (Even line red)"}, \ + {15, "unexpected"}) + +#define show_colorimetry(col) \ + __print_symbolic(col, { 0, "SMPTE 170M/BT.601"}, \ + { 1, "BT.701"}, \ + { 2, "xvYCC601"}, \ + { 3, "xvYCC709"}, \ + { 4, "sYCC601"}, \ + { 5, "AdobeYCC601"}, \ + { 6, "BT.2020 (c)"}, \ + { 7, "BT.2020 (nc)"}, \ + { 8, "DolbyVision VSVDB"}, \ + { 9, "BT.2020 (RGB)"}, \ + {10, "sRGB"}, \ + {11, "scRGB"}, \ + {12, "scRGBfixed"}, \ + {13, "AdobeRGB"}, \ + {14, "DCI-P3 (D65)"}, \ + {15, "DCI-P3 (Theater)"}, \ + {16, "Default RGB"}, \ + {17, "unexpected"}) + +#define show_range(range) \ + __print_symbolic(range, { 0, "Full"}, \ + { 1, "Limited"}, \ + { 2, "unexpected"}) + +TRACE_EVENT(iomfb_color_mode, + TP_PROTO(struct apple_dcp *dcp, u32 id, u32 score, u32 depth, + u32 colorimetry, u32 eotf, u32 range, u32 pixel_enc), + TP_ARGS(dcp, id, score, depth, colorimetry, eotf, range, pixel_enc), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, id) + __field(u32, score) + __field(u32, depth) + __field(u32, colorimetry) + __field(u32, eotf) + __field(u32, range) + __field(u32, pixel_enc) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->id = id; + __entry->score = score; + __entry->depth = depth; + __entry->colorimetry = min_t(u32, colorimetry, 17U); + __entry->eotf = min_t(u32, eotf, 4U); + __entry->range = min_t(u32, range, 2U); + __entry->pixel_enc = min_t(u32, pixel_enc, 15U); + ), + TP_printk("dcp=%llx, id=%u, score=%u, depth=%u, colorimetry=%s, eotf=%s, range=%s, pixel_enc=%s", + __entry->dcp, + __entry->id, + __entry->score, + __entry->depth, + show_colorimetry(__entry->colorimetry), + show_eotf(__entry->eotf), + show_range(__entry->range), + show_encoding(__entry->pixel_enc) + ) +); + +TRACE_EVENT(iomfb_timing_mode, + TP_PROTO(struct apple_dcp *dcp, u32 id, u32 score, u32 width, + u32 height, u32 clock, u32 color_mode), + TP_ARGS(dcp, id, score, width, height, clock, color_mode), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, id) + __field(u32, score) + __field(u32, width) + __field(u32, height) + __field(u32, clock) + __field(u32, color_mode) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->id = id; + __entry->score = score; + __entry->width = width; + __entry->height = height; + __entry->clock = clock; + __entry->color_mode = color_mode; + ), + TP_printk("dcp=%llx, id=%u, score=%u, %ux%u@%u.%u, color_mode=%u", + __entry->dcp, + __entry->id, + __entry->score, + __entry->width, + __entry->height, + __entry->clock >> 16, + ((__entry->clock & 0xffff) * 1000) >> 16, + __entry->color_mode + ) +); + +TRACE_EVENT(avep_sound_mode, + TP_PROTO(struct apple_dcp *dcp, u32 rates, u64 formats, unsigned int nchans), + TP_ARGS(dcp, rates, formats, nchans), + TP_STRUCT__entry( + __field(u64, dcp) + __field(u32, rates) + __field(u64, formats) + __field(unsigned int, nchans) + ), + TP_fast_assign( + __entry->dcp = (u64)dcp; + __entry->rates = rates; + __entry->formats = formats; + __entry->nchans = nchans; + ), + TP_printk("dcp=%llx, rates=%#x, formats=%#llx, nchans=%#x", + __entry->dcp, + __entry->rates, + __entry->formats, + __entry->nchans + ) +); + +#endif /* _TRACE_DCP_H */ + +/* This part must be outside protection */ + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#include diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h new file mode 100644 index 00000000000000..5a33ce1db61c47 --- /dev/null +++ b/drivers/gpu/drm/apple/version_utils.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_VERSION_UTILS_H__ +#define __APPLE_VERSION_UTILS_H__ + +#include +#include + +#define DCP_FW_UNION(u) (u).DCP_FW +#define DCP_FW_SUFFIX CONCATENATE(_, DCP_FW) +#define DCP_FW_NAME(name) CONCATENATE(name, DCP_FW_SUFFIX) +#define DCP_FW_VERSION(x, y, z) ( ((x) << 16) | ((y) << 8) | (z) ) + +#endif /*__APPLE_VERSION_UTILS_H__*/ diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c index 6ee909f8d53496..50e86f352838f4 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c @@ -4,6 +4,8 @@ * Author: James.Qian.Wang * */ +#include + #include #include #include @@ -93,7 +95,9 @@ komeda_fb_afbc_size_check(struct komeda_fb *kfb, struct drm_file *file, kfb->afbc_size = kfb->offset_payload + n_blocks * ALIGN(bpp * AFBC_SUPERBLK_PIXELS / 8, AFBC_SUPERBLK_ALIGNMENT); - min_size = kfb->afbc_size + fb->offsets[0]; + if (check_add_overflow(kfb->afbc_size, fb->offsets[0], &min_size)) { + goto check_failed; + } if (min_size > obj->size) { DRM_DEBUG_KMS("afbc size check failed, obj_size: 0x%zx. min_size 0x%llx.\n", obj->size, min_size); diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig new file mode 100644 index 00000000000000..dfc8987d2ced36 --- /dev/null +++ b/drivers/gpu/drm/asahi/Kconfig @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0 + +config RUST_DRM_SCHED + bool + select DRM_SCHED + +config RUST_DRM_GPUVM + bool + select DRM_GPUVM + +config DRM_ASAHI + tristate "Asahi (DRM support for Apple AGX GPUs)" + depends on RUST + depends on DRM=y + depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) + depends on MMU + depends on IOMMU_SUPPORT + depends on PAGE_SIZE_16KB + select APPLE_PMP_REPORT + select RUST_DRM_SCHED + select RUST_DRM_GEM_SHMEM_HELPER + select RUST_DRM_GPUVM + select RUST_APPLE_RTKIT + select WANT_DEV_COREDUMP + help + DRM driver for Apple AGX GPUs (G13x, found in the M1 SoC family) + +config DRM_ASAHI_DEBUG_ALLOCATOR + bool "Use debug allocator" + depends on DRM_ASAHI + help + Use an alternate, simpler allocator which significantly reduces + performance, but can help find firmware- or GPU-side memory safety + issues. However, it can also trigger firmware bugs more easily, + so expect GPU crashes. + + Say N unless you are debugging firmware structures or porting to a + new firmware version. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile new file mode 100644 index 00000000000000..e6724866798760 --- /dev/null +++ b/drivers/gpu/drm/asahi/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DRM_ASAHI) += asahi.o diff --git a/drivers/gpu/drm/asahi/alloc.rs b/drivers/gpu/drm/asahi/alloc.rs new file mode 100644 index 00000000000000..2711b784843300 --- /dev/null +++ b/drivers/gpu/drm/asahi/alloc.rs @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU kernel object allocator. +//! +//! This kernel driver needs to manage a large number of GPU objects, in both firmware/kernel +//! address space and user address space. This module implements a simple grow-only heap allocator +//! based on the DRM MM range allocator, and a debug allocator that allocates each object as a +//! separate GEM object. +//! +//! Allocations may optionally have debugging enabled, which adds preambles that store metadata +//! about the allocation. This is useful for live debugging using the hypervisor or postmortem +//! debugging with a GPU memory snapshot, since it makes it easier to identify use-after-free and +//! caching issues. + +use kernel::{ + drm::mm, + error::Result, + prelude::*, + str::CString, // +}; + +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::types::Zeroable; +use crate::mmu; +use crate::object::{ + GpuArray, + GpuObject, + GpuOnlyArray, + GpuStruct, + GpuWeakPointer, // +}; +use crate::util::RangeExt; + +use core::cmp::Ordering; +use core::fmt; +use core::fmt::{ + Debug, + Formatter, // +}; +use core::marker::PhantomData; +use core::mem; +use core::ops::Range; +use core::ptr::NonNull; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Alloc; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = HeapAllocator; + +#[cfg(not(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR))] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = HeapAllocation; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocator type +pub(crate) type DefaultAllocator = SimpleAllocator; + +#[cfg(CONFIG_DRM_ASAHI_DEBUG_ALLOCATOR)] +/// The driver-global allocation type +pub(crate) type DefaultAllocation = SimpleAllocation; + +/// Represents a raw allocation (without any type information). +pub(crate) trait RawAllocation { + /// Returns the CPU-side pointer (if CPU mapping is enabled) as a byte non-null pointer. + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// Represents a typed allocation. +pub(crate) trait Allocation: Debug { + /// Returns the typed CPU-side pointer (if CPU mapping is enabled). + fn ptr(&self) -> Option>; + /// Returns the GPU VA pointer as a u64. + fn gpu_ptr(&self) -> u64; + /// Returns the size of the allocation in bytes. + fn size(&self) -> usize; + /// Returns the AsahiDevice that owns this allocation. + fn device(&self) -> &AsahiDevice; +} + +/// A generic typed allocation wrapping a RawAllocation. +/// +/// This is currently the only Allocation implementation, since it is shared by all allocators. +/// +/// # Invariants +/// The alloaction at `alloc` must have a size equal or greater than `alloc_size` plus `debug_offset` plus `padding`. +pub(crate) struct GenericAlloc { + alloc: U, + alloc_size: usize, + debug_offset: usize, + padding: usize, + tag: u32, + pad_word: u32, + _p: PhantomData, +} + +impl Allocation for GenericAlloc { + /// Returns a pointer to the inner (usable) part of the allocation. + fn ptr(&self) -> Option> { + // SAFETY: self.debug_offset is always within the allocation per the invariant, so is safe to add + // to the base pointer. + unsafe { self.alloc.ptr().map(|p| p.add(self.debug_offset).cast()) } + } + /// Returns the GPU pointer to the inner (usable) part of the allocation. + fn gpu_ptr(&self) -> u64 { + self.alloc.gpu_ptr() + self.debug_offset as u64 + } + /// Returns the size of the inner (usable) part of the allocation. + fn size(&self) -> usize { + self.alloc_size + } + fn device(&self) -> &AsahiDevice { + self.alloc.device() + } +} + +impl Debug for GenericAlloc { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::>()) + .field("ptr", &format_args!("{:?}", self.ptr())) + .field("gpu_ptr", &format_args!("{:#X?}", self.gpu_ptr())) + .field("size", &format_args!("{:#X?}", self.size())) + .finish() + } +} + +/// Debugging data associated with an allocation, when debugging is enabled. +#[repr(C)] +struct AllocDebugData { + state: u32, + tag: u32, + size: u64, + base_gpuva: u64, + obj_gpuva: u64, + name: [u8; 0x20], +} + +/// Magic flag indicating a live allocation. +const STATE_LIVE: u32 = u32::from_le_bytes(*b"LIVE"); +/// Magic flag indicating a freed allocation. +const STATE_DEAD: u32 = u32::from_le_bytes(*b"DEAD"); + +/// Marker byte to identify when firmware/GPU write beyond the end of an allocation. +const GUARD_MARKER: u32 = 0x93939393; + +impl Drop for GenericAlloc { + fn drop(&mut self) { + let debug_len = mem::size_of::(); + if self.debug_offset >= debug_len { + if let Some(p) = self.alloc.ptr() { + // SAFETY: self.debug_offset is always greater than the alloc size per + // the invariant, and greater than debug_len as checked above. + unsafe { + let p = p.as_ptr().add(self.debug_offset - debug_len); + (p as *mut u32).write(STATE_DEAD); + } + } + } + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = self.ptr() { + // SAFETY: Writing to our inner base pointer with our known inner size is safe. + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; + } + } + if self.padding != 0 { + if let Some(p) = self.ptr() { + // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing + // the inner base pointer, after `size()` bytes. + let guard = unsafe { + core::slice::from_raw_parts( + (p.as_ptr() as *mut u8 as *const u8).add(self.size()), + self.padding, + ) + }; + let mut first_err = None; + let mut last_err = 0; + for (i, p) in guard.iter().enumerate() { + if *p != (self.pad_word >> (8 * (i & 3))) as u8 { + if first_err.is_none() { + first_err = Some(i); + } + last_err = i; + } + } + if let Some(start) = first_err { + dev_warn!( + self.device().as_ref(), + "Allocator: Corruption after object of type {}/{:#x} at {:#x}:{:#x} + {:#x}..={:#x}\n", + core::any::type_name::(), + self.tag, + self.gpu_ptr(), + self.size(), + start, + last_err, + ); + } + } + } + } +} + +static_assert!(mem::size_of::() == 0x40); + +/// A trait representing an allocator. +pub(crate) trait Allocator { + /// The raw allocation type used by this allocator. + type Raw: RawAllocation; + // TODO: Needs associated_type_defaults + // type Allocation = GenericAlloc; + + /// Returns whether CPU-side mapping is enabled. + fn cpu_maps(&self) -> bool; + /// Returns the minimum alignment for allocations. + fn min_align(&self) -> usize; + /// Allocate an object of the given size in bytes with the given alignment. + fn alloc(&mut self, size: usize, align: usize) -> Result; + + /// Returns a tuple of (count, size) of how much garbage (freed but not yet reusable objects) + /// exists in this allocator. Optional. + fn garbage(&self) -> (usize, usize) { + (0, 0) + } + /// Collect garbage for this allocator, up to the given object count. Optional. + fn collect_garbage(&mut self, _count: usize) {} + + /// Allocate a new GpuStruct object. See [`GpuObject::new`]. + #[inline(never)] + fn new_object( + &mut self, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result>> { + GpuObject::>::new(self.alloc_object()?, inner, callback) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_default`]. + #[inline(never)] + fn new_default( + &mut self, + ) -> Result>> + where + for<'a> ::Raw<'a>: Default + Zeroable, + { + GpuObject::>::new_default(self.alloc_object()?) + } + + /// Allocate a new GpuStruct object. See [`GpuObject::new_init`]. + #[inline(never)] + fn new_init<'a, T: GpuStruct, R: PinInit, F>, E, F>( + &mut self, + inner_init: impl Init, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result>> + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + GpuObject::>::new_init_prealloc( + self.alloc_object()?, + |_p| inner_init, + raw_init, + ) + } + + /// Allocate a generic buffer of the given size and alignment, applying the debug features if + /// enabled to tag it and detect overflows. + fn alloc_generic( + &mut self, + size: usize, + align: usize, + tag: Option, + ) -> Result> { + let padding = if debug_enabled(DebugFlags::DetectOverflows) { + size + } else { + 0 + }; + + let ret: GenericAlloc = + if self.cpu_maps() && debug_enabled(debug::DebugFlags::DebugAllocations) { + let debug_align = self.min_align().max(align); + let debug_len = mem::size_of::(); + let debug_offset = (debug_len * 2 + debug_align - 1) & !(debug_align - 1); + + let alloc = self.alloc(size + debug_offset + padding, align)?; + + let mut debug = AllocDebugData { + state: STATE_LIVE, + tag: tag.unwrap_or(0), + size: size as u64, + base_gpuva: alloc.gpu_ptr(), + obj_gpuva: alloc.gpu_ptr() + debug_offset as u64, + name: [0; 0x20], + }; + + let name = core::any::type_name::().as_bytes(); + let len = name.len().min(debug.name.len() - 1); + debug.name[..len].copy_from_slice(&name[..len]); + + if let Some(p) = alloc.ptr() { + // SAFETY: Per the size calculations above, this pointer math and the + // writes never exceed the allocation size. + unsafe { + let p = p.as_ptr(); + p.write_bytes(0x42, debug_offset - 2 * debug_len); + let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; + let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; + prev.copy_from(cur, 1); + cur.copy_from(&debug, 1); + }; + } + + GenericAlloc { + alloc, + alloc_size: size, + debug_offset, + tag: tag.unwrap_or(0), + pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, + padding, + _p: PhantomData, + } + } else { + GenericAlloc { + alloc: self.alloc(size + padding, align)?, + alloc_size: size, + debug_offset: 0, + tag: tag.unwrap_or(0), + pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, + padding, + _p: PhantomData, + } + }; + + if debug_enabled(DebugFlags::FillAllocations) { + if let Some(p) = ret.ptr() { + // SAFETY: Writing to our inner base pointer with our known inner size is safe. + unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; + } + } + + if padding != 0 { + if let Some(p) = ret.ptr() { + // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing + // the inner base pointer, after `size()` bytes. + let guard = unsafe { + core::slice::from_raw_parts_mut( + (p.as_ptr() as *mut u8).add(ret.size()), + padding, + ) + }; + for (i, p) in guard.iter_mut().enumerate() { + *p = (ret.pad_word >> (8 * (i & 3))) as u8; + } + } + } + + Ok(ret) + } + + /// Allocate an object of a given type, without actually initializing the allocation. + /// + /// This is useful to directly call [`GpuObject::new_*`], without borrowing a reference to the + /// allocator for the entire duration (e.g. if further allocations need to happen inside the + /// callbacks). + fn alloc_object(&mut self) -> Result> { + let size = mem::size_of::>(); + let align = mem::align_of::>(); + + self.alloc_generic(size, align, None) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, None)?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuArray` of a given type and length. + fn array_empty_tagged( + &mut self, + count: usize, + tag: &[u8; 4], + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, Some(u32::from_le_bytes(*tag)))?; + GpuArray::>::empty(alloc, count) + } + + /// Allocate an empty `GpuOnlyArray` of a given type and length. + fn array_gpuonly( + &mut self, + count: usize, + ) -> Result>> { + let size = mem::size_of::() * count; + let align = mem::align_of::(); + + let alloc = self.alloc_generic(size, align, None)?; + GpuOnlyArray::>::new(alloc, count) + } +} + +/// A simple allocation backed by a separate GEM object. +/// +/// # Invariants +/// `ptr` is either None or a valid, non-null pointer to the CPU view of the object. +/// `gpu_ptr` is the GPU-side VA of the object. +pub(crate) struct SimpleAllocation { + dev: AsahiDevRef, + ptr: Option>, + gpu_ptr: u64, + _mapping: mmu::KernelMapping, + obj: crate::gem::ObjectRef, +} + +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to send across threads. +unsafe impl Send for SimpleAllocation {} +/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to share across threads. +unsafe impl Sync for SimpleAllocation {} + +impl Drop for SimpleAllocation { + fn drop(&mut self) { + mod_dev_dbg!( + self.device(), + "SimpleAllocator: drop object @ {:#x}\n", + self.gpu_ptr() + ); + if debug_enabled(DebugFlags::FillAllocations) { + if let Ok(mut vmap) = self.obj.vmap() { + vmap.memset(0x42); + } + } + } +} + +impl RawAllocation for SimpleAllocation { + fn ptr(&self) -> Option> { + self.ptr + } + fn gpu_ptr(&self) -> u64 { + self.gpu_ptr + } + fn device(&self) -> &AsahiDevice { + &self.dev + } +} + +/// A simple allocator that allocates each object as its own GEM object, aligned to the end of a +/// page. +/// +/// This is very slow, but it has the advantage that over-reads by the firmware or GPU will fault on +/// the guard page after the allocation, which can be useful to validate that the firmware's or +/// GPU's idea of object size what we expect. +pub(crate) struct SimpleAllocator { + dev: AsahiDevRef, + range: Range, + prot: mmu::Prot, + vm: mmu::Vm, + min_align: usize, + cpu_maps: bool, +} + +impl SimpleAllocator { + /// Create a new `SimpleAllocator` for a given address range and `Vm`. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + min_align: usize, + prot: mmu::Prot, + _block_size: usize, + mut cpu_maps: bool, + _name: fmt::Arguments<'_>, + _keep_garbage: bool, + ) -> Result { + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + Ok(SimpleAllocator { + dev: dev.into(), + vm: vm.clone(), + range, + prot, + min_align, + cpu_maps, + }) + } +} + +impl Allocator for SimpleAllocator { + type Raw = SimpleAllocation; + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + #[inline(never)] + fn alloc(&mut self, size: usize, align: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + let align = self.min_align.max(align); + let offset = (size_aligned - size) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new: size={:#x} size_al={:#x} al={:#x} off={:#x}\n", + size, + size_aligned, + align, + offset + ); + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + let p = obj.vmap()?.as_mut_ptr() as *mut u8; + if debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.memset(0xde); + } + let mapping = obj.map_into_range( + &self.vm, + self.range.clone(), + self.min_align.max(mmu::UAT_PGSZ) as u64, + self.prot, + true, + )?; + + let iova = mapping.iova(); + + // SAFETY: Per the math above to calculate `size_aligned`, this can never overflow. + let ptr = unsafe { p.add(offset) }; + let gpu_ptr = iova + offset as u64; + + mod_dev_dbg!( + &self.dev, + "SimpleAllocator::new -> {:#?} / {:#?} | {:#x} / {:#x}\n", + p, + ptr, + iova, + gpu_ptr + ); + + Ok(SimpleAllocation { + dev: self.dev.clone(), + ptr: NonNull::new(ptr), + gpu_ptr, + _mapping: mapping, + obj, + }) + } +} + +/// Inner data for an allocation from the heap allocator. +/// +/// This is wrapped in an `mm::Node`. +pub(crate) struct HeapAllocationInner { + dev: AsahiDevRef, + ptr: Option>, + real_size: usize, +} + +/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to send across threads. +unsafe impl Send for HeapAllocationInner {} +/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to share between threads. +unsafe impl Sync for HeapAllocationInner {} + +/// Outer view of a heap allocation. +/// +/// This uses an Option<> so we can move the internal `Node` into the garbage pool when it gets +/// dropped. +/// +/// # Invariants +/// The `Option` must always be `Some(...)` while this object is alive. +pub(crate) struct HeapAllocation(Option>); + +impl Drop for HeapAllocation { + fn drop(&mut self) { + let node = self.0.take().unwrap(); + let size = node.size(); + let alloc = node.alloc_ref(); + + alloc.with(|a| { + if let Some(garbage) = a.garbage.as_mut() { + if garbage.push(node, GFP_KERNEL).is_err() { + dev_err!( + &a.dev.as_ref(), + "HeapAllocation[{}]::drop: Failed to keep garbage\n", + &*a.name, + ); + } + a.total_garbage += size as usize; + None + } else { + // We need to ensure node survives this scope, since dropping it + // will try to take the mm lock and deadlock us + Some(node) + } + }); + } +} + +impl mm::AllocInner for HeapAllocatorInner { + fn drop_object( + &mut self, + start: u64, + _size: u64, + _color: usize, + obj: &mut HeapAllocationInner, + ) { + /* real_size == 0 means it's a guard node */ + if obj.real_size > 0 { + mod_dev_dbg!( + obj.dev, + "HeapAllocator[{}]: drop object @ {:#x} ({} bytes)\n", + &*self.name, + start, + obj.real_size, + ); + self.allocated -= obj.real_size; + } + } +} + +impl RawAllocation for HeapAllocation { + // SAFETY: This function must always return a valid pointer. + // Since the HeapAllocation contains a reference to the + // backing_objects array that contains the object backing this pointer, + // and objects are only ever added to it, this pointer is guaranteed to + // remain valid for the lifetime of the HeapAllocation. + fn ptr(&self) -> Option> { + self.0.as_ref().unwrap().ptr + } + // SAFETY: This function must always return a valid GPU pointer. + // See the explanation in ptr(). + fn gpu_ptr(&self) -> u64 { + self.0.as_ref().unwrap().start() + } + fn device(&self) -> &AsahiDevice { + &self.0.as_ref().unwrap().dev + } +} + +/// Inner data for a heap allocator which uses the DRM MM range allocator to manage the heap. +/// +/// This is wrapped by an `mm::Allocator`. +struct HeapAllocatorInner { + dev: AsahiDevRef, + allocated: usize, + backing_objects: KVec<(crate::gem::ObjectRef, mmu::KernelMapping, u64)>, + garbage: Option>>, + total_garbage: usize, + name: CString, +} + +/// A heap allocator which uses the DRM MM range allocator to manage its objects. +/// +/// The heap is composed of a series of GEM objects. This implementation only ever grows the heap, +/// never shrinks it. +pub(crate) struct HeapAllocator { + dev: AsahiDevRef, + range: Range, + top: u64, + prot: mmu::Prot, + vm: mmu::Vm, + min_align: usize, + block_size: usize, + cpu_maps: bool, + guard_nodes: KVec>, + mm: mm::Allocator, + name: CString, + garbage: Option>>, +} + +impl HeapAllocator { + /// Create a new HeapAllocator for a given `Vm` and address range. + #[allow(dead_code)] + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + range: Range, + min_align: usize, + prot: mmu::Prot, + block_size: usize, + mut cpu_maps: bool, + name: fmt::Arguments<'_>, + keep_garbage: bool, + ) -> Result { + if !min_align.is_power_of_two() { + return Err(EINVAL); + } + if debug_enabled(DebugFlags::ForceCPUMaps) { + cpu_maps = true; + } + + let name = CString::try_from_fmt(name)?; + + let inner = HeapAllocatorInner { + dev: dev.into(), + allocated: 0, + backing_objects: KVec::new(), + // TODO: This clearly needs a try_clone() or similar + name: CString::try_from_fmt(fmt!("{}", &*name))?, + garbage: if keep_garbage { + Some(KVec::new()) + } else { + None + }, + total_garbage: 0, + }; + + let mm = mm::Allocator::new(range.start, range.range(), inner)?; + + Ok(HeapAllocator { + dev: dev.into(), + vm: vm.clone(), + top: range.start, + range, + prot, + min_align, + block_size: block_size.max(min_align), + cpu_maps, + guard_nodes: KVec::new(), + mm, + name, + garbage: if keep_garbage { + Some({ + let mut v = KVec::new(); + v.reserve(128, GFP_KERNEL)?; + v + }) + } else { + None + }, + }) + } + + /// Add a new backing block of the given size to this heap. + /// + /// If CPU mapping is enabled, this also adds a guard node to the range allocator to ensure that + /// objects cannot straddle backing block boundaries, since we cannot easily create a contiguous + /// CPU VA mapping for them. This can create some fragmentation. If CPU mapping is disabled, we + /// skip the guard blocks, since the GPU view of the heap is always contiguous. + #[inline(never)] + fn add_block(&mut self, size: usize) -> Result { + let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + if self.top.saturating_add(size_aligned as u64) > self.range.end { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Exhausted VA space\n", + &*self.name, + ); + } + + let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; + if self.cpu_maps && debug_enabled(DebugFlags::FillAllocations) { + obj.vmap()?.memset(0xde); + } + + let gpu_ptr = self.top; + let mapping = obj + .map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps) + .inspect_err(|err| { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n", + &*self.name, + gpu_ptr, + err + ); + })?; + + if self.cpu_maps { + // Create virtual mapping here ahead of time so that the vmap() in + // alloc_inner() does not take the the object's dma_resv lock while + // the mm lock is locked. mmu::Vm requires the opposite lock order. + obj.vmap()?; + } + + self.mm + .with_inner(|inner| inner.backing_objects.reserve(1, GFP_KERNEL))?; + + let mut new_top = self.top + size_aligned as u64; + if self.cpu_maps { + let guard = self.min_align.max(mmu::UAT_PGSZ); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::add_block: Adding guard node {:#x}:{:#x}\n", + &*self.name, + new_top, + guard + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: 0, + }; + + let node = match self.mm.reserve_node(inner, new_top, guard as u64, 0) { + Ok(a) => a, + Err(a) => { + dev_err!( + self.dev.as_ref(), + "HeapAllocator[{}]::add_block: Failed to reserve guard node {:#x}:{:#x}: {:?}\n", + &*self.name, + guard, + new_top, + a + ); + return Err(EIO); + } + }; + + self.guard_nodes.push(node, GFP_KERNEL)?; + + new_top += guard as u64; + } + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::add_block: top={:#x}\n", + &*self.name, + new_top + ); + + self.mm.with_inner(|inner| { + inner + .backing_objects + .push((obj, mapping, gpu_ptr), GFP_KERNEL) + })?; + + self.top = new_top; + + cls_dev_dbg!( + MemStats, + &self.dev, + "{} Heap: grow to {} bytes\n", + &*self.name, + self.top - self.range.start + ); + + Ok(()) + } + + /// Find the backing object index that backs a given GPU address. + fn find_obj(&mut self, addr: u64) -> Result { + self.mm.with_inner(|inner| { + inner + .backing_objects + .binary_search_by(|obj| { + let start = obj.2; + let end = obj.2 + obj.0.size() as u64; + if start > addr { + Ordering::Greater + } else if end <= addr { + Ordering::Less + } else { + Ordering::Equal + } + }) + .or(Err(ENOENT)) + }) + } + + fn alloc_inner(&mut self, size: usize, align: usize) -> Result { + if align != 0 && !align.is_power_of_two() { + return Err(EINVAL); + } + let align = self.min_align.max(align); + let size_aligned = (size + align - 1) & !(align - 1); + + mod_dev_dbg!( + &self.dev, + "HeapAllocator[{}]::new: size={:#x} size_al={:#x}\n", + &*self.name, + size, + size_aligned, + ); + + let inner = HeapAllocationInner { + dev: self.dev.clone(), + ptr: None, + real_size: size, + }; + + let mut node = match self.mm.insert_node_generic( + inner, + size_aligned as u64, + align as u64, + 0, + mm::InsertMode::Best, + ) { + Ok(a) => a, + Err(a) => { + dev_err!( + &self.dev.as_ref(), + "HeapAllocator[{}]::new: Failed to insert node of size {:#x} / align {:#x}: {:?}\n", + &*self.name, size_aligned, align, a + ); + return Err(a); + } + }; + + self.mm.with_inner(|inner| inner.allocated += size); + + let mut new_object = false; + let start = node.start(); + let end = start + node.size(); + if end > self.top { + if start > self.top { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: top={:#x}, start={:#x}\n", + &*self.name, + self.top, + start + ); + } + let block_size = self.block_size.max((end - self.top) as usize); + self.add_block(block_size)?; + new_object = true; + } + assert!(end <= self.top); + + if self.cpu_maps { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: mapping to CPU\n", + &*self.name + ); + + let idx = if new_object { + None + } else { + Some(match self.find_obj(start) { + Ok(a) => a, + Err(_) => { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: Failed to find object at {:#x}\n", + &*self.name, + start + ); + return Err(EIO); + } + }) + }; + let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { + let idx = idx.unwrap_or(inner.backing_objects.len() - 1); + let obj = &mut inner.backing_objects[idx]; + let p = obj.0.vmap()?.as_mut_ptr() as *mut u8; + Ok((obj.2, obj.0.size(), p)) + })?; + assert!(obj_start <= start); + assert!(obj_start + obj_size as u64 >= end); + node.as_mut().inner_mut().ptr = + // SAFETY: Per the asserts above, this offset is always within the allocation. + NonNull::new(unsafe { p.add((start - obj_start) as usize) }); + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: CPU pointer = {:?}\n", + &*self.name, + node.ptr + ); + } + + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]::alloc: Allocated {:#x} bytes @ {:#x}\n", + &*self.name, + end - start, + start + ); + + Ok(HeapAllocation(Some(node))) + } +} + +impl Allocator for HeapAllocator { + type Raw = HeapAllocation; + + fn cpu_maps(&self) -> bool { + self.cpu_maps + } + + fn min_align(&self) -> usize { + self.min_align + } + + fn alloc(&mut self, size: usize, align: usize) -> Result { + let ret = self.alloc_inner(size, align); + + if ret.is_err() { + dev_warn!( + self.dev.as_ref(), + "HeapAllocator[{}]::alloc: Allocation of {:#x}({:#x}) size object failed\n", + &*self.name, + size, + align + ); + } + ret + } + + fn garbage(&self) -> (usize, usize) { + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_ref() { + (g.len(), inner.total_garbage) + } else { + (0, 0) + } + }) + } + + fn collect_garbage(&mut self, mut count: usize) { + if let Some(garbage) = self.garbage.as_mut() { + garbage.clear(); + + while count > 0 { + let block = count.min(garbage.capacity()); + assert!(block > 0); + + // Take the garbage out of the inner block, so we can safely drop it without deadlocking + self.mm.with_inner(|inner| { + if let Some(g) = inner.garbage.as_mut() { + for node in g.drain(0..block) { + inner.total_garbage -= node.size() as usize; + garbage + .push(node, GFP_KERNEL) + .expect("push() failed after reserve()"); + } + } + }); + + count -= block; + // Now drop it + garbage.clear(); + } + } + } +} + +impl Drop for HeapAllocatorInner { + fn drop(&mut self) { + mod_dev_dbg!( + self.dev, + "HeapAllocator[{}]: dropping allocator\n", + &*self.name + ); + if self.allocated > 0 { + // This should never happen + dev_crit!( + self.dev.as_ref(), + "HeapAllocator[{}]: dropping with {} bytes allocated\n", + &*self.name, + self.allocated + ); + } + } +} diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs new file mode 100644 index 00000000000000..296091dced554d --- /dev/null +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. + +mod alloc; +mod buffer; +mod channel; +#[cfg(CONFIG_DEV_COREDUMP)] +mod crashdump; +mod debug; +mod driver; +mod event; +mod file; +mod float; +mod fw; +mod gem; +mod gpu; +mod hw; +mod initdata; +mod mem; +mod microseq; +mod mmu; +mod object; +mod pgtable; +mod queue; +mod regs; +mod slotalloc; +mod util; +#[cfg(CONFIG_DRM_ASAHI_MAPLE_TREE)] +mod vm; +mod workqueue; + +kernel::module_platform_driver! { + type: driver::AsahiDriver, + name: "asahi", + description: "AGX GPU driver for Apple silicon SoCs", + license: "Dual MIT/GPL", + params: { + debug_flags: u64 { + default: 0, + // permissions: 0o644, + description: "Debug flags", + }, + fault_control: u32 { + default: 0xb, + // permissions: 0, + description: "Fault control (0x0: hard faults, 0xb: macOS default)", + }, + initial_tvb_size: usize { + default: 0x8, + // permissions: 0o644, + description: "Initial TVB size in blocks", + }, + robust_isolation: u32 { + default: 0, + // permissions: 0o644, + description: "Fully isolate GPU contexts (limits performance)", + }, + }, +} diff --git a/drivers/gpu/drm/asahi/buffer.rs b/drivers/gpu/drm/asahi/buffer.rs new file mode 100644 index 00000000000000..309482441062d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/buffer.rs @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tiled Vertex Buffer management +//! +//! This module manages the Tiled Vertex Buffer, also known as the Parameter Buffer (in imgtec +//! parlance) or the tiler heap (on other architectures). This buffer holds transformed primitive +//! data between the vertex/tiling stage and the fragment stage. +//! +//! On AGX, the buffer is a heap of 128K blocks split into 32K pages (which must be aligned to a +//! multiple of 32K in VA space). The buffer can be shared between multiple render jobs, and each +//! will allocate pages from it during vertex processing and return them during fragment processing. +//! +//! If the buffer runs out of free pages, the vertex pass stops and a partial fragment pass occurs, +//! spilling the intermediate render target state to RAM (a partial render). This is all managed +//! transparently by the firmware. Since partial renders are less efficient, the kernel must grow +//! the heap in response to feedback from the firmware to avoid partial renders in the future. +//! Currently, we only ever grow the heap, and never shrink it. +//! +//! AGX also supports memoryless render targets, which can be used for intermediate results within +//! a render pass. To support partial renders, it seems the GPU/firmware has the ability to borrow +//! pages from the TVB buffer as a temporary render target buffer. Since this happens during a +//! partial render itself, if the buffer runs out of space, it requires synchronous growth in +//! response to a firmware interrupt. This is not currently supported, but may be in the future, +//! though it is unclear whether it is worth the effort. +//! +//! This module is also in charge of managing the temporary objects associated with a single render +//! pass, which includes the top-level tile array, the tail pointer cache, preemption buffers, and +//! other miscellaneous structures collectively managed as a "scene". +//! +//! To avoid runaway memory usage, there is a maximum size for buffers (at that point it's unlikely +//! that partial renders will incur much overhead over the buffer data access itself). This is +//! different depending on whether memoryless render targets are in use, and is currently hardcoded. +//! to the most common value used by macOS. + +use crate::debug::*; +use crate::fw::buffer; +use crate::fw::types::*; +use crate::util::*; +use crate::{ + alloc, + fw, + gpu, + hw, + mmu, + slotalloc, // +}; +use core::sync::atomic::Ordering; +use kernel::new_mutex; +use kernel::prelude::*; +use kernel::sync::{ + Arc, + Mutex, // +}; +use kernel::{ + c_str, + static_lock_class, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; + +/// There are 127 GPU/firmware-side buffer manager slots (yes, 127, not 128). +const NUM_BUFFERS: u32 = 127; + +/// Page size bits for buffer pages (32K). VAs must be aligned to this size. +pub(crate) const PAGE_SHIFT: usize = 15; +/// Page size for buffer pages. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_SHIFT; +/// Number of pages in a buffer block, which should be contiguous in VA space. +pub(crate) const PAGES_PER_BLOCK: usize = 4; +/// Size of a buffer block. +pub(crate) const BLOCK_SIZE: usize = PAGE_SIZE * PAGES_PER_BLOCK; + +/// Metadata about the tiling configuration for a scene. This is computed in the `render` module. +/// based on dimensions, tile size, and other info. +pub(crate) struct TileInfo { + /// Tile count in the X dimension. Tiles are always 32x32. + pub(crate) tiles_x: u32, + /// Tile count in the Y dimension. Tiles are always 32x32. + pub(crate) tiles_y: u32, + /// Total tile count. + pub(crate) tiles: u32, + /// Micro-tile width (16 or 32). + pub(crate) utile_width: u32, + /// Micro-tile height (16 or 32). + pub(crate) utile_height: u32, + // Macro-tiles in the X dimension. Always 4. + //pub(crate) mtiles_x: u32, + // Macro-tiles in the Y dimension. Always 4. + //pub(crate) mtiles_y: u32, + /// Tiles per macro-tile in the X dimension. + pub(crate) tiles_per_mtile_x: u32, + /// Tiles per macro-tile in the Y dimension. + pub(crate) tiles_per_mtile_y: u32, + // Total tiles per macro-tile. + //pub(crate) tiles_per_mtile: u32, + /// Micro-tiles per macro-tile in the X dimension. + pub(crate) utiles_per_mtile_x: u32, + /// Micro-tiles per macro-tile in the Y dimension. + pub(crate) utiles_per_mtile_y: u32, + // Total micro-tiles per macro-tile. + //pub(crate) utiles_per_mtile: u32, + /// Size of the top-level tilemap, in bytes (for all layers, one cluster). + pub(crate) tilemap_size: usize, + /// Size of the Tail Pointer Cache, in bytes (for all layers * clusters). + pub(crate) tpc_size: usize, + /// Number of blocks in the clustering meta buffer (for clustering) per layer. + pub(crate) meta1_layer_stride: u32, + /// Number of blocks in the clustering meta buffer (for clustering). + pub(crate) meta1_blocks: u32, + /// Layering metadata size. + pub(crate) layermeta_size: usize, + /// Minimum number of TVB blocks for this render. + pub(crate) min_tvb_blocks: usize, + /// Tiling parameter structure passed to firmware. + pub(crate) params: fw::vertex::raw::TilingParameters, +} + +/// A single scene, representing a render pass and its required buffers. +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Scene { + object: GpuObject, + slot: u32, + rebind: bool, + preempt2_off: usize, + preempt3_off: usize, + // Note: these are dead code only on some version variants. + // It's easier to do this than to propagate the version conditionals everywhere. + #[allow(dead_code)] + meta1_off: usize, + #[allow(dead_code)] + meta2_off: usize, + #[allow(dead_code)] + meta3_off: usize, + #[allow(dead_code)] + meta4_off: usize, +} + +#[versions(AGX)] +impl Scene::ver { + /// Returns true if the buffer was bound to a fresh manager slot, and therefore needs an init + /// command before a render. + pub(crate) fn rebind(&self) -> bool { + self.rebind + } + + /// Returns the buffer manager slot this scene's buffer was bound to. + pub(crate) fn slot(&self) -> u32 { + self.slot + } + + /// Returns the GPU pointer to the [`buffer::Scene::ver`]. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, buffer::Scene::ver> { + self.object.gpu_pointer() + } + + /// Returns the GPU weak pointer to the [`buffer::Scene::ver`]. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + self.object.weak_pointer() + } + + /// Returns the GPU weak pointer to the kernel-side temp buffer. + /// (purpose unknown...) + pub(crate) fn kernel_buffer_pointer(&self) -> GpuWeakPointer<[u8]> { + self.object.buffer.inner.lock().kernel_buffer.weak_pointer() + } + + /// Returns the GPU pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn buffer_pointer(&self) -> GpuPointer<'_, buffer::Info::ver> { + // SAFETY: We can't return the strong pointer directly since its lifetime crosses a lock, + // but we know its lifetime will be valid as long as &self since we hold a reference to the + // buffer, so just construct the strong pointer with the right lifetime here. + unsafe { self.weak_buffer_pointer().upgrade() } + } + + /// Returns the GPU weak pointer to the `buffer::Info::ver` object associated with this Scene. + pub(crate) fn weak_buffer_pointer(&self) -> GpuWeakPointer { + self.object.buffer.inner.lock().info.weak_pointer() + } + + /// Returns the GPU pointer to the TVB heap metadata buffer. + pub(crate) fn tvb_heapmeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_pointer() + } + + /// Returns the GPU pointer to the layer metadata buffer. + pub(crate) fn tvb_layermeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_heapmeta.gpu_offset_pointer(0x200) + } + + /// Returns the GPU pointer to the top-level TVB tilemap buffer. + pub(crate) fn tvb_tilemap_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tvb_tilemap.gpu_pointer() + } + + /// Returns the GPU pointer to the Tail Pointer Cache buffer. + pub(crate) fn tpc_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.tpc.gpu_pointer() + } + + /// Returns the GPU pointer to the first preemption scratch buffer. + pub(crate) fn preempt_buf_1_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object.preempt_buf.gpu_pointer() + } + + /// Returns the GPU pointer to the second preemption scratch buffer. + pub(crate) fn preempt_buf_2_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt2_off) + } + + /// Returns the GPU pointer to the third preemption scratch buffer. + pub(crate) fn preempt_buf_3_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { + self.object + .preempt_buf + .gpu_offset_pointer(self.preempt3_off) + } + + /// Returns the GPU pointer to the per-cluster tilemap buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn cluster_tilemaps_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.tilemaps.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering layer metadata buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn tvb_cluster_layermeta_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_pointer()) + } + + /// Returns the GPU pointer to the clustering metadata 1 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_1_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta1_off)) + } + + /// Returns the GPU pointer to the clustering metadata 2 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_2_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta2_off)) + } + + /// Returns the GPU pointer to the clustering metadata 3 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_3_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta3_off)) + } + + /// Returns the GPU pointer to the clustering metadata 4 buffer, if clustering is enabled. + #[allow(dead_code)] + pub(crate) fn meta_4_pointer(&self) -> Option> { + self.object + .clustering + .as_ref() + .map(|c| c.meta.gpu_offset_pointer(self.meta4_off)) + } +} + +#[versions(AGX)] +impl Drop for Scene::ver { + fn drop(&mut self) { + let mut inner = self.object.buffer.inner.lock(); + assert_ne!(inner.active_scenes, 0); + inner.active_scenes -= 1; + + if inner.active_scenes == 0 { + mod_pr_debug!( + "Buffer: no scenes left, dropping slot {}", + inner.active_slot.take().unwrap().slot() + ); + inner.active_slot = None; + } + } +} + +/// Inner data for a single TVB buffer object. +#[versions(AGX)] +struct BufferInner { + info: GpuObject, + ualloc: Arc>, + ualloc_priv: Arc>, + blocks: KVec>, + max_blocks: usize, + max_blocks_nomemless: usize, + mgr: BufferManager::ver, + active_scenes: usize, + active_slot: Option>, + last_token: Option, + tpc: Option>>, + kernel_buffer: GpuArray, + stats: GpuObject, + cfg: &'static hw::HwConfig, + preempt1_size: usize, + preempt2_size: usize, + preempt3_size: usize, + num_clusters: usize, +} + +/// Locked and reference counted TVB buffer. +#[versions(AGX)] +pub(crate) struct Buffer { + inner: Arc>, +} + +#[versions(AGX)] +impl Buffer::ver { + /// Create a new Buffer for a given VM, given the per-VM allocators. + pub(crate) fn new( + gpu: &dyn gpu::GpuManager, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + mgr: &BufferManager::ver, + ) -> Result { + // These are the typical max numbers on macOS. + // 8GB machines have this halved. + let max_size: usize = 862_322_688; // bytes + let max_size_nomemless = max_size / 3; + + let max_blocks = max_size / BLOCK_SIZE; + let max_blocks_nomemless = max_size_nomemless / BLOCK_SIZE; + let max_pages = max_blocks * PAGES_PER_BLOCK; + let max_pages_nomemless = max_blocks_nomemless * PAGES_PER_BLOCK; + + let num_clusters = gpu.get_dyncfg().id.num_clusters as usize; + let num_clusters_adj = if num_clusters > 1 { + num_clusters + 1 + } else { + 1 + }; + + let preempt1_size = num_clusters_adj * gpu.get_cfg().preempt1_size; + let preempt2_size = num_clusters_adj * gpu.get_cfg().preempt2_size; + let preempt3_size = num_clusters_adj * gpu.get_cfg().preempt3_size; + + let shared = &mut alloc.shared; + let info = alloc.private.new_init( + { + let ualloc_priv = &ualloc_priv; + try_init!(buffer::Info::ver { + block_ctl: shared.new_default::()?, + counter: shared.new_default::()?, + page_list: ualloc_priv.lock().array_empty_tagged(max_pages, b"PLST")?, + block_list: ualloc_priv + .lock() + .array_empty_tagged(max_blocks * 2, b"BLST")?, + }) + }, + |inner, _p| { + try_init!(buffer::raw::Info::ver { + gpu_counter: 0x0, + unk_4: 0, + last_id: 0x0, + cur_id: -1, + unk_10: 0x0, + gpu_counter2: 0x0, + unk_18: 0x0, + #[ver(V < V13_0B4 || G >= G14X)] + unk_1c: 0x0, + page_list: inner.page_list.gpu_pointer(), + page_list_size: (4 * max_pages).try_into()?, + page_count: AtomicU32::new(0), + max_blocks: max_blocks.try_into()?, + block_count: AtomicU32::new(0), + unk_38: 0x0, + block_list: inner.block_list.gpu_pointer(), + block_ctl: inner.block_ctl.gpu_pointer(), + last_page: AtomicU32::new(0), + gpu_page_ptr1: 0x0, + gpu_page_ptr2: 0x0, + unk_58: 0x0, + block_size: BLOCK_SIZE as u32, + unk_60: U64(0x0), + counter: inner.counter.gpu_pointer(), + unk_70: 0x0, + unk_74: 0x0, + unk_78: 0x0, + unk_7c: 0x0, + unk_80: 0x1, + max_pages: max_pages.try_into()?, + max_pages_nomemless: max_pages_nomemless.try_into()?, + unk_8c: 0x0, + unk_90: Default::default(), + }) + }, + )?; + + // Technically similar to Scene below, let's play it safe. + let kernel_buffer = alloc.shared.array_empty_tagged(0x40, b"KBUF")?; + let stats = alloc + .shared + .new_object(Default::default(), |_inner| buffer::raw::Stats { + reset: AtomicU32::from(1), + ..Default::default() + })?; + + Ok(Buffer::ver { + inner: Arc::pin_init( + new_mutex!(BufferInner::ver { + info, + ualloc, + ualloc_priv, + blocks: KVec::new(), + max_blocks, + max_blocks_nomemless, + mgr: mgr.clone(), + active_scenes: 0, + active_slot: None, + last_token: None, + tpc: None, + kernel_buffer, + stats, + cfg: gpu.get_cfg(), + preempt1_size, + preempt2_size, + preempt3_size, + num_clusters, + }), + GFP_KERNEL, + )?, + }) + } + + /// Returns the total block count allocated to this Buffer. + pub(crate) fn block_count(&self) -> u32 { + self.inner.lock().blocks.len() as u32 + } + + /// Automatically grow the Buffer based on feedback from the statistics. + pub(crate) fn auto_grow(&self) -> Result { + let inner = self.inner.lock(); + + let used_pages = inner.stats.with(|raw, _inner| { + let used = raw.max_pages.load(Ordering::Relaxed); + raw.reset.store(1, Ordering::Release); + used as usize + }); + + let need_blocks = (used_pages * 2) + .div_ceil(PAGES_PER_BLOCK) + .min(inner.max_blocks_nomemless); + let want_blocks = (used_pages * 3) + .div_ceil(PAGES_PER_BLOCK) + .min(inner.max_blocks_nomemless); + + let cur_count = inner.blocks.len(); + + if need_blocks <= cur_count { + Ok(false) + } else { + // Grow to 3x requested size (same logic as macOS) + core::mem::drop(inner); + self.ensure_blocks(want_blocks)?; + Ok(true) + } + } + + /// Synchronously grow the Buffer. + pub(crate) fn sync_grow(&self) { + let inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + core::mem::drop(inner); + if self.ensure_blocks(cur_count + 10).is_err() { + pr_err!("BufferManager: Failed to grow buffer synchronously\n"); + } + } + + /// Ensure that the buffer has at least a certain minimum size in blocks. + pub(crate) fn ensure_blocks(&self, min_blocks: usize) -> Result { + let mut inner = self.inner.lock(); + + let cur_count = inner.blocks.len(); + if cur_count >= min_blocks { + return Ok(false); + } + if min_blocks > inner.max_blocks { + return Err(ENOMEM); + } + + let add_blocks = min_blocks - cur_count; + let new_count = min_blocks; + + let mut new_blocks: KVec> = KVec::new(); + + // Allocate the new blocks first, so if it fails they will be dropped + let mut ualloc = inner.ualloc.lock(); + for _i in 0..add_blocks { + new_blocks.push(ualloc.array_gpuonly(BLOCK_SIZE)?, GFP_KERNEL)?; + } + core::mem::drop(ualloc); + + // Then actually commit them + inner.blocks.reserve(add_blocks, GFP_KERNEL)?; + + for (i, block) in new_blocks.into_iter().enumerate() { + let page_num = (block.gpu_va().get() >> PAGE_SHIFT) as u32; + + inner + .blocks + .push(block, GFP_KERNEL) + .expect("push() failed after reserve()"); + inner.info.block_list[2 * (cur_count + i)] = page_num; + for j in 0..PAGES_PER_BLOCK { + inner.info.page_list[(cur_count + i) * PAGES_PER_BLOCK + j] = page_num + j as u32; + } + } + + inner.info.block_ctl.with(|raw, _inner| { + raw.total.store(new_count as u32, Ordering::SeqCst); + raw.wptr.store(new_count as u32, Ordering::SeqCst); + }); + + /* Only do this update if the buffer manager is idle (which means we own it) */ + if inner.active_scenes == 0 { + let page_count = (new_count * PAGES_PER_BLOCK) as u32; + inner.info.with(|raw, _inner| { + raw.page_count.store(page_count, Ordering::Relaxed); + raw.block_count.store(new_count as u32, Ordering::Relaxed); + raw.last_page.store(page_count - 1, Ordering::Relaxed); + }); + } + + Ok(true) + } + + /// Create a new [`Scene::ver`] (render pass) using this buffer. + pub(crate) fn new_scene( + &self, + alloc: &mut gpu::KernelAllocators, + tile_info: &TileInfo, + ) -> Result { + let mut inner = self.inner.lock(); + + let tilemap_size = tile_info.tilemap_size; + let tpc_size = tile_info.tpc_size; + + // TODO: what is this exactly? + mod_pr_debug!("Buffer: Allocating TVB buffers\n"); + + // This seems to be a list, with 4x2 bytes of headers and 8 bytes per entry. + // On single-cluster devices, the used length always seems to be 1. + // On M1 Ultra, it can grow and usually doesn't exceed 64 entries. + // macOS allocates a whole 64K * 0x80 for this, so let's go with + // that to be safe... + let user_buffer = inner.ualloc.lock().array_empty_tagged( + if inner.num_clusters > 1 { + 0x10080 + } else { + 0x80 + }, + b"UBUF", + )?; + + let tvb_heapmeta = inner + .ualloc + .lock() + .array_empty_tagged(0x200 + tile_info.layermeta_size, b"HMTA")?; + let tvb_tilemap = inner + .ualloc + .lock() + .array_empty_tagged(tilemap_size, b"TMAP")?; + + mod_pr_debug!("Buffer: Allocating misc buffers\n"); + let preempt_buf = inner.ualloc.lock().array_empty_tagged( + inner.preempt1_size + inner.preempt2_size + inner.preempt3_size, + b"PRMT", + )?; + + let tpc = match inner.tpc.as_ref() { + Some(buf) if buf.len() >= tpc_size => buf.clone(), + _ => { + // MacOS allocates this as shared GPU+FW, but + // priv seems to work and might be faster? + // Needs to be FW-writable anyway, so ualloc + // won't work. + let buf = Arc::new( + inner.ualloc_priv.lock().array_empty_tagged( + (tpc_size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK, + b"TPC ", + )?, + GFP_KERNEL, + )?; + inner.tpc = Some(buf.clone()); + buf + } + }; + + let mut clmeta_size = 0; + let mut meta1_size = 0; + let mut meta2_size = 0; + let mut meta3_size = 0; + + let clustering = if inner.num_clusters > 1 { + let cfg = inner.cfg.clustering.as_ref().unwrap(); + + clmeta_size = tile_info.layermeta_size * cfg.max_splits; + // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) + // Unused on t602x? + meta1_size = align(tile_info.meta1_blocks as usize * cfg.meta1_blocksize, 0x80); + meta2_size = align(cfg.meta2_size, 0x80); + meta3_size = align(cfg.meta3_size, 0x80); + let meta4_size = cfg.meta4_size; + + let meta_size = clmeta_size + meta1_size + meta2_size + meta3_size + meta4_size; + + mod_pr_debug!("Buffer: Allocating clustering buffers\n"); + let tilemaps = inner + .ualloc + .lock() + .array_empty_tagged(cfg.max_splits * tilemap_size, b"CTMP")?; + let meta = inner.ualloc.lock().array_empty_tagged(meta_size, b"CMTA")?; + Some(buffer::ClusterBuffers { tilemaps, meta }) + } else { + None + }; + + // Could be made strong, but we wind up with a deadlock if we try to grab the + // pointer through the inner.buffer path inside the closure. + let stats_pointer = inner.stats.weak_pointer(); + + let _gpu = &mut alloc.gpu; + + // macOS allocates this as private. However, the firmware does not + // DC CIVAC this before reading it (like it does most other things), + // which causes odd cache incoherency bugs when combined with + // speculation on the firmware side (maybe). This doesn't happen + // on macOS because these structs are a circular pool that is mapped + // already initialized. Just mark this shared for now. + let scene = alloc.shared.new_init( + try_init!(buffer::Scene::ver { + user_buffer: user_buffer, + buffer: self.clone(), + tvb_heapmeta: tvb_heapmeta, + tvb_tilemap: tvb_tilemap, + tpc: tpc, + clustering: clustering, + preempt_buf: preempt_buf, + #[ver(G >= G14X)] + control_word: _gpu.array_empty_tagged(1, b"CWRD")?, + }), + |inner, _p| { + try_init!(buffer::raw::Scene::ver { + #[ver(G >= G14X)] + control_word: inner.control_word.gpu_pointer(), + #[ver(G >= G14X)] + control_word2: inner.control_word.gpu_pointer(), + pass_page_count: AtomicU32::new(0), + unk_4: 0, + unk_8: U64(0), + unk_10: U64(0), + user_buffer: inner.user_buffer.gpu_pointer(), + unk_20: 0, + #[ver(V >= V13_3)] + unk_28: U64(0), + stats: stats_pointer, + total_page_count: AtomicU32::new(0), + #[ver(G < G14X)] + unk_30: U64(0), + #[ver(G < G14X)] + unk_38: U64(0), + }) + }, + )?; + + let mut rebind = false; + + if inner.active_slot.is_none() { + assert_eq!(inner.active_scenes, 0); + + let slot = inner.mgr.0.get_inner(inner.last_token, |inner, mgr| { + inner.owners[mgr.slot() as usize] = Some(self.clone()); + Ok(()) + })?; + rebind = slot.changed(); + + mod_pr_debug!("Buffer: assigning slot {} (rebind={})", slot.slot(), rebind); + + inner.last_token = Some(slot.token()); + inner.active_slot = Some(slot); + } + + inner.active_scenes += 1; + + Ok(Scene::ver { + object: scene, + slot: inner.active_slot.as_ref().unwrap().slot(), + rebind, + preempt2_off: inner.preempt1_size, + preempt3_off: inner.preempt1_size + inner.preempt2_size, + meta1_off: clmeta_size, + meta2_off: clmeta_size + meta1_size, + meta3_off: clmeta_size + meta1_size + meta2_size, + meta4_off: clmeta_size + meta1_size + meta2_size + meta3_size, + }) + } + + /// Increment the buffer manager usage count. Should we done once we know the Scene is ready + /// to be committed and used in commands submitted to the GPU. + pub(crate) fn increment(&self) { + let inner = self.inner.lock(); + inner.info.counter.with(|raw, _inner| { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + // We have inner locked anyway, so this is not racy. + let v = raw.count.load(Ordering::Relaxed); + raw.count.store(v + 1, Ordering::Relaxed); + }); + } + + pub(crate) fn any_ref(&self) -> Arc { + self.inner.clone() + } +} + +#[versions(AGX)] +impl Clone for Buffer::ver { + fn clone(&self) -> Self { + Buffer::ver { + inner: self.inner.clone(), + } + } +} + +#[versions(AGX)] +struct BufferSlotInner(); + +#[versions(AGX)] +impl slotalloc::SlotItem for BufferSlotInner::ver { + type Data = BufferManagerInner::ver; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("BufferManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the buffer manager, to be protected by the SlotAllocator lock. +#[versions(AGX)] +pub(crate) struct BufferManagerInner { + owners: KVec>, +} + +/// The GPU-global buffer manager, used to allocate and release buffer slots from the pool. +#[versions(AGX)] +pub(crate) struct BufferManager(slotalloc::SlotAllocator); + +#[versions(AGX)] +impl BufferManager::ver { + pub(crate) fn new() -> Result { + let mut owners = KVec::new(); + for _i in 0..(NUM_BUFFERS as usize) { + owners.push(None, GFP_KERNEL)?; + } + Ok(BufferManager::ver(slotalloc::SlotAllocator::new( + NUM_BUFFERS, + BufferManagerInner::ver { owners }, + |_inner, _slot| Some(BufferSlotInner::ver()), + c_str!("BufferManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?)) + } + + /// Signals a Buffer to synchronously grow. + pub(crate) fn grow(&self, slot: u32) { + match self + .0 + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + pr_err!( + "BufferManager: Unexpected grow request for slot {}. This might deadlock. Please report this bug.\n", + slot + ); + owner.sync_grow(); + } + None => { + pr_err!( + "BufferManager: Received grow request for empty slot {}\n", + slot + ); + } + } + } +} + +#[versions(AGX)] +impl Clone for BufferManager::ver { + fn clone(&self) -> Self { + BufferManager::ver(self.0.clone()) + } +} diff --git a/drivers/gpu/drm/asahi/channel.rs b/drivers/gpu/drm/asahi/channel.rs new file mode 100644 index 00000000000000..30cc0efbf3ce22 --- /dev/null +++ b/drivers/gpu/drm/asahi/channel.rs @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU ring buffer channels +//! +//! The GPU firmware use a set of ring buffer channels to receive commands from the driver and send +//! it notifications and status messages. +//! +//! These ring buffers mostly follow uniform conventions, so they share the same base +//! implementation. + +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::channels::*; +use crate::fw::initdata::{ + raw, + ChannelRing, // +}; +use crate::fw::types::*; +use crate::{ + buffer, + event, + gpu, + mem, // +}; +use kernel::{ + c_str, + prelude::*, + sync::Arc, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, +}; + +pub(crate) use crate::fw::channels::PipeType; + +/// A receive (FW->driver) channel. +pub(crate) struct RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + ring: ChannelRing, + // FIXME: needs feature(generic_const_exprs) + //rptr: [u32; T::SUB_CHANNELS], + rptr: [u32; 6], + count: u32, +} + +impl RxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + /// Allocates a new receive channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(RxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(T::SUB_CHANNELS * count)?, + }, + rptr: Default::default(), + count: count as u32, + }) + } + + /// Receives a message on the specified sub-channel index, optionally leaving in the ring + /// buffer. + /// + /// Returns None if the channel is empty. + fn get_or_peek(&mut self, index: usize, peek: bool) -> Option { + self.ring.state.with(|raw, _inner| { + let wptr = T::wptr(raw, index); + let rptr = &mut self.rptr[index]; + if wptr == *rptr { + None + } else { + let off = self.count as usize * index; + let msg = self.ring.ring[off + *rptr as usize]; + if !peek { + *rptr = (*rptr + 1) % self.count; + T::set_rptr(raw, index, *rptr); + } + Some(msg) + } + }) + } + + /// Receives a message on the specified sub-channel index, and dequeues it from the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn get(&mut self, index: usize) -> Option { + self.get_or_peek(index, false) + } + + /// Peeks a message on the specified sub-channel index, leaving it in the ring buffer. + /// + /// Returns None if the channel is empty. + pub(crate) fn peek(&mut self, index: usize) -> Option { + self.get_or_peek(index, true) + } +} + +/// A transmit (driver->FW) channel. +pub(crate) struct TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + ring: ChannelRing, + wptr: u32, + count: u32, +} + +impl TxChannel +where + for<'a> ::Raw<'a>: Debug + Default + Zeroable, +{ + /// Allocates a new cached transmit channel with a given message count. + pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.private.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Allocates a new uncached transmit channel with a given message count. + pub(crate) fn new_uncached( + alloc: &mut gpu::KernelAllocators, + count: usize, + ) -> Result> { + Ok(TxChannel { + ring: ChannelRing { + state: alloc.shared.new_default()?, + ring: alloc.shared.array_empty(count)?, + }, + wptr: 0, + count: count as u32, + }) + } + + /// Send a message to the ring, returning a cookie with the ring buffer position. + /// + /// This will poll/block if the ring is full, which we don't really expect to happen. + pub(crate) fn put(&mut self, msg: &U) -> u32 { + self.ring.state.with(|raw, _inner| { + let next_wptr = (self.wptr + 1) % self.count; + let mut rptr = T::rptr(raw); + if next_wptr == rptr { + pr_err!( + "TX ring buffer is full! Waiting... ({}, {})\n", + next_wptr, + rptr + ); + // TODO: block properly on incoming messages? + while next_wptr == rptr { + fsleep(Delta::from_millis(8)); + rptr = T::rptr(raw); + } + } + self.ring.ring[self.wptr as usize] = *msg; + mem::sync(); + T::set_wptr(raw, next_wptr); + self.wptr = next_wptr; + }); + self.wptr + } + + /// Wait for a previously submitted message to be popped off of the ring by the GPU firmware. + /// + /// This busy-loops, and is intended to be used for rare cases when we need to block for + /// completion of a cache management or invalidation operation synchronously (which + /// the firmware normally completes fast enough not to be worth sleeping for). + /// If the poll takes longer than 10ms, this switches to sleeping between polls. + pub(crate) fn wait_for(&mut self, wptr: u32, timeout_ms: i64) -> Result { + const MAX_FAST_POLL: i64 = 10; + let start = Instant::::now(); + let timeout_ms = timeout_ms.max(1); + let timeout_fast = Delta::from_millis(timeout_ms.min(MAX_FAST_POLL)); + let timeout_slow = Delta::from_millis(timeout_ms); + self.ring.state.with(|raw, _inner| { + while start.elapsed() < timeout_fast { + if T::rptr(raw) == wptr { + return Ok(()); + } + mem::sync(); + } + while start.elapsed() < timeout_slow { + if T::rptr(raw) == wptr { + return Ok(()); + } + fsleep(Delta::from_millis(5)); + mem::sync(); + } + Err(ETIMEDOUT) + }) + } +} + +/// Device Control channel for global device management commands. +#[versions(AGX)] +pub(crate) struct DeviceControlChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +#[versions(AGX)] +impl DeviceControlChannel::ver { + const COMMAND_TIMEOUT_MS: i64 = 1000; + + /// Allocate a new Device Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(DeviceControlChannel::ver { + dev: dev.into(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Device Control command. + pub(crate) fn send(&mut self, msg: &DeviceControlMsg::ver) -> u32 { + cls_dev_dbg!(DeviceControlCh, self.dev, "DeviceControl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Device Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Pipe channel to submit WorkQueue execution requests. +#[versions(AGX)] +pub(crate) struct PipeChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +#[versions(AGX)] +impl PipeChannel::ver { + /// Allocate a new Pipe submission channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(PipeChannel::ver { + dev: dev.into(), + ch: TxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Pipe kick command to the firmware. + pub(crate) fn send(&mut self, msg: &PipeMsg::ver) { + cls_dev_dbg!(PipeCh, self.dev, "Pipe: {:?}\n", msg); + self.ch.put(msg); + } +} + +/// Firmware Control channel, used for secure cache flush requests. +pub(crate) struct FwCtlChannel { + dev: AsahiDevRef, + ch: TxChannel, +} + +impl FwCtlChannel { + const COMMAND_TIMEOUT_MS: i64 = 1000; + + /// Allocate a new Firmware Control channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwCtlChannel { + dev: dev.into(), + ch: TxChannel::::new_uncached(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Submits a Firmware Control command to the firmware. + pub(crate) fn send(&mut self, msg: &FwCtlMsg) -> u32 { + cls_dev_dbg!(FwCtlCh, self.dev, "FwCtl: {:?}\n", msg); + self.ch.put(msg) + } + + /// Waits for a previously submitted Firmware Control command to complete. + pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { + self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) + } +} + +/// Event channel, used to notify the driver of command completions, GPU faults and errors, and +/// other events. +#[versions(AGX)] +pub(crate) struct EventChannel { + dev: AsahiDevRef, + ch: RxChannel, + ev_mgr: Arc, + buf_mgr: buffer::BufferManager::ver, + gpu: Option>, +} + +#[versions(AGX)] +impl EventChannel::ver { + /// Allocate a new Event channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ev_mgr: Arc, + buf_mgr: buffer::BufferManager::ver, + ) -> Result { + Ok(EventChannel::ver { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x100)?, + ev_mgr, + buf_mgr, + gpu: None, + }) + } + + /// Registers the managing `Gpu` instance that will handle events on this channel. + pub(crate) fn set_manager(&mut self, gpu: Arc) { + self.gpu = Some(gpu); + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new Event messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + // SAFETY: The raw view is always valid for all bit patterns. + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=EVENT_MAX => { + // SAFETY: Since we have checked the tag to be in range, + // accessing the enum view is valid. + let msg = unsafe { msg.msg }; + + cls_dev_dbg!(EventCh, self.dev, "Event: {:?}\n", msg); + match msg { + EventMsg::Fault => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_fault(), + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::Timeout { + counter, + unk_8, + event_slot, + } => match self.gpu.as_ref() { + Some(gpu) => gpu.handle_timeout(counter, event_slot, unk_8), + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::Flag { firing, .. } => { + for (i, flags) in firing.iter().enumerate() { + for j in 0..32 { + if flags & (1u32 << j) != 0 { + self.ev_mgr.signal((i * 32 + j) as u32); + } + } + } + } + EventMsg::GrowTVB { + vm_slot, + buffer_slot, + counter, + } => match self.gpu.as_ref() { + Some(gpu) => { + self.buf_mgr.grow(buffer_slot); + gpu.ack_grow(buffer_slot, vm_slot, counter); + } + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + EventMsg::ChannelError { + error_type, + pipe_type, + event_slot, + event_value, + } => match self.gpu.as_ref() { + Some(gpu) => { + let error_type = match error_type { + 0 => ChannelErrorType::MemoryError, + 1 => ChannelErrorType::DMKill, + 2 => ChannelErrorType::Aborted, + 3 => ChannelErrorType::Unk3, + a => ChannelErrorType::Unknown(a), + }; + gpu.handle_channel_error( + error_type, + pipe_type, + event_slot, + event_value, + ); + } + None => { + dev_crit!( + self.dev.as_ref(), + "EventChannel: No GPU manager available!\n" + ) + } + }, + msg => { + dev_crit!(self.dev.as_ref(), "Unknown event message: {:?}\n", msg); + } + } + } + _ => { + // SAFETY: The raw view is always valid for all bit patterns. + dev_warn!(self.dev.as_ref(), "Unknown event message: {:?}\n", unsafe { + msg.raw + }); + } + } + } + } +} + +/// Firmware Log channel. This one is pretty special, since it has 6 sub-channels (for different log +/// levels), and it also uses a side buffer to actually hold the log messages, only passing around +/// pointers in the main buffer. +pub(crate) struct FwLogChannel { + dev: AsahiDevRef, + ch: RxChannel, + payload_buf: GpuArray, +} + +impl FwLogChannel { + const RING_SIZE: usize = 0x100; + const BUF_SIZE: usize = 0x100; + + /// Allocate a new Firmware Log channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(FwLogChannel { + dev: dev.into(), + ch: RxChannel::::new(alloc, Self::RING_SIZE)?, + payload_buf: alloc + .shared + .array_empty(Self::BUF_SIZE * FwLogChannelState::SUB_CHANNELS)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Returns the GPU pointers to the firmware log payload buffer. + pub(crate) fn get_buf(&self) -> GpuWeakPointer<[RawFwLogPayloadMsg]> { + self.payload_buf.weak_pointer() + } + + /// Polls for new log messages on all sub-rings. + pub(crate) fn poll(&mut self) { + for i in 0..=FwLogChannelState::SUB_CHANNELS - 1 { + while let Some(msg) = self.ch.peek(i) { + cls_dev_dbg!(FwLogCh, self.dev, "FwLog{}: {:?}\n", i, msg); + if msg.msg_type != 2 { + dev_warn!(self.dev.as_ref(), "Unknown FWLog{} message: {:?}\n", i, msg); + self.ch.get(i); + continue; + } + if msg.msg_index.0 as usize >= Self::BUF_SIZE { + dev_warn!( + self.dev.as_ref(), + "FWLog{} message index out of bounds: {:?}\n", + i, + msg + ); + self.ch.get(i); + continue; + } + let index = Self::BUF_SIZE * i + msg.msg_index.0 as usize; + let payload = &self.payload_buf.as_slice()[index]; + if payload.msg_type != 3 { + dev_warn!( + self.dev.as_ref(), + "Unknown FWLog{} payload: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + } + let msg = if let Some(end) = payload.msg.iter().position(|&r| r == 0) { + CStr::from_bytes_with_nul(&(*payload.msg)[..end + 1]) + .unwrap_or(c_str!("cstr_err")) + } else { + dev_warn!( + self.dev.as_ref(), + "FWLog{} payload not NUL-terminated: {:?}\n", + i, + payload + ); + self.ch.get(i); + continue; + }; + match i { + 0 => dev_dbg!(self.dev.as_ref(), "FWLog: {}\n", msg), + 1 => dev_info!(self.dev.as_ref(), "FWLog: {}\n", msg), + 2 => dev_notice!(self.dev.as_ref(), "FWLog: {}\n", msg), + 3 => dev_warn!(self.dev.as_ref(), "FWLog: {}\n", msg), + 4 => dev_err!(self.dev.as_ref(), "FWLog: {}\n", msg), + 5 => dev_crit!(self.dev.as_ref(), "FWLog: {}\n", msg), + _ => (), + }; + self.ch.get(i); + } + } + } +} + +pub(crate) struct KTraceChannel { + dev: AsahiDevRef, + ch: RxChannel, +} + +/// KTrace channel, used to receive detailed execution trace markers from the firmware. +/// We currently disable this in initdata, so no messages are expected here at this time. +impl KTraceChannel { + /// Allocate a new KTrace channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(KTraceChannel { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x200)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new KTrace messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + cls_dev_dbg!(KTraceCh, self.dev, "KTrace: {:?}\n", msg); + } + } +} + +/// Statistics channel, reporting power-related statistics to the driver. +/// Not really implemented other than debug logs yet... +#[versions(AGX)] +pub(crate) struct StatsChannel { + dev: AsahiDevRef, + ch: RxChannel, +} + +#[versions(AGX)] +impl StatsChannel::ver { + /// Allocate a new Statistics channel. + pub(crate) fn new( + dev: &AsahiDevice, + alloc: &mut gpu::KernelAllocators, + ) -> Result { + Ok(StatsChannel::ver { + dev: dev.into(), + ch: RxChannel::::new(alloc, 0x100)?, + }) + } + + /// Returns the raw `ChannelRing` structure to pass to firmware. + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + self.ch.ring.to_raw() + } + + /// Polls for new statistics messages on this ring. + pub(crate) fn poll(&mut self) { + while let Some(msg) = self.ch.get(0) { + // SAFETY: The raw view is always valid for all bit patterns. + let tag = unsafe { msg.raw.0 }; + match tag { + 0..=STATS_MAX::ver => { + // SAFETY: Since we have checked the tag to be in range, + // accessing the enum view is valid. + let msg = unsafe { msg.msg }; + cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); + } + _ => { + // SAFETY: The raw view is always valid for all bit patterns. + pr_warn!("Unknown stats message: {:?}\n", unsafe { msg.raw }); + } + } + } + } +} diff --git a/drivers/gpu/drm/asahi/crashdump.rs b/drivers/gpu/drm/asahi/crashdump.rs new file mode 100644 index 00000000000000..bd9f2f1649584f --- /dev/null +++ b/drivers/gpu/drm/asahi/crashdump.rs @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU crash dump formatter +//! +//! Takes a raw dump of firmware/kernel mapped pages from `pgtable` and formats it into +//! an ELF core dump suitable for dumping into userspace. + +use core::mem::size_of; + +use kernel::{ + devcoredump::DevCoreDump, + error::Result, + page::{Page, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE}, + prelude::*, + types::Owned, + uapi, +}; + +use crate::hw; +use crate::pgtable::{self, DumpedPage, Prot, UAT_PGSZ}; +use crate::util::align; + +pub(crate) struct CrashDump { + headers: KVVec, + pages: KVVec>, +} + +const NOTE_NAME_AGX: &str = "AGX"; +const NOTE_AGX_DUMP_INFO: u32 = 1; + +const NOTE_NAME_RTKIT: &str = "RTKIT"; +const NOTE_RTKIT_CRASHLOG: u32 = 1; + +#[repr(C)] +pub(crate) struct AGXDumpInfo { + initdata_address: u64, + chip_id: u32, + gpu_gen: hw::GpuGen, + gpu_variant: hw::GpuVariant, + gpu_rev: hw::GpuRevision, + total_active_cores: u32, + firmware_version: [u32; 6], +} + +struct ELFNote { + name: &'static str, + ty: u32, + data: KVVec, +} + +pub(crate) struct CrashDumpBuilder { + page_dump: KVVec, + notes: KVec, +} + +/// Helper to convert ELF headers into byte slices +/// TODO: Hook this up into kernel::AsBytes somehow +/// +/// # Safety +/// +/// Types implementing this trait must have no padding bytes. +unsafe trait AsBytes: Sized { + fn as_bytes(&self) -> &[u8] { + // SAFETY: This trait is only implemented for types with no padding bytes + unsafe { core::slice::from_raw_parts(self as *const _ as *const u8, size_of::()) } + } + fn slice_as_bytes(slice: &[Self]) -> &[u8] { + // SAFETY: This trait is only implemented for types with no padding bytes + unsafe { + core::slice::from_raw_parts(slice.as_ptr() as *const u8, core::mem::size_of_val(slice)) + } + } +} + +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Ehdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Phdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for uapi::Elf64_Nhdr {} +// SAFETY: This type has no padding +unsafe impl AsBytes for AGXDumpInfo {} + +const FIRMWARE_ENTRYPOINT: u64 = 0xFFFFFF8000000000u64; + +impl CrashDumpBuilder { + pub(crate) fn new(page_dump: KVVec) -> Result { + Ok(CrashDumpBuilder { + page_dump, + notes: KVec::new(), + }) + } + + pub(crate) fn add_agx_info( + &mut self, + cfg: &hw::HwConfig, + dyncfg: &hw::DynConfig, + initdata_address: u64, + ) -> Result { + let mut info = AGXDumpInfo { + chip_id: cfg.chip_id, + gpu_gen: dyncfg.id.gpu_gen, + gpu_variant: dyncfg.id.gpu_variant, + gpu_rev: dyncfg.id.gpu_rev, + total_active_cores: dyncfg.id.total_active_cores, + firmware_version: [0; 6], + initdata_address, + }; + info.firmware_version[..dyncfg.firmware_version.len().min(6)] + .copy_from_slice(&dyncfg.firmware_version); + + let mut data = KVVec::new(); + data.extend_from_slice(info.as_bytes(), GFP_KERNEL)?; + + self.notes.push( + ELFNote { + name: NOTE_NAME_AGX, + ty: NOTE_AGX_DUMP_INFO, + data, + }, + GFP_KERNEL, + )?; + Ok(()) + } + + pub(crate) fn add_crashlog(&mut self, crashlog: &[u8]) -> Result { + let mut data = KVVec::new(); + data.extend_from_slice(crashlog, GFP_KERNEL)?; + + self.notes.push( + ELFNote { + name: NOTE_NAME_RTKIT, + ty: NOTE_RTKIT_CRASHLOG, + data, + }, + GFP_KERNEL, + )?; + + Ok(()) + } + + pub(crate) fn finalize(self) -> Result { + let CrashDumpBuilder { page_dump, notes } = self; + + let mut ehdr: uapi::Elf64_Ehdr = Default::default(); + + ehdr.e_ident[uapi::EI_MAG0 as usize..=uapi::EI_MAG3 as usize].copy_from_slice(b"\x7fELF"); + ehdr.e_ident[uapi::EI_CLASS as usize] = uapi::ELFCLASS64 as u8; + ehdr.e_ident[uapi::EI_DATA as usize] = uapi::ELFDATA2LSB as u8; + ehdr.e_ident[uapi::EI_VERSION as usize] = uapi::EV_CURRENT as u8; + ehdr.e_type = uapi::ET_CORE as u16; + ehdr.e_machine = uapi::EM_AARCH64 as u16; + ehdr.e_version = uapi::EV_CURRENT; + ehdr.e_entry = FIRMWARE_ENTRYPOINT; + ehdr.e_ehsize = core::mem::size_of::() as u16; + ehdr.e_phentsize = core::mem::size_of::() as u16; + + let phdr_offset = core::mem::size_of::(); + + // PHDRs come after the ELF header + ehdr.e_phoff = phdr_offset as u64; + + let mut phdrs = KVVec::new(); + + // First PHDR is the NOTE section + phdrs.push( + uapi::Elf64_Phdr { + p_type: uapi::PT_NOTE, + p_flags: uapi::PF_R, + p_align: 1, + ..Default::default() + }, + GFP_KERNEL, + )?; + + // Generate the page phdrs. The offset will be fixed up later. + let mut off: usize = 0; + let mut next = None; + let mut pages: KVVec> = KVVec::new(); + + for mut page in page_dump { + let vaddr = page.iova; + let paddr = page.pte & pgtable::PTE_ADDR_BITS; + let flags = Prot::from_pte(page.pte).elf_flags(); + let valid = page.data.is_some(); + let cur = (vaddr, paddr, flags, valid); + if Some(cur) != next { + phdrs.push( + uapi::Elf64_Phdr { + p_type: uapi::PT_LOAD, + p_offset: if valid { off as u64 } else { 0 }, + p_vaddr: vaddr, + p_paddr: paddr, + p_filesz: if valid { UAT_PGSZ as u64 } else { 0 }, + p_memsz: UAT_PGSZ as u64, + p_flags: flags, + p_align: UAT_PGSZ as u64, + }, + GFP_KERNEL, + )?; + if valid { + off += UAT_PGSZ; + } + } else { + let ph = phdrs.last_mut().unwrap(); + ph.p_memsz += UAT_PGSZ as u64; + if valid { + ph.p_filesz += UAT_PGSZ as u64; + off += UAT_PGSZ; + } + } + if let Some(data_page) = page.data.take() { + pages.push(data_page, GFP_KERNEL)?; + } + next = Some(( + vaddr + UAT_PGSZ as u64, + paddr + UAT_PGSZ as u64, + flags, + valid, + )); + } + + ehdr.e_phnum = phdrs.len() as u16; + + let note_offset = phdr_offset + size_of::() * phdrs.len(); + + let mut note_data: KVVec = KVVec::new(); + + for note in notes { + let hdr = uapi::Elf64_Nhdr { + n_namesz: note.name.len() as u32 + 1, + n_descsz: note.data.len() as u32, + n_type: note.ty, + }; + note_data.extend_from_slice(hdr.as_bytes(), GFP_KERNEL)?; + note_data.extend_from_slice(note.name.as_bytes(), GFP_KERNEL)?; + note_data.push(0, GFP_KERNEL)?; + while note_data.len() & 3 != 0 { + note_data.push(0, GFP_KERNEL)?; + } + note_data.extend_from_slice(¬e.data, GFP_KERNEL)?; + while note_data.len() & 3 != 0 { + note_data.push(0, GFP_KERNEL)?; + } + } + + // NOTE section comes after the PHDRs + phdrs[0].p_offset = note_offset as u64; + phdrs[0].p_filesz = note_data.len() as u64; + + // Align data section to the page size + let data_offset = align(note_offset + note_data.len(), UAT_PGSZ); + + // Fix up data PHDR offsets + for phdr in &mut phdrs[1..] { + phdr.p_offset += data_offset as u64; + } + + // Build ELF header buffer + let mut headers: KVVec = KVVec::from_elem(0, data_offset, GFP_KERNEL)?; + + headers[0..size_of::()].copy_from_slice(ehdr.as_bytes()); + headers[phdr_offset..phdr_offset + phdrs.len() * size_of::()] + .copy_from_slice(AsBytes::slice_as_bytes(&phdrs)); + headers[note_offset..note_offset + note_data.len()].copy_from_slice(¬e_data); + + Ok(CrashDump { headers, pages }) + } +} + +impl DevCoreDump for CrashDump { + fn read(&self, buf: &mut [u8], mut offset: usize) -> Result { + let mut read = 0; + let mut left = buf.len(); + if offset < self.headers.len() { + let block = left.min(self.headers.len() - offset); + buf[..block].copy_from_slice(&self.headers[offset..offset + block]); + read += block; + offset += block; + left -= block; + } + if left == 0 { + return Ok(read); + } + offset -= self.headers.len(); // Offset from the page area + + while left > 0 { + let page_index = offset >> PAGE_SHIFT; + let page_offset = offset & !PAGE_MASK; + let block = left.min(PAGE_SIZE - page_offset); + let Some(page) = self.pages.get(page_index) else { + break; + }; + let slice = &mut buf[read..read + block]; + // SAFETY: We own the page, and the slice guarantees the + // dst length is sufficient. + unsafe { page.read_raw(slice.as_mut_ptr(), page_offset, slice.len())? }; + read += block; + offset += block; + left -= block; + } + + Ok(read) + } +} diff --git a/drivers/gpu/drm/asahi/debug.rs b/drivers/gpu/drm/asahi/debug.rs new file mode 100644 index 00000000000000..50628ade5ab8e4 --- /dev/null +++ b/drivers/gpu/drm/asahi/debug.rs @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(dead_code)] + +//! Debug enable/disable flags and convenience macros + +#[allow(unused_imports)] +pub(crate) use super::{ + cls_dev_dbg, + cls_pr_debug, + debug, + mod_dev_dbg, + mod_pr_debug, // +}; +use crate::module_parameters; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +static DEBUG_FLAGS: AtomicU64 = AtomicU64::new(0); + +/// Debug flag bit indices +pub(crate) enum DebugFlags { + // 0-4: Memory-related debug + Mmu = 0, + PgTable = 1, + Alloc = 2, + Gem = 3, + Object = 4, + + // 5-7: Firmware objects and resources + Event = 5, + Buffer = 6, + WorkQueue = 7, + + // 8-13: DRM interface, rendering, compute, GPU globals + Gpu = 8, + File = 9, + Queue = 10, + Render = 11, + Compute = 12, + Errors = 13, + + // 14-15: Misc stats + MemStats = 14, + TVBStats = 15, + + // 16-22: Channels + FwLogCh = 16, + KTraceCh = 17, + StatsCh = 18, + EventCh = 19, + PipeCh = 20, + DeviceControlCh = 21, + FwCtlCh = 22, + + // 32-35: Allocator debugging + FillAllocations = 32, + DebugAllocations = 33, + DetectOverflows = 34, + ForceCPUMaps = 35, + + // 36-: Behavior flags + ConservativeTlbi = 36, + KeepGpuPowered = 37, + WaitForPowerOff = 38, + NoGpuRecovery = 39, + DisableClustering = 40, + + // 48-: Misc + Debug0 = 48, + Debug1 = 49, + Debug2 = 50, + Debug3 = 51, + Debug4 = 52, + Debug5 = 53, + Debug6 = 54, + Debug7 = 55, + + VerboseFaults = 61, + AllowUnknownOverrides = 62, + OopsOnGpuCrash = 63, +} + +/// Update the cached global debug flags from the module parameter +pub(crate) fn update_debug_flags() { + let flags = *module_parameters::debug_flags.value(); + + DEBUG_FLAGS.store(flags, Ordering::Relaxed); +} + +/// Check whether debug is enabled for a given flag +#[inline(always)] +pub(crate) fn debug_enabled(flag: DebugFlags) -> bool { + DEBUG_FLAGS.load(Ordering::Relaxed) & 1 << (flag as usize) != 0 +} + +/// Run some code only if debug is enabled for the calling module +#[macro_export] +macro_rules! debug { + ($($arg:tt)*) => { + if $crate::debug::debug_enabled(DEBUG_CLASS) { + $($arg)* + } + }; +} + +/// pr_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_pr_debug ( + ($($arg:tt)*) => ( + $crate::debug! { ::kernel::pr_info! ( $($arg)* ); } + ) +); + +/// dev_info!() if debug is enabled for the calling module +#[macro_export] +macro_rules! mod_dev_dbg ( + ($dev:expr, $($arg:tt)*) => ( + $crate::debug! { ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); } + ) +); + +/// pr_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_pr_debug ( + ($cls:ident, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::pr_info! ( $($arg)* ); + } + ) +); + +/// dev_info!() if debug is enabled for a specific module +#[macro_export] +macro_rules! cls_dev_dbg ( + ($cls:ident, $dev:expr, $($arg:tt)*) => ( + if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { + ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); + } + ) +); diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs new file mode 100644 index 00000000000000..03245a4826be71 --- /dev/null +++ b/drivers/gpu/drm/asahi/driver.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU driver implementation. + +use kernel::{ + c_str, + device::Core, + dma::{ + Device, + DmaMask, // + }, + drm, + drm::ioctl, + of, + platform, + prelude::*, + sync::Arc, // +}; + +use crate::{ + debug, + file, + gem::AsahiObject, + gpu, + hw, + regs, // +}; + +use kernel::macros::vtable; +use kernel::types::ARef; + +/// Holds a reference to the top-level `GpuManager` object. +#[pin_data] +pub(crate) struct AsahiData { + #[pin] + pub(crate) gpu: Arc, + pub(crate) pdev: ARef, + pub(crate) resources: regs::Resources, +} + +unsafe impl Send for AsahiData {} +unsafe impl Sync for AsahiData {} + +pub(crate) struct AsahiDriver { + #[expect(unused)] + drm: ARef>, +} + +unsafe impl Send for AsahiDriver {} +unsafe impl Sync for AsahiDriver {} + +/// Convenience type alias for the DRM device type for this driver. +pub(crate) type AsahiDevice = drm::device::Device; +pub(crate) type AsahiDevRef = ARef; + +/// DRM Driver metadata +const INFO: drm::driver::DriverInfo = drm::driver::DriverInfo { + major: 0, + minor: 0, + patchlevel: 0, + name: c_str!("asahi"), + desc: c_str!("Apple AGX Graphics"), +}; + +/// DRM Driver implementation for `AsahiDriver`. +#[vtable] +impl drm::driver::Driver for AsahiDriver { + /// Our `DeviceData` type, reference-counted + type Data = AsahiData; + /// Our `File` type. + type File = file::File; + /// Our `Object` type. + type Object = drm::gem::shmem::Object; + + const INFO: drm::driver::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM + | drm::driver::FEAT_RENDER + | drm::driver::FEAT_SYNCOBJ + | drm::driver::FEAT_SYNCOBJ_TIMELINE + | drm::driver::FEAT_GEM_GPUVA; + + kernel::declare_drm_ioctls! { + (ASAHI_GET_PARAMS, drm_asahi_get_params, + ioctl::RENDER_ALLOW, crate::file::File::get_params), + (ASAHI_GET_TIME, drm_asahi_get_time, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::get_time), + (ASAHI_VM_CREATE, drm_asahi_vm_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_create), + (ASAHI_VM_DESTROY, drm_asahi_vm_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_destroy), + (ASAHI_VM_BIND, drm_asahi_vm_bind, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_bind), + (ASAHI_GEM_CREATE, drm_asahi_gem_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_create), + (ASAHI_GEM_MMAP_OFFSET, drm_asahi_gem_mmap_offset, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_mmap_offset), + (ASAHI_GEM_BIND_OBJECT, drm_asahi_gem_bind_object, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_bind_object), + (ASAHI_QUEUE_CREATE, drm_asahi_queue_create, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_create), + (ASAHI_QUEUE_DESTROY, drm_asahi_queue_destroy, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), + (ASAHI_SUBMIT, drm_asahi_submit, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), + } +} + +// OF Device ID table.s +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + ( + of::DeviceId::new(c_str!("apple,agx-t8103")), + &hw::t8103::HWCONFIG + ), + ( + of::DeviceId::new(c_str!("apple,agx-t8112")), + &hw::t8112::HWCONFIG + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6000")), + &hw::t600x::HWCONFIG_T6000 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6001")), + &hw::t600x::HWCONFIG_T6001 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6002")), + &hw::t600x::HWCONFIG_T6002 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6020")), + &hw::t602x::HWCONFIG_T6020 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6021")), + &hw::t602x::HWCONFIG_T6021 + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6022")), + &hw::t602x::HWCONFIG_T6022 + ), + ] +); + +/// Platform Driver implementation for `AsahiDriver`. +impl platform::Driver for AsahiDriver { + type IdInfo = &'static hw::HwConfig; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + /// Device probe function. + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { + debug::update_debug_flags(); + + dev_info!(pdev.as_ref(), "Probing...\n"); + + let cfg = info.ok_or(ENODEV)?; + + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + + let res = regs::Resources::new(pdev)?; + + // Initialize misc MMIO + res.init_mmio()?; + + // Start the coprocessor CPU, so UAT can initialize the handoff + regs::Resources::start_cpu(pdev)?; + + let fwnode = pdev.as_ref().fwnode().ok_or(EIO)?; + let compat: KVec = fwnode + .property_read_array_vec(c_str!("apple,firmware-compat"), 3)? + .required_by(pdev.as_ref())?; + + // TODO: This is very temporary + // SAFETY: This should be safe as data is not touched by the driver + // untill it gets fully initialised. + // Additionally drm::device::Device::release() will not drop data and + // leaks instead. + let uninit = unsafe { + pin_init::pin_init_from_closure::(|_slot| Ok(())) + }; + let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + + let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { + (hw::GpuGen::G13, _, &[12, 3, 0]) => { + gpu::GpuManagerG13V12_3::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { + gpu::GpuManagerG14V12_4::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G13, _, &[13, 5, 0]) => { + gpu::GpuManagerG13V13_5::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 5, 0]) => { + gpu::GpuManagerG14V13_5::new(&drm.clone(), &res, cfg)? as Arc + } + (hw::GpuGen::G14, _, &[13, 5, 0]) => { + gpu::GpuManagerG14XV13_5::new(&drm.clone(), &res, cfg)? as Arc + } + _ => { + dev_info!( + pdev.as_ref(), + "Unsupported GPU/firmware combination ({:?}, {:?}, {:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + compat + ); + return Err(ENODEV); + } + }; + + let data = try_pin_init!(AsahiData { + gpu, + pdev: pdev.into(), + resources: res, + }); + + let ptr: *const AsahiData = &raw const **drm; + unsafe { + data.__pinned_init(ptr as *mut AsahiData)?; + } + + (*drm).gpu.init()?; + + drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0)?; + + Ok(Self { drm }) + } +} diff --git a/drivers/gpu/drm/asahi/event.rs b/drivers/gpu/drm/asahi/event.rs new file mode 100644 index 00000000000000..edd7d701e665cd --- /dev/null +++ b/drivers/gpu/drm/asahi/event.rs @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU event manager +//! +//! The GPU firmware manages work completion by using event objects (Apple calls them "stamps"), +//! which are monotonically incrementing counters. There are a fixed number of objects, and +//! they are managed with a `SlotAllocator`. +//! +//! This module manages the set of available events and lets users compute expected values. +//! It also manages signaling owners when the GPU firmware reports that an event fired. + +use crate::debug::*; +use crate::fw::types::*; +use crate::{ + gpu, + slotalloc, + workqueue, // +}; +use core::cmp; +use core::sync::atomic::Ordering; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::{ + c_str, + static_lock_class, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Event; + +/// Number of events managed by the firmware. +const NUM_EVENTS: u32 = 128; + +/// Inner data associated with a given event slot. +pub(crate) struct EventInner { + /// CPU pointer to the driver notification event stamp + stamp: *const AtomicU32, + /// GPU pointer to the driver notification event stamp + gpu_stamp: GpuWeakPointer, + /// GPU pointer to the firmware-internal event stamp + gpu_fw_stamp: GpuWeakPointer, +} + +/// SAFETY: The event slots are safe to send across threads. +unsafe impl Send for EventInner {} + +/// Alias for an event token, which allows requesting the same event. +pub(crate) type Token = slotalloc::SlotToken; +/// Alias for an allocated `Event` that has a slot. +pub(crate) type Event = slotalloc::Guard; + +/// Represents a given stamp value for an event. +#[derive(Eq, PartialEq, Copy, Clone, Debug)] +#[repr(transparent)] +pub(crate) struct EventValue(u32); + +impl EventValue { + /// Returns the `EventValue` that succeeds this one. + pub(crate) fn next(&self) -> EventValue { + EventValue(self.0.wrapping_add(0x100)) + } + + /// Increments this `EventValue` in place. + pub(crate) fn increment(&mut self) { + self.0 = self.0.wrapping_add(0x100); + } + + /* Not used + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn add(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_add(val.checked_mul(0x100).expect("Adding too many events")); + } + */ + + /// Increments this `EventValue` in place by a certain count. + pub(crate) fn sub(&mut self, val: u32) { + self.0 = self + .0 + .wrapping_sub(val.checked_mul(0x100).expect("Subtracting too many events")); + } + + /// Computes the delta between this event and another event. + pub(crate) fn delta(&self, other: &EventValue) -> i32 { + (self.0.wrapping_sub(other.0) as i32) >> 8 + } +} + +impl PartialOrd for EventValue { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for EventValue { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.delta(other).cmp(&0) + } +} + +impl EventInner { + /// Returns the GPU pointer to the driver notification stamp + pub(crate) fn stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_stamp + } + + /// Returns the GPU pointer to the firmware internal stamp + pub(crate) fn fw_stamp_pointer(&self) -> GpuWeakPointer { + self.gpu_fw_stamp + } + + /// Fetches the current event value from shared memory + pub(crate) fn current(&self) -> EventValue { + // SAFETY: The pointer is always valid as constructed in + // EventManager below, and outside users cannot construct + // new EventInners, nor move or copy them, and Guards as + // returned by the SlotAllocator hold a reference to the + // SlotAllocator containing the EventManagerInner, which + // keeps the GpuObject the stamp is contained within alive. + EventValue(unsafe { &*self.stamp }.load(Ordering::Acquire)) + } +} + +impl slotalloc::SlotItem for EventInner { + type Data = EventManagerInner; + + fn release(&mut self, data: &mut Self::Data, slot: u32) { + mod_pr_debug!("EventManager: Released slot {}\n", slot); + data.owners[slot as usize] = None; + } +} + +/// Inner data for the event manager, to be protected by the SlotAllocator lock. +pub(crate) struct EventManagerInner { + stamps: GpuArray, + fw_stamps: GpuArray, + // Note: Use dyn to avoid having to version this entire module. + owners: KVec>>, +} + +/// Top-level EventManager object. +pub(crate) struct EventManager { + alloc: slotalloc::SlotAllocator, +} + +impl EventManager { + /// Create a new EventManager. + #[inline(never)] + pub(crate) fn new(alloc: &mut gpu::KernelAllocators) -> Result { + let mut owners = KVec::new(); + for _i in 0..(NUM_EVENTS as usize) { + owners.push(None, GFP_KERNEL)?; + } + let inner = EventManagerInner { + stamps: alloc.shared.array_empty(NUM_EVENTS as usize)?, + fw_stamps: alloc.private.array_empty(NUM_EVENTS as usize)?, + owners, + }; + + for slot in 0..NUM_EVENTS { + inner.stamps[slot as usize] + .0 + .store(slot << 24, Ordering::Relaxed); + } + + Ok(EventManager { + alloc: slotalloc::SlotAllocator::new( + NUM_EVENTS, + inner, + |inner: &mut EventManagerInner, slot| { + Some(EventInner { + stamp: &inner.stamps[slot as usize].0, + gpu_stamp: inner.stamps.weak_item_pointer(slot as usize), + gpu_fw_stamp: inner.fw_stamps.weak_item_pointer(slot as usize), + }) + }, + c_str!("EventManager::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }) + } + + /// Gets a free `Event`, optionally trying to reuse the last one allocated by this caller. + pub(crate) fn get( + &self, + token: Option, + owner: Arc, + ) -> Result { + let ev = self.alloc.get_inner(token, |inner, ev| { + mod_pr_debug!( + "EventManager: Registered owner {:p} on slot {}\n", + &*owner, + ev.slot() + ); + inner.owners[ev.slot() as usize] = Some(owner); + Ok(()) + })?; + Ok(ev) + } + + /// Signals an event by slot, indicating completion (of one or more commands). + pub(crate) fn signal(&self, slot: u32) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.signal(); + } + None => { + mod_pr_debug!("EventManager: Received event for empty slot {}\n", slot); + } + } + } + + /// Marks the owner of an event as having lost its work due to a GPU error. + pub(crate) fn mark_error(&self, slot: u32, wait_value: u32, error: workqueue::WorkError) { + match self + .alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + { + Some(owner) => { + owner.mark_error(EventValue(wait_value), error); + } + None => { + pr_err!("Received error for empty slot {}\n", slot); + } + } + } + + /// Returns a reference to the workqueue owning an event. + pub(crate) fn get_owner( + &self, + slot: u32, + ) -> Option> { + self.alloc + .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) + } + + /// Fail all commands, used when the GPU crashes. + pub(crate) fn fail_all(&self, error: workqueue::WorkError) { + let mut owners: KVec> = KVec::new(); + + self.alloc.with_inner(|inner| { + for wq in inner.owners.iter().filter_map(|o| o.as_ref()).cloned() { + if owners.push(wq, GFP_KERNEL).is_err() { + pr_err!("Failed to signal failure to WorkQueue\n"); + } + } + }); + + for wq in owners { + wq.fail_all(error); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs new file mode 100644 index 00000000000000..02a4ce874a92c8 --- /dev/null +++ b/drivers/gpu/drm/asahi/file.rs @@ -0,0 +1,1092 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! File implementation, which represents a single DRM client. +//! +//! This is in charge of managing the resources associated with one GPU client, including an +//! arbitrary number of submission queues and Vm objects, and reporting hardware/driver +//! information to userspace and accepting submissions. + +use crate::debug::*; +use crate::driver::AsahiDevice; +use crate::{ + alloc, + buffer, + driver, + gem, + mmu, + module_parameters, + queue, + util::{ + align, + align_down, + gcd, + AnyBitPattern, + RangeExt, + Reader, // + }, // +}; +use core::mem::MaybeUninit; +use core::ops::Deref; +use core::ops::Range; +use core::ptr::addr_of_mut; +use kernel::bindings; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::gem::BaseObject; +use kernel::error::code::*; +use kernel::new_mutex; +use kernel::prelude::*; +use kernel::sync::{ + Arc, + Mutex, // +}; +use kernel::time::NSEC_PER_SEC; +use kernel::uaccess::{ + UserPtr, + UserSlice, // +}; +use kernel::{ + dma_fence, + drm, + uapi, + xarray, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::File; + +pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; + +/// A client instance of an `mmu::Vm` address space. +struct Vm { + ualloc: Arc>, + ualloc_priv: Arc>, + vm: mmu::Vm, + kernel_range: Range, + _dummy_mapping: mmu::KernelMapping, +} + +impl Drop for Vm { + fn drop(&mut self) { + // When the user Vm is dropped, unmap everything in the user range + let left_range = VM_USER_RANGE.start..self.kernel_range.start; + let right_range = self.kernel_range.end..VM_USER_RANGE.end; + + if !left_range.is_empty() + && self + .vm + .unmap_range(left_range.start, left_range.range()) + .is_err() + { + pr_err!("Vm::Drop: vm.unmap_range() failed\n"); + } + if !right_range.is_empty() + && self + .vm + .unmap_range(right_range.start, right_range.range()) + .is_err() + { + pr_err!("Vm::Drop: vm.unmap_range() failed\n"); + } + } +} + +/// Sync object from userspace. +pub(crate) struct SyncItem { + pub(crate) syncobj: drm::syncobj::SyncObj, + pub(crate) fence: Option, + pub(crate) chain_fence: Option, + pub(crate) timeline_value: u64, +} + +impl SyncItem { + fn parse_one(file: &DrmFile, data: uapi::drm_asahi_sync, out: bool) -> Result { + match data.sync_type { + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_SYNCOBJ => { + if data.timeline_value != 0 { + cls_pr_debug!(Errors, "Non-timeline sync object with a nonzero value\n"); + return Err(EINVAL); + } + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + + Ok(SyncItem { + fence: if out { + None + } else { + Some(syncobj.fence_get().ok_or_else(|| { + cls_pr_debug!(Errors, "Failed to get fence from sync object\n"); + EINVAL + })?) + }, + syncobj, + chain_fence: None, + timeline_value: data.timeline_value, + }) + } + uapi::drm_asahi_sync_type_DRM_ASAHI_SYNC_TIMELINE_SYNCOBJ => { + let syncobj = drm::syncobj::SyncObj::lookup_handle(file, data.handle)?; + let fence = if out { + None + } else { + syncobj + .fence_get() + .ok_or_else(|| { + cls_pr_debug!( + Errors, + "Failed to get fence from timeline sync object\n" + ); + EINVAL + })? + .chain_find_seqno(data.timeline_value)? + }; + + Ok(SyncItem { + fence, + syncobj, + chain_fence: if out { + Some(dma_fence::FenceChain::new()?) + } else { + None + }, + timeline_value: data.timeline_value, + }) + } + _ => { + cls_pr_debug!(Errors, "Invalid sync type {}\n", data.sync_type); + Err(EINVAL) + } + } + } + + fn parse_array( + file: &DrmFile, + ptr: u64, + in_count: u32, + out_count: u32, + ) -> Result> { + let count = in_count + out_count; + let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + + const STRIDE: usize = core::mem::size_of::(); + let size = STRIDE * count as usize; + + // SAFETY: We only read this once, so there are no TOCTOU issues. + let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); + + for i in 0..count { + let mut sync: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: The size of `sync` is STRIDE + reader.read_raw(unsafe { + core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) + })?; + + // SAFETY: All bit patterns in the struct are valid + let sync = unsafe { sync.assume_init() }; + + vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; + } + + Ok(vec) + } +} + +#[derive(Clone)] +pub(crate) enum Object { + TimestampBuffer(Arc), +} + +/// State associated with a client. +// #[pin_data] +pub(crate) struct File { + id: u64, + // #[pin] + vms: xarray::XArray>, + // #[pin] + queues: xarray::XArray>>>, + // #[pin] + objects: xarray::XArray>, +} + +/// Convenience type alias for our DRM `File` type. +pub(crate) type DrmFile = drm::File; + +/// Available VM range for the user +const VM_USER_RANGE: Range = mmu::IOVA_USER_USABLE_RANGE; + +/// Minimum reserved AS for kernel mappings +const VM_KERNEL_MIN_SIZE: u64 = 0x20000000; + +impl drm::file::DriverFile for File { + type Driver = driver::AsahiDriver; + + /// Create a new `File` instance for a fresh client. + fn open(device: &AsahiDevice) -> Result>> { + debug::update_debug_flags(); + + let gpu = &device.gpu; + let id = gpu.ids().file.next(); + + mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); + Ok(KBox::pin_init(File::new(id), GFP_KERNEL)?) + } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } +} + +// SAFETY: All bit patterns are valid by construction. +unsafe impl AnyBitPattern for uapi::drm_asahi_gem_bind_op {} + +impl File { + fn new(id: u64) -> impl PinInit { + unsafe { + pin_init::pin_init_from_closure(move |slot: *mut Self| { + let raw_vms = addr_of_mut!((*slot).vms); + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_vms)?; + + let raw_queues = addr_of_mut!((*slot).queues); + xarray::XArray::>>>::new( + xarray::AllocKind::Alloc1, + ) + .__pinned_init(raw_queues)?; + + let raw_objects = addr_of_mut!((*slot).objects); + xarray::XArray::>::new(xarray::AllocKind::Alloc1) + .__pinned_init(raw_objects)?; + + (*slot).id = id; + Ok(()) + }) + } + } + + fn vms(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for vms. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.vms) } + } + + #[allow(clippy::type_complexity)] + fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { + // SAFETY: Structural pinned projection for queues. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.queues) } + } + + fn objects(self: Pin<&Self>) -> Pin<&xarray::XArray>> { + // SAFETY: Structural pinned projection for objects. + // We never move out of this field. + unsafe { self.map_unchecked(|s| &s.objects) } + } + + /// IOCTL: get_param: Get a driver parameter value. + pub(crate) fn get_params( + device: &AsahiDevice, + data: &uapi::drm_asahi_get_params, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!(device, "[File {}]: IOCTL: get_params\n", file.inner().id); + + let gpu = &device.gpu; + + if data.param_group != 0 || data.pad != 0 { + cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); + return Err(EINVAL); + } + + if gpu.is_crashed() { + return Err(ENODEV); + } + + let mut params = uapi::drm_asahi_params_global { + features: 0, + + gpu_generation: gpu.get_dyncfg().id.gpu_gen as u32, + gpu_variant: gpu.get_dyncfg().id.gpu_variant as u32, + gpu_revision: gpu.get_dyncfg().id.gpu_rev as u32, + chip_id: gpu.get_cfg().chip_id, + + num_dies: gpu.get_cfg().num_dies, + num_clusters_total: gpu.get_dyncfg().id.num_clusters, + num_cores_per_cluster: gpu.get_dyncfg().id.num_cores, + core_masks: [0; uapi::DRM_ASAHI_MAX_CLUSTERS as usize], + + vm_start: VM_USER_RANGE.start, + vm_end: VM_USER_RANGE.end, + vm_kernel_min_size: VM_KERNEL_MIN_SIZE, + + max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, + max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, + max_frequency_khz: gpu.get_dyncfg().pwr.max_frequency_khz(), + + command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds + }; + + for (i, mask) in gpu.get_dyncfg().id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + if *module_parameters::fault_control.value() == 0xb { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + } + + let size = core::mem::size_of::().min(data.size.try_into()?); + + // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. + let mut params_writer = + UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); + + // SAFETY: `size` is at most the sizeof of `params` + params_writer.write_slice(unsafe { + core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) + })?; + + Ok(0) + } + + /// IOCTL: vm_create: Create a new `Vm`. + pub(crate) fn vm_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_vm_create, + file: &DrmFile, + ) -> Result { + let kernel_range = data.kernel_start..data.kernel_end; + + // Validate requested kernel range + if !VM_USER_RANGE.is_superset(kernel_range.clone()) + || kernel_range.range() < VM_KERNEL_MIN_SIZE + || kernel_range.start & (mmu::UAT_PGMSK as u64) != 0 + || kernel_range.end & (mmu::UAT_PGMSK as u64) != 0 + { + cls_pr_debug!(Errors, "vm_create: Invalid kernel range\n"); + return Err(EINVAL); + } + + // Align to buffer::PAGE_SIZE so the allocators are happy + let kernel_range = align(kernel_range.start, buffer::PAGE_SIZE as u64) + ..align_down(kernel_range.end, buffer::PAGE_SIZE as u64); + + let kernel_half_size = align_down(kernel_range.range() >> 1, buffer::PAGE_SIZE as u64); + let kernel_gpu_range = kernel_range.start..(kernel_range.start + kernel_half_size); + let kernel_gpufw_range = kernel_gpu_range.end..kernel_range.end; + + let gpu = &device.gpu; + let file_id = file.inner().id; + let vm = gpu.new_vm(kernel_range.clone())?; + + let vm_xa = file.inner().vms(); + let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let id: u32 = resv.index().try_into()?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM Create\n", file_id, id); + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating allocators\n", + file_id, + id + ); + let ualloc = Arc::pin_init( + new_mutex!(alloc::DefaultAllocator::new( + device, + &vm, + kernel_gpu_range, + buffer::PAGE_SIZE, + mmu::PROT_GPU_SHARED_RW, + 512 * 1024, + true, + fmt!("File {} VM {} GPU Shared", file_id, id), + false, + )?), + GFP_KERNEL, + )?; + let ualloc_priv = Arc::pin_init( + new_mutex!(alloc::DefaultAllocator::new( + device, + &vm, + kernel_gpufw_range, + buffer::PAGE_SIZE, + mmu::PROT_GPU_FW_PRIV_RW, + 64 * 1024, + true, + fmt!("File {} VM {} GPU FW Private", file_id, id), + false, + )?), + GFP_KERNEL, + )?; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating dummy object\n", + file_id, + id + ); + let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?; + dummy_obj.vmap()?.memset(0); + let dummy_mapping = + dummy_obj.map_at(&vm, mmu::IOVA_UNK_PAGE, mmu::PROT_GPU_SHARED_RW, true)?; + + mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); + resv.fill(KBox::new( + Vm { + ualloc, + ualloc_priv, + vm, + kernel_range, + _dummy_mapping: dummy_mapping, + }, + GFP_KERNEL, + )?)?; + + data.vm_id = id; + + Ok(0) + } + + /// IOCTL: vm_destroy: Destroy a `Vm`. + pub(crate) fn vm_destroy( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_vm_destroy, + file: &DrmFile, + ) -> Result { + let vm = file.inner().vms().remove(data.vm_id as usize); + if vm.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: gem_create: Create a new GEM object. + pub(crate) fn gem_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_create, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x?}\n", + file.inner().id, + data.size + ); + + if (data.flags + & !(uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK + | uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE)) + != 0 + || (data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 + && data.vm_id != 0) + { + cls_pr_debug!(Errors, "gem_create: Invalid arguments\n"); + return Err(EINVAL); + } + + let resv_gem; + let resv_obj = if data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 { + resv_gem = file + .inner() + .vms() + .lock() + .get(data.vm_id.try_into()?) + .ok_or(ENOENT)? + .vm + .get_resv_obj(); + Some(resv_gem.deref()) + } else { + None + }; + + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj)?; + + let handle = gem.create_handle(file)?; + data.handle = handle; + + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_create size={:#x} handle={:#x?}\n", + file.inner().id, + data.size, + data.handle + ); + + Ok(0) + } + + /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. + pub(crate) fn gem_mmap_offset( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_mmap_offset, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {}]: IOCTL: gem_mmap_offset handle={:#x?}\n", + file.inner().id, + data.handle + ); + + if data.flags != 0 { + cls_pr_debug!(Errors, "gem_mmap_offset: Unexpected flags\n"); + return Err(EINVAL); + } + + let gem = gem::Object::lookup_handle(file, data.handle)?; + data.offset = gem.create_mmap_offset()?; + Ok(0) + } + + /// IOCTL: vm_bind: Map or unmap memory into a Vm. + pub(crate) fn vm_bind( + device: &AsahiDevice, + data: &uapi::drm_asahi_vm_bind, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: vm_bind\n", + file.inner().id, + data.vm_id, + ); + + if data.stride == 0 || data.pad != 0 { + cls_pr_debug!(Errors, "vm_bind: Unexpected headers\n"); + return Err(EINVAL); + } + + let vm_id = data.vm_id.try_into()?; + + let mut vec = KVec::new(); + let size = (data.stride * data.num_binds) as usize; + let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); + reader.read_all(&mut vec, GFP_KERNEL)?; + let mut reader = Reader::new(&vec); + + for _i in 0..data.num_binds { + let bind: uapi::drm_asahi_gem_bind_op = reader.read_up_to(data.stride as usize)?; + Self::do_gem_bind_unbind(vm_id, &bind, file)?; + } + + Ok(0) + } + + pub(crate) fn do_gem_bind_unbind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { + Self::do_gem_unbind(vm_id, data, file) + } else { + Self::do_gem_bind(vm_id, data, file) + } + } + + pub(crate) fn do_gem_bind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if (data.addr | data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", + data.addr, + data.range + ); + return Err(EINVAL); // Must be page aligned + } + + if (data.flags + & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE)) + != 0 + { + cls_pr_debug!(Errors, "gem_bind: Invalid flags {:#x}\n", data.flags); + return Err(EINVAL); + } + + let single_page = data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + + let bo = gem::Object::lookup_handle(file, data.handle)?; + + let start = data.addr; + let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; + let range = start..end; + + let bo_accessed_size = if single_page { + mmu::UAT_PGMSK as u64 + } else { + data.range + }; + let end_off = data.offset.checked_add(bo_accessed_size).ok_or(EINVAL)?; + if end_off as usize > bo.size() { + return Err(EINVAL); + } + + if !VM_USER_RANGE.is_superset(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid map range {:#x}..{:#x} (not contained in user range)\n", + start, + end + ); + return Err(EINVAL); // Invalid map range + } + + let prot = if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0 { + if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_RW + } else { + mmu::PROT_GPU_SHARED_RO + } + } else if data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0 { + mmu::PROT_GPU_SHARED_WO + } else { + cls_pr_debug!( + Errors, + "gem_bind: Must specify read or write (flags: {:#x})\n", + data.flags + ); + return Err(EINVAL); // Must specify one of DRM_ASAHI_BIND_{READ,WRITE} + }; + + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; + + // Clone it immediately so we aren't holding the XArray lock + let vm = guarded_vm.vm.clone(); + let kernel_range = guarded_vm.kernel_range.clone(); + let _ = guarded_vm; + core::mem::drop(guard); + + if kernel_range.overlaps(range) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid map range {:#x}..{:#x} (intrudes in kernel range)\n", + start, + end + ); + return Err(EINVAL); + } + + vm.bind_object(&bo, data.addr, data.range, data.offset, prot, single_page)?; + + vm.bo_deferred_cleanup(); + + Ok(0) + } + + pub(crate) fn do_gem_unbind( + vm_id: usize, + data: &uapi::drm_asahi_gem_bind_op, + file: &DrmFile, + ) -> Result { + if data.offset != 0 + || data.flags != uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND + || data.handle != 0 + { + cls_pr_debug!(Errors, "gem_unbind: offset/flags/handle not zero\n"); + return Err(EINVAL); + } + + if (data.addr | data.range) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", + data.addr, + data.range + ); + return Err(EINVAL); // Must be page aligned + } + + let start = data.addr; + let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; + let range = start..end; + + if !VM_USER_RANGE.is_superset(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid unmap range {:#x}..{:#x} (not contained in user range)\n", + start, + end + ); + return Err(EINVAL); // Invalid map range + } + + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; + + // Clone it immediately so we aren't holding the XArray lock + let vm = guarded_vm.vm.clone(); + let kernel_range = guarded_vm.kernel_range.clone(); + let _ = guarded_vm; + core::mem::drop(guard); + + if kernel_range.overlaps(range.clone()) { + cls_pr_debug!( + Errors, + "gem_bind: Invalid unmap range {:#x}..{:#x} (intrudes in kernel range)\n", + start, + end + ); + return Err(EINVAL); + } + + vm.unmap_range(range.start, range.range())?; + + vm.bo_deferred_cleanup(); + + Ok(0) + } + + pub(crate) fn unbind_gem_object(file: &DrmFile, bo: &gem::Object) -> Result { + // TODO: use iter() + let mut index = 0; + loop { + let vms = file.inner().vms(); + let item = vms.find(index, usize::MAX); + match item { + Some((idx, file_vm)) => { + // Clone since we can't hold the xarray spinlock while + // calling drop_mappings() + let vm = file_vm.borrow().vm.clone(); + core::mem::drop(file_vm); + vm.drop_mappings(bo)?; + if idx == usize::MAX { + break; + } + index = idx + 1; + } + None => break, + } + } + Ok(()) + } + + /// IOCTL: gem_bind_object: Map or unmap a GEM object as a special object. + pub(crate) fn gem_bind_object( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + mod_dev_dbg!( + device, + "[File {} VM {}]: IOCTL: gem_bind_object op={:?} handle={:#x?} flags={:#x?} {:#x?}:{:#x?} object_handle={:#x?}\n", + file.inner().id, + data.vm_id, + data.op, + data.handle, + data.flags, + data.offset, + data.range, + data.object_handle + ); + + if data.pad != 0 { + cls_pr_debug!(Errors, "gem_bind_object: Unexpected pad\n"); + return Err(EINVAL); + } + + if data.vm_id != 0 { + cls_pr_debug!(Errors, "gem_bind_object: Unexpected vm_id\n"); + return Err(EINVAL); + } + + match data.op { + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_BIND => { + Self::do_gem_bind_object(device, data, file) + } + uapi::drm_asahi_bind_object_op_DRM_ASAHI_BIND_OBJECT_OP_UNBIND => { + Self::do_gem_unbind_object(device, data, file) + } + _ => { + cls_pr_debug!(Errors, "gem_bind_object: Invalid op {}\n", data.op); + Err(EINVAL) + } + } + } + + pub(crate) fn do_gem_bind_object( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if (data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { + cls_pr_debug!( + Errors, + "gem_bind_object: Range/offset not page aligned: {:#x} {:#x}\n", + data.range, + data.offset + ); + return Err(EINVAL); // Must be page aligned + } + + if data.flags != uapi::drm_asahi_bind_object_flags_DRM_ASAHI_BIND_OBJECT_USAGE_TIMESTAMPS { + cls_pr_debug!(Errors, "gem_bind_object: Invalid flags {:#x}\n", data.flags); + return Err(EINVAL); + } + + let offset = data.offset.try_into()?; + let end_offset = data + .offset + .checked_add(data.range) + .ok_or(EINVAL)? + .try_into()?; + let bo = gem::ObjectRef::new(gem::Object::lookup_handle(file, data.handle)?); + + let mapping = Arc::new( + device.gpu.map_timestamp_buffer(bo, offset..end_offset)?, + GFP_KERNEL, + )?; + let obj = KBox::new(Object::TimestampBuffer(mapping), GFP_KERNEL)?; + let handle = file + .inner() + .objects() + .lock() + .insert_limit(1..=u32::MAX, obj, GFP_KERNEL)? as u64; + + data.object_handle = handle as u32; + Ok(0) + } + + pub(crate) fn do_gem_unbind_object( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_bind_object, + file: &DrmFile, + ) -> Result { + if data.range != 0 || data.offset != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Range/offset not zero: {:#x} {:#x}\n", + data.range, + data.offset + ); + return Err(EINVAL); + } + + if data.flags != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Invalid flags {:#x}\n", + data.flags + ); + return Err(EINVAL); + } + + if data.handle != 0 { + cls_pr_debug!( + Errors, + "gem_unbind_object: Invalid handle {}\n", + data.handle + ); + return Err(EINVAL); + } + + let object = file.inner().objects().remove(data.object_handle as usize); + if object.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: queue_create: Create a new command submission queue of a given type. + pub(crate) fn queue_create( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_queue_create, + file: &DrmFile, + ) -> Result { + let file_id = file.inner().id; + + mod_dev_dbg!( + device, + "[File {} VM {}]: Creating queue prio={:?} flags={:#x?}\n", + file_id, + data.vm_id, + data.priority, + data.flags, + ); + + if data.flags != 0 || data.priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + cls_pr_debug!(Errors, "queue_create: Invalid arguments\n"); + return Err(EINVAL); + } + + // TODO: Allow with CAP_SYS_NICE + if data.priority >= uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_HIGH { + cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); + return Err(EINVAL); + } + + let queues_xa = file.inner().queues(); + let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; + let vms_xa = file.inner().vms(); + let guard = vms_xa.lock(); + let file_vm = guard.get(data.vm_id.try_into()?).ok_or(ENOENT)?; + let vm = file_vm.vm.clone(); + let ualloc = file_vm.ualloc.clone(); + let ualloc_priv = file_vm.ualloc_priv.clone(); + // Drop the vms lock eagerly + let _ = file_vm; + core::mem::drop(guard); + + let queue = device.gpu.new_queue( + vm, + ualloc, + ualloc_priv, + // TODO: Plumb deeper the enum + uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME - data.priority, + data.usc_exec_base, + )?; + + data.queue_id = resv.index().try_into()?; + resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; + + Ok(0) + } + + /// IOCTL: queue_destroy: Destroy a command submission queue. + pub(crate) fn queue_destroy( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_queue_destroy, + file: &DrmFile, + ) -> Result { + // grab the queue so the xarray spinlock is dropped first + let queue = file.inner().queues().remove(data.queue_id as usize); + if queue.is_none() { + Err(ENOENT) + } else { + Ok(0) + } + } + + /// IOCTL: submit: Submit GPU work to a command submission queue. + pub(crate) fn submit( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_submit, + file: &DrmFile, + ) -> Result { + debug::update_debug_flags(); + + if data.flags != 0 || data.pad != 0 { + cls_pr_debug!(Errors, "submit: Invalid arguments\n"); + return Err(EINVAL); + } + + let gpu = &device.gpu; + gpu.update_globals(); + + // Upgrade to Arc to drop the XArray lock early + let queue: Arc>> = file + .inner() + .queues() + .lock() + .get(data.queue_id.try_into()?) + .ok_or(ENOENT)? + .into(); + + let id = gpu.ids().submission.next(); + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit (submission ID: {})\n", + file.inner().id, + data.queue_id, + id + ); + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing syncs\n", + file.inner().id, + data.queue_id, + id + ); + let syncs = + SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; + + mod_dev_dbg!( + device, + "[File {} Queue {}]: IOCTL: submit({}): Parsing commands\n", + file.inner().id, + data.queue_id, + id + ); + + let mut vec = KVec::new(); + + // Copy the command buffer into the kernel. Because we need to iterate + // the command buffer twice, we do this in one big copy_from_user to + // avoid TOCTOU issues. + let reader = UserSlice::new( + UserPtr::from_addr(data.cmdbuf as _), + data.cmdbuf_size as usize, + ) + .reader(); + reader.read_all(&mut vec, GFP_KERNEL)?; + + let objects = file.inner().objects(); + let ret = queue + .lock() + .submit(id, syncs, data.in_sync_count as usize, &vec, objects); + + match ret { + Err(ERESTARTSYS) => Err(ERESTARTSYS), + Err(e) => { + dev_info!( + device.as_ref(), + "[File {} Queue {}]: IOCTL: submit failed! (submission ID: {} err: {:?})\n", + file.inner().id, + data.queue_id, + id, + e + ); + Err(e) + } + Ok(()) => Ok(0), + } + } + + /// IOCTL: get_time: Get the current GPU timer value. + pub(crate) fn get_time( + device: &AsahiDevice, + data: &mut uapi::drm_asahi_get_time, + _file: &DrmFile, + ) -> Result { + if data.flags != 0 { + cls_pr_debug!(Errors, "get_time: Unexpected flags\n"); + return Err(EINVAL); + } + + // TODO: Do this on device-init for perf. + let gpu = &device.gpu; + let frequency_hz = gpu.get_cfg().base_clock_hz as u64; + let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); + + let num = (NSEC_PER_SEC as u64) / ts_gcd; + let den = frequency_hz / ts_gcd; + + let raw: u64; + + // SAFETY: Assembly only loads the timer + unsafe { + core::arch::asm!( + "mrs {x}, CNTPCT_EL0", + x = out(reg) raw + ); + } + + data.gpu_timestamp = (raw * num) / den; + + Ok(0) + } +} + +impl Drop for File { + fn drop(&mut self) { + mod_pr_debug!("[File {}]: Closing...\n", self.id); + } +} diff --git a/drivers/gpu/drm/asahi/float.rs b/drivers/gpu/drm/asahi/float.rs new file mode 100644 index 00000000000000..d58a3d284da124 --- /dev/null +++ b/drivers/gpu/drm/asahi/float.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Basic soft floating-point support +//! +//! The GPU firmware requires a large number of power-related configuration values, many of which +//! are IEEE 754 32-bit floating point values. These values change not only between GPU/SoC +//! variants, but also between specific hardware platforms using these SoCs, so they must be +//! derived from device tree properties. There are many redundant values computed from the same +//! inputs with simple add/sub/mul/div calculations, plus a few values that are actually specific +//! to each individual device depending on its binning and fused voltage configuration, so it +//! doesn't make sense to store the final values to be passed to the firmware in the device tree. +//! +//! Therefore, we need a way to perform floating-point calculations in the kernel. +//! +//! Using the actual FPU from kernel mode is asking for trouble, since there is no way to bound +//! the execution of FPU instructions to a controlled section of code without outright putting it +//! in its own compilation unit, which is quite painful for Rust. Since these calculations only +//! have to happen at initialization time and there is no need for performance, let's use a simple +//! software float implementation instead. +//! +//! This implementation makes no attempt to be fully IEEE754 compliant, but it's good enough and +//! gives bit-identical results to macOS in the vast majority of cases, with one or two exceptions +//! related to slightly non-compliant rounding. + +use core::ops; +use kernel::{ + of, + prelude::*, // +}; + +/// An IEEE754-compatible floating point number implemented in software. +#[derive(Default, Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct F32(u32); + +// SAFETY: F32 is a transparent repr of `u32` and therefore zeroable +unsafe impl Zeroable for F32 {} + +#[derive(Default, Debug, Copy, Clone)] +struct F32U { + sign: bool, + exp: i32, + frac: i64, +} + +impl F32 { + /// Convert a raw 32-bit representation into an F32 + pub(crate) const fn from_bits(u: u32) -> F32 { + F32(u) + } + + // Convert a `f32` value into an F32 + // + // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. + #[doc(hidden)] + pub(crate) const fn from_f32(v: f32) -> F32 { + // Replace with to_bits() after kernel Rust minreq is >= 1.83.0 + #[allow(clippy::transmute_float_to_int)] + #[allow(unnecessary_transmutes)] + // SAFETY: Transmuting f32 to u32 is always safe + F32(unsafe { core::mem::transmute::(v) }) + } + + // Convert an F32 into a `f32` value + // + // For testing only. + #[doc(hidden)] + #[cfg(test)] + pub(crate) fn to_f32(self) -> f32 { + f32::from_bits(self.0) + } + + const fn unpack(&self) -> F32U { + F32U { + sign: self.0 & (1 << 31) != 0, + exp: ((self.0 >> 23) & 0xff) as i32 - 127, + frac: (((self.0 & 0x7fffff) | 0x800000) as i64) << 9, + } + .norm() + } +} + +/// Safely construct an `F32` out of a constant floating-point value. +/// +/// This ensures that the conversion happens in const context, so no floating point operations are +/// emitted. +#[macro_export] +macro_rules! f32 { + ([$($val:expr),*]) => {{ + [$(f32!($val)),*] + }}; + ($val:expr) => {{ + const _K: $crate::float::F32 = $crate::float::F32::from_f32($val); + _K + }}; +} + +impl ops::Neg for F32 { + type Output = F32; + + fn neg(self) -> F32 { + F32(self.0 ^ (1 << 31)) + } +} + +impl ops::Add for F32 { + type Output = F32; + + fn add(self, rhs: F32) -> F32 { + self.unpack().add(rhs.unpack()).pack() + } +} + +impl ops::Sub for F32 { + type Output = F32; + + fn sub(self, rhs: F32) -> F32 { + self.unpack().add((-rhs).unpack()).pack() + } +} + +impl ops::Mul for F32 { + type Output = F32; + + fn mul(self, rhs: F32) -> F32 { + self.unpack().mul(rhs.unpack()).pack() + } +} + +impl ops::Div for F32 { + type Output = F32; + + fn div(self, rhs: F32) -> F32 { + self.unpack().div(rhs.unpack()).pack() + } +} + +macro_rules! from_ints { + ($u:ty, $i:ty) => { + impl From<$i> for F32 { + fn from(v: $i) -> F32 { + F32U::from_i64(v as i64).pack() + } + } + impl From<$u> for F32 { + fn from(v: $u) -> F32 { + F32U::from_u64(v as u64).pack() + } + } + }; +} + +from_ints!(u8, i8); +from_ints!(u16, i16); +from_ints!(u32, i32); +from_ints!(u64, i64); + +impl F32U { + const INFINITY: F32U = f32!(f32::INFINITY).unpack(); + const NEG_INFINITY: F32U = f32!(f32::NEG_INFINITY).unpack(); + + fn from_i64(v: i64) -> F32U { + F32U { + sign: v < 0, + exp: 32, + frac: v.abs(), + } + .norm() + } + + fn from_u64(mut v: u64) -> F32U { + let mut exp = 32; + if v >= (1 << 63) { + exp = 31; + v >>= 1; + } + F32U { + sign: false, + exp, + frac: v as i64, + } + .norm() + } + + fn shr(&mut self, shift: i32) { + if shift > 63 { + self.exp = 0; + self.frac = 0; + } else { + self.frac >>= shift; + } + } + + fn align(a: &mut F32U, b: &mut F32U) { + if a.exp > b.exp { + b.shr(a.exp - b.exp); + b.exp = a.exp; + } else { + a.shr(b.exp - a.exp); + a.exp = b.exp; + } + } + + fn mul(self, other: F32U) -> F32U { + F32U { + sign: self.sign != other.sign, + exp: self.exp + other.exp, + frac: ((self.frac >> 8) * (other.frac >> 8)) >> 16, + } + } + + fn div(self, other: F32U) -> F32U { + if other.frac == 0 || self.is_inf() { + if self.sign { + F32U::NEG_INFINITY + } else { + F32U::INFINITY + } + } else { + F32U { + sign: self.sign != other.sign, + exp: self.exp - other.exp, + frac: ((self.frac << 24) / (other.frac >> 8)), + } + } + } + + fn add(mut self, mut other: F32U) -> F32U { + F32U::align(&mut self, &mut other); + if self.sign == other.sign { + self.frac += other.frac; + } else { + self.frac -= other.frac; + } + if self.frac < 0 { + self.sign = !self.sign; + self.frac = -self.frac; + } + self + } + + const fn norm(mut self) -> F32U { + let lz = self.frac.leading_zeros() as i32; + if lz > 31 { + self.frac <<= lz - 31; + self.exp -= lz - 31; + } else if lz < 31 { + self.frac >>= 31 - lz; + self.exp += 31 - lz; + } + + if self.is_zero() { + return F32U { + sign: self.sign, + frac: 0, + exp: 0, + }; + } + self + } + + const fn is_zero(&self) -> bool { + self.frac == 0 || self.exp < -126 + } + + const fn is_inf(&self) -> bool { + self.exp > 127 + } + + const fn pack(mut self) -> F32 { + self = self.norm(); + if !self.is_zero() { + self.frac += 0x100; + self = self.norm(); + } + + if self.is_inf() { + if self.sign { + return f32!(f32::NEG_INFINITY); + } else { + return f32!(f32::INFINITY); + } + } else if self.is_zero() { + if self.sign { + return f32!(-0.0); + } else { + return f32!(0.0); + } + } + + F32(if self.sign { 1u32 << 31 } else { 0u32 } + | ((self.exp + 127) as u32) << 23 + | ((self.frac >> 9) & 0x7fffff) as u32) + } +} + +impl<'a> TryFrom> for F32 { + type Error = Error; + + fn try_from(p: of::Property<'_>) -> core::result::Result { + let bits: u32 = p.try_into()?; + Ok(F32::from_bits(bits)) + } +} + +impl of::PropertyUnit for F32 { + const UNIT_SIZE: usize = 4; + + fn from_bytes(data: &[u8]) -> Result { + Ok(F32::from_bits(::from_bytes(data)?)) + } +} + +// TODO: Make this an actual test and figure out how to make it run. +#[cfg(test)] +mod tests { + #[test] + fn test_all() { + fn add(a: f32, b: f32) { + println!( + "{} + {} = {} {}", + a, + b, + (F32::from_f32(a) + F32::from_f32(b)).to_f32(), + a + b + ); + } + fn sub(a: f32, b: f32) { + println!( + "{} - {} = {} {}", + a, + b, + (F32::from_f32(a) - F32::from_f32(b)).to_f32(), + a - b + ); + } + fn mul(a: f32, b: f32) { + println!( + "{} * {} = {} {}", + a, + b, + (F32::from_f32(a) * F32::from_f32(b)).to_f32(), + a * b + ); + } + fn div(a: f32, b: f32) { + println!( + "{} / {} = {} {}", + a, + b, + (F32::from_f32(a) / F32::from_f32(b)).to_f32(), + a / b + ); + } + + fn test(a: f32, b: f32) { + add(a, b); + sub(a, b); + mul(a, b); + div(a, b); + } + + test(1.123, 7.567); + test(1.123, 1.456); + test(7.567, 1.123); + test(1.123, -7.567); + test(1.123, -1.456); + test(7.567, -1.123); + test(-1.123, -7.567); + test(-1.123, -1.456); + test(-7.567, -1.123); + test(1000.123, 0.001); + test(1000.123, 0.0000001); + test(0.0012, 1000.123); + test(0.0000001, 1000.123); + test(0., 0.); + test(0., 1.); + test(1., 0.); + test(1., 1.); + test(2., f32::INFINITY); + test(2., f32::NEG_INFINITY); + test(f32::INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::NEG_INFINITY, 2.); + test(f32::MAX, 2.); + test(f32::MIN, 2.); + test(f32::MIN_POSITIVE, 2.); + test(2., f32::MAX); + test(2., f32::MIN); + test(2., f32::MIN_POSITIVE); + } +} diff --git a/drivers/gpu/drm/asahi/fw/buffer.rs b/drivers/gpu/drm/asahi/fw/buffer.rs new file mode 100644 index 00000000000000..b1f4974fd02902 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/buffer.rs @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU tiled vertex buffer control firmware structures + +use super::types::*; +use super::workqueue; +use crate::{ + default_zeroed, + no_debug, + trivial_gpustruct, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct BlockControl { + pub(crate) total: AtomicU32, + pub(crate) wptr: AtomicU32, + pub(crate) unk: AtomicU32, + pub(crate) pad: Pad<0x34>, + } + default_zeroed!(BlockControl); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Counter { + pub(crate) count: AtomicU32, + __pad: Pad<0x3c>, + } + default_zeroed!(Counter); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct Stats { + pub(crate) max_pages: AtomicU32, + pub(crate) max_b: AtomicU32, + pub(crate) overflow_count: AtomicU32, + pub(crate) gpu_c: AtomicU32, + pub(crate) __pad0: Pad<0x10>, + pub(crate) reset: AtomicU32, + pub(crate) __pad1: Pad<0x1c>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Info<'a> { + pub(crate) gpu_counter: u32, + pub(crate) unk_4: u32, + pub(crate) last_id: i32, + pub(crate) cur_id: i32, + pub(crate) unk_10: u32, + pub(crate) gpu_counter2: u32, + pub(crate) unk_18: u32, + + #[ver(V < V13_0B4 || G >= G14X)] + pub(crate) unk_1c: u32, + + pub(crate) page_list: GpuPointer<'a, &'a [u32]>, + pub(crate) page_list_size: u32, + pub(crate) page_count: AtomicU32, + pub(crate) max_blocks: u32, + pub(crate) block_count: AtomicU32, + pub(crate) unk_38: u32, + pub(crate) block_list: GpuPointer<'a, &'a [u32]>, + pub(crate) block_ctl: GpuPointer<'a, super::BlockControl>, + pub(crate) last_page: AtomicU32, + pub(crate) gpu_page_ptr1: u32, + pub(crate) gpu_page_ptr2: u32, + pub(crate) unk_58: u32, + pub(crate) block_size: u32, + pub(crate) unk_60: U64, + pub(crate) counter: GpuPointer<'a, super::Counter>, + pub(crate) unk_70: u32, + pub(crate) unk_74: u32, + pub(crate) unk_78: u32, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) max_pages: u32, + pub(crate) max_pages_nomemless: u32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: Array<0x30, u8>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Scene<'a> { + #[ver(G >= G14X)] + pub(crate) control_word: GpuPointer<'a, &'a [u32]>, + #[ver(G >= G14X)] + pub(crate) control_word2: GpuPointer<'a, &'a [u32]>, + pub(crate) pass_page_count: AtomicU32, + pub(crate) unk_4: u32, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) user_buffer: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_20: u32, + #[ver(V >= V13_3)] + pub(crate) unk_28: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) total_page_count: AtomicU32, + #[ver(G < G14X)] + pub(crate) unk_30: U64, // pad + #[ver(G < G14X)] + pub(crate) unk_38: U64, // pad + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitBuffer<'a> { + pub(crate) tag: workqueue::CommandType, + pub(crate) vm_slot: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + pub(crate) block_count: u32, + pub(crate) buffer: GpuPointer<'a, super::Info::ver>, + pub(crate) stamp_value: EventValue, + } +} + +trivial_gpustruct!(BlockControl); +trivial_gpustruct!(Counter); +trivial_gpustruct!(Stats); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Info { + pub(crate) block_ctl: GpuObject, + pub(crate) counter: GpuObject, + pub(crate) page_list: GpuArray, + pub(crate) block_list: GpuArray, +} + +#[versions(AGX)] +impl GpuStruct for Info::ver { + type Raw<'a> = raw::Info::ver<'a>; +} + +pub(crate) struct ClusterBuffers { + pub(crate) tilemaps: GpuArray, + pub(crate) meta: GpuArray, +} + +#[versions(AGX)] +pub(crate) struct Scene { + pub(crate) user_buffer: GpuArray, + pub(crate) buffer: crate::buffer::Buffer::ver, + pub(crate) tvb_heapmeta: GpuArray, + pub(crate) tvb_tilemap: GpuArray, + pub(crate) tpc: Arc>, + pub(crate) clustering: Option, + pub(crate) preempt_buf: GpuArray, + #[ver(G >= G14X)] + pub(crate) control_word: GpuArray, +} + +#[versions(AGX)] +no_debug!(Scene::ver); + +#[versions(AGX)] +impl GpuStruct for Scene::ver { + type Raw<'a> = raw::Scene::ver<'a>; +} + +#[versions(AGX)] +pub(crate) struct InitBuffer { + pub(crate) scene: Arc, +} + +#[versions(AGX)] +no_debug!(InitBuffer::ver); + +#[versions(AGX)] +impl workqueue::Command for InitBuffer::ver {} + +#[versions(AGX)] +impl GpuStruct for InitBuffer::ver { + type Raw<'a> = raw::InitBuffer::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/channels.rs b/drivers/gpu/drm/asahi/fw/channels.rs new file mode 100644 index 00000000000000..c1a7ec82aad1e2 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/channels.rs @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU communication channel firmware structures (ring buffers) + +use super::types::*; +use crate::default_zeroed; +use core::sync::atomic::Ordering; +use kernel::static_assert; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0x1c>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, ChannelState<'a>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct FwCtlChannelState<'a> { + pub(crate) read_ptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) write_ptr: AtomicU32, + __pad1: Pad<0xc>, + _p: PhantomData<&'a ()>, + } + default_zeroed!(<'a>, FwCtlChannelState<'a>); +} + +pub(crate) trait RxChannelState: GpuStruct + Debug + Default +where + for<'a> ::Raw<'a>: Default + Zeroable, +{ + const SUB_CHANNELS: usize; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32; + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32); +} + +#[derive(Debug, Default)] +pub(crate) struct ChannelState {} + +impl GpuStruct for ChannelState { + type Raw<'a> = raw::ChannelState<'a>; +} + +impl RxChannelState for ChannelState { + const SUB_CHANNELS: usize = 1; + + fn wptr(raw: &Self::Raw<'_>, _index: usize) -> u32 { + raw.write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, _index: usize, rptr: u32) { + raw.read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwLogChannelState {} + +impl GpuStruct for FwLogChannelState { + type Raw<'a> = Array<6, raw::ChannelState<'a>>; +} + +impl RxChannelState for FwLogChannelState { + const SUB_CHANNELS: usize = 6; + + fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32 { + raw[index].write_ptr.load(Ordering::Acquire) + } + + fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32) { + raw[index].read_ptr.store(rptr, Ordering::Release); + } +} + +#[derive(Debug, Default)] +pub(crate) struct FwCtlChannelState {} + +impl GpuStruct for FwCtlChannelState { + type Raw<'a> = raw::FwCtlChannelState<'a>; +} + +pub(crate) trait TxChannelState: GpuStruct + Debug + Default { + fn rptr(raw: &Self::Raw<'_>) -> u32; + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32); +} + +impl TxChannelState for ChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +impl TxChannelState for FwCtlChannelState { + fn rptr(raw: &Self::Raw<'_>) -> u32 { + raw.read_ptr.load(Ordering::Acquire) + } + + fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { + raw.write_ptr.store(wptr, Ordering::Release); + } +} + +#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum PipeType { + #[default] + Vertex = 0, + Fragment = 1, + Compute = 2, +} + +#[versions(AGX)] +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RunWorkQueueMsg { + pub(crate) pipe_type: PipeType, + pub(crate) work_queue: Option>, + pub(crate) wptr: u32, + pub(crate) event_slot: u32, + pub(crate) is_new: bool, + #[ver(V >= V13_2 && G == G14)] + pub(crate) __pad: Pad<0x2b>, + #[ver(V < V13_2 || G != G14)] + pub(crate) __pad: Pad<0x1b>, +} + +#[versions(AGX)] +pub(crate) type PipeMsg = RunWorkQueueMsg::ver; + +#[versions(AGX)] +pub(crate) const DEVICECONTROL_SZ: usize = { + #[ver(V < V13_2 || G != G14)] + { + 0x2c + } + #[ver(V >= V13_2 && G == G14)] + { + 0x3c + } +}; + +// TODO: clean up when arbitrary_enum_discriminant is stable +// https://github.com/rust-lang/rust/issues/60553 + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum DeviceControlMsg { + Unk00(Array), + Unk01(Array), + Unk02(Array), + Unk03(Array), + Unk04(Array), + Unk05(Array), + Unk06(Array), + Unk07(Array), + Unk08(Array), + Unk09(Array), + Unk0a(Array), + Unk0b(Array), + Unk0c(Array), + #[ver(V >= V13_3)] + Unk0d(Array), + GrowTVBAck { + unk_4: u32, + buffer_slot: u32, + vm_slot: u32, + counter: u32, + subpipe: u32, + halt_count: U64, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x1c }>, + }, + RecoverChannel { + pipe_type: u32, + work_queue: GpuWeakPointer, + event_value: u32, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x10 }>, + }, + IdlePowerOff { + val: u32, + __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x4 }>, + }, + Unk10(Array), + Unk11(Array), + Unk12(Array), + Unk13(Array), + Unk14(Array), // Init? + Unk15(Array), // Enable something + Unk16(Array), // Disable something + DestroyContext { + unk_4: u32, + ctx_23: u8, + #[ver(V < V13_3)] + __pad0: Pad<3>, + unk_c: U32, + unk_10: U32, + ctx_0: u8, + ctx_1: u8, + ctx_4: u8, + #[ver(V < V13_3)] + __pad1: Pad<1>, + #[ver(V < V13_3)] + unk_18: u32, + gpu_context: Option>, + #[ver(V < V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x20 }>, + #[ver(V >= V13_3)] + __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x18 }>, + }, + Unk18(Array), + Initialize(Pad), // Update RegionC +} + +#[versions(AGX)] +static_assert!(core::mem::size_of::() == 4 + DEVICECONTROL_SZ::ver); + +#[versions(AGX)] +default_zeroed!(DeviceControlMsg::ver); + +#[derive(Copy, Clone, Default, Debug)] +#[repr(C)] +#[allow(dead_code)] +pub(crate) struct FwCtlMsg { + pub(crate) addr: U64, + pub(crate) unk_8: u32, + pub(crate) slot: u32, + pub(crate) page_count: u16, + pub(crate) unk_12: u16, +} + +pub(crate) const EVENT_SZ: usize = 0x34; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum ChannelErrorType { + MemoryError, + DMKill, + Aborted, + Unk3, + Unknown(u32), +} + +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum EventMsg { + Fault, + Flag { + firing: [u32; 4], + unk_14: u16, + }, + Unk2(Array), + Unk3(Array), + Timeout { + counter: u32, + unk_8: u32, + event_slot: i32, + }, + Unk5(Array), + Unk6(Array), + GrowTVB { + vm_slot: u32, + buffer_slot: u32, + counter: u32, + }, + ChannelError { + error_type: u32, + pipe_type: u32, + event_slot: u32, + event_value: u32, + }, + // Max discriminant: 0x8 +} + +static_assert!(core::mem::size_of::() == 4 + EVENT_SZ); + +pub(crate) const EVENT_MAX: u32 = 0x8; + +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawEventMsg { + pub(crate) raw: (u32, Array), + pub(crate) msg: EventMsg, +} + +default_zeroed!(RawEventMsg); + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogMsg { + pub(crate) msg_type: u32, + __pad0: u32, + pub(crate) msg_index: U64, + __pad1: Pad<0x28>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawFwLogPayloadMsg { + pub(crate) msg_type: u32, + pub(crate) seq_no: u32, + pub(crate) timestamp: U64, + pub(crate) msg: Array<0xc8, u8>, +} + +#[derive(Debug, Copy, Clone, Default)] +#[repr(C)] +pub(crate) struct RawKTraceMsg { + pub(crate) msg_type: u32, + pub(crate) timestamp: U64, + pub(crate) args: Array<4, U64>, + pub(crate) code: u8, + pub(crate) channel: u8, + __pad: Pad<1>, + pub(crate) thread: u8, + pub(crate) unk_flag: U64, +} + +#[versions(AGX)] +pub(crate) const STATS_SZ: usize = { + #[ver(V < V13_0B4)] + { + 0x2c + } + #[ver(V >= V13_0B4)] + { + 0x3c + } +}; + +#[versions(AGX)] +#[derive(Debug, Copy, Clone)] +#[repr(C, u32)] +#[allow(dead_code)] +pub(crate) enum StatsMsg { + Power { + // 0x00 + __pad: Pad<0x18>, + power: U64, + }, + Unk1(Array<{ STATS_SZ::ver }, u8>), + PowerOn { + // 0x02 + off_time: U64, + }, + PowerOff { + // 0x03 + on_time: U64, + }, + Utilization { + // 0x04 + timestamp: U64, + util1: u32, + util2: u32, + util3: u32, + util4: u32, + }, + Unk5(Array<{ STATS_SZ::ver }, u8>), + Unk6(Array<{ STATS_SZ::ver }, u8>), + Unk7(Array<{ STATS_SZ::ver }, u8>), + Unk8(Array<{ STATS_SZ::ver }, u8>), + AvgPower { + // 0x09 + active_cs: U64, + unk2: u32, + unk3: u32, + unk4: u32, + avg_power: u32, + }, + Temperature { + // 0x0a + __pad: Pad<0x8>, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, + PowerState { + // 0x0b + timestamp: U64, + last_busy_ts: U64, + active: u32, + poweroff: u32, + unk1: u32, + pstate: u32, + unk2: u32, + unk3: u32, + }, + FwBusy { + // 0x0c + timestamp: U64, + busy: u32, + }, + PState { + // 0x0d + __pad: Pad<0x8>, + ps_min: u32, + unk1: u32, + ps_max: u32, + unk2: u32, + }, + TempSensor { + // 0x0e + __pad: Pad<0x4>, + sensor_id: u32, + raw_value: u32, + scale: u32, + tmin: u32, + tmax: u32, + }, // Max discriminant: 0xe +} + +#[versions(AGX)] +static_assert!(core::mem::size_of::() == 4 + STATS_SZ::ver); + +#[versions(AGX)] +pub(crate) const STATS_MAX: u32 = 0xe; + +#[versions(AGX)] +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) union RawStatsMsg { + pub(crate) raw: (u32, Array<{ STATS_SZ::ver }, u8>), + pub(crate) msg: StatsMsg::ver, +} + +#[versions(AGX)] +default_zeroed!(RawStatsMsg::ver); diff --git a/drivers/gpu/drm/asahi/fw/compute.rs b/drivers/gpu/drm/asahi/fw/compute.rs new file mode 100644 index 00000000000000..f5f6ffa9d8d0d8 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/compute.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU compute job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) cdm_ctrl_stream_base: U64, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf4: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf5: GpuPointer<'a, &'a [u8]>, + pub(crate) usc_exec_base_cp: U64, + pub(crate) unk_38: U64, + pub(crate) helper_program: u32, + pub(crate) unk_44: u32, + pub(crate) helper_arg: U64, + pub(crate) helper_cfg: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) iogpu_unk_40: u32, + pub(crate) __pad: Pad<0xfc>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + #[ver(V >= V13_0B4)] + pub(crate) unk_0_0: u32, + pub(crate) unk_0: Array<0x24, u8>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) cdm_ctrl_stream_end: U64, + pub(crate) unk_34: Array<0x20, u8>, + pub(crate) unk_g14x: u32, + pub(crate) unk_58: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_5c: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunCompute<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) unk_4: u32, + pub(crate) vm_slot: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) unk_pointee: u32, + #[ver(G < G14X)] + pub(crate) __pad0: Array<0x50, u8>, + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1<'a>, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + pub(crate) __pad1: Array<0x20, u8>, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) job_params2: JobParameters2::ver<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) meta: job::raw::JobMeta, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_2d1: Array<3, u8>, + pub(crate) unk_2d4: u32, + pub(crate) unk_2d8: u8, + #[ver(V >= V13_0B4)] + pub(crate) context_store_req: U64, + #[ver(V >= V13_0B4)] + pub(crate) context_store_compl: U64, + #[ver(V >= V13_0B4)] + pub(crate) unk_2e9: Array<0x14, u8>, + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: U32, + #[ver(V >= V13_0B4)] + pub(crate) unk_pad: Array<0x10, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunCompute { + pub(crate) notifier: Arc>, + pub(crate) preempt_buf: GpuArray, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunCompute::ver { + type Raw<'a> = raw::RunCompute::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunCompute::ver {} diff --git a/drivers/gpu/drm/asahi/fw/event.rs b/drivers/gpu/drm/asahi/fw/event.rs new file mode 100644 index 00000000000000..52bc456f58707d --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/event.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU events control structures & stamps + +use super::types::*; +use crate::{ + default_zeroed, + trivial_gpustruct, // +}; +use core::sync::atomic::Ordering; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct LinkedListHead { + pub(crate) prev: Option>, + pub(crate) next: Option>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierList { + pub(crate) list_head: LinkedListHead, + pub(crate) unkptr_10: U64, + } + default_zeroed!(NotifierList); + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct NotifierState { + unk_14: u32, + unk_18: U64, + unk_20: u32, + vm_slot: u32, + has_vtx: u32, + pstamp_vtx: Array<4, U64>, + has_frag: u32, + pstamp_frag: Array<4, U64>, + has_comp: u32, + pstamp_comp: Array<4, U64>, + #[ver(G >= G14 && V < V13_0B4)] + unk_98_g14_0: Array<0x14, u8>, + in_list: u32, + list_head: LinkedListHead, + #[ver(G >= G14 && V < V13_0B4)] + unk_a8_g14_0: Pad<4>, + #[ver(V >= V13_0B4)] + pub(crate) unk_buf: Array<0x8, u8>, // Init to all-ff + } + + #[versions(AGX)] + impl Default for NotifierState::ver { + fn default() -> Self { + #[allow(unused_mut)] + // SAFETY: All bit patterns are valid for this type. + let mut s: Self = unsafe { core::mem::zeroed() }; + #[ver(V >= V13_0B4)] + s.unk_buf = Array::new([0xff; 0x8]); + s + } + } + + #[derive(Debug)] + #[repr(transparent)] + pub(crate) struct Threshold(AtomicU64); + default_zeroed!(Threshold); + + impl Threshold { + pub(crate) fn increase(&self, amount: u32) { + // We could use fetch_add, but the non-LSE atomic + // sequence Rust produces confuses the hypervisor. + let v = self.0.load(Ordering::Relaxed); + self.0.store(v + (amount as u64), Ordering::Relaxed); + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Notifier<'a> { + pub(crate) threshold: GpuPointer<'a, super::Threshold>, + pub(crate) generation: AtomicU32, + pub(crate) cur_count: AtomicU32, + pub(crate) unk_10: AtomicU32, + pub(crate) state: NotifierState::ver, + } +} + +trivial_gpustruct!(Threshold); +trivial_gpustruct!(NotifierList); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Notifier { + pub(crate) threshold: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for Notifier::ver { + type Raw<'a> = raw::Notifier::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/fragment.rs b/drivers/gpu/drm/asahi/fw/fragment.rs new file mode 100644 index 00000000000000..3daad1ae4db671 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/fragment.rs @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU fragment job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + buffer, + fw, + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct BackgroundProgram { + pub(crate) rsrc_spec: U64, + pub(crate) address: U64, + } + + #[derive(Debug, Clone, Copy, Default)] + #[repr(C)] + pub(crate) struct EotProgram { + pub(crate) unk_0: U64, + pub(crate) unk_8: u32, + pub(crate) rsrc_spec: u32, + pub(crate) unk_10: u32, + pub(crate) address: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c_padding: u32, + } + + impl EotProgram { + pub(crate) fn new(rsrc_spec: u32, address: u32) -> EotProgram { + EotProgram { + rsrc_spec, + address, + ..Default::default() + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct ArrayAddr { + pub(crate) ptr: U64, + pub(crate) unk_padding: U64, + } + + #[versions(AGX)] + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct AuxFBInfo { + pub(crate) isp_ctl: u32, + pub(crate) unk2: u32, + pub(crate) width: u32, + pub(crate) height: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk3: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) utile_config: u32, + pub(crate) unk_4: u32, + pub(crate) bg: BackgroundProgram, + pub(crate) ppp_multisamplectl: U64, + pub(crate) isp_scissor_base: U64, + pub(crate) isp_dbias_base: U64, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) isp_zls_pixels: U64, + pub(crate) isp_oclqry_base: U64, + pub(crate) zls_ctrl: U64, + + #[ver(G >= G14)] + pub(crate) unk_58_g14_0: U64, + #[ver(G >= G14)] + pub(crate) unk_58_g14_8: U64, + + pub(crate) z_load: U64, + pub(crate) z_store: U64, + pub(crate) s_load: U64, + pub(crate) s_store: U64, + + #[ver(G >= G14)] + pub(crate) unk_68_g14_0: Array<0x20, u8>, + + pub(crate) z_load_stride: U64, + pub(crate) z_store_stride: U64, + pub(crate) s_load_stride: U64, + pub(crate) s_store_stride: U64, + pub(crate) z_load_comp: U64, + pub(crate) z_load_comp_stride: U64, + pub(crate) z_store_comp: U64, + pub(crate) z_store_comp_stride: U64, + pub(crate) s_load_comp: U64, + pub(crate) s_load_comp_stride: U64, + pub(crate) s_store_comp: U64, + pub(crate) s_store_comp_stride: U64, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, + pub(crate) mtile_stride_dwords: U64, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) tile_config: U64, + pub(crate) aux_fb: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_108: Array<0x6, U64>, + pub(crate) usc_exec_base_isp: U64, + pub(crate) unk_140: U64, + pub(crate) helper_program: u32, + pub(crate) unk_14c: u32, + pub(crate) helper_arg: U64, + pub(crate) unk_158: U64, + pub(crate) unk_160: U64, + + #[ver(G < G14)] + pub(crate) __pad: Pad<0x1d8>, + #[ver(G >= G14)] + pub(crate) __pad: Pad<0x1a8>, + #[ver(V < V13_0B4)] + pub(crate) __pad1: Pad<0x8>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2 { + pub(crate) eot_rsrc_spec: u32, + pub(crate) eot_usc: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) isp_merge_upper_x: F32, + pub(crate) isp_merge_upper_y: F32, + pub(crate) unk_18: U64, + pub(crate) utiles_per_mtile_y: u16, + pub(crate) utiles_per_mtile_x: u16, + pub(crate) unk_24: u32, + pub(crate) tile_counts: u32, + pub(crate) tib_blocks: u32, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) helper_cfg: u32, + pub(crate) __pad: Pad<0xac>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters3 { + pub(crate) isp_dbias_base: ArrayAddr, + pub(crate) isp_scissor_base: ArrayAddr, + pub(crate) isp_oclqry_base: U64, + pub(crate) unk_118: U64, + pub(crate) unk_120: Array<0x25, U64>, + pub(crate) unk_partial_bg: BackgroundProgram, + pub(crate) unk_258: U64, + pub(crate) unk_260: U64, + pub(crate) unk_268: U64, + pub(crate) unk_270: U64, + pub(crate) partial_bg: BackgroundProgram, + pub(crate) zls_ctrl: U64, + pub(crate) unk_290: U64, + pub(crate) z_load: U64, + pub(crate) z_partial_stride: U64, + pub(crate) z_partial_comp_stride: U64, + pub(crate) z_store: U64, + pub(crate) z_partial: U64, + pub(crate) z_partial_comp: U64, + pub(crate) s_load: U64, + pub(crate) s_partial_stride: U64, + pub(crate) s_partial_comp_stride: U64, + pub(crate) s_store: U64, + pub(crate) s_partial: U64, + pub(crate) s_partial_comp: U64, + pub(crate) unk_2f8: Array<2, U64>, + pub(crate) tib_blocks: u32, + pub(crate) unk_30c: u32, + pub(crate) aux_fb_info: AuxFBInfo::ver, + pub(crate) tile_config: U64, + pub(crate) unk_328_padding: Array<0x8, u8>, + pub(crate) unk_partial_eot: EotProgram, + pub(crate) partial_eot: EotProgram, + pub(crate) isp_bgobjdepth: u32, + pub(crate) isp_bgobjvals: u32, + pub(crate) sample_size: u32, + pub(crate) unk_37c: u32, + pub(crate) unk_380: U64, + pub(crate) unk_388: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_390_0: U64, + + pub(crate) isp_zls_pixels: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunFragment<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + pub(crate) ppp_multisamplectl: U64, + pub(crate) samples: u32, + pub(crate) tiles_per_mtile_y: u16, + pub(crate) tiles_per_mtile_x: u16, + pub(crate) unk_50: U64, + pub(crate) unk_58: U64, + pub(crate) isp_merge_upper_x: F32, + pub(crate) isp_merge_upper_y: F32, + pub(crate) unk_68: U64, + pub(crate) tile_count: U64, + + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] + pub(crate) job_params2: JobParameters2, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + + pub(crate) job_params3: JobParameters3::ver, + pub(crate) unk_758_flag: u32, + pub(crate) unk_75c_flag: u32, + pub(crate) unk_buf: Array<0x110, u8>, + pub(crate) busy_flag: u32, + pub(crate) tvb_overflow_count: u32, + pub(crate) unk_878: u32, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) process_empty_tiles: u32, + pub(crate) no_clear_pipeline_textures: u32, + pub(crate) msaa_zs: u32, + pub(crate) unk_pointee: u32, + #[ver(V >= V13_3)] + pub(crate) unk_v13_3: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_925: Array<3, u8>, + pub(crate) unk_928: u32, + pub(crate) unk_92c: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_92d_8: Array<0x1b, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunFragment { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) aux_fb: GpuArray, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunFragment::ver { + type Raw<'a> = raw::RunFragment::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunFragment::ver {} diff --git a/drivers/gpu/drm/asahi/fw/initdata.rs b/drivers/gpu/drm/asahi/fw/initdata.rs new file mode 100644 index 00000000000000..7c7488e950acef --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/initdata.rs @@ -0,0 +1,1352 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU initialization / global structures + +use super::channels; +use super::types::*; +use crate::{ + default_zeroed, + gem, + mmu, + no_debug, + trivial_gpustruct, // +}; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct ChannelRing { + pub(crate) state: Option>, + pub(crate) ring: Option>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PipeChannels { + pub(crate) vtx: ChannelRing, + pub(crate) frag: ChannelRing, + pub(crate) comp: ChannelRing, + } + #[versions(AGX)] + default_zeroed!(PipeChannels::ver); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatusFlags { + pub(crate) halt_count: AtomicU64, + __pad0: Pad<0x8>, + pub(crate) halted: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) resume: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) unk_40: u32, + __pad3: Pad<0xc>, + pub(crate) unk_ctr: u32, + __pad4: Pad<0xc>, + pub(crate) unk_60: u32, + __pad5: Pad<0xc>, + pub(crate) unk_70: u32, + __pad6: Pad<0xc>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct FwStatus { + pub(crate) fwctl_channel: ChannelRing, + pub(crate) flags: FwStatusFlags, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared1 { + pub(crate) table: Array<16, i32>, + pub(crate) unk_44: Array<0x60, u8>, + pub(crate) unk_a4: u32, + pub(crate) unk_a8: u32, + } + default_zeroed!(HwDataShared1); + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2Curve { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) t1: Array<16, u16>, + pub(crate) t2: Array<16, i16>, + pub(crate) t3: Array<8, Array<16, i32>>, + } + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct HwDataShared2G14 { + pub(crate) unk_0: Array<5, u32>, + pub(crate) unk_14: u32, + pub(crate) unk_18: Array<8, u32>, + pub(crate) curve1: HwDataShared2Curve, + pub(crate) curve2: HwDataShared2Curve, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared2 { + pub(crate) table: Array<10, i32>, + pub(crate) unk_28: Array<0x10, u8>, + pub(crate) g14: HwDataShared2G14, + pub(crate) unk_500: u32, + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + } + default_zeroed!(HwDataShared2); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataShared3 { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) table: Array<16, u32>, + pub(crate) unk_4c: u32, + } + default_zeroed!(HwDataShared3); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataA130Extra { + pub(crate) unk_0: Array<0x38, u8>, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) gpu_se_inactive_threshold: u32, + pub(crate) unk_44: u32, + pub(crate) gpu_se_engagement_criteria: i32, + pub(crate) gpu_se_reset_criteria: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + pub(crate) unk_5c: u32, + pub(crate) gpu_se_filter_a_neg: F32, + pub(crate) gpu_se_filter_1_a_neg: F32, + pub(crate) gpu_se_filter_a: F32, + pub(crate) gpu_se_filter_1_a: F32, + pub(crate) gpu_se_ki_dt: F32, + pub(crate) gpu_se_ki_1_dt: F32, + pub(crate) unk_78: F32, + pub(crate) unk_7c: F32, + pub(crate) gpu_se_kp: F32, + pub(crate) gpu_se_kp_1: F32, + pub(crate) unk_88: u32, + pub(crate) unk_8c: u32, + pub(crate) max_pstate_scaled_1: u32, + pub(crate) unk_94: u32, + pub(crate) unk_98: u32, + pub(crate) unk_9c: F32, + pub(crate) unk_a0: u32, + pub(crate) unk_a4: u32, + pub(crate) gpu_se_filter_time_constant_ms: u32, + pub(crate) gpu_se_filter_time_constant_1_ms: u32, + pub(crate) gpu_se_filter_time_constant_clks: U64, + pub(crate) gpu_se_filter_time_constant_1_clks: U64, + pub(crate) unk_c0: u32, + pub(crate) unk_c4: F32, + pub(crate) unk_c8: Array<0x4c, u8>, + pub(crate) unk_114: F32, + pub(crate) unk_118: u32, + pub(crate) unk_11c: u32, + pub(crate) unk_120: u32, + pub(crate) unk_124: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_12c: Array<0x8c, u8>, + } + default_zeroed!(HwDataA130Extra); + + #[repr(C)] + pub(crate) struct T81xxData { + pub(crate) unk_d8c: u32, + pub(crate) unk_d90: u32, + pub(crate) unk_d94: u32, + pub(crate) unk_d98: u32, + pub(crate) unk_d9c: F32, + pub(crate) unk_da0: u32, + pub(crate) unk_da4: F32, + pub(crate) unk_da8: u32, + pub(crate) unk_dac: F32, + pub(crate) unk_db0: u32, + pub(crate) unk_db4: u32, + pub(crate) unk_db8: F32, + pub(crate) unk_dbc: F32, + pub(crate) unk_dc0: u32, + pub(crate) unk_dc4: u32, + pub(crate) unk_dc8: u32, + pub(crate) max_pstate_scaled: u32, + } + default_zeroed!(T81xxData); + + #[versions(AGX)] + #[derive(Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct PowerZone { + pub(crate) val: F32, + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc_x4: u32, + pub(crate) filter_tc_xperiod: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_10: u32, + #[ver(V >= V13_0B4)] + pub(crate) unk_14: u32, + pub(crate) filter_a_neg: F32, + pub(crate) filter_a: F32, + pub(crate) pad: u32, + } + + #[versions(AGX)] + const MAX_CORES_PER_CLUSTER: usize = { + #[ver(G >= G14X)] + { + 16 + } + #[ver(G < G14X)] + { + 8 + } + }; + + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct AuxLeakCoef { + pub(crate) afr_1: Array<2, F32>, + pub(crate) cs_1: Array<2, F32>, + pub(crate) afr_2: Array<2, F32>, + pub(crate) cs_2: Array<2, F32>, + } + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct HwDataA { + pub(crate) unk_0: u32, + pub(crate) clocks_per_period: u32, + + #[ver(V >= V13_0B4)] + pub(crate) clocks_per_period_2: u32, + + pub(crate) unk_8: u32, + pub(crate) pwr_status: AtomicU32, + pub(crate) unk_10: F32, + pub(crate) unk_14: u32, + pub(crate) unk_18: u32, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) actual_pstate: u32, + pub(crate) tgt_pstate: u32, + pub(crate) unk_30: u32, + pub(crate) cur_pstate: u32, + pub(crate) unk_38: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3c_0: u32, + + pub(crate) base_pstate_scaled: u32, + pub(crate) unk_40: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) unk_48: u32, + pub(crate) min_pstate_scaled: u32, + pub(crate) freq_mhz: F32, + pub(crate) unk_54: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_74_0: u32, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_b4: Array<0x100, u8>, + pub(crate) unk_1b4: u32, + pub(crate) temp_c: u32, + pub(crate) avg_power_mw: u32, + pub(crate) update_ts: U64, + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: Array<0x478, u8>, + pub(crate) pad_644: Pad<0x8>, + pub(crate) unk_64c: u32, + pub(crate) unk_650: u32, + pub(crate) pad_654: u32, + pub(crate) pwr_filter_a_neg: F32, + pub(crate) pad_65c: u32, + pub(crate) pwr_filter_a: F32, + pub(crate) pad_664: u32, + pub(crate) pwr_integral_gain: F32, + pub(crate) pad_66c: u32, + pub(crate) pwr_integral_min_clamp: F32, + pub(crate) max_power_1: F32, + pub(crate) pwr_proportional_gain: F32, + pub(crate) pad_67c: u32, + pub(crate) pwr_pstate_related_k: F32, + pub(crate) pwr_pstate_max_dc_offset: i32, + pub(crate) unk_688: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) pad_690: u32, + pub(crate) unk_694: u32, + pub(crate) max_power_2: u32, + pub(crate) pad_69c: Pad<0x18>, + pub(crate) unk_6b4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_6b8_0: Array<0x10, u8>, + + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_6bc: u32, + pub(crate) pad_6c0: Pad<0x14>, + pub(crate) ppm_filter_tc_periods_x4: u32, + pub(crate) unk_6d8: u32, + pub(crate) pad_6dc: u32, + pub(crate) ppm_filter_a_neg: F32, + pub(crate) pad_6e4: u32, + pub(crate) ppm_filter_a: F32, + pub(crate) pad_6ec: u32, + pub(crate) ppm_ki_dt: F32, + pub(crate) pad_6f4: u32, + pub(crate) pwr_integral_min_clamp_2: u32, + pub(crate) unk_6fc: F32, + pub(crate) ppm_kp: F32, + pub(crate) pad_704: u32, + pub(crate) unk_708: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_4: u32, + pub(crate) unk_714: u32, + pub(crate) pad_718: u32, + pub(crate) unk_71c: F32, + pub(crate) max_power_3: u32, + pub(crate) cur_power_mw_2: u32, + pub(crate) ppm_filter_tc_ms: u32, + pub(crate) unk_72c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) ppm_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_730_c: u32, + + pub(crate) unk_730: F32, + pub(crate) unk_734: u32, + pub(crate) unk_738: u32, + pub(crate) unk_73c: u32, + pub(crate) unk_740: u32, + pub(crate) unk_744: u32, + pub(crate) unk_748: Array<0x4, F32>, + pub(crate) unk_758: u32, + pub(crate) perf_tgt_utilization: u32, + pub(crate) pad_760: u32, + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_boost_ce_step: u32, + pub(crate) perf_reset_iters: u32, + pub(crate) pad_770: u32, + pub(crate) unk_774: u32, + pub(crate) unk_778: u32, + pub(crate) perf_filter_drop_threshold: u32, + pub(crate) perf_filter_a_neg: F32, + pub(crate) perf_filter_a2_neg: F32, + pub(crate) perf_filter_a: F32, + pub(crate) perf_filter_a2: F32, + pub(crate) perf_ki: F32, + pub(crate) perf_ki2: F32, + pub(crate) perf_integral_min_clamp: F32, + pub(crate) unk_79c: F32, + pub(crate) perf_kp: F32, + pub(crate) perf_kp2: F32, + pub(crate) boost_state_unk_k: F32, + pub(crate) base_pstate_scaled_2: u32, + pub(crate) max_pstate_scaled_5: u32, + pub(crate) base_pstate_scaled_3: u32, + pub(crate) pad_7b8: u32, + pub(crate) perf_cur_utilization: F32, + pub(crate) perf_tgt_utilization_2: u32, + pub(crate) pad_7c4: Pad<0x18>, + pub(crate) unk_7dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_7e0_0: Array<0x10, u8>, + + pub(crate) base_pstate_scaled_4: u32, + pub(crate) pad_7e4: u32, + pub(crate) unk_7e8: Array<0x14, u8>, + pub(crate) unk_7fc: F32, + pub(crate) pwr_min_duty_cycle_2: F32, + pub(crate) max_pstate_scaled_6: F32, + pub(crate) max_freq_mhz: u32, + pub(crate) pad_80c: u32, + pub(crate) unk_810: u32, + pub(crate) pad_814: u32, + pub(crate) pwr_min_duty_cycle_3: u32, + pub(crate) unk_81c: u32, + pub(crate) pad_820: u32, + pub(crate) min_pstate_scaled_4: F32, + pub(crate) max_pstate_scaled_7: u32, + pub(crate) unk_82c: u32, + pub(crate) unk_alpha_neg: F32, + pub(crate) unk_alpha: F32, + pub(crate) unk_838: u32, + pub(crate) unk_83c: u32, + pub(crate) pad_840: Pad<0x2c>, + pub(crate) unk_86c: u32, + pub(crate) fast_die0_sensor_mask: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask: U64, + pub(crate) fast_die0_release_temp_cc: u32, + pub(crate) unk_87c: i32, + pub(crate) unk_880: u32, + pub(crate) unk_884: u32, + pub(crate) pad_888: u32, + pub(crate) unk_88c: u32, + pub(crate) pad_890: u32, + pub(crate) unk_894: F32, + pub(crate) pad_898: u32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) pad_8a0: u32, + pub(crate) unk_8a4: u32, + pub(crate) unk_8a8: F32, + pub(crate) fast_die0_kp: F32, + pub(crate) pad_8b0: u32, + pub(crate) unk_8b4: u32, + pub(crate) pwr_min_duty_cycle_4: u32, + pub(crate) max_pstate_scaled_8: u32, + pub(crate) max_pstate_scaled_9: u32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) unk_8c8: u32, + pub(crate) unk_8cc: u32, + pub(crate) pad_8d0: Pad<0x14>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_8e4_0: Array<0x10, u8>, + + pub(crate) unk_8e4: u32, + pub(crate) unk_8e8: u32, + pub(crate) max_pstate_scaled_10: u32, + pub(crate) unk_8f0: u32, + pub(crate) unk_8f4: u32, + pub(crate) pad_8f8: u32, + pub(crate) pad_8fc: u32, + pub(crate) unk_900: Array<0x24, u8>, + + pub(crate) unk_coef_a1: Array<8, Array>, + pub(crate) unk_coef_a2: Array<8, Array>, + + pub(crate) pad_b24: Pad<0x70>, + pub(crate) max_pstate_scaled_11: u32, + pub(crate) freq_with_off: u32, + pub(crate) unk_b9c: u32, + pub(crate) unk_ba0: U64, + pub(crate) unk_ba8: U64, + pub(crate) unk_bb0: u32, + pub(crate) unk_bb4: u32, + + #[ver(V >= V13_3)] + pub(crate) pad_bb8_0: Pad<0x200>, + #[ver(V >= V13_5)] + pub(crate) pad_bb8_200: Pad<0x8>, + + pub(crate) pad_bb8: Pad<0x74>, + pub(crate) unk_c2c: u32, + pub(crate) power_zone_count: u32, + pub(crate) max_power_4: u32, + pub(crate) max_power_5: u32, + pub(crate) max_power_6: u32, + pub(crate) unk_c40: u32, + pub(crate) unk_c44: F32, + pub(crate) avg_power_target_filter_a_neg: F32, + pub(crate) avg_power_target_filter_a: F32, + pub(crate) avg_power_target_filter_tc_x4: u32, + pub(crate) avg_power_target_filter_tc_xperiod: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_target_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c58_4: u32, + + pub(crate) power_zones: Array<5, PowerZone::ver>, + pub(crate) avg_power_filter_tc_periods_x4: u32, + pub(crate) unk_cfc: u32, + pub(crate) unk_d00: u32, + pub(crate) avg_power_filter_a_neg: F32, + pub(crate) unk_d08: u32, + pub(crate) avg_power_filter_a: F32, + pub(crate) unk_d10: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) unk_d18: u32, + pub(crate) unk_d1c: u32, + pub(crate) unk_d20: F32, + pub(crate) avg_power_kp: F32, + pub(crate) unk_d28: u32, + pub(crate) unk_d2c: u32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) max_pstate_scaled_12: u32, + pub(crate) max_pstate_scaled_13: u32, + pub(crate) unk_d3c: u32, + pub(crate) max_power_7: F32, + pub(crate) max_power_8: u32, + pub(crate) unk_d48: u32, + pub(crate) avg_power_filter_tc_ms: u32, + pub(crate) unk_d50: u32, + + #[ver(V >= V13_0B4)] + pub(crate) avg_power_filter_tc_clks: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_d54_4: Array<0xc, u8>, + + pub(crate) unk_d54: Array<0x10, u8>, + pub(crate) max_pstate_scaled_14: u32, + pub(crate) unk_d68: Array<0x24, u8>, + + pub(crate) t81xx_data: T81xxData, + + pub(crate) unk_dd0: Array<0x40, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_e10_pad: Array<0x10, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_e10_0: HwDataA130Extra, + + pub(crate) unk_e10: Array<0xc, u8>, + + pub(crate) fast_die0_sensor_mask_2: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_2: U64, + + pub(crate) unk_e24: u32, + pub(crate) unk_e28: u32, + pub(crate) unk_e2c: Pad<0x1c>, + pub(crate) unk_coef_b1: Array<8, Array>, + pub(crate) unk_coef_b2: Array<8, Array>, + + #[ver(G >= G14X)] + pub(crate) pad_1048_0: Pad<0x600>, + + pub(crate) pad_1048: Pad<0x5e4>, + + pub(crate) fast_die0_sensor_mask_alt: U64, + #[ver(G >= G14X)] + pub(crate) fast_die1_sensor_mask_alt: U64, + #[ver(V < V13_0B4)] + pub(crate) fast_die0_sensor_present: U64, + + pub(crate) unk_163c: u32, + + pub(crate) unk_1640: Array<0x2000, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_3640_0: Array<0x2000, u8>, + + pub(crate) unk_3640: u32, + pub(crate) unk_3644: u32, + pub(crate) hws1: HwDataShared1, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2: Array<16, u16>, + + pub(crate) hws2: HwDataShared2, + pub(crate) unk_3c00: u32, + pub(crate) unk_3c04: u32, + pub(crate) hws3: HwDataShared3, + pub(crate) unk_3c58: Array<0x3c, u8>, + pub(crate) unk_3c94: u32, + pub(crate) unk_3c98: U64, + pub(crate) unk_3ca0: U64, + pub(crate) unk_3ca8: U64, + pub(crate) unk_3cb0: U64, + pub(crate) ts_last_idle: U64, + pub(crate) ts_last_poweron: U64, + pub(crate) ts_last_poweroff: U64, + pub(crate) unk_3cd0: U64, + pub(crate) unk_3cd8: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_3ce0_0: u32, + + pub(crate) unk_3ce0: u32, + pub(crate) unk_3ce4: u32, + pub(crate) unk_3ce8: u32, + pub(crate) unk_3cec: u32, + pub(crate) unk_3cf0: u32, + pub(crate) core_leak_coef: Array<8, F32>, + pub(crate) sram_leak_coef: Array<8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) aux_leak_coef: AuxLeakCoef, + #[ver(V >= V13_0B4)] + pub(crate) unk_3d34_0: Array<0x18, u8>, + + pub(crate) unk_3d34: Array<0x38, u8>, + } + #[versions(AGX)] + default_zeroed!(HwDataA::ver); + #[versions(AGX)] + no_debug!(HwDataA::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C)] + pub(crate) struct IOMapping { + pub(crate) phys_addr: U64, + pub(crate) virt_addr: U64, + pub(crate) total_size: u32, + pub(crate) element_size: u32, + pub(crate) readwrite: U64, + } + + #[versions(AGX)] + const IO_MAPPING_COUNT: usize = { + #[ver(V < V13_0B4)] + { + 0x14 + } + #[ver(V >= V13_0B4 && V < V13_3)] + { + 0x17 + } + #[ver(V >= V13_3 && V < V13_5)] + { + 0x18 + } + #[ver(V >= V13_5)] + { + 0x19 + } + }; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataBAuxPStates { + pub(crate) cs_max_pstate: u32, + pub(crate) cs_frequencies: Array<0x10, u32>, + pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, + pub(crate) cs_unkpad: u32, + pub(crate) afr_max_pstate: u32, + pub(crate) afr_frequencies: Array<0x8, u32>, + pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, + pub(crate) afr_unkpad: u32, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct HwDataB { + #[ver(V < V13_0B4)] + pub(crate) unk_0: U64, + + pub(crate) unk_8: U64, + + #[ver(V < V13_0B4)] + pub(crate) unk_10: U64, + + pub(crate) unk_18: U64, + pub(crate) unk_20: U64, + pub(crate) unk_28: U64, + pub(crate) unk_30: U64, + pub(crate) timestamp_area_base: U64, + pub(crate) pad_40: Pad<0x20>, + + #[ver(V < V13_0B4)] + pub(crate) yuv_matrices: Array<0xf, Array<3, Array<4, i16>>>, + + #[ver(V >= V13_0B4)] + pub(crate) yuv_matrices: Array<0x3f, Array<3, Array<4, i16>>>, + + pub(crate) pad_1c8: Pad<0x8>, + pub(crate) io_mappings: Array, + + #[ver(V >= V13_0B4)] + pub(crate) sgx_sram_ptr: U64, + + pub(crate) chip_id: u32, + pub(crate) unk_454: u32, + pub(crate) unk_458: u32, + pub(crate) unk_45c: u32, + pub(crate) unk_460: u32, + pub(crate) unk_464: u32, + pub(crate) unk_468: u32, + pub(crate) unk_46c: u32, + pub(crate) unk_470: u32, + pub(crate) unk_474: u32, + pub(crate) unk_478: u32, + pub(crate) unk_47c: u32, + pub(crate) unk_480: u32, + pub(crate) unk_484: u32, + pub(crate) unk_488: u32, + pub(crate) unk_48c: u32, + pub(crate) base_clock_khz: u32, + pub(crate) power_sample_period: u32, + pub(crate) pad_498: Pad<0x4>, + pub(crate) unk_49c: u32, + pub(crate) unk_4a0: u32, + pub(crate) unk_4a4: u32, + pub(crate) pad_4a8: Pad<0x4>, + pub(crate) unk_4ac: u32, + pub(crate) pad_4b0: Pad<0x8>, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: Array<0x4, u8>, + pub(crate) unk_4c0: u32, + pub(crate) unk_4c4: u32, + pub(crate) unk_4c8: u32, + pub(crate) unk_4cc: u32, + pub(crate) unk_4d0: u32, + pub(crate) unk_4d4: u32, + pub(crate) unk_4d8: Array<0x4, u8>, + pub(crate) unk_4dc: u32, + pub(crate) unk_4e0: U64, + pub(crate) unk_4e8: u32, + pub(crate) unk_4ec: u32, + pub(crate) unk_4f0: u32, + pub(crate) unk_4f4: u32, + pub(crate) unk_4f8: u32, + pub(crate) unk_4fc: u32, + pub(crate) unk_500: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_504_0: u32, + + pub(crate) unk_504: u32, + pub(crate) unk_508: u32, + pub(crate) unk_50c: u32, + pub(crate) unk_510: u32, + pub(crate) unk_514: u32, + pub(crate) unk_518: u32, + pub(crate) unk_51c: u32, + pub(crate) unk_520: u32, + pub(crate) unk_524: u32, + pub(crate) unk_528: u32, + pub(crate) unk_52c: u32, + pub(crate) unk_530: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_534_0: u32, + + pub(crate) unk_534: u32, + pub(crate) unk_538: u32, + + pub(crate) num_frags: u32, + pub(crate) unk_540: u32, + pub(crate) unk_544: u32, + pub(crate) unk_548: u32, + pub(crate) unk_54c: u32, + pub(crate) unk_550: u32, + pub(crate) unk_554: u32, + pub(crate) uat_ttb_base: U64, + pub(crate) gpu_core_id: u32, + pub(crate) gpu_rev_id: u32, + pub(crate) num_cores: u32, + pub(crate) max_pstate: u32, + + #[ver(V < V13_0B4)] + pub(crate) num_pstates: u32, + + pub(crate) frequencies: Array<0x10, u32>, + pub(crate) voltages: Array<0x10, [u32; 0x8]>, + pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, + + #[ver(V >= V13_3)] + pub(crate) unk_9f4_0: Pad<64>, + + pub(crate) sram_k: Array<0x10, F32>, + pub(crate) unk_9f4: Array<0x10, u32>, + pub(crate) rel_max_powers: Array<0x10, u32>, + pub(crate) rel_boost_freqs: Array<0x10, u32>, + + #[ver(V >= V13_3)] + pub(crate) unk_arr_0: Array<32, u32>, + + #[ver(V < V13_0B4)] + pub(crate) min_sram_volt: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ab8: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_abc: u32, + + #[ver(V < V13_0B4)] + pub(crate) unk_ac0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) aux_ps: HwDataBAuxPStates, + + #[ver(V >= V13_3)] + pub(crate) pad_ac4_0: Array<0x44c, u8>, + + pub(crate) pad_ac4: Pad<0x8>, + pub(crate) unk_acc: u32, + pub(crate) unk_ad0: u32, + pub(crate) pad_ad4: Pad<0x10>, + pub(crate) unk_ae4: Array<0x4, u32>, + pub(crate) pad_af4: Pad<0x4>, + pub(crate) unk_af8: u32, + pub(crate) pad_afc: Pad<0x8>, + pub(crate) unk_b04: u32, + pub(crate) unk_b08: u32, + pub(crate) unk_b0c: u32, + + #[ver(G >= G14X)] + pub(crate) pad_b10_0: Array<0x8, u8>, + + pub(crate) unk_b10: u32, + pub(crate) timer_offset: U64, + pub(crate) unk_b1c: u32, + pub(crate) unk_b20: u32, + pub(crate) unk_b24: u32, + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: u32, + pub(crate) unk_b30: u32, + pub(crate) unk_b34: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b38_4: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_b38_8: u32, + + pub(crate) unk_b38: Array<0xc, u32>, + pub(crate) unk_b68: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_b6c: Array<0xd0, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_c3c_0: Array<0x8, u8>, + + #[ver(G < G14X && V >= V13_5)] + pub(crate) unk_c3c_8: Array<0x10, u8>, + + #[ver(V >= V13_5)] + pub(crate) unk_c3c_18: Array<0x20, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_c3c: u32, + } + #[versions(AGX)] + default_zeroed!(HwDataB::ver); + + #[derive(Debug)] + #[repr(C, packed)] + pub(crate) struct GpuStatsVtx { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsVtx); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsFrag { + // This changes all the time and we don't use it, let's just make it a big buffer + // except for these two fields which may need init. + #[ver(G >= G14X)] + pub(crate) unk1_0: Array<0x910, u8>, + pub(crate) unk1: Array<0x100, u8>, + pub(crate) cur_stamp_id: i32, + pub(crate) unk2: Array<0x14, u8>, + pub(crate) unk_id: i32, + pub(crate) unk3: Array<0x1000, u8>, + } + + #[versions(AGX)] + impl Default for GpuStatsFrag::ver { + fn default() -> Self { + Self { + #[ver(G >= G14X)] + unk1_0: Default::default(), + unk1: Default::default(), + cur_stamp_id: -1, + unk2: Default::default(), + unk_id: -1, + unk3: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsVtx { + pub(crate) total_cmds: u32, + pub(crate) stats: GpuStatsVtx, + } + default_zeroed!(GpuGlobalStatsVtx); + + #[versions(AGX)] + #[derive(Debug, Default)] + #[repr(C)] + pub(crate) struct GpuGlobalStatsFrag { + pub(crate) total_cmds: u32, + pub(crate) unk_4: u32, + pub(crate) stats: GpuStatsFrag::ver, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct GpuStatsComp { + // This changes all the time and we don't use it, let's just make it a big buffer + pub(crate) opaque: Array<0x3000, u8>, + } + default_zeroed!(GpuStatsComp); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RuntimeScratch { + pub(crate) unk_280: Array<0x6800, u8>, + pub(crate) unk_6a80: u32, + pub(crate) gpu_idle: u32, + pub(crate) unkpad_6a88: Pad<0x14>, + pub(crate) unk_6a9c: u32, + pub(crate) unk_ctr0: u32, + pub(crate) unk_ctr1: u32, + pub(crate) unk_6aa8: u32, + pub(crate) unk_6aac: u32, + pub(crate) unk_ctr2: u32, + pub(crate) unk_6ab4: u32, + pub(crate) unk_6ab8: u32, + pub(crate) unk_6abc: u32, + pub(crate) unk_6ac0: u32, + pub(crate) unk_6ac4: u32, + pub(crate) unk_ctr3: u32, + pub(crate) unk_6acc: u32, + pub(crate) unk_6ad0: u32, + pub(crate) unk_6ad4: u32, + pub(crate) unk_6ad8: u32, + pub(crate) unk_6adc: u32, + pub(crate) unk_6ae0: u32, + pub(crate) unk_6ae4: u32, + pub(crate) unk_6ae8: u32, + pub(crate) unk_6aec: u32, + pub(crate) unk_6af0: u32, + pub(crate) unk_ctr4: u32, + pub(crate) unk_ctr5: u32, + pub(crate) unk_6afc: u32, + pub(crate) pad_6b00: Pad<0x38>, + + #[ver(G >= G14X)] + pub(crate) pad_6b00_extra: Array<0x4800, u8>, + + pub(crate) unk_6b38: u32, + pub(crate) pad_6b3c: Pad<0x84>, + } + #[versions(AGX)] + default_zeroed!(RuntimeScratch::ver); + + #[versions(AGX)] + #[repr(C)] + pub(crate) struct RuntimePointers<'a> { + pub(crate) pipes: Array<4, PipeChannels::ver>, + + pub(crate) device_control: + ChannelRing, + pub(crate) event: ChannelRing, + pub(crate) fw_log: ChannelRing, + pub(crate) ktrace: ChannelRing, + pub(crate) stats: ChannelRing, + + pub(crate) __pad0: Pad<0x50>, + pub(crate) unk_160: U64, + pub(crate) unk_168: U64, + pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, + pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, + pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, + pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, + pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, + pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, + pub(crate) hwdata_b: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) hwdata_b_2: GpuPointer<'a, super::HwDataB::ver>, + pub(crate) fwlog_buf: Option>, + pub(crate) unkptr_1b8: GpuPointer<'a, &'a [u8]>, + + #[ver(G < G14X)] + pub(crate) unkptr_1c0: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14X)] + pub(crate) unkptr_1c8: GpuPointer<'a, &'a [u8]>, + + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: u32, + pub(crate) unk_1d8: Array<0x3c, u8>, + pub(crate) buffer_mgr_ctl_gpu_addr: U64, + pub(crate) buffer_mgr_ctl_fw_addr: U64, + pub(crate) __pad1: Pad<0x5c>, + pub(crate) gpu_scratch: RuntimeScratch::ver, + } + #[versions(AGX)] + no_debug!(RuntimePointers::ver<'_>); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct PendingStamp { + pub(crate) info: AtomicU32, + pub(crate) wait_value: AtomicU32, + } + default_zeroed!(PendingStamp); + + #[derive(Debug, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct FaultInfo { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) queue_uuid: u32, + pub(crate) unk_c: u32, + pub(crate) unk_10: u32, + pub(crate) unk_14: u32, + } + default_zeroed!(FaultInfo); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct PowerZoneGlobal { + pub(crate) target: u32, + pub(crate) target_off: u32, + pub(crate) filter_tc: u32, + } + default_zeroed!(PowerZoneGlobal); + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Globals { + pub(crate) ktrace_enable: u32, + pub(crate) unk_4: Array<0x20, u8>, + + #[ver(V >= V13_2)] + pub(crate) unk_24_0: u32, + + pub(crate) unk_24: u32, + + #[ver(V >= V13_0B4)] + pub(crate) debug: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_28_4: u32, + + pub(crate) unk_28: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_2c_0: u32, + + pub(crate) unk_2c: u32, + pub(crate) unk_30: u32, + pub(crate) unk_34: u32, + pub(crate) unk_38: Array<0x1c, u8>, + + // pub(crate) sub: GlobalsSub::ver, + pub(crate) unk_54: u16, + pub(crate) unk_56: u16, + pub(crate) unk_58: u16, + pub(crate) unk_5a: U32, + pub(crate) unk_5e: U32, + pub(crate) unk_62: U32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_66_0: Array<0xc, u8>, + + pub(crate) unk_66: U32, + pub(crate) unk_6a: Array<0x16, u8>, + // end GlobalsSub::ver + + pub(crate) unk_80: Array<0xf80, u8>, + pub(crate) unk_1000: Array<0x7000, u8>, + pub(crate) unk_8000: Array<0x900, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_8900_pad: Array<0x484c, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_8900_pad2: Array<0x54, u8>, + + pub(crate) unk_8900: u32, + pub(crate) pending_submissions: AtomicU32, + pub(crate) max_power: u32, + pub(crate) max_pstate_scaled: u32, + pub(crate) max_pstate_scaled_2: u32, + pub(crate) unk_8914: u32, + pub(crate) unk_8918: u32, + pub(crate) max_pstate_scaled_3: u32, + pub(crate) unk_8920: u32, + pub(crate) power_zone_count: u32, + pub(crate) avg_power_filter_tc_periods: u32, + pub(crate) avg_power_ki_dt: F32, + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + pub(crate) avg_power_target_filter_tc: u32, + pub(crate) power_zones: Array<5, PowerZoneGlobal>, + pub(crate) unk_8978: Array<0x44, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89bc_0: Array<0x3c, u8>, + + pub(crate) unk_89bc: u32, + pub(crate) fast_die0_release_temp: u32, + pub(crate) unk_89c4: i32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_kp: F32, + pub(crate) fast_die0_ki_dt: F32, + pub(crate) unk_89d4: Array<0xc, u8>, + pub(crate) unk_89e0: u32, + pub(crate) max_power_2: u32, + pub(crate) ppm_kp: F32, + pub(crate) ppm_ki_dt: F32, + pub(crate) unk_89f0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_0: Array<0x8, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_89f4_c: Array<0x50, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_89f4_5c: Array<0xc, u8>, + + pub(crate) unk_89f4: u32, + pub(crate) hws1: HwDataShared1, + pub(crate) hws2: HwDataShared2, + + #[ver(V >= V13_0B4)] + pub(crate) idle_off_standby_timer: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_4: Array<0x8, F32>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_hws2_24: u32, + + pub(crate) unk_hws2_28: u32, + + pub(crate) hws3: HwDataShared3, + pub(crate) unk_9004: Array<8, u8>, + pub(crate) unk_900c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_9010_4: Array<0x14, u8>, + + pub(crate) unk_9010: Array<0x2c, u8>, + pub(crate) unk_903c: u32, + pub(crate) unk_9040: Array<0xc0, u8>, + pub(crate) unk_9100: Array<0x6f00, u8>, + pub(crate) unk_10000: Array<0xe50, u8>, + pub(crate) unk_10e50: u32, + pub(crate) unk_10e54: Array<0x2c, u8>, + + #[ver((G >= G14X && V < V13_3) || (G <= G14 && V >= V13_3))] + pub(crate) unk_x_pad: Array<0x4, u8>, + + // bit 0: sets sgx_reg 0x17620 + // bit 1: sets sgx_reg 0x17630 + pub(crate) fault_control: u32, + pub(crate) do_init: u32, + pub(crate) unk_10e88: Array<0x188, u8>, + pub(crate) idle_ts: U64, + pub(crate) idle_unk: U64, + pub(crate) progress_check_interval_3d: u32, + pub(crate) progress_check_interval_ta: u32, + pub(crate) progress_check_interval_cl: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_0: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_4: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_8: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_c: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_1102c_10: u32, + + pub(crate) unk_1102c: u32, + pub(crate) idle_off_delay_ms: AtomicU32, + pub(crate) fender_idle_off_delay_ms: u32, + pub(crate) fw_early_wake_timeout_ms: u32, + #[ver(V == V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) pending_stamps: Array<0x100, PendingStamp>, + #[ver(V != V13_3)] + pub(crate) ps_pad_0: Pad<0x8>, + pub(crate) unkpad_ps: Pad<0x78>, + pub(crate) unk_117bc: u32, + pub(crate) fault_info: FaultInfo, + pub(crate) counter: u32, + pub(crate) unk_118dc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_118e0_0: Array<0x9c, u8>, + + #[ver(G >= G14X)] + pub(crate) unk_118e0_9c: Array<0x580, u8>, + + #[ver(V >= V13_3)] + pub(crate) unk_118e0_9c_x: Array<0x8, u8>, + + pub(crate) cl_context_switch_timeout_ms: u32, + + #[ver(V >= V13_0B4)] + pub(crate) cl_kill_timeout_ms: u32, + + pub(crate) cdm_context_store_latency_threshold: u32, + pub(crate) unk_118e8: u32, + pub(crate) unk_118ec: Array<0x400, u8>, + pub(crate) unk_11cec: Array<0x54, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11d40: Array<0x19c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11edc: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11ee0: Array<0x1c, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_11efc: u32, + + #[ver(V >= V13_3)] + pub(crate) unk_11f00: Array<0x280, u8>, + } + #[versions(AGX)] + default_zeroed!(Globals::ver); + + #[derive(Debug, Default, Clone, Copy)] + #[repr(C, packed)] + pub(crate) struct UatLevelInfo { + pub(crate) unk_3: u8, + pub(crate) unk_1: u8, + pub(crate) unk_2: u8, + pub(crate) index_shift: u8, + pub(crate) num_entries: u16, + pub(crate) unk_4: u16, + pub(crate) unk_8: U64, + pub(crate) unk_10: U64, + pub(crate) index_mask: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct InitData<'a> { + #[ver(V >= V13_0B4)] + pub(crate) ver_info: Array<0x4, u16>, + + pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, + pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointers::ver>, + pub(crate) globals: GpuPointer<'a, super::Globals::ver>, + pub(crate) fw_status: GpuPointer<'a, super::FwStatus>, + pub(crate) uat_page_size: u16, + pub(crate) uat_page_bits: u8, + pub(crate) uat_num_levels: u8, + pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, + pub(crate) __pad0: Pad<0x14>, + pub(crate) host_mapped_fw_allocations: u32, + pub(crate) unk_ac: u32, + pub(crate) unk_b0: u32, + pub(crate) unk_b4: u32, + pub(crate) unk_b8: u32, + } +} + +#[derive(Debug)] +pub(crate) struct ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, +} + +impl ChannelRing +where + for<'a> ::Raw<'a>: Debug, +{ + pub(crate) fn to_raw(&self) -> raw::ChannelRing { + raw::ChannelRing { + state: Some(self.state.weak_pointer()), + ring: Some(self.ring.weak_pointer()), + } + } +} + +trivial_gpustruct!(FwStatus); +trivial_gpustruct!(GpuGlobalStatsVtx); +#[versions(AGX)] +trivial_gpustruct!(GpuGlobalStatsFrag::ver); +trivial_gpustruct!(GpuStatsComp); + +#[versions(AGX)] +trivial_gpustruct!(HwDataA::ver); + +#[versions(AGX)] +trivial_gpustruct!(HwDataB::ver); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct Stats { + pub(crate) vtx: GpuObject, + pub(crate) frag: GpuObject, + pub(crate) comp: GpuObject, +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RuntimePointers { + pub(crate) stats: Stats::ver, + + pub(crate) hwdata_a: GpuObject, + pub(crate) unkptr_190: GpuArray, + pub(crate) unkptr_198: GpuArray, + pub(crate) hwdata_b: GpuObject, + + pub(crate) unkptr_1b8: GpuArray, + pub(crate) unkptr_1c0: GpuArray, + pub(crate) unkptr_1c8: GpuArray, + + pub(crate) buffer_mgr_ctl: gem::ObjectRef, + pub(crate) buffer_mgr_ctl_low_mapping: Option, + pub(crate) buffer_mgr_ctl_high_mapping: Option, +} + +#[versions(AGX)] +impl GpuStruct for RuntimePointers::ver { + type Raw<'a> = raw::RuntimePointers::ver<'a>; +} + +#[versions(AGX)] +trivial_gpustruct!(Globals::ver); + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct InitData { + pub(crate) unk_buf: GpuArray, + pub(crate) runtime_pointers: GpuObject, + pub(crate) globals: GpuObject, + pub(crate) fw_status: GpuObject, +} + +#[versions(AGX)] +impl GpuStruct for InitData::ver { + type Raw<'a> = raw::InitData::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/fw/job.rs b/drivers/gpu/drm/asahi/fw/job.rs new file mode 100644 index 00000000000000..e4f2f9225ea050 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/job.rs @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common GPU job firmware structures + +use super::types::*; +use crate::{ + default_zeroed, + mmu, + trivial_gpustruct, // +}; +use kernel::prelude::Result; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct JobMeta { + pub(crate) unk_0: u16, + pub(crate) unk_2: u8, + pub(crate) no_preemption: u8, + pub(crate) stamp: GpuWeakPointer, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) stamp_slot: u32, + pub(crate) evctl_index: u32, + pub(crate) flush_stamps: u32, + pub(crate) uuid: u32, + pub(crate) event_seq: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct EncoderParams { + pub(crate) unk_8: u32, + pub(crate) sync_grow: u32, + pub(crate) unk_10: u32, + pub(crate) encoder_id: u32, + pub(crate) unk_18: u32, + pub(crate) unk_mask: u32, + pub(crate) sampler_array: U64, + pub(crate) sampler_count: u32, + pub(crate) sampler_max: u32, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobTimestamps { + pub(crate) start: AtomicU64, + pub(crate) end: AtomicU64, + } + default_zeroed!(JobTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RenderTimestamps { + pub(crate) vtx: JobTimestamps, + pub(crate) frag: JobTimestamps, + } + default_zeroed!(RenderTimestamps); + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Register { + pub(crate) number: u32, + pub(crate) value: U64, + } + default_zeroed!(Register); + + impl Register { + fn new(number: u32, value: u64) -> Register { + Register { + number, + value: U64(value), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RegisterArray { + pub(crate) registers: Array<128, Register>, + pub(crate) pad: Array<0x100, u8>, + + pub(crate) addr: GpuWeakPointer>, + pub(crate) count: u16, + pub(crate) length: u16, + pub(crate) unk_pad: u32, + } + + impl RegisterArray { + pub(crate) fn new( + self_ptr: GpuWeakPointer>, + cb: impl FnOnce(&mut RegisterArray), + ) -> RegisterArray { + let mut array = RegisterArray { + registers: Default::default(), + pad: Default::default(), + addr: self_ptr, + count: 0, + length: 0, + unk_pad: 0, + }; + + cb(&mut array); + + array + } + + pub(crate) fn add(&mut self, number: u32, value: u64) { + self.registers[self.count as usize] = Register::new(number, value); + self.count += 1; + self.length += core::mem::size_of::() as u16; + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct TimestampPointers<'a> { + pub(crate) start_addr: Option>, + pub(crate) end_addr: Option>, + } +} + +trivial_gpustruct!(JobTimestamps); +trivial_gpustruct!(RenderTimestamps); + +#[derive(Debug)] +pub(crate) struct UserTimestamp { + pub(crate) mapping: Arc, + pub(crate) offset: usize, +} + +#[derive(Debug, Default)] +pub(crate) struct UserTimestamps { + pub(crate) start: Option, + pub(crate) end: Option, +} + +impl UserTimestamps { + pub(crate) fn any(&self) -> bool { + self.start.is_some() || self.end.is_some() + } + + pub(crate) fn pointers(&self) -> Result> { + Ok(raw::TimestampPointers { + start_addr: self + .start + .as_ref() + .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) + .transpose()?, + end_addr: self + .end + .as_ref() + .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) + .transpose()?, + }) + } +} diff --git a/drivers/gpu/drm/asahi/fw/microseq.rs b/drivers/gpu/drm/asahi/fw/microseq.rs new file mode 100644 index 00000000000000..ff59deda6be615 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/microseq.rs @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU firmware microsequence operations + +use super::types::*; +use super::{ + buffer, + compute, + fragment, + initdata, + job, + vertex, + workqueue, // +}; +use crate::default_zeroed; + +pub(crate) trait Operation {} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +enum OpCode { + WaitForIdle = 0x01, + WaitForIdle2 = 0x02, + RetireStamp = 0x18, + #[allow(dead_code)] + Timestamp = 0x19, + StartVertex = 0x22, + FinalizeVertex = 0x23, + StartFragment = 0x24, + FinalizeFragment = 0x25, + StartCompute = 0x29, + FinalizeCompute = 0x2a, +} + +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum Pipe { + Vertex = 1 << 0, + Fragment = 1 << 8, + Compute = 1 << 15, +} + +pub(crate) const MAX_ATTACHMENTS: usize = 16; + +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub(crate) struct Attachment { + pub(crate) address: U64, + pub(crate) size: u32, + pub(crate) unk_c: u16, + pub(crate) unk_e: u16, +} +default_zeroed!(Attachment); + +#[derive(Debug, Clone, Copy, Default)] +#[repr(C)] +pub(crate) struct Attachments { + pub(crate) list: Array, + pub(crate) count: u32, +} + +#[derive(Debug, Copy, Clone)] +#[repr(transparent)] +pub(crate) struct OpHeader(u32); + +impl OpHeader { + const fn new(opcode: OpCode) -> OpHeader { + OpHeader(opcode as u32) + } + const fn with_args(opcode: OpCode, args: u32) -> OpHeader { + OpHeader(opcode as u32 | args) + } +} + +macro_rules! simple_op { + ($name:ident) => { + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct $name(OpHeader); + + impl $name { + pub(crate) const HEADER: $name = $name(OpHeader::new(OpCode::$name)); + } + }; +} + +pub(crate) mod op { + use super::*; + + simple_op!(StartVertex); + simple_op!(FinalizeVertex); + simple_op!(StartFragment); + simple_op!(FinalizeFragment); + simple_op!(StartCompute); + simple_op!(FinalizeCompute); + simple_op!(WaitForIdle2); + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct RetireStamp(OpHeader); + impl RetireStamp { + pub(crate) const HEADER: RetireStamp = + RetireStamp(OpHeader::with_args(OpCode::RetireStamp, 0x40000000)); + } + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct WaitForIdle(OpHeader); + impl WaitForIdle { + pub(crate) const fn new(pipe: Pipe) -> WaitForIdle { + WaitForIdle(OpHeader::with_args(OpCode::WaitForIdle, (pipe as u32) << 8)) + } + } + + #[allow(dead_code)] + #[derive(Debug, Copy, Clone)] + pub(crate) struct Timestamp(OpHeader); + impl Timestamp { + #[allow(dead_code)] + pub(crate) const fn new(flag: bool) -> Timestamp { + Timestamp(OpHeader::with_args(OpCode::Timestamp, (flag as u32) << 31)) + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle { + pub(crate) header: op::WaitForIdle, +} + +impl Operation for WaitForIdle {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct WaitForIdle2 { + pub(crate) header: op::WaitForIdle2, +} + +impl Operation for WaitForIdle2 {} + +#[derive(Debug)] +#[repr(C)] +pub(crate) struct RetireStamp { + pub(crate) header: op::RetireStamp, +} + +impl Operation for RetireStamp {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct Timestamp<'a> { + pub(crate) header: op::Timestamp, + pub(crate) command_time: GpuWeakPointer, + pub(crate) ts_pointers: GpuWeakPointer>, + // Unused? + pub(crate) update_ts: GpuWeakPointer>>, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) user_ts_pointers: GpuWeakPointer>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: GpuWeakPointer, + + pub(crate) uuid: u32, + pub(crate) unk_30_padding: u32, +} + +#[versions(AGX)] +impl<'a> Operation for Timestamp::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartVertex<'a> { + pub(crate) header: op::StartVertex, + pub(crate) tiling_params: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) scene: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_38: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) unk_44: u32, + pub(crate) event_seq: U64, + pub(crate) unk_50: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, + + pub(crate) unk_178: u32, +} + +#[versions(AGX)] +impl<'a> Operation for StartVertex::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeVertex { + pub(crate) header: op::FinalizeVertex, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) unk_34: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_48: U64, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: U64, + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, + pub(crate) unk_68: u32, + + #[ver(G >= G14 && V < V13_0B4)] + pub(crate) unk_68_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_74: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeVertex::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartFragment<'a> { + pub(crate) header: op::StartFragment, + pub(crate) job_params2: Option>, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, + pub(crate) stats: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) tvb_overflow_count: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_50: u32, + pub(crate) event_generation: u32, + pub(crate) buffer_slot: u32, + pub(crate) sync_grow: u32, + pub(crate) event_seq: U64, + pub(crate) unk_68: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + pub(crate) unk_job_buf: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_7c_0: U64, + pub(crate) unk_7c: u32, + pub(crate) unk_80: u32, + pub(crate) unk_84: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartFragment::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeFragment { + pub(crate) header: op::FinalizeFragment, + pub(crate) uuid: u32, + pub(crate) unk_8: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_18: u32, + pub(crate) scene: GpuWeakPointer, + pub(crate) buffer: GpuWeakPointer, + pub(crate) unk_2c: U64, + pub(crate) stats: GpuWeakPointer, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) busy_flag: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) work_item: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_60: u32, + pub(crate) unk_758_flag: GpuWeakPointer, + #[ver(V >= V13_3)] + pub(crate) unk_6c_0: U64, + pub(crate) unk_6c: U64, + pub(crate) unk_74: U64, + pub(crate) unk_7c: U64, + pub(crate) unk_84: U64, + pub(crate) unk_8c: U64, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_8c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_9c: Array<0x10, u8>, +} + +#[versions(AGX)] +impl Operation for FinalizeFragment::ver {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct StartCompute<'a> { + pub(crate) header: op::StartCompute, + pub(crate) unk_pointer: GpuWeakPointer, + pub(crate) job_params1: Option>>, + #[ver(G >= G14X)] + pub(crate) registers: GpuWeakPointer, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + pub(crate) unk_28: u32, + pub(crate) event_generation: u32, + pub(crate) event_seq: U64, + pub(crate) unk_38: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_44: u32, + pub(crate) uuid: u32, + pub(crate) attachments: Attachments, + pub(crate) padding: u32, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + #[ver(V >= V13_0B4)] + pub(crate) notifier_buf: GpuWeakPointer>, +} + +#[versions(AGX)] +impl<'a> Operation for StartCompute::ver<'a> {} + +#[versions(AGX)] +#[derive(Debug)] +#[repr(C)] +pub(crate) struct FinalizeCompute<'a> { + pub(crate) header: op::FinalizeCompute, + pub(crate) stats: GpuWeakPointer, + pub(crate) work_queue: GpuWeakPointer, + pub(crate) vm_slot: u32, + #[ver(V < V13_0B4)] + pub(crate) unk_18: u32, + pub(crate) job_params2: GpuWeakPointer>, + pub(crate) unk_24: u32, + pub(crate) uuid: u32, + pub(crate) fw_stamp: GpuWeakPointer, + pub(crate) stamp_value: EventValue, + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + pub(crate) unk_48: u32, + pub(crate) unk_4c: u32, + pub(crate) unk_50: u32, + pub(crate) unk_54: u32, + pub(crate) unk_58: u32, + + #[ver(G == G14 && V < V13_0B4)] + pub(crate) unk_5c_g14: U64, + + pub(crate) restart_branch_offset: i32, + pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic + + #[ver(V >= V13_0B4)] + pub(crate) unk_64: Array<0xd, u8>, + + #[ver(V >= V13_0B4)] + pub(crate) unk_flag: GpuWeakPointer, + + #[ver(V >= V13_0B4)] + pub(crate) unk_79: Array<0x7, u8>, +} + +#[versions(AGX)] +impl<'a> Operation for FinalizeCompute::ver<'a> {} diff --git a/drivers/gpu/drm/asahi/fw/mod.rs b/drivers/gpu/drm/asahi/fw/mod.rs new file mode 100644 index 00000000000000..a5649aa20d3a8e --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware structures for Apple AGX GPUs + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod event; +pub(crate) mod fragment; +pub(crate) mod initdata; +pub(crate) mod job; +pub(crate) mod microseq; +pub(crate) mod types; +pub(crate) mod vertex; +pub(crate) mod workqueue; diff --git a/drivers/gpu/drm/asahi/fw/types.rs b/drivers/gpu/drm/asahi/fw/types.rs new file mode 100644 index 00000000000000..f55d6cec6b8ca3 --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/types.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common types for firmware structure definitions + +use crate::{alloc, object}; +use core::fmt; +use core::ops::{Deref, DerefMut, Index, IndexMut}; + +pub(crate) use crate::event::EventValue; +pub(crate) use crate::object::{ + GpuPointer, + GpuStruct, + GpuWeakPointer, // +}; +pub(crate) use crate::{ + f32, + float::F32, // +}; + +pub(crate) use core::fmt::Debug; +pub(crate) use core::marker::PhantomData; +pub(crate) use core::sync::atomic::{ + AtomicI32, + AtomicU32, + AtomicU64, // +}; +pub(crate) use kernel::macros::versions; +pub(crate) use kernel::prelude::Zeroable; + +// Make the trait visible +pub(crate) use crate::alloc::Allocator as _Allocator; + +/// General allocator type used for the driver +pub(crate) type Allocator = alloc::DefaultAllocator; + +/// General GpuObject type used for the driver +pub(crate) type GpuObject = + object::GpuObject>; + +/// General GpuArray type used for the driver +pub(crate) type GpuArray = object::GpuArray>; + +/// General GpuOnlyArray type used for the driver +pub(crate) type GpuOnlyArray = + object::GpuOnlyArray>; + +/// A stamp slot that is shared between firmware and the driver. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct Stamp(pub(crate) AtomicU32); + +/// A stamp slot that is for private firmware use. +/// +/// This is a separate type to guard against pointer type confusion. +#[derive(Debug, Default)] +#[repr(transparent)] +pub(crate) struct FwStamp(pub(crate) AtomicU32); + +/// An unaligned u64 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U64(pub(crate) u64); + +// SAFETY: U64 is zeroable just like u64 +unsafe impl Zeroable for U64 {} + +impl fmt::Debug for U64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +// SAFETY: U32 is zeroable just like u32 +unsafe impl Zeroable for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +/// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by +/// hand or not very useful. +#[macro_export] +macro_rules! no_debug { + ($type:ty) => { + impl ::core::fmt::Debug for $type { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "...") + } + } + }; +} + +/// Implement Zeroable for a given type (and Default along with it). +/// +/// # Safety +/// +/// This macro must only be used if a type only contains primitive types which can be +/// zero-initialized, FFI structs intended to be zero-initialized, or other types which +/// impl Zeroable. +#[macro_export] +macro_rules! default_zeroed { + (<$($lt:lifetime),*>, $type:ty) => { + impl<$($lt),*> Default for $type { + fn default() -> $type { + unsafe { core::mem::zeroed() } + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl<$($lt),*> ::pin_init::Zeroable for $type {} + }; + ($type:ty) => { + impl Default for $type { + fn default() -> $type { + unsafe { core::mem::zeroed() } + } + } + // SAFETY: The user is responsible for ensuring this is safe. + unsafe impl ::pin_init::Zeroable for $type {} + }; +} + +/// A convenience type for a number of padding bytes. Hidden from Debug formatting. +#[derive(Copy, Clone)] +#[repr(C, packed)] +pub(crate) struct Pad([u8; N]); + +/// SAFETY: Primitive type, safe to zero-init. +unsafe impl Zeroable for Pad {} + +impl Default for Pad { + fn default() -> Self { + unsafe { core::mem::zeroed() } + } +} + +impl fmt::Debug for Pad { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(format_args!("")) + } +} + +/// A convenience type for a fixed-sized array with Default/Zeroable impls. +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) struct Array([T; N]); + +impl Array { + pub(crate) fn new(data: [T; N]) -> Self { + Self(data) + } +} + +// SAFETY: Arrays of Zeroable values can be safely Zeroable. +unsafe impl Zeroable for Array {} + +impl Default for Array { + fn default() -> Self { + unsafe { core::mem::zeroed() } + } +} + +impl Index for Array { + type Output = T; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl IndexMut for Array { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.0[index] + } +} + +impl Deref for Array { + type Target = [T; N]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Array { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl fmt::Debug for Array { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// Convenience macro to define an identically-named trivial GpuStruct with no inner fields for a +/// given raw type name. +#[macro_export] +macro_rules! trivial_gpustruct { + ($type:ident) => { + #[derive(Debug)] + pub(crate) struct $type {} + + impl GpuStruct for $type { + type Raw<'a> = raw::$type; + } + $crate::default_zeroed!($type); + }; +} diff --git a/drivers/gpu/drm/asahi/fw/vertex.rs b/drivers/gpu/drm/asahi/fw/vertex.rs new file mode 100644 index 00000000000000..07a0e05c72112c --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/vertex.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU vertex job firmware structures + +use super::types::*; +use super::{ + event, + job, + workqueue, // +}; +use crate::{ + buffer, + fw, + microseq, + mmu, // +}; +use kernel::sync::Arc; + +pub(crate) mod raw { + use super::*; + + #[derive(Debug, Default, Copy, Clone)] + #[repr(C)] + pub(crate) struct TilingParameters { + pub(crate) rgn_size: u32, + pub(crate) unk_4: u32, + pub(crate) ppp_ctrl: u32, + pub(crate) x_max: u16, + pub(crate) y_max: u16, + pub(crate) te_screen: u32, + pub(crate) te_mtile1: u32, + pub(crate) te_mtile2: u32, + pub(crate) tiles_per_mtile: u32, + pub(crate) tpc_stride: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + pub(crate) helper_cfg: u32, + pub(crate) __pad: Pad<0x70>, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters1<'a> { + pub(crate) unk_0: U64, + pub(crate) unk_8: F32, + pub(crate) unk_c: F32, + pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_tilemaps: Option>, + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, + pub(crate) iogpu_unk_54: U64, + pub(crate) iogpu_unk_56: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta1: Option>, + pub(crate) utile_config: u32, + pub(crate) unk_4c: u32, + pub(crate) ppp_multisamplectl: U64, + pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, + #[ver(G < G14)] + pub(crate) tvb_cluster_layermeta: Option>, + #[ver(G < G14)] + pub(crate) core_mask: Array<2, u32>, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_80: U64, + pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, + pub(crate) vdm_ctrl_stream_base: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta2: Option>, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta3: Option>, + #[ver(G < G14)] + pub(crate) tiling_control: u32, + #[ver(G < G14)] + pub(crate) unk_ac: u32, + pub(crate) unk_b0: Array<6, U64>, + pub(crate) usc_exec_base_ta: U64, + #[ver(G < G14)] + pub(crate) tvb_cluster_meta4: Option>, + #[ver(G < G14)] + pub(crate) unk_f0: U64, + pub(crate) unk_f8: U64, + pub(crate) helper_program: u32, + pub(crate) unk_104: u32, + pub(crate) helper_arg: U64, + pub(crate) unk_110: U64, + pub(crate) unk_118: u32, + #[ver(G >= G14)] + pub(crate) __pad: Pad<{ 8 * 9 + 0x268 }>, + #[ver(G < G14)] + pub(crate) __pad: Pad<0x268>, + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct JobParameters2<'a> { + pub(crate) unk_480: Array<4, u32>, + pub(crate) unk_498: U64, + pub(crate) unk_4a0: u32, + pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, + pub(crate) unk_4ac: u32, + pub(crate) unk_4b0: U64, + pub(crate) unk_4b8: u32, + pub(crate) unk_4bc: U64, + pub(crate) unk_4c4_padding: Array<0x48, u8>, + pub(crate) unk_50c: u32, + pub(crate) unk_510: U64, + pub(crate) unk_518: U64, + pub(crate) unk_520: U64, + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RunVertex<'a> { + pub(crate) tag: workqueue::CommandType, + + #[ver(V >= V13_0B4)] + pub(crate) counter: U64, + + pub(crate) vm_slot: u32, + pub(crate) unk_8: u32, + pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, + pub(crate) buffer_slot: u32, + pub(crate) unk_1c: u32, + pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, + pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, + pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, + pub(crate) unk_34: u32, + + #[ver(G < G14X)] + pub(crate) job_params1: JobParameters1::ver<'a>, + #[ver(G < G14X)] + pub(crate) tiling_params: TilingParameters, + #[ver(G >= G14X)] + pub(crate) registers: job::raw::RegisterArray, + + pub(crate) tpc: GpuPointer<'a, &'a [u8]>, + pub(crate) tpc_size: U64, + pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, + pub(crate) microsequence_size: u32, + pub(crate) fragment_stamp_slot: u32, + pub(crate) fragment_stamp_value: EventValue, + pub(crate) unk_pointee: u32, + pub(crate) unk_pad: u32, + pub(crate) job_params2: JobParameters2<'a>, + pub(crate) encoder_params: job::raw::EncoderParams, + pub(crate) unk_55c: u32, + pub(crate) unk_560: u32, + pub(crate) sync_grow: u32, + pub(crate) unk_568: u32, + pub(crate) uses_scratch: u32, + pub(crate) meta: job::raw::JobMeta, + pub(crate) unk_after_meta: u32, + pub(crate) unk_buf_0: U64, + pub(crate) unk_buf_8: U64, + pub(crate) unk_buf_10: U64, + pub(crate) command_time: U64, + pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, + pub(crate) client_sequence: u8, + pub(crate) pad_5d5: Array<3, u8>, + pub(crate) unk_5d8: u32, + pub(crate) unk_5dc: u8, + + #[ver(V >= V13_0B4)] + pub(crate) unk_ts: U64, + + #[ver(V >= V13_0B4)] + pub(crate) unk_5dd_8: Array<0x1b, u8>, + } +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct RunVertex { + pub(crate) notifier: Arc>, + pub(crate) scene: Arc, + pub(crate) micro_seq: microseq::MicroSequence, + pub(crate) vm_bind: mmu::VmBind, + pub(crate) timestamps: Arc>, + pub(crate) user_timestamps: job::UserTimestamps, +} + +#[versions(AGX)] +impl GpuStruct for RunVertex::ver { + type Raw<'a> = raw::RunVertex::ver<'a>; +} + +#[versions(AGX)] +impl workqueue::Command for RunVertex::ver {} diff --git a/drivers/gpu/drm/asahi/fw/workqueue.rs b/drivers/gpu/drm/asahi/fw/workqueue.rs new file mode 100644 index 00000000000000..b86bd3ff3757af --- /dev/null +++ b/drivers/gpu/drm/asahi/fw/workqueue.rs @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU work queue firmware structes + +use super::event; +use super::types::*; +use crate::event::EventValue; +use crate::{ + default_zeroed, + trivial_gpustruct, // +}; +use kernel::sync::Arc; + +#[derive(Debug)] +#[repr(u32)] +pub(crate) enum CommandType { + RunVertex = 0, + RunFragment = 1, + #[allow(dead_code)] + RunBlitter = 2, + RunCompute = 3, + Barrier = 4, + InitBuffer = 6, +} + +pub(crate) trait Command: GpuStruct + Send + Sync {} + +pub(crate) mod raw { + use super::*; + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct Barrier { + pub(crate) tag: CommandType, + pub(crate) wait_stamp: GpuWeakPointer, + pub(crate) wait_value: EventValue, + pub(crate) wait_slot: u32, + pub(crate) stamp_self: EventValue, + pub(crate) uuid: u32, + pub(crate) external_barrier: u32, + // G14X addition + pub(crate) internal_barrier_type: u32, + pub(crate) padding: Pad<0x1c>, + } + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct GpuContextData { + pub(crate) unk_0: u8, + pub(crate) unk_1: u8, + unk_2: Array<0x2, u8>, + pub(crate) unk_4: u8, + pub(crate) unk_5: u8, + unk_6: Array<0x18, u8>, + pub(crate) unk_1e: u8, + pub(crate) unk_1f: u8, + unk_20: Array<0x3, u8>, + pub(crate) unk_23: u8, + unk_24: Array<0x1c, u8>, + } + + impl Default for GpuContextData { + fn default() -> Self { + Self { + unk_0: 0xff, + unk_1: 0xff, + unk_2: Default::default(), + unk_4: 0, + unk_5: 1, + unk_6: Default::default(), + unk_1e: 0xff, + unk_1f: 0, + unk_20: Default::default(), + unk_23: 2, + unk_24: Default::default(), + } + } + } + + #[derive(Debug)] + #[repr(C)] + pub(crate) struct RingState { + pub(crate) gpu_doneptr: AtomicU32, + __pad0: Pad<0xc>, + pub(crate) unk_10: AtomicU32, + __pad1: Pad<0xc>, + pub(crate) unk_20: AtomicU32, + __pad2: Pad<0xc>, + pub(crate) gpu_rptr: AtomicU32, + __pad3: Pad<0xc>, + pub(crate) cpu_wptr: AtomicU32, + __pad4: Pad<0xc>, + pub(crate) rb_size: u32, + __pad5: Pad<0xc>, + // This isn't part of the structure, but it's here as a + // debugging hack so we can inspect what ring position + // the driver considered complete and freeable. + pub(crate) cpu_freeptr: AtomicU32, + __pad6: Pad<0xc>, + } + default_zeroed!(RingState); + + #[derive(Debug, Clone, Copy)] + #[repr(C)] + pub(crate) struct Priority( + pub(crate) u32, + pub(crate) u32, + pub(crate) U64, + pub(crate) u32, + pub(crate) u32, + pub(crate) u32, + ); + + pub(crate) const PRIORITY: [Priority; 4] = [ + Priority(0, 0, U64(0xffff_ffff_ffff_0000), 1, 0, 1), + Priority(1, 1, U64(0xffff_ffff_0000_0000), 0, 0, 0), + Priority(2, 2, U64(0xffff_0000_0000_0000), 0, 0, 2), + Priority(3, 3, U64(0x0000_0000_0000_0000), 0, 0, 3), + ]; + + impl Default for Priority { + fn default() -> Priority { + PRIORITY[2] + } + } + + #[versions(AGX)] + #[derive(Debug)] + #[repr(C)] + pub(crate) struct QueueInfo<'a> { + pub(crate) state: GpuPointer<'a, super::RingState>, + pub(crate) ring: GpuPointer<'a, &'a [u64]>, + pub(crate) notifier_list: GpuPointer<'a, event::NotifierList>, + pub(crate) gpu_buf: GpuPointer<'a, &'a [u8]>, + pub(crate) gpu_rptr1: AtomicU32, + pub(crate) gpu_rptr2: AtomicU32, + pub(crate) gpu_rptr3: AtomicU32, + pub(crate) event_id: AtomicI32, + pub(crate) priority: Priority, + pub(crate) unk_4c: i32, + pub(crate) uuid: u32, + pub(crate) unk_54: i32, + pub(crate) unk_58: U64, + pub(crate) busy: AtomicU32, + pub(crate) __pad: Pad<0x20>, + #[ver(V >= V13_2 && G < G14X)] + pub(crate) unk_84_0: u32, + pub(crate) unk_84_state: AtomicU32, + pub(crate) error_count: AtomicU32, + pub(crate) unk_8c: u32, + pub(crate) unk_90: u32, + pub(crate) unk_94: u32, + pub(crate) pending: AtomicU32, + pub(crate) unk_9c: u32, + pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, + pub(crate) unk_a8: U64, + #[ver(V >= V13_2 && G < G14X)] + pub(crate) unk_b0: u32, + } +} + +trivial_gpustruct!(Barrier); +trivial_gpustruct!(RingState); + +impl Command for Barrier {} + +pub(crate) struct GpuContextData { + pub(crate) _buffer: Arc, +} +impl GpuStruct for GpuContextData { + type Raw<'a> = raw::GpuContextData; +} + +#[versions(AGX)] +#[derive(Debug)] +pub(crate) struct QueueInfo { + pub(crate) state: GpuObject, + pub(crate) ring: GpuArray, + pub(crate) gpu_buf: GpuArray, + pub(crate) notifier_list: Arc>, + pub(crate) gpu_context: Arc, +} + +#[versions(AGX)] +impl GpuStruct for QueueInfo::ver { + type Raw<'a> = raw::QueueInfo::ver<'a>; +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs new file mode 100644 index 00000000000000..8affba257d956f --- /dev/null +++ b/drivers/gpu/drm/asahi/gem.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi driver GEM object implementation +//! +//! Basic wrappers and adaptations between generic GEM shmem objects and this driver's +//! view of what a GPU buffer object is. It is in charge of keeping track of all mappings for +//! each GEM object so we can remove them when a client (File) or a Vm are destroyed, as well as +//! implementing RTKit buffers on top of GEM objects for firmware use. + +use kernel::{ + drm, + drm::gem::{ + shmem, + shmem::VMap, + BaseObject, + DriverObject, // + }, + error::Result, + prelude::*, + types::ARef, + uapi, +}; + +use core::ops::Range; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use crate::{ + debug::*, + driver::{ + AsahiDevice, + AsahiDriver, // + }, + file, + mmu, + util::*, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; + +/// Represents the inner data of a GEM object for this driver. +#[pin_data] +pub(crate) struct AsahiObject { + /// ID for debug + id: u64, + /// Object creation flags. + flags: u32, + /// Whether this object can be exported. + exportable: bool, + /// Whether this is a kernel-created object. + kernel: bool, +} + +/// Type alias for the shmem GEM object type for this driver. +pub(crate) type Object = shmem::Object; + +unsafe impl Send for AsahiObject {} +unsafe impl Sync for AsahiObject {} + +// /// Type alias for the SGTable type for this driver. +// pub(crate) type SGTable = shmem::SGTable; + +/// A shared reference to a GEM object for this driver. +pub(crate) struct ObjectRef { + /// The underlying GEM object reference + pub(crate) gem: ARef, + /// The kernel-side VMap of this object, if needed + vmap: Option>, +} + +crate::no_debug!(ObjectRef); + +static GEM_ID: AtomicU64 = AtomicU64::new(0); + +impl ObjectRef { + /// Create a new wrapper for a raw GEM object reference. + pub(crate) fn new(gem: ARef) -> ObjectRef { + ObjectRef { gem, vmap: None } + } + + /// Return the `VMap` for this object, creating it if necessary. + pub(crate) fn vmap(&mut self) -> Result> { + if self.vmap.is_none() { + self.vmap = Some(self.gem.owned_vmap()?); + } + self.gem.vmap() + } + + /// Returns the size of an object in bytes + pub(crate) fn size(&self) -> usize { + self.gem.size() + } + + /// Maps an object into a given `Vm` at any free address within a given range. + pub(crate) fn map_into_range( + &mut self, + vm: &crate::mmu::Vm, + range: Range, + alignment: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + // Only used for kernel objects now + if !self.gem.kernel { + return Err(EINVAL); + } + vm.map_in_range(&self.gem, 0..self.gem.size(), alignment, range, prot, guard) + } + + /// Maps a range within an object into a given `Vm` at any free address within a given range. + pub(crate) fn map_range_into_range( + &mut self, + vm: &crate::mmu::Vm, + obj_range: Range, + range: Range, + alignment: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + if obj_range.end > self.gem.size() { + return Err(EINVAL); + } + if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(&*self.gem) + { + return Err(EINVAL); + } + vm.map_in_range(&self.gem, obj_range, alignment, range, prot, guard) + } + + /// Maps an object into a given `Vm` at a specific address. + /// + /// Returns Err(ENOSPC) if the requested address is already busy. + pub(crate) fn map_at( + &mut self, + vm: &crate::mmu::Vm, + addr: u64, + prot: mmu::Prot, + guard: bool, + ) -> Result { + if self.gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(&*self.gem) + { + return Err(EINVAL); + } + + vm.map_at(addr, self.gem.size(), self.gem.clone(), prot, guard) + } +} + +pub(crate) struct AsahiObjConfig { + flags: u32, + exportable: bool, + kernel: bool, +} + +/// Create a new kernel-owned GEM object. +pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { + let gem = shmem::Object::::new( + dev, + align(size, mmu::UAT_PGSZ), + shmem::ObjectConfig:: { + map_wc: false, + parent_resv_obj: None, + }, + AsahiObjConfig { + flags: 0, + exportable: false, + kernel: true, + }, + )?; + + mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); + Ok(ObjectRef::new(gem)) +} + +/// Create a new user-owned GEM object with the given flags. +pub(crate) fn new_object( + dev: &AsahiDevice, + size: usize, + flags: u32, + parent_object: Option<&shmem::Object>, +) -> Result> { + if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() + { + return Err(EINVAL); + } + + let gem = shmem::Object::::new( + dev, + align(size, mmu::UAT_PGSZ), + shmem::ObjectConfig:: { + map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, + parent_resv_obj: parent_object, + }, + AsahiObjConfig { + flags, + exportable: parent_object.is_none(), + kernel: false, + }, + )?; + + mod_pr_debug!("AsahiObject new user object: id={}\n", gem.id); + Ok(gem) +} + +#[vtable] +impl DriverObject for AsahiObject { + type Driver = AsahiDriver; + type Args = AsahiObjConfig; + + const HAS_EXPORT: bool = true; + + /// Callback to create the inner data of a GEM object + fn new(_dev: &AsahiDevice, _size: usize, args: Self::Args) -> impl PinInit { + let id = GEM_ID.fetch_add(1, Ordering::Relaxed); + mod_pr_debug!("AsahiObject::new id={}\n", id); + try_pin_init!(AsahiObject { + id, + flags: args.flags, + exportable: args.exportable, + kernel: args.kernel, + }) + } + + /// Callback to drop all mappings for a GEM object owned by a given `File` + fn close(obj: &::Object, file: &drm::gem::DriverFile) { + // fn close(obj: &Object, file: &DrmFile) { + mod_pr_debug!("AsahiObject::close id={}\n", obj.id); + if file::File::unbind_gem_object(file, obj).is_err() { + pr_err!("AsahiObject::close: Failed to unbind GEM object\n"); + } + } + + /// Optional handle for exporting a gem object. + fn export( + obj: &::Object, + flags: u32, + ) -> Result> { + if !obj.exportable { + return Err(EINVAL); + } + + obj.prime_export(flags) + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs new file mode 100644 index 00000000000000..b6c1c64c503671 --- /dev/null +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -0,0 +1,1642 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Top-level GPU manager +//! +//! This module is the root of all GPU firmware management for a given driver instance. It is +//! responsible for initialization, owning the top-level managers (events, UAT, etc.), and +//! communicating with the raw RtKit endpoints to send and receive messages to/from the GPU +//! firmware. +//! +//! It is also the point where diverging driver firmware/GPU variants (using the versions macro) +//! are unified, so that the top level of the driver itself (in `driver`) does not have to concern +//! itself with version dependence. + +use core::any::Any; +use core::ops::Range; +use core::slice; +use core::sync::atomic::{ + AtomicBool, + AtomicU64, + Ordering, // +}; + +use kernel::{ + c_str, + drm::gem::shmem, + error::code::*, + io::mem::{ + Mem, + MemFlag, // + }, + iosys_map::IoSysMapRef, + macros::versions, + new_mutex, + prelude::*, + soc::apple::rtkit, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, + Mutex, + UniqueArc, // + }, + time::{ + Delta, + Instant, + Monotonic, // + }, + types::ForeignOwnable, // +}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + devcoredump, + time::msecs_to_jiffies, // +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::fw::channels::{ + ChannelErrorType, + PipeType, // +}; +use crate::fw::types::{ + U32, + U64, // +}; +use crate::{ + alloc, + buffer, + channel, + event, + fw, + gem, + hw, + initdata, + mem, + mmu, + queue, + regs, + workqueue, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Gpu; + +/// Firmware endpoint for init & incoming notifications. +const EP_FIRMWARE: u8 = 0x20; + +/// Doorbell endpoint for work/message submissions. +const EP_DOORBELL: u8 = 0x21; + +/// Initialize the GPU firmware. +const MSG_INIT: u64 = 0x81 << 48; +const INIT_DATA_MASK: u64 = (1 << 44) - 1; + +/// TX channel doorbell. +const MSG_TX_DOORBELL: u64 = 0x83 << 48; +/// Firmware control channel doorbell. +const MSG_FWCTL: u64 = 0x84 << 48; +// /// Halt the firmware (?). +// const MSG_HALT: u64 = 0x85 << 48; + +/// Receive channel doorbell notification. +const MSG_RX_DOORBELL: u64 = 0x42 << 48; + +/// Doorbell number for firmware kicks/wakeups. +const DOORBELL_KICKFW: u64 = 0x10; +/// Doorbell number for device control channel kicks. +const DOORBELL_DEVCTRL: u64 = 0x11; + +// Upper kernel half VA address ranges. +/// Private (cached) firmware structure VA range base. +const IOVA_KERN_PRIV_RANGE: Range = 0xffffffa000000000..0xffffffa600000000; +/// Private (cached) GPU-RO firmware structure VA range base. +const IOVA_KERN_GPU_RO_RANGE: Range = 0xffffffa600000000..0xffffffa800000000; +/// Shared (uncached) firmware structure VA range base. +const IOVA_KERN_SHARED_RANGE: Range = 0xffffffa800000000..0xffffffaa00000000; +/// Shared (uncached) read-only firmware structure VA range base. +const IOVA_KERN_SHARED_RO_RANGE: Range = 0xffffffaa00000000..0xffffffac00000000; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_GPU_RANGE: Range = 0xffffffac00000000..0xffffffae00000000; +/// GPU/FW shared structure VA range base. +const IOVA_KERN_RTKIT_RANGE: Range = 0xffffffae00000000..0xffffffae10000000; +/// Shared (uncached) timestamp region. +pub(crate) const IOVA_KERN_TIMESTAMP_RANGE: Range = 0xffffffae10000000..0xffffffae14000000; +/// FW MMIO VA range base. +const IOVA_KERN_MMIO_RANGE: Range = 0xffffffaf00000000..0xffffffb000000000; + +/// GPU/FW buffer manager control address (context 0 low) +pub(crate) const IOVA_KERN_GPU_BUFMGR_LOW: u64 = 0x20_0000_0000; +/// GPU/FW buffer manager control address (context 0 high) +pub(crate) const IOVA_KERN_GPU_BUFMGR_HIGH: u64 = 0xffffffaeffff0000; + +/// Timeout for entering the halt state after a fault or request. +const HALT_ENTER_TIMEOUT: Delta = Delta::from_millis(100); + +/// Maximum amount of firmware-private memory garbage allowed before collection. +/// Collection flushes the FW cache and is expensive, so this needs to be +/// reasonably high. +const MAX_FW_ALLOC_GARBAGE_BYTES: usize = 16 * 1024 * 1024; +/// Maximum count of firmware-private memory garbage objects allowed before collection. +/// This works out to 16K of memory in the garbage list (8 bytes each), which keeps us +/// within the safe range for kmalloc (on 16K page systems). +const MAX_FW_ALLOC_GARBAGE_OBJECTS: usize = 2048; + +/// Global allocators used for kernel-half structures. +pub(crate) struct KernelAllocators { + pub(crate) private: alloc::DefaultAllocator, + pub(crate) shared: alloc::DefaultAllocator, + pub(crate) shared_ro: alloc::DefaultAllocator, + #[allow(dead_code)] + pub(crate) gpu: alloc::DefaultAllocator, + pub(crate) gpu_ro: alloc::DefaultAllocator, +} + +/// Receive (GPU->driver) ring buffer channels. +#[versions(AGX)] +#[pin_data] +struct RxChannels { + event: channel::EventChannel::ver, + fw_log: channel::FwLogChannel, + ktrace: channel::KTraceChannel, + stats: channel::StatsChannel::ver, +} + +/// GPU work submission pipe channels (driver->GPU). +#[versions(AGX)] +struct PipeChannels { + pub(crate) vtx: KVec>>>, + pub(crate) frag: KVec>>>, + pub(crate) comp: KVec>>>, +} + +/// Misc command transmit (driver->GPU) channels. +#[versions(AGX)] +#[pin_data] +struct TxChannels { + pub(crate) device_control: channel::DeviceControlChannel::ver, +} + +/// Number of work submission pipes per type, one for each priority level. +const NUM_PIPES: usize = 4; + +/// A generic monotonically incrementing ID used to uniquely identify object instances within the +/// driver. +pub(crate) struct ID(AtomicU64); + +impl ID { + /// Create a new ID counter with a given value. + fn new(val: u64) -> ID { + ID(AtomicU64::new(val)) + } + + /// Fetch the next unique ID. + pub(crate) fn next(&self) -> u64 { + self.0.fetch_add(1, Ordering::Relaxed) + } +} + +impl Default for ID { + /// IDs default to starting at 2, as 0/1 are considered reserved for the system. + fn default() -> Self { + Self::new(2) + } +} + +/// A guard representing one active submission on the GPU. When dropped, decrements the active +/// submission count. +pub(crate) struct OpGuard(Arc); + +impl Drop for OpGuard { + fn drop(&mut self) { + self.0.end_op(); + } +} + +/// Set of global sequence IDs used in the driver. +#[derive(Default)] +pub(crate) struct SequenceIDs { + /// `File` instance ID. + pub(crate) file: ID, + /// `Vm` instance ID. + pub(crate) vm: ID, + /// Submission instance ID. + pub(crate) submission: ID, + /// `Queue` instance ID. + pub(crate) queue: ID, +} + +/// Top-level GPU manager that owns all the global state relevant to the driver instance. +#[versions(AGX)] +#[pin_data] +pub(crate) struct GpuManager { + dev: AsahiDevRef, + cfg: &'static hw::HwConfig, + dyncfg: hw::DynConfig, + pub(crate) initdata: fw::types::GpuObject, + uat: mmu::Uat, + crashed: AtomicBool, + #[pin] + alloc: Mutex, + io_mappings: KVec, + next_mmio_iova: u64, + #[pin] + rtkit: Mutex>>, + #[pin] + rx_channels: Mutex, + #[pin] + tx_channels: Mutex, + #[pin] + fwctl_channel: Mutex, + pipes: PipeChannels::ver, + event_manager: Arc, + buffer_mgr: buffer::BufferManager::ver, + ids: SequenceIDs, + #[allow(clippy::vec_box)] + #[pin] + garbage_contexts: Mutex>>>, +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + /// Cast Arc as an Any type. + fn arc_as_any(self: Arc) -> Arc; + /// Initialize the GPU. + fn init(&self) -> Result; + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Create a new `Vm` given a unique `File` ID. + fn new_vm(&self, kernel_range: Range) -> Result; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; + /// Create a new user command queue. + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result>; + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Kick the firmware (wake it up if asleep). + /// + /// This should be useful to reduce latency on work submission, so we can ask the firmware to + /// wake up while we do some preparatory work for the work submission. + fn kick_firmware(&self) -> Result; + /// Flush the entire firmware cache. + /// + /// TODO: Does this actually work? + fn flush_fw_cache(&self) -> Result; + /// Handle a GPU work timeout event. + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32); + /// Handle a GPU fault event. + fn handle_fault(&self); + /// Handle a channel error event. + fn handle_channel_error( + &self, + error_type: ChannelErrorType, + pipe_type: u32, + event_slot: u32, + event_value: u32, + ); + /// Acknowledge a Buffer grow op. + fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32); + /// Send a firmware control command (secure cache flush). + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result; + /// Get the static GPU configuration for this SoC. + fn get_cfg(&self) -> &'static hw::HwConfig; + /// Get the dynamic GPU configuration for this SoC. + fn get_dyncfg(&self) -> &hw::DynConfig; + /// Register an unused context as garbage + fn free_context(&self, data: KBox>); + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; +} + +/// Private generic trait for functions that don't need to escape this module. +trait GpuManagerPriv { + /// Decrement the pending submission counter. + fn end_op(&self); +} + +pub(crate) struct RtkitObject { + vmap: shmem::VMap, + mapping: mmu::KernelMapping, +} + +impl rtkit::Buffer for RtkitObject { + fn iova(&self) -> Result { + Ok(self.mapping.iova() as usize) + } + fn buf(&mut self) -> Result> { + Ok(self.vmap.get()) + } +} + +#[versions(AGX)] +#[vtable] +impl rtkit::Operations for GpuManager::ver { + type Data = Arc; + type Buffer = RtkitObject; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let dev = &data.dev; + //dev_info!(dev.as_ref(), "RtKit message: {:#x}:{:#x}\n", ep, msg); + + if ep != EP_FIRMWARE || msg != MSG_RX_DOORBELL { + dev_err!(dev.as_ref(), "Unknown message: {:#x}:{:#x}\n", ep, msg); + return; + } + + let mut ch = data.rx_channels.lock(); + + ch.fw_log.poll(); + ch.ktrace.poll(); + ch.stats.poll(); + ch.event.poll(); + } + + fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { + let dev = &data.dev; + + data.crashed.store(true, Ordering::Relaxed); + + #[cfg(CONFIG_DEV_COREDUMP)] + if let Err(e) = data.generate_crashdump(crashlog) { + dev_err!(dev.as_ref(), "Could not generate crashdump: {:?}\n", e); + } + #[cfg(not(CONFIG_DEV_COREDUMP))] + let _ = crashlog; + + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware crashed"); + } else { + dev_err!(dev.as_ref(), "GPU firmware crashed, failing all jobs\n"); + data.event_manager.fail_all(workqueue::WorkError::NoDevice); + } + } + + fn shmem_alloc( + data: ::Borrowed<'_>, + size: usize, + ) -> Result { + let dev = &data.dev; + mod_dev_dbg!(dev, "shmem_alloc() {:#x} bytes\n", size); + + let mut obj = gem::new_kernel_object(dev, size)?; + let vmap = obj.gem.owned_vmap()?; + let mapping = obj.map_into_range( + data.uat.kernel_vm(), + IOVA_KERN_RTKIT_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + true, + )?; + mod_dev_dbg!(dev, "shmem_alloc() -> VA {:#x}\n", mapping.iova()); + Ok(RtkitObject { vmap, mapping }) + } +} + +#[versions(AGX)] +impl GpuManager::ver { + /// Create a new GpuManager of this version/GPU combination. + #[inline(never)] + pub(crate) fn new( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + ) -> Result> { + let uat = Self::make_uat(dev, cfg)?; + let dyncfg = Self::make_dyncfg(dev, res, cfg, &uat)?; + + let mut alloc = KernelAllocators { + private: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_PRIV_RANGE, + 0x80, + mmu::PROT_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel Private"), + true, + )?, + shared: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RANGE, + 0x80, + mmu::PROT_FW_SHARED_RW, + 1024 * 1024, + true, + fmt!("Kernel Shared"), + false, + )?, + shared_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_SHARED_RO_RANGE, + 0x80, + mmu::PROT_FW_SHARED_RO, + 64 * 1024, + true, + fmt!("Kernel RO Shared"), + false, + )?, + gpu: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RANGE, + 0x80, + mmu::PROT_GPU_FW_SHARED_RW, + 64 * 1024, + true, + fmt!("Kernel GPU Shared"), + false, + )?, + gpu_ro: alloc::DefaultAllocator::new( + dev, + uat.kernel_vm(), + IOVA_KERN_GPU_RO_RANGE, + 0x80, + mmu::PROT_GPU_RO_FW_PRIV_RW, + 1024 * 1024, + true, + fmt!("Kernel GPU RO Shared"), + true, + )?, + }; + + let event_manager = Self::make_event_manager(&mut alloc)?; + let mut initdata = Self::make_initdata(dev, cfg, &dyncfg, &mut alloc)?; + + initdata.runtime_pointers.buffer_mgr_ctl_low_mapping = + Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( + uat.kernel_lower_vm(), + IOVA_KERN_GPU_BUFMGR_LOW, + mmu::PROT_GPU_SHARED_RW, + false, + )?); + initdata.runtime_pointers.buffer_mgr_ctl_high_mapping = + Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( + uat.kernel_vm(), + IOVA_KERN_GPU_BUFMGR_HIGH, + mmu::PROT_FW_SHARED_RW, + false, + )?); + + let mut mgr = Self::make_mgr(dev, cfg, dyncfg, uat, alloc, event_manager, initdata)?; + + { + let fwctl = mgr.fwctl_channel.lock(); + let p_fwctl = fwctl.to_raw(); + core::mem::drop(fwctl); + + mgr.as_mut() + .initdata_mut() + .fw_status + .with_mut(|raw, _inner| { + raw.fwctl_channel = p_fwctl; + }); + } + + { + let txc = mgr.tx_channels.lock(); + let p_device_control = txc.device_control.to_raw(); + core::mem::drop(txc); + + let rxc = mgr.rx_channels.lock(); + let p_event = rxc.event.to_raw(); + let p_fw_log = rxc.fw_log.to_raw(); + let p_ktrace = rxc.ktrace.to_raw(); + let p_stats = rxc.stats.to_raw(); + let p_fwlog_buf = rxc.fw_log.get_buf(); + core::mem::drop(rxc); + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + raw.device_control = p_device_control; + raw.event = p_event; + raw.fw_log = p_fw_log; + raw.ktrace = p_ktrace; + raw.stats = p_stats; + raw.fwlog_buf = Some(p_fwlog_buf); + }); + } + + let mut p_pipes: KVec = KVec::new(); + + for ((v, f), c) in mgr + .pipes + .vtx + .iter() + .zip(&mgr.pipes.frag) + .zip(&mgr.pipes.comp) + { + p_pipes.push( + fw::initdata::raw::PipeChannels::ver { + vtx: v.lock().to_raw(), + frag: f.lock().to_raw(), + comp: c.lock().to_raw(), + }, + GFP_KERNEL, + )?; + } + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .with_mut(|raw, _inner| { + for (i, p) in p_pipes.into_iter().enumerate() { + raw.pipes[i].vtx = p.vtx; + raw.pipes[i].frag = p.frag; + raw.pipes[i].comp = p.comp; + } + }); + + for (i, map) in cfg.io_mappings.iter().enumerate() { + if let Some(map) = map.as_ref() { + Self::iomap(&mut mgr, cfg, i, map)?; + } + } + + #[ver(V >= V13_0B4)] + if let Some(base) = cfg.sram_base { + let size = cfg.sram_size.unwrap(); + let iova = mgr.as_mut().alloc_mmio_iova(size); + + let mapping = mgr + .uat + .kernel_vm() + .map_io(iova, base, size, mmu::PROT_FW_SHARED_RW)?; + + mgr.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.sgx_sram_ptr = U64(mapping.iova()); + }); + + mgr.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; + } + + let mgr = Arc::from(mgr); + + let rtkit = rtkit::RtKit::::new(dev.as_ref(), None, 0, mgr.clone())?; + + *mgr.rtkit.lock() = Some(rtkit); + + { + let mut rxc = mgr.rx_channels.lock(); + rxc.event.set_manager(mgr.clone()); + } + + Ok(mgr) + } + + /// Return a mutable reference to the initdata member + fn initdata_mut( + self: Pin<&mut Self>, + ) -> &mut fw::types::GpuObject { + // SAFETY: initdata does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().initdata } + } + + /// Return a mutable reference to the io_mappings member + fn io_mappings_mut(self: Pin<&mut Self>) -> &mut KVec { + // SAFETY: io_mappings does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().io_mappings } + } + + /// Allocate an MMIO iova range + fn alloc_mmio_iova(self: Pin<&mut Self>, size: usize) -> u64 { + // SAFETY: next_mmio_iova does not require structural pinning. + let next_ref = unsafe { &mut self.get_unchecked_mut().next_mmio_iova }; + + let addr = *next_ref; + let next = addr + (size + mmu::UAT_PGSZ) as u64; + + assert!(next <= IOVA_KERN_MMIO_RANGE.end); + + *next_ref = next; + + addr + } + + /// Build the entire GPU InitData structure tree and return it as a boxed GpuObject. + fn make_initdata( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: &hw::DynConfig, + alloc: &mut KernelAllocators, + ) -> Result>> { + let mut builder = initdata::InitDataBuilder::ver::new(dev, alloc, cfg, dyncfg); + builder.build() + } + + /// Create a fresh boxed Uat instance. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_uat(dev: &AsahiDevice, cfg: &'static hw::HwConfig) -> Result> { + // G14X has a new thing in the Scene structure that unfortunately requires + // write access from user contexts. Hopefully it's not security-sensitive. + #[ver(G >= G14X)] + let map_kernel_to_user = true; + #[ver(G < G14X)] + let map_kernel_to_user = false; + + Ok(KBox::new( + mmu::Uat::new(dev, cfg, map_kernel_to_user)?, + GFP_KERNEL, + )?) + } + + /// Actually create the final GpuManager instance, as a UniqueArc. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_mgr( + dev: &AsahiDevice, + cfg: &'static hw::HwConfig, + dyncfg: KBox, + uat: KBox, + mut alloc: KernelAllocators, + event_manager: Arc, + initdata: KBox>, + ) -> Result>> { + let mut pipes = PipeChannels::ver { + vtx: KVec::new(), + frag: KVec::new(), + comp: KVec::new(), + }; + + for _i in 0..=NUM_PIPES - 1 { + pipes.vtx.push( + KBox::pin_init( + new_mutex!(channel::PipeChannel::ver::new(dev, &mut alloc)?, "pipe_vtx",), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + pipes.frag.push( + KBox::pin_init( + new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_frag", + ), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + pipes.comp.push( + KBox::pin_init( + new_mutex!( + channel::PipeChannel::ver::new(dev, &mut alloc)?, + "pipe_comp", + ), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + } + + let fwctl_channel = channel::FwCtlChannel::new(dev, &mut alloc)?; + + let buffer_mgr = buffer::BufferManager::ver::new()?; + let event_manager_clone = event_manager.clone(); + let buffer_mgr_clone = buffer_mgr.clone(); + let alloc_ref = &mut alloc; + let rx_channels = KBox::init( + try_init!(RxChannels::ver { + event: channel::EventChannel::ver::new( + dev, + alloc_ref, + event_manager_clone, + buffer_mgr_clone, + )?, + fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, + ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, + stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, + }), + GFP_KERNEL, + )?; + + let alloc_ref = &mut alloc; + let tx_channels = KBox::init( + try_init!(TxChannels::ver { + device_control: channel::DeviceControlChannel::ver::new(dev, alloc_ref)?, + }), + GFP_KERNEL, + )?; + + let x = UniqueArc::pin_init( + try_pin_init!(GpuManager::ver { + dev: dev.into(), + cfg, + dyncfg: KBox::::into_inner(dyncfg), + initdata: KBox::>::into_inner(initdata), + uat: KBox::::into_inner(uat), + io_mappings: KVec::new(), + next_mmio_iova: IOVA_KERN_MMIO_RANGE.start, + rtkit <- new_mutex!(None, "rtkit"), + crashed: AtomicBool::new(false), + event_manager, + alloc <- new_mutex!(alloc, "alloc"), + fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), + rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), + tx_channels <- new_mutex!(KBox::::into_inner(tx_channels), "tx_channels"), + pipes, + buffer_mgr, + ids: Default::default(), + garbage_contexts <- new_mutex!(KVec::new(), "garbage_contexts"), + }), + GFP_KERNEL, + )?; + + Ok(x) + } + + fn load_hwdata_blob(dev: &AsahiDevice, name: &CStr, size_name: &CStr) -> Result> { + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let size: usize = dev + .as_ref() + .fwnode() + .ok_or(ENOENT)? + .property_read::(size_name) + .or(0) + .try_into()?; + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + // SAFETY: No dma here, just loading init data. + let mem = unsafe { Mem::try_new(res, (MemFlag::WB).into())? }; + if size > mem.size() { + return Err(ENOENT); + } + // SAFETY: trusting the bootloader to fill it out correctly + let blob_sl = unsafe { slice::from_raw_parts(mem.ptr(), size) }; + let mut blob = KVVec::new(); + blob.extend_from_slice(blob_sl, GFP_KERNEL)?; + Ok(blob) + } + + /// Fetch and validate the GPU dynamic configuration from the device tree and hardware. + /// + /// Force disable inlining to avoid blowing up the stack. + #[inline(never)] + fn make_dyncfg( + dev: &AsahiDevice, + res: ®s::Resources, + cfg: &'static hw::HwConfig, + uat: &mmu::Uat, + ) -> Result> { + let gpu_id = res.get_gpu_id()?; + + dev_info!(dev.as_ref(), "GPU Information:\n"); + dev_info!( + dev.as_ref(), + " Type: {:?}{:?}\n", + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + dev_info!(dev.as_ref(), " Clusters: {}\n", gpu_id.num_clusters); + dev_info!( + dev.as_ref(), + " Cores: {} ({})\n", + gpu_id.num_cores, + gpu_id.num_cores * gpu_id.num_clusters + ); + dev_info!( + dev.as_ref(), + " Frags: {} ({})\n", + gpu_id.num_frags, + gpu_id.num_frags * gpu_id.num_clusters + ); + dev_info!( + dev.as_ref(), + " GPs: {} ({})\n", + gpu_id.num_gps, + gpu_id.num_gps * gpu_id.num_clusters + ); + dev_info!(dev.as_ref(), " Core masks: {:#x?}\n", gpu_id.core_masks); + dev_info!( + dev.as_ref(), + " Active cores: {}\n", + gpu_id.total_active_cores + ); + + dev_info!(dev.as_ref(), "Getting configuration from device tree...\n"); + let pwr_cfg = hw::PwrConfig::load(dev, cfg)?; + dev_info!(dev.as_ref(), "Dynamic configuration fetched\n"); + + if gpu_id.gpu_gen != cfg.gpu_gen || gpu_id.gpu_variant != cfg.gpu_variant { + dev_err!( + dev.as_ref(), + "GPU type mismatch (expected {:?}{:?}, found {:?}{:?})\n", + cfg.gpu_gen, + cfg.gpu_variant, + gpu_id.gpu_gen, + gpu_id.gpu_variant + ); + return Err(EIO); + } + if gpu_id.num_clusters > cfg.max_num_clusters { + dev_err!( + dev.as_ref(), + "Too many clusters ({} > {})\n", + gpu_id.num_clusters, + cfg.max_num_clusters + ); + return Err(EIO); + } + if gpu_id.num_cores > cfg.max_num_cores { + dev_err!( + dev.as_ref(), + "Too many cores ({} > {})\n", + gpu_id.num_cores, + cfg.max_num_cores + ); + return Err(EIO); + } + if gpu_id.num_frags > cfg.max_num_frags { + dev_err!( + dev.as_ref(), + "Too many frags ({} > {})\n", + gpu_id.num_frags, + cfg.max_num_frags + ); + return Err(EIO); + } + if gpu_id.num_gps > cfg.max_num_gps { + dev_err!( + dev.as_ref(), + "Too many GPs ({} > {})\n", + gpu_id.num_gps, + cfg.max_num_gps + ); + return Err(EIO); + } + + let fwnode = dev.as_ref().fwnode().ok_or(ENOENT)?; + + Ok(KBox::new( + hw::DynConfig { + pwr: pwr_cfg, + uat_ttb_base: uat.ttb_base(), + id: gpu_id, + firmware_version: fwnode + .property_read_array_vec(c_str!("apple,firmware-version"), 3)? + .or(kernel::kvec![0; 3]?), + + hw_data_a: Self::load_hwdata_blob( + dev, + c_str!("hw-cal-a"), + c_str!("debug,hw-cal-a-size"), + ) + .unwrap_or(KVVec::new()), + hw_data_b: Self::load_hwdata_blob( + dev, + c_str!("hw-cal-b"), + c_str!("debug,hw-cal-b-size"), + ) + .unwrap_or(KVVec::new()), + hw_globals: Self::load_hwdata_blob( + dev, + c_str!("globals"), + c_str!("debug,globals-size"), + ) + .unwrap_or(KVVec::new()), + }, + GFP_KERNEL, + )?) + } + + /// Create the global GPU event manager, and return an `Arc<>` to it. + fn make_event_manager(alloc: &mut KernelAllocators) -> Result> { + Ok(Arc::new(event::EventManager::new(alloc)?, GFP_KERNEL)?) + } + + /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified + /// index. + fn iomap( + this: &mut Pin>, + cfg: &'static hw::HwConfig, + index: usize, + map: &hw::IOMapping, + ) -> Result { + let dies = if map.per_die { + cfg.num_dies as usize + } else { + 1 + }; + + let off = map.base & mmu::UAT_PGMSK; + let base = map.base - off; + let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; + let map_size = end - base; + + // Array mappings must be aligned + assert!((off == 0 && map_size == map.size) || (map.count == 1 && !map.per_die)); + assert!(map.count > 0); + + let iova = this.as_mut().alloc_mmio_iova(map_size * map.count * dies); + let mut cur_iova = iova; + + for die in 0..dies { + for i in 0..map.count { + let phys_off = die * 0x20_0000_0000 + i * map.stride; + + let mapping = this.uat.kernel_vm().map_io( + cur_iova, + base + phys_off, + map_size, + if map.writable { + mmu::PROT_FW_MMIO_RW + } else { + mmu::PROT_FW_MMIO_RO + }, + )?; + + this.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; + cur_iova += map_size as u64; + } + } + + this.as_mut() + .initdata_mut() + .runtime_pointers + .hwdata_b + .with_mut(|raw, _| { + raw.io_mappings[index] = fw::initdata::raw::IOMapping { + phys_addr: U64(map.base as u64), + virt_addr: U64(iova + off as u64), + total_size: (map.size * map.count * dies) as u32, + element_size: map.size as u32, + readwrite: U64(map.writable as u64), + }; + }); + + Ok(()) + } + + /// Mark work associated with currently in-progress event slots as failed, after a fault or + /// timeout. + fn mark_pending_events(&self, culprit_slot: Option, error: workqueue::WorkError) { + dev_err!(self.dev.as_ref(), " Pending events:\n"); + + self.initdata.globals.with(|raw, _inner| { + for (index, i) in raw.pending_stamps.iter().enumerate() { + let info = i.info.load(Ordering::Relaxed); + let wait_value = i.wait_value.load(Ordering::Relaxed); + + if info & 1 != 0 { + #[ver(V >= V13_5)] + let slot = (info >> 4) & 0x7f; + #[ver(V < V13_5)] + let slot = (info >> 3) & 0x7f; + #[ver(V >= V13_5)] + let flags = info & 0xf; + #[ver(V < V13_5)] + let flags = info & 0x7; + dev_err!( + self.dev.as_ref(), + " [{}:{}] flags={} value={:#x}\n", + index, + slot, + flags, + wait_value + ); + let error = if culprit_slot.is_some() && culprit_slot != Some(slot) { + workqueue::WorkError::Killed + } else { + error + }; + self.event_manager.mark_error(slot, wait_value, error); + i.info.store(0, Ordering::Relaxed); + i.wait_value.store(0, Ordering::Relaxed); + } + } + }); + } + + /// Fetch the GPU MMU fault information from the hardware registers. + fn get_fault_info(&self) -> Option { + let res = &(*self.dev).resources; + + let info = res.get_fault_info(self.cfg); + if info.is_some() { + dev_err!( + self.dev.as_ref(), + " Fault info: {:#x?}\n", + info.as_ref().unwrap() + ); + } + info + } + + /// Resume the GPU firmware after it halts (due to a timeout, fault, or request). + fn recover(&self) { + self.initdata.fw_status.with(|raw, _inner| { + let halt_count = raw.flags.halt_count.load(Ordering::Relaxed); + let mut halted = raw.flags.halted.load(Ordering::Relaxed); + dev_err!(self.dev.as_ref(), " Halt count: {}\n", halt_count); + dev_err!(self.dev.as_ref(), " Halted: {}\n", halted); + + if halted == 0 { + let start = Instant::::now(); + while start.elapsed() < HALT_ENTER_TIMEOUT { + halted = raw.flags.halted.load(Ordering::Relaxed); + if halted != 0 { + break; + } + mem::sync(); + } + halted = raw.flags.halted.load(Ordering::Relaxed); + } + + if debug_enabled(DebugFlags::NoGpuRecovery) { + dev_crit!( + self.dev.as_ref(), + " GPU recovery is disabled, wedging forever!\n" + ); + } else if halted != 0 { + dev_err!(self.dev.as_ref(), " Attempting recovery...\n"); + raw.flags.halted.store(0, Ordering::SeqCst); + raw.flags.resume.store(1, Ordering::SeqCst); + } else { + dev_err!(self.dev.as_ref(), " Cannot recover.\n"); + } + }); + } + + /// Return the packed GPU enabled core masks. + // Only used for some versions + #[allow(dead_code)] + pub(crate) fn core_masks_packed(&self) -> &[u32] { + self.dyncfg.id.core_masks_packed.as_slice() + } + + /// Kick a submission pipe for a submitted job to tell the firmware to start processing it. + pub(crate) fn run_job(&self, job: workqueue::JobSubmission::ver<'_>) -> Result { + mod_dev_dbg!(self.dev, "GPU: run_job\n"); + + let pipe_type = job.pipe_type(); + mod_dev_dbg!(self.dev, "GPU: run_job: pipe_type={:?}\n", pipe_type); + + let pipes = match pipe_type { + PipeType::Vertex => &self.pipes.vtx, + PipeType::Fragment => &self.pipes.frag, + PipeType::Compute => &self.pipes.comp, + }; + + let index: usize = job.priority() as usize; + let mut pipe = pipes.get(index).ok_or(EIO)?.lock(); + + mod_dev_dbg!(self.dev, "GPU: run_job: run()\n"); + job.run(&mut pipe); + mod_dev_dbg!(self.dev, "GPU: run_job: ring doorbell\n"); + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message( + EP_DOORBELL, + MSG_TX_DOORBELL | pipe_type as u64 | ((index as u64) << 2), + )?; + mod_dev_dbg!(self.dev, "GPU: run_job: done\n"); + + Ok(()) + } + + pub(crate) fn start_op(self: &Arc) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); + + mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); + self.kick_firmware()?; + Ok(OpGuard(self.clone())) + } + + fn invalidate_context( + &self, + context: &fw::types::GpuObject, + ) -> Result { + mod_dev_dbg!( + self.dev, + "Invalidating GPU context @ {:?}\n", + context.weak_pointer() + ); + + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.alloc.lock(); + let (garbage_count, _) = guard.private.garbage(); + let (garbage_count_gpuro, _) = guard.gpu_ro.garbage(); + + let dc = context.with( + |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + ctx_23: raw.unk_23, + #[ver(V < V13_3)] + __pad0: Default::default(), + unk_c: U32(0), + unk_10: U32(0), + ctx_0: raw.unk_0, + ctx_1: raw.unk_1, + ctx_4: raw.unk_4, + #[ver(V < V13_3)] + __pad1: Default::default(), + #[ver(V < V13_3)] + unk_18: 0, + gpu_context: Some(context.weak_pointer()), + __pad2: Default::default(), + }, + ); + + mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + + mod_dev_dbg!( + self.dev, + "GPU context invalidated: {:?}\n", + context.weak_pointer() + ); + + // The invalidation does a cache flush, so it is okay to collect garbage + guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(garbage_count_gpuro); + + Ok(()) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + fn generate_crashdump(&self, crashlog: Option<&[u8]>) -> Result { + // Lock the allocators, to block kernel/FW memory mutations (mostly) + let kalloc = self.alloc(); + let pages = self.uat.dump_kernel_pages()?; + core::mem::drop(kalloc); + + let mut crashdump = crate::crashdump::CrashDumpBuilder::new(pages)?; + let initdata_addr = self.initdata.gpu_va().get(); + crashdump.add_agx_info(self.cfg, &self.dyncfg, initdata_addr)?; + if let Some(crashlog) = crashlog { + crashdump.add_crashlog(crashlog)?; + } + let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; + + devcoredump::dev_coredump( + self.dev.as_ref(), + &crate::THIS_MODULE, + crashdump, + GFP_KERNEL, + msecs_to_jiffies(60 * 60 * 1000), + ); + + Ok(()) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn init(&self) -> Result { + self.tx_channels.lock().device_control.send( + &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), + ); + + let initdata = self.initdata.gpu_va().get(); + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + + rtk.as_mut().boot()?; + rtk.as_mut().start_endpoint(EP_FIRMWARE)?; + rtk.as_mut().start_endpoint(EP_DOORBELL)?; + rtk.as_mut() + .send_message(EP_FIRMWARE, MSG_INIT | (initdata & INIT_DATA_MASK))?; + rtk.as_mut() + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + core::mem::drop(guard); + + self.kick_firmware()?; + Ok(()) + } + + fn update_globals(&self) { + let mut timeout: u32 = 2; + if debug_enabled(DebugFlags::WaitForPowerOff) { + timeout = 0; + } else if debug_enabled(DebugFlags::KeepGpuPowered) { + timeout = 5000; + } + + self.initdata.globals.with(|raw, _inner| { + raw.idle_off_delay_ms.store(timeout, Ordering::Relaxed); + }); + } + + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend> { + /* Clean up idle contexts */ + let mut garbage_ctx = KVec::new(); + core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); + + for ctx in garbage_ctx { + if self.invalidate_context(&ctx).is_err() { + dev_err!( + self.dev.as_ref(), + "GpuContext: Failed to invalidate GPU context!\n" + ); + if debug_enabled(DebugFlags::OopsOnGpuCrash) { + panic!("GPU firmware timed out"); + } + } + } + + let mut guard = self.alloc.lock(); + let (garbage_count, garbage_bytes) = guard.private.garbage(); + let (ro_garbage_count, ro_garbage_bytes) = guard.gpu_ro.garbage(); + + if garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES + || ro_garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES + || garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS + || ro_garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS + { + mod_dev_dbg!( + self.dev, + "Collecting kalloc garbage (private: {} objects, {} bytes, gpuro: {} objects, {} bytes)\n", + garbage_count, + garbage_bytes, + ro_garbage_count, + ro_garbage_bytes + ); + if self.flush_fw_cache().is_err() { + dev_err!(self.dev.as_ref(), "Failed to flush FW cache\n"); + } else { + guard.private.collect_garbage(garbage_count); + guard.gpu_ro.collect_garbage(ro_garbage_count); + } + } + + guard + } + + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + + fn bind_vm(&self, vm: &mmu::Vm) -> Result { + self.uat.bind(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + &self.dev, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + + fn kick_firmware(&self) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_KICKFW)?; + + Ok(()) + } + + fn flush_fw_cache(&self) -> Result { + mod_dev_dbg!(self.dev, "Flushing coprocessor data cache\n"); + + if self.is_crashed() { + return Err(ENODEV); + } + + // ctx_0 == 0xff or ctx_1 == 0xff cause no effect on context, + // but this command does a full cache flush too, so abuse it + // for that. + + let dc = fw::channels::DeviceControlMsg::ver::DestroyContext { + unk_4: 0, + + ctx_23: 0, + #[ver(V < V13_3)] + __pad0: Default::default(), + unk_c: U32(0), + unk_10: U32(0), + ctx_0: 0xff, + ctx_1: 0xff, + ctx_4: 0, + #[ver(V < V13_3)] + __pad1: Default::default(), + #[ver(V < V13_3)] + unk_18: 0, + gpu_context: None, + __pad2: Default::default(), + }; + + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; + } + + txch.device_control.wait_for(token)?; + Ok(()) + } + + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "** GPU timeout nya~!!!!! **\n"); + dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); + dev_err!(self.dev.as_ref(), " Timeout count: {}\n", counter); + dev_err!(self.dev.as_ref(), " Unk: {}\n", unk); + + // If we have fault info, consider it a fault. + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Timeout, + }; + self.mark_pending_events(event_slot.try_into().ok(), error); + self.recover(); + } + + fn handle_fault(&self) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "GPU fault nya~!!!!!\n"); + let error = match self.get_fault_info() { + Some(info) => workqueue::WorkError::Fault(info), + None => workqueue::WorkError::Unknown, + }; + self.mark_pending_events(None, error); + self.recover(); + } + + fn handle_channel_error( + &self, + error_type: ChannelErrorType, + pipe_type: u32, + event_slot: u32, + event_value: u32, + ) { + dev_err!(self.dev.as_ref(), " (\\________/) \n"); + dev_err!(self.dev.as_ref(), " | | \n"); + dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); + dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); + dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); + dev_err!(self.dev.as_ref(), " |________| \n"); + dev_err!(self.dev.as_ref(), "GPU channel error nya~!!!!!\n"); + dev_err!(self.dev.as_ref(), " Error type: {:?}\n", error_type); + dev_err!(self.dev.as_ref(), " Pipe type: {}\n", pipe_type); + dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); + dev_err!(self.dev.as_ref(), " Event value: {:#x?}\n", event_value); + + self.event_manager.mark_error( + event_slot, + event_value, + workqueue::WorkError::ChannelError(error_type), + ); + + let wq = match self.event_manager.get_owner(event_slot) { + Some(wq) => wq, + None => { + dev_err!( + self.dev.as_ref(), + "Workqueue not found for this event slot!\n" + ); + return; + } + }; + + let wq = match wq.as_any().downcast_ref::() { + Some(wq) => wq, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with WorkQueue!\n"); + return; + } + }; + + if debug_enabled(DebugFlags::VerboseFaults) { + wq.dump_info(); + } + + let dc = fw::channels::DeviceControlMsg::ver::RecoverChannel { + pipe_type, + work_queue: wq.info_pointer(), + event_value, + __pad: Default::default(), + }; + + mod_dev_dbg!(self.dev, "Recover Channel command: {:?}\n", &dc); + let mut txch = self.tx_channels.lock(); + + let token = txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + if rtk + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "Failed to send Recover Channel command\n" + ); + } + } + + if txch.device_control.wait_for(token).is_err() { + dev_err!( + self.dev.as_ref(), + "Timed out waiting for Recover Channel command\n" + ); + } + + if debug_enabled(DebugFlags::VerboseFaults) { + wq.dump_info(); + } + } + + fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32) { + let halt_count = self + .initdata + .fw_status + .with(|raw, _inner| raw.flags.halt_count.load(Ordering::Relaxed)); + + let dc = fw::channels::DeviceControlMsg::ver::GrowTVBAck { + unk_4: 1, + buffer_slot, + vm_slot, + counter, + subpipe: 0, // TODO + halt_count: U64(halt_count), + __pad: Default::default(), + }; + + mod_dev_dbg!(self.dev, "TVB Grow Ack command: {:?}\n", &dc); + + let mut txch = self.tx_channels.lock(); + + txch.device_control.send(&dc); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + if rtk + .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) + .is_err() + { + dev_err!(self.dev.as_ref(), "Failed to send TVB Grow Ack command\n"); + } + } + } + + fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result { + if self.is_crashed() { + return Err(ENODEV); + } + + let mut fwctl = self.fwctl_channel.lock(); + let token = fwctl.send(&msg); + { + let mut guard = self.rtkit.lock(); + let rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(EP_DOORBELL, MSG_FWCTL)?; + } + fwctl.wait_for(token)?; + Ok(()) + } + + fn get_cfg(&self) -> &'static hw::HwConfig { + self.cfg + } + + fn get_dyncfg(&self) -> &hw::DynConfig { + &self.dyncfg + } + + fn free_context(&self, ctx: KBox>) { + let mut garbage = self.garbage_contexts.lock(); + + if garbage.push(ctx, GFP_KERNEL).is_err() { + dev_err!( + self.dev.as_ref(), + "Failed to reserve space for freed context, deadlock possible.\n" + ); + } + } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + bo.map_range_into_range( + self.uat.kernel_vm(), + range, + IOVA_KERN_TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +#[versions(AGX)] +impl GpuManagerPriv for GpuManager::ver { + fn end_op(&self) { + let val = self + .initdata + .globals + .with(|raw, _inner| raw.pending_submissions.fetch_sub(1, Ordering::Release)); + + mod_dev_dbg!(self.dev, "OP end (pending: {})\n", val - 1); + } +} diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs new file mode 100644 index 00000000000000..611764b5463b59 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -0,0 +1,657 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-SoC hardware configuration structures +//! +//! This module contains the definitions used to store per-GPU and per-SoC configuration data. + +use crate::driver::AsahiDevice; +use crate::fw::types::*; +use kernel::c_str; +use kernel::prelude::*; + +const MAX_POWERZONES: usize = 5; + +pub(crate) mod t600x; +pub(crate) mod t602x; +pub(crate) mod t8103; +pub(crate) mod t8112; + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} + +/// GPU revision enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevision { + A0 = 0x00, + A1 = 0x01, + B0 = 0x10, + B1 = 0x11, + C0 = 0x20, + C1 = 0x21, +} + +/// GPU core type enumeration. Note: Part of the firmware ABI. +#[derive(Debug, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuCore { + // Unknown = 0, + // G5P = 1, + // G5G = 2, + // G9P = 3, + // G9G = 4, + // G10P = 5, + // G11P = 6, + // G11M = 7, + // G11G = 8, + // G12P = 9, + // G13P = 10, + G13G = 11, + G13S = 12, + G13C = 13, + // G14P = 14, + G14G = 15, + G14S = 16, + G14C = 17, + G14D = 18, // Split out, unlike G13D +} + +/// GPU revision ID. Note: Part of the firmware ABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuRevisionID { + // Unknown = 0, + A0 = 1, + A1 = 2, + B0 = 3, + B1 = 4, + C0 = 5, + C1 = 6, +} + +/// A single performance state of the GPU. +#[derive(Debug)] +pub(crate) struct PState { + /// Voltage in millivolts, per GPU cluster. + pub(crate) volt_mv: KVec, + /// Frequency in hertz. + pub(crate) freq_hz: u32, + /// Maximum power consumption of the GPU at this pstate, in milliwatts. + pub(crate) pwr_mw: u32, +} + +impl PState { + pub(crate) fn max_volt_mv(&self) -> u32 { + *self.volt_mv.iter().max().expect("No voltages") + } +} + +/// A power zone definition (we have no idea what this is but Apple puts them in the DT). +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct PowerZone { + pub(crate) target: u32, + pub(crate) target_offset: u32, + pub(crate) filter_tc: u32, +} + +/// An MMIO mapping used by the firmware. +#[derive(Debug, Copy, Clone)] +pub(crate) struct IOMapping { + /// Base physical address of the mapping. + pub(crate) base: usize, + /// Whether this mapping should be replicated to all dies + pub(crate) per_die: bool, + /// Number of mappings. + pub(crate) count: usize, + /// Size of one mapping. + pub(crate) size: usize, + /// Stride between mappings. + pub(crate) stride: usize, + /// Whether the mapping should be writable. + pub(crate) writable: bool, +} + +impl IOMapping { + /// Convenience constructor for a new IOMapping. + pub(crate) const fn new( + base: usize, + per_die: bool, + count: usize, + size: usize, + stride: usize, + writable: bool, + ) -> IOMapping { + IOMapping { + base, + per_die, + count, + size, + stride, + writable, + } + } +} + +/// Unknown HwConfigA fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigA { + pub(crate) unk_87c: i32, + pub(crate) unk_8cc: u32, + pub(crate) unk_e24: u32, +} + +/// Unknown HwConfigB fields that vary from SoC to SoC. +#[allow(missing_docs)] +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwConfigB { + pub(crate) unk_454: u32, + pub(crate) unk_4e0: u64, + pub(crate) unk_534: u32, + pub(crate) unk_ab8: u32, + pub(crate) unk_abc: u32, + pub(crate) unk_b30: u32, +} + +/// Render command configs that vary from SoC to SoC. +#[derive(Debug, Copy, Clone)] +pub(crate) struct HwRenderConfig { + /// Vertex/tiling-related configuration register (lsb: disable clustering) + pub(crate) tiling_control: u32, +} + +#[derive(Debug)] +pub(crate) struct HwConfigShared2Curves { + pub(crate) t1_coef: u32, + pub(crate) t2: &'static [i16], + pub(crate) t3_coefs: &'static [u32], + pub(crate) t3_scales: &'static [u32], +} + +/// Static hardware clustering configuration for multi-cluster SoCs. +#[derive(Debug)] +pub(crate) struct HwClusteringConfig { + pub(crate) meta1_blocksize: usize, + pub(crate) meta2_size: usize, + pub(crate) meta3_size: usize, + pub(crate) meta4_size: usize, + pub(crate) max_splits: usize, +} + +/// Static hardware configuration for a given SoC model. +#[derive(Debug)] +pub(crate) struct HwConfig { + /// Chip ID in hex format (e.g. 0x8103 for t8103). + pub(crate) chip_id: u32, + /// GPU generation. + pub(crate) gpu_gen: GpuGen, + /// GPU variant type. + pub(crate) gpu_variant: GpuVariant, + /// GPU core type ID (as known by the firmware). + pub(crate) gpu_core: GpuCore, + + /// Base clock used used for timekeeping. + pub(crate) base_clock_hz: u32, + /// Output address space for the UAT on this SoC. + pub(crate) uat_oas: u32, + /// Number of dies on this SoC. + pub(crate) num_dies: u32, + /// Maximum number of clusters on this SoC. + pub(crate) max_num_clusters: u32, + /// Maximum number of cores per cluster for this GPU. + pub(crate) max_num_cores: u32, + /// Maximum number of frags per cluster for this GPU. + pub(crate) max_num_frags: u32, + /// Maximum number of GPs per cluster for this GPU. + pub(crate) max_num_gps: u32, + + /// Required size of the first preemption buffer. + pub(crate) preempt1_size: usize, + /// Required size of the second preemption buffer. + pub(crate) preempt2_size: usize, + /// Required size of the third preemption buffer. + pub(crate) preempt3_size: usize, + + /// Required size of the compute preemption buffer. + pub(crate) compute_preempt1_size: usize, + + pub(crate) clustering: Option, + + /// Rendering-relevant configuration. + pub(crate) render: HwRenderConfig, + + /// Misc HWDataA field values. + pub(crate) da: HwConfigA, + /// Misc HWDataB field values. + pub(crate) db: HwConfigB, + /// HwDataShared1.table. + pub(crate) shared1_tab: &'static [i32], + /// HwDataShared1.unk_a4. + pub(crate) shared1_a4: u32, + /// HwDataShared2.table. + pub(crate) shared2_tab: &'static [i32], + /// HwDataShared2.unk_508. + pub(crate) shared2_unk_508: u32, + /// HwDataShared2.unk_508. + pub(crate) shared2_curves: Option, + + /// HwDataShared3.unk_8. + pub(crate) shared3_unk: u32, + /// HwDataShared3.table. + pub(crate) shared3_tab: &'static [u32], + + /// Globals.idle_off_standby_timer. + pub(crate) idle_off_standby_timer_default: u32, + /// Globals.unk_hws2_4. + pub(crate) unk_hws2_4: Option<[F32; 8]>, + /// Globals.unk_hws2_24. + pub(crate) unk_hws2_24: u32, + /// Globals.unk_54 + pub(crate) global_unk_54: u16, + + /// Constant related to SRAM voltages. + pub(crate) sram_k: F32, + /// Unknown per-cluster coefficients 1. + pub(crate) unk_coef_a: &'static [&'static [F32]], + /// Unknown per-cluster coefficients 2. + pub(crate) unk_coef_b: &'static [&'static [F32]], + /// Unknown table in Global struct. + pub(crate) global_tab: Option<&'static [u8]>, + /// Whether this GPU has CS/AFR performance states + pub(crate) has_csafr: bool, + + /// Temperature sensor list (8 bits per sensor). + pub(crate) fast_sensor_mask: [u64; 2], + /// Temperature sensor list (alternate). + pub(crate) fast_sensor_mask_alt: [u64; 2], + /// Temperature sensor present bitmask. + pub(crate) fast_die0_sensor_present: u32, + /// Required MMIO mappings for this GPU/firmware. + pub(crate) io_mappings: &'static [Option], + /// SRAM base + pub(crate) sram_base: Option, + /// SRAM size + pub(crate) sram_size: Option, +} + +/// Dynamic (fetched from hardware/DT) configuration. +#[derive(Debug)] +pub(crate) struct DynConfig { + /// Base physical address of the UAT TTB (from DT reserved memory region). + pub(crate) uat_ttb_base: u64, + /// GPU ID configuration read from hardware. + pub(crate) id: GpuIdConfig, + /// Power calibration configuration for this specific chip/device. + pub(crate) pwr: PwrConfig, + /// Firmware version. + #[allow(dead_code)] + pub(crate) firmware_version: KVec, + + pub(crate) hw_data_a: KVVec, + pub(crate) hw_data_b: KVVec, + pub(crate) hw_globals: KVVec, +} + +/// Specific GPU ID configuration fetched from SGX MMIO registers. +#[derive(Debug)] +pub(crate) struct GpuIdConfig { + /// GPU generation (should match static config). + pub(crate) gpu_gen: GpuGen, + /// GPU variant type (should match static config). + pub(crate) gpu_variant: GpuVariant, + /// GPU silicon revision. + pub(crate) gpu_rev: GpuRevision, + /// GPU silicon revision ID (firmware enum). + pub(crate) gpu_rev_id: GpuRevisionID, + /// Total number of GPU clusters. + pub(crate) num_clusters: u32, + /// Maximum number of GPU cores per cluster. + pub(crate) num_cores: u32, + /// Number of frags per cluster. + pub(crate) num_frags: u32, + /// Number of GPs per cluster. + pub(crate) num_gps: u32, + /// Total number of active cores for the whole GPU. + pub(crate) total_active_cores: u32, + /// Mask of active cores per cluster. + pub(crate) core_masks: KVec, + /// Packed mask of all active cores. + pub(crate) core_masks_packed: KVec, +} + +/// Configurable CS/AFR GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct CsAfrPwrConfig { + /// GPU CS performance state list. + pub(crate) perf_states_cs: KVec, + /// GPU AFR performance state list. + pub(crate) perf_states_afr: KVec, + + /// CS leakage coefficient per die. + pub(crate) leak_coef_cs: KVec, + /// AFR leakage coefficient per die. + pub(crate) leak_coef_afr: KVec, + + /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, +} + +/// Configurable GPU power settings from the device tree. +#[derive(Debug)] +pub(crate) struct PwrConfig { + /// GPU performance state list. + pub(crate) perf_states: KVec, + /// GPU power zone list. + pub(crate) power_zones: KVec, + + /// Core leakage coefficient per cluster. + pub(crate) core_leak_coef: KVec, + /// SRAM leakage coefficient per cluster. + pub(crate) sram_leak_coef: KVec, + + pub(crate) csafr: Option, + + /// Maximum total power of the GPU in milliwatts. + pub(crate) max_power_mw: u32, + /// Maximum frequency of the GPU in megahertz. + pub(crate) max_freq_mhz: u32, + + /// Minimum performance state to start at. + pub(crate) perf_base_pstate: u32, + /// Maximum enabled performance state. + pub(crate) perf_max_pstate: u32, + + /// Minimum voltage for the SRAM power domain in microvolts. + pub(crate) min_sram_microvolt: u32, + + // Most of these fields are just named after Apple ADT property names and we don't fully + // understand them. They configure various power-related PID loops and filters. + /// Average power filter time constant in milliseconds. + pub(crate) avg_power_filter_tc_ms: u32, + /// Average power filter PID integral gain? + pub(crate) avg_power_ki_only: F32, + /// Average power filter PID proportional gain? + pub(crate) avg_power_kp: F32, + pub(crate) avg_power_min_duty_cycle: u32, + /// Average power target filter time constant in periods. + pub(crate) avg_power_target_filter_tc: u32, + /// "Fast die0" (temperature?) PID integral gain. + pub(crate) fast_die0_integral_gain: F32, + /// "Fast die0" (temperature?) PID proportional gain. + pub(crate) fast_die0_proportional_gain: F32, + pub(crate) fast_die0_prop_tgt_delta: u32, + pub(crate) fast_die0_release_temp: u32, + /// Delay from the fender (?) becoming idle to powerdown + pub(crate) fender_idle_off_delay_ms: u32, + /// Timeout from firmware early wake to sleep if no work was submitted (?) + pub(crate) fw_early_wake_timeout_ms: u32, + /// Delay from the GPU becoming idle to powerdown + pub(crate) idle_off_delay_ms: u32, + /// Related to the above? + pub(crate) idle_off_standby_timer: u32, + /// Percent? + pub(crate) perf_boost_ce_step: u32, + /// Minimum utilization before performance state is increased in %. + pub(crate) perf_boost_min_util: u32, + pub(crate) perf_filter_drop_threshold: u32, + /// Performance PID filter time constant? (periods?) + pub(crate) perf_filter_time_constant: u32, + /// Performance PID filter time constant 2? (periods?) + pub(crate) perf_filter_time_constant2: u32, + /// Performance PID integral gain. + pub(crate) perf_integral_gain: F32, + /// Performance PID integral gain 2 (?). + pub(crate) perf_integral_gain2: F32, + pub(crate) perf_integral_min_clamp: u32, + /// Performance PID proportional gain. + pub(crate) perf_proportional_gain: F32, + /// Performance PID proportional gain 2 (?). + pub(crate) perf_proportional_gain2: F32, + pub(crate) perf_reset_iters: u32, + /// Target GPU utilization for the performance controller in %. + pub(crate) perf_tgt_utilization: u32, + /// Power sampling period in milliseconds. + pub(crate) power_sample_period: u32, + /// PPM (?) filter time constant in milliseconds. + pub(crate) ppm_filter_time_constant_ms: u32, + /// PPM (?) filter PID integral gain. + pub(crate) ppm_ki: F32, + /// PPM (?) filter PID proportional gain. + pub(crate) ppm_kp: F32, + /// Power consumption filter time constant (periods?) + pub(crate) pwr_filter_time_constant: u32, + /// Power consumption filter PID integral gain. + pub(crate) pwr_integral_gain: F32, + pub(crate) pwr_integral_min_clamp: u32, + pub(crate) pwr_min_duty_cycle: u32, + pub(crate) pwr_proportional_gain: F32, + /// Power sample period in base clocks, used when not an integer number of ms + pub(crate) pwr_sample_period_aic_clks: u32, + + pub(crate) se_engagement_criteria: i32, + pub(crate) se_filter_time_constant: u32, + pub(crate) se_filter_time_constant_1: u32, + pub(crate) se_inactive_threshold: u32, + pub(crate) se_ki: F32, + pub(crate) se_ki_1: F32, + pub(crate) se_kp: F32, + pub(crate) se_kp_1: F32, + pub(crate) se_reset_criteria: u32, +} + +impl PwrConfig { + fn load_opp( + dev: &AsahiDevice, + name: &CStr, + cfg: &HwConfig, + is_main: bool, + ) -> Result> { + let mut perf_states = KVec::new(); + + let node = dev.as_ref().of_node().ok_or(EIO)?; + let opps = node.parse_phandle(name, 0).ok_or(EIO)?; + + for opp in opps.children() { + let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let pwr_uw: u32 = if is_main { + opp.get_property(c_str!("opp-microwatt"))? + } else { + 0 + }; + + let voltage_count = if is_main { + cfg.max_num_clusters + } else { + cfg.num_dies + }; + + if volt_uv.len() != voltage_count as usize { + dev_err!( + dev.as_ref(), + "Invalid opp-microvolt length (expected {}, got {})\n", + voltage_count, + volt_uv.len() + ); + return Err(EINVAL); + } + + volt_uv.iter_mut().for_each(|a| *a /= 1000); + let volt_mv = volt_uv; + + let pwr_mw = pwr_uw / 1000; + + perf_states.push( + PState { + freq_hz: freq_hz.try_into()?, + volt_mv, + pwr_mw, + }, + GFP_KERNEL, + )?; + } + + if perf_states.is_empty() { + Err(EINVAL) + } else { + Ok(perf_states) + } + } + + /// Load the GPU power configuration from the device tree. + pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { + let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; + let node = dev.as_ref().of_node().ok_or(EIO)?; + + macro_rules! prop { + ($prop:expr, $default:expr) => {{ + node.get_opt_property(c_str!($prop)) + .map_err(|e| { + dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); + e + })? + .unwrap_or($default) + }}; + ($prop:expr) => {{ + node.get_property(c_str!($prop)).map_err(|e| { + dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); + e + })? + }}; + } + + let pz_data = prop!("apple,power-zones", KVec::new()); + + if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { + dev_err!(dev.as_ref(), "Invalid apple,power-zones value\n"); + return Err(EINVAL); + } + + let pz_count = pz_data.len() / 3; + let mut power_zones = KVec::new(); + for i in (0..pz_count).step_by(3) { + power_zones.push( + PowerZone { + target: pz_data[i], + target_offset: pz_data[i + 1], + filter_tc: pz_data[i + 2], + }, + GFP_KERNEL, + )?; + } + + let core_leak_coef: KVec = prop!("apple,core-leak-coef"); + let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); + + if core_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev.as_ref(), "Invalid apple,core-leak-coef\n"); + return Err(EINVAL); + } + if sram_leak_coef.len() != cfg.max_num_clusters as usize { + dev_err!(dev.as_ref(), "Invalid apple,sram_leak_coef\n"); + return Err(EINVAL); + } + + let csafr = if cfg.has_csafr { + Some(CsAfrPwrConfig { + perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, + perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, + leak_coef_cs: prop!("apple,cs-leak-coef"), + leak_coef_afr: prop!("apple,afr-leak-coef"), + min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), + }) + } else { + None + }; + + let power_sample_period: u32 = prop!("apple,power-sample-period"); + + Ok(PwrConfig { + core_leak_coef, + sram_leak_coef, + + max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), + max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, + + perf_base_pstate: prop!("apple,perf-base-pstate", 1), + perf_max_pstate: perf_states.len() as u32 - 1, + min_sram_microvolt: prop!("apple,min-sram-microvolt"), + + avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), + avg_power_ki_only: prop!("apple,avg-power-ki-only"), + avg_power_kp: prop!("apple,avg-power-kp"), + avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), + avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), + fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), + fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), + fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), + fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), + fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), + fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), + idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), + idle_off_standby_timer: prop!( + "apple,idleoff-standby-timer", + cfg.idle_off_standby_timer_default + ), + perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), + perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), + perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), + perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), + perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), + perf_integral_gain2: prop!("apple,perf-integral-gain2"), + perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), + perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), + perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), + perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), + perf_reset_iters: prop!("apple,perf-reset-iters", 6), + perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), + power_sample_period, + ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), + ppm_ki: prop!("apple,ppm-ki"), + ppm_kp: prop!("apple,ppm-kp"), + pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), + pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), + pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), + pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), + pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), + pwr_sample_period_aic_clks: prop!( + "apple,pwr-sample-period-aic-clks", + cfg.base_clock_hz / 1000 * power_sample_period + ), + se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), + se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), + se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), + se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), + se_ki: prop!("apple,se-ki", f32!(-50.0)), + se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), + se_kp: prop!("apple,se-kp", f32!(-5.0)), + se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), + se_reset_criteria: prop!("apple,se-reset-criteria", 50), + + perf_states, + power_zones, + csafr, + }) + } + + pub(crate) fn max_frequency_khz(&self) -> u32 { + self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 + } +} diff --git a/drivers/gpu/drm/asahi/hw/t600x.rs b/drivers/gpu/drm/asahi/hw/t600x.rs new file mode 100644 index 00000000000000..58665f985ec38e --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t600x.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(mcc_count: usize, has_die1: bool) -> [Option; 20] { + [ + Some(IOMapping::new(0x404d00000, false, 1, 0x1c000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x28e104000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x28e494000, true, 1, 0x4000, 0, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + true, + mcc_count, + 0xd8000, + 0x1000000, + true, + )), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + Some(IOMapping::new(0x2643c4000, false, 1, 0x1000, 0, true)), // NIA Special agent idle register die 0 + if has_die1 { + // NIA Special agent idle register die 1 + Some(IOMapping::new(0x22643c4000, false, 1, 0x1000, 0, true)) + } else { + None + }, + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, false, 1, 0x1000, 0, true)), // ? + Some(IOMapping::new(0x28e3c0000, false, 1, 0x2000, 0, false)), // ? + ] +} + +pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { + chip_id: 0x6002, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G13C, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x3bd00, + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 8, + meta3_size: 0x280 * 8, + meta4_size: 0x30 * 16, + max_splits: 16, + }), + + render: HwRenderConfig { + tiling_control: 0xa540, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 1, + unk_ab8: 0x2084, + unk_abc: 0x80, + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0xffff, + shared2_tab: &[-1, -1, -1, -1, 0x2aa, 0xaaa, -1, -1, 0, 0], + shared2_unk_508: 0xcc00001, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([9.838]), + &f32!([9.819]), + &f32!([9.826]), + &f32!([9.799]), + &f32!([9.799]), + &f32!([9.826]), + &f32!([9.819]), + &f32!([9.838]), + ], + unk_coef_b: &[ + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + &f32!([13.0]), + ], + global_tab: Some(&[ + 0, 1, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: false, + fast_sensor_mask: [0x8080808080808080, 0], + fast_sensor_mask_alt: [0x9090909090909090, 0], + fast_die0_sensor_present: 0xff, + io_mappings: &iomaps(8, true), + sram_base: None, + sram_size: None, +}; + +pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { + chip_id: 0x6001, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G13C, + + num_dies: 1, + max_num_clusters: 4, + fast_sensor_mask: [0x80808080, 0], + fast_sensor_mask_alt: [0x90909090, 0], + fast_die0_sensor_present: 0x0f, + io_mappings: &iomaps(8, false), + ..HWCONFIG_T6002 +}; + +pub(crate) const HWCONFIG_T6000: super::HwConfig = HwConfig { + chip_id: 0x6000, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G13S, + + max_num_clusters: 2, + fast_sensor_mask: [0x8080, 0], + fast_sensor_mask_alt: [0x9090, 0], + fast_die0_sensor_present: 0x03, + io_mappings: &iomaps(4, false), + ..HWCONFIG_T6001 +}; diff --git a/drivers/gpu/drm/asahi/hw/t602x.rs b/drivers/gpu/drm/asahi/hw/t602x.rs new file mode 100644 index 00000000000000..98a7ac2b76e571 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t602x.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. + +use crate::f32; + +use super::*; + +const fn iomaps(chip_id: u32, mcc_count: usize) -> [Option; 24] { + [ + Some(IOMapping::new(0x404d00000, false, 1, 0x144000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x28e106000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(match chip_id { + 0x6020 => IOMapping::new(0x28e460000, true, 1, 0x4000, 0, false), + _ => IOMapping::new(0x28e478000, true, 1, 0x4000, 0, false), + }), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x404e08000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + None, // GMGIFAFRegs + Some(IOMapping::new( + 0x200000000, + true, + mcc_count, + 0xd8000, + 0x1000000, + true, + )), // MCache registers + Some(IOMapping::new(0x28e118000, false, 1, 0x4000, 0, false)), // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + Some(IOMapping::new(0x28e3d0000, false, 1, 0x4000, 0, true)), // ? + Some(IOMapping::new(0x28e3c0000, false, 1, 0x4000, 0, false)), // ? + Some(IOMapping::new(0x28e3d8000, false, 1, 0x4000, 0, true)), // ? + Some(IOMapping::new(0x404eac000, true, 1, 0x4000, 0, true)), // ? + None, + None, + ] +} + +// TODO: Tentative +pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { + chip_id: 0x6022, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::D, + gpu_core: GpuCore::G14D, + + base_clock_hz: 24_000_000, + uat_oas: 42, + num_dies: 2, + max_num_clusters: 8, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x40, + compute_preempt1_size: 0x25980 * 2, // Conservative guess + clustering: Some(HwClusteringConfig { + meta1_blocksize: 0x44, + meta2_size: 0xc0 * 16, + meta3_size: 0x280 * 16, + meta4_size: 0x10 * 128, + max_splits: 64, + }), + + render: HwRenderConfig { + tiling_control: 0x180340, + }, + + da: HwConfigA { + unk_87c: 500, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0, // Unused + unk_abc: 0, // Unused + unk_b30: 0, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0xaaaaa, 0], + shared2_unk_508: 0xc00007, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 11000, + t2: &[ + 0xf07, 0x4c0, 0x680, 0x8c0, 0xa80, 0xc40, 0xd80, 0xec0, 0xf40, + ], + t3_coefs: &[0, 20, 27, 36, 43, 50, 55, 60, 62], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 8, + shared3_tab: &[ + 125, 125, 125, 125, 125, 125, 125, 125, 7500, 125, 125, 125, 125, 125, 125, 125, + ], + idle_off_standby_timer_default: 700, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.5, 0.9])), + unk_hws2_24: 6, + global_unk_54: 4000, + sram_k: f32!(1.02), + unk_coef_a: &[ + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), + &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), + &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), + &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), + ], + unk_coef_b: &[ + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), + &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), + &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), + ], + global_tab: Some(&[ + 0, 2, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, + ]), + has_csafr: true, + fast_sensor_mask: [0x40005000c000d00, 0xd000c0005000400], + // Apple typo? Should probably be 0x140015001c001d00 + fast_sensor_mask_alt: [0x140015001d001d00, 0x1d001c0015001400], + fast_die0_sensor_present: 0, // Unused + io_mappings: &iomaps(0x6022, 8), + sram_base: Some(0x404d60000), + sram_size: Some(0x20000), +}; + +pub(crate) const HWCONFIG_T6021: super::HwConfig = HwConfig { + chip_id: 0x6021, + gpu_variant: GpuVariant::C, + gpu_core: GpuCore::G14C, + + num_dies: 1, + max_num_clusters: 4, + compute_preempt1_size: 0x25980, + unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), + fast_sensor_mask: [0x40005000c000d00, 0], + fast_sensor_mask_alt: [0x140015001d001d00, 0], + io_mappings: &iomaps(0x6021, 8), + ..HWCONFIG_T6022 +}; + +pub(crate) const HWCONFIG_T6020: super::HwConfig = HwConfig { + chip_id: 0x6020, + gpu_variant: GpuVariant::S, + gpu_core: GpuCore::G14S, + + db: HwConfigB { + unk_454: 0, + ..HWCONFIG_T6021.db + }, + + max_num_clusters: 2, + fast_sensor_mask: [0xc000d00, 0], + fast_sensor_mask_alt: [0x1d001d00, 0], + io_mappings: &iomaps(0x6020, 4), + ..HWCONFIG_T6021 +}; diff --git a/drivers/gpu/drm/asahi/hw/t8103.rs b/drivers/gpu/drm/asahi/hw/t8103.rs new file mode 100644 index 00000000000000..484bf6c3414f2f --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8103.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8103 platforms (M1). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8103, + gpu_gen: GpuGen::G13, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G13G, + + base_clock_hz: 24_000_000, + uat_oas: 40, + num_dies: 1, + max_num_clusters: 1, + max_num_cores: 8, + max_num_frags: 8, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x7f80, + clustering: None, + + render: HwRenderConfig { + // bit 0: disable clustering (always) + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: -220, + unk_8cc: 9880, + unk_e24: 112, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 0, + unk_534: 0, + unk_ab8: 0x48, + unk_abc: 0x8, + unk_b30: 0, + }, + shared1_tab: &[ + -1, 0x7282, 0x50ea, 0x370a, 0x25be, 0x1c1f, 0x16fb, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ], + shared1_a4: 0xffff, + shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0, 0], + shared2_unk_508: 0xc00007, + shared2_curves: None, + shared3_unk: 0, + shared3_tab: &[], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + sram_k: f32!(1.02), + unk_coef_a: &[], + unk_coef_b: &[], + global_tab: None, + has_csafr: false, + fast_sensor_mask: [0x12, 0], + fast_sensor_mask_alt: [0x12, 0], + fast_die0_sensor_present: 0x01, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, false, 1, 0x1c000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x23b104000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2e8000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs + Some(IOMapping::new(0x23bc00000, false, 1, 0x1000, 0, true)), // PMPDoorbell + Some(IOMapping::new(0x204d80000, false, 1, 0x5000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers + None, // AICBankedRegisters + Some(IOMapping::new(0x23b738000, false, 1, 0x1000, 0, true)), // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + None, // CRE registers + None, // Streaming codec registers + None, // + None, // + ], + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/hw/t8112.rs b/drivers/gpu/drm/asahi/hw/t8112.rs new file mode 100644 index 00000000000000..3eba0457d76ac9 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8112.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8112 platforms (M2). + +use crate::f32; + +use super::*; + +pub(crate) const HWCONFIG: super::HwConfig = HwConfig { + chip_id: 0x8112, + gpu_gen: GpuGen::G14, + gpu_variant: GpuVariant::G, + gpu_core: GpuCore::G14G, + + base_clock_hz: 24_000_000, + uat_oas: 40, + num_dies: 1, + max_num_clusters: 1, + max_num_cores: 10, + max_num_frags: 10, + max_num_gps: 4, + + preempt1_size: 0x540, + preempt2_size: 0x280, + preempt3_size: 0x20, + compute_preempt1_size: 0x10000, // TODO: Check + clustering: None, + + render: HwRenderConfig { + // TODO: this is unused here, may be present in newer FW + tiling_control: 0xa041, + }, + + da: HwConfigA { + unk_87c: 900, + unk_8cc: 11000, + unk_e24: 125, + }, + db: HwConfigB { + unk_454: 1, + unk_4e0: 4, + unk_534: 0, + unk_ab8: 0x2048, + unk_abc: 0x4000, + unk_b30: 1, + }, + shared1_tab: &[ + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, + ], + shared1_a4: 0, + shared2_tab: &[-1, -1, -1, -1, -1, -1, -1, -1, 0xaa5aa, 0], + shared2_unk_508: 0xc00000, + shared2_curves: Some(HwConfigShared2Curves { + t1_coef: 7200, + t2: &[ + 0xf07, 0x4c0, 0x6c0, 0x8c0, 0xac0, 0xc40, 0xdc0, 0xec0, 0xf80, + ], + t3_coefs: &[0, 20, 28, 36, 44, 50, 56, 60, 63], + t3_scales: &[9, 3209, 10400], + }), + shared3_unk: 5, + shared3_tab: &[ + 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, 10700, + 10700, 10700, 10700, + ], + idle_off_standby_timer_default: 0, + unk_hws2_4: None, + unk_hws2_24: 0, + global_unk_54: 0xffff, + + sram_k: f32!(1.02), + // 13.2: last coef changed from 6.6 to 5.3, assuming that was a fix we can backport + unk_coef_a: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + unk_coef_b: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], + global_tab: None, + has_csafr: false, + fast_sensor_mask: [0x6800, 0], + fast_sensor_mask_alt: [0x6800, 0], + fast_die0_sensor_present: 0x02, + io_mappings: &[ + Some(IOMapping::new(0x204d00000, false, 1, 0x14000, 0, true)), // Fender + Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer + Some(IOMapping::new(0x23b0c4000, false, 1, 0x4000, 0, true)), // AICSWInt + Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX + None, // UVD + None, // unused + None, // DisplayUnderrunWA + Some(IOMapping::new(0x23b2c0000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs + None, // PMPDoorbell + Some(IOMapping::new(0x204d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs + Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs + Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers + None, // AICBankedRegisters + None, // PMGRScratch + None, // NIA Special agent idle register die 0 + None, // NIA Special agent idle register die 1 + Some(IOMapping::new(0x204e00000, false, 1, 0x10000, 0, true)), // CRE registers + Some(IOMapping::new(0x27d050000, false, 1, 0x4000, 0, true)), // Streaming codec registers + Some(IOMapping::new(0x23b3d0000, false, 1, 0x1000, 0, true)), // + Some(IOMapping::new(0x23b3c0000, false, 1, 0x1000, 0, false)), // + ], + sram_base: None, + sram_size: None, +}; diff --git a/drivers/gpu/drm/asahi/initdata.rs b/drivers/gpu/drm/asahi/initdata.rs new file mode 100644 index 00000000000000..b190a7da6fe85f --- /dev/null +++ b/drivers/gpu/drm/asahi/initdata.rs @@ -0,0 +1,1021 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! GPU initialization data builder. +//! +//! The root of all interaction between the GPU firmware and the host driver is a complex set of +//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration +//! and the pointers to the ring buffers and global data fields that are used for communication at +//! runtime. +//! +//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values +//! derived from observing the InitData structures that macOS generates. + +use crate::f32; +use crate::fw::initdata::*; +use crate::fw::types::*; +use crate::module_parameters; +use crate::{ + driver::AsahiDevice, + gem, + gpu, + hw, + mmu, // +}; +use kernel::error::{ + Error, + Result, // +}; +use kernel::macros::versions; +use kernel::prelude::*; +use kernel::try_init; + +use ::pin_init; +use ::pin_init::Init; + +/// Builder helper for the global GPU InitData. +#[versions(AGX)] +pub(crate) struct InitDataBuilder<'a> { + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, +} + +#[versions(AGX)] +impl<'a> InitDataBuilder::ver<'a> { + /// Create a new InitData builder + pub(crate) fn new( + dev: &'a AsahiDevice, + alloc: &'a mut gpu::KernelAllocators, + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> InitDataBuilder::ver<'a> { + InitDataBuilder::ver { + dev, + alloc, + cfg, + dyncfg, + } + } + + /// Create the HwDataShared1 structure, which is used in two places in InitData. + fn hw_shared1(cfg: &'static hw::HwConfig) -> impl Init { + init!(raw::HwDataShared1 { + unk_a4: cfg.shared1_a4, + ..Zeroable::init_zeroed() + }) + .chain(|ret| { + for (i, val) in cfg.shared1_tab.iter().enumerate() { + ret.table[i] = *val; + } + Ok(()) + }) + } + + fn init_curve( + curve: &mut raw::HwDataShared2Curve, + unk_0: u32, + unk_4: u32, + t1: &[u16], + t2: &[i16], + t3: &[KVec], + ) { + curve.unk_0 = unk_0; + curve.unk_4 = unk_4; + (*curve.t1)[..t1.len()].copy_from_slice(t1); + (*curve.t1)[t1.len()..].fill(t1[0]); + (*curve.t2)[..t2.len()].copy_from_slice(t2); + (*curve.t2)[t2.len()..].fill(t2[0]); + for (i, a) in curve.t3.iter_mut().enumerate() { + a.fill(0x3ffffff); + if i < t3.len() { + let b = &t3[i]; + (**a)[..b.len()].copy_from_slice(b); + } + } + } + + /// Create the HwDataShared2 structure, which is used in two places in InitData. + fn hw_shared2( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init + 'a { + try_init!(raw::HwDataShared2 { + unk_28: Array::new([0xff; 16]), + g14: Default::default(), + unk_508: cfg.shared2_unk_508, + ..Zeroable::init_zeroed() + }) + .chain(|ret| { + for (i, val) in cfg.shared2_tab.iter().enumerate() { + ret.table[i] = *val; + } + + let curve_cfg = match cfg.shared2_curves.as_ref() { + None => return Ok(()), + Some(a) => a, + }; + + let mut t1 = KVec::new(); + let mut t3 = KVec::new(); + + for _ in 0..curve_cfg.t3_scales.len() { + t3.push(KVec::new(), GFP_KERNEL)?; + } + + for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { + let t3_coef = curve_cfg.t3_coefs[i]; + if t3_coef == 0 { + t1.push(0xffff, GFP_KERNEL)?; + for j in t3.iter_mut() { + j.push(0x3ffffff, GFP_KERNEL)?; + } + continue; + } + + let f_khz = (ps.freq_hz / 1000) as u64; + let v_max = ps.max_volt_mv() as u64; + + t1.push( + (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) + .try_into() + .unwrap(), + GFP_KERNEL, + )?; + + for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { + t3[j].push( + (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) + .try_into() + .unwrap(), + GFP_KERNEL, + )?; + } + } + + ret.g14.unk_14 = 0x6000000; + Self::init_curve( + &mut ret.g14.curve1, + 0, + 0x20000000, + &[0xffff], + &[0x0f07], + &[], + ); + Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); + + Ok(()) + }) + } + + /// Create the HwDataShared3 structure, which is used in two places in InitData. + fn hw_shared3(cfg: &'static hw::HwConfig) -> impl Init { + pin_init::init_zeroed::().chain(|ret| { + if !cfg.shared3_tab.is_empty() { + ret.unk_0 = 1; + ret.unk_4 = 500; + ret.unk_8 = cfg.shared3_unk; + ret.table.copy_from_slice(cfg.shared3_tab); + ret.unk_4c = 1; + } + Ok(()) + }) + } + + /// Create an unknown T81xx-specific data structure. + fn t81xx_data( + cfg: &'static hw::HwConfig, + dyncfg: &'a hw::DynConfig, + ) -> impl Init { + let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; + + pin_init::init_zeroed::().chain(move |_ret| { + match cfg.chip_id { + 0x8103 | 0x8112 => { + #[ver(V < V13_3)] + { + _ret.unk_d8c = 0x80000000; + _ret.unk_d90 = 4; + _ret.unk_d9c = f32!(0.6); + _ret.unk_da4 = f32!(0.4); + _ret.unk_dac = f32!(0.38552); + _ret.unk_db8 = f32!(65536.0); + _ret.unk_dbc = f32!(13.56); + _ret.max_pstate_scaled = 100 * _perf_max_pstate; + } + } + _ => (), + } + Ok(()) + }) + } + + /// Create the HwDataA structure. This mostly contains power-related configuration. + fn hwdata_a(&mut self) -> Result> { + let pwr = &self.dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; + #[ver(V >= V13_0B4)] + let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; + let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); + let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); + let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); + let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + #[ver(V >= V13_0B4)] + let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; + let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); + let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); + + let base_ps = pwr.perf_base_pstate; + let base_ps_scaled = 100 * base_ps; + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + let boost_ps_count = max_ps - base_ps; + + #[allow(unused_variables)] + let base_clock_khz = self.cfg.base_clock_hz / 1000; + let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; + + #[allow(unused_variables)] + let clocks_per_period_coarse = self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; + + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + try_init!(raw::HwDataA::ver { + clocks_per_period: v_clocks_per_period, + #[ver(V >= V13_0B4)] + clocks_per_period_2: v_clocks_per_period, + pwr_status: AtomicU32::new(4), + unk_10: f32!(1.0), + actual_pstate: 1, + tgt_pstate: 1, + base_pstate_scaled: base_ps_scaled, + unk_40: 1, + max_pstate_scaled: max_ps_scaled, + min_pstate_scaled: 100, + unk_64c: 625, + pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, + pwr_filter_a: pwr_filter_a, + pwr_integral_gain: pwr.pwr_integral_gain, + pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), + max_power_1: pwr.max_power_mw.into(), + pwr_proportional_gain: pwr.pwr_proportional_gain, + pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), + pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, + max_pstate_scaled_2: max_ps_scaled, + max_power_2: pwr.max_power_mw, + max_pstate_scaled_3: max_ps_scaled, + ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, + ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, + ppm_filter_a: ppm_filter_a, + ppm_ki_dt: pwr.ppm_ki * period_s, + unk_6fc: f32!(65536.0), + ppm_kp: pwr.ppm_kp, + pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, + max_pstate_scaled_4: max_ps_scaled, + unk_71c: f32!(0.0), + max_power_3: pwr.max_power_mw, + cur_power_mw_2: 0x0, + ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, + #[ver(V >= V13_0B4)] + ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, + perf_tgt_utilization: pwr.perf_tgt_utilization, + perf_boost_min_util: pwr.perf_boost_min_util, + perf_boost_ce_step: pwr.perf_boost_ce_step, + perf_reset_iters: pwr.perf_reset_iters, + unk_774: 6, + unk_778: 1, + perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, + perf_filter_a_neg: f32!(1.0) - perf_filter_a, + perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, + perf_filter_a: perf_filter_a, + perf_filter_a2: perf_filter_a2, + perf_ki: pwr.perf_integral_gain, + perf_ki2: pwr.perf_integral_gain2, + perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), + unk_79c: f32!(95.0), + perf_kp: pwr.perf_proportional_gain, + perf_kp2: pwr.perf_proportional_gain2, + boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), + base_pstate_scaled_2: base_ps_scaled, + max_pstate_scaled_5: max_ps_scaled, + base_pstate_scaled_3: base_ps_scaled, + perf_tgt_utilization_2: pwr.perf_tgt_utilization, + base_pstate_scaled_4: base_ps_scaled, + unk_7fc: f32!(65536.0), + pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), + max_pstate_scaled_6: max_ps_scaled.into(), + max_freq_mhz: pwr.max_freq_mhz, + pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, + min_pstate_scaled_4: f32!(100.0), + max_pstate_scaled_7: max_ps_scaled, + unk_alpha_neg: f32!(0.8), + unk_alpha: f32!(0.2), + fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask: U64(cfg.fast_sensor_mask[1]), + fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, + unk_87c: cfg.da.unk_87c, + unk_880: 0x4, + unk_894: f32!(1.0), + + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_8a8: f32!(65536.0), + fast_die0_kp: pwr.fast_die0_proportional_gain, + pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, + max_pstate_scaled_8: max_ps_scaled, + max_pstate_scaled_9: max_ps_scaled, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + unk_8cc: cfg.da.unk_8cc, + max_pstate_scaled_10: max_ps_scaled, + max_pstate_scaled_11: max_ps_scaled, + unk_c2c: 1, + power_zone_count: pwr.power_zones.len() as u32, + max_power_4: pwr.max_power_mw, + max_power_5: pwr.max_power_mw, + max_power_6: pwr.max_power_mw, + avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, + avg_power_target_filter_a: avg_power_target_filter_a, + avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, + avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, + #[ver(V >= V13_0B4)] + avg_power_target_filter_tc_clks: period_ms + * pwr.avg_power_target_filter_tc + * base_clock_khz, + avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, + avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, + avg_power_filter_a: avg_power_filter_a, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + unk_d20: f32!(65536.0), + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + max_pstate_scaled_12: max_ps_scaled, + max_pstate_scaled_13: max_ps_scaled, + max_power_7: pwr.max_power_mw.into(), + max_power_8: pwr.max_power_mw, + avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, + #[ver(V >= V13_0B4)] + avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, + max_pstate_scaled_14: max_ps_scaled, + t81xx_data <- Self::t81xx_data(cfg, dyncfg), + #[ver(V >= V13_0B4)] + unk_e10_0 <- { + let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); + let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); + try_init!(raw::HwDataA130Extra { + unk_38: 4, + unk_3c: 8000, + gpu_se_inactive_threshold: pwr.se_inactive_threshold, + gpu_se_engagement_criteria: pwr.se_engagement_criteria, + gpu_se_reset_criteria: pwr.se_reset_criteria, + unk_54: 50, + unk_58: 0x1, + gpu_se_filter_a_neg: f32!(1.0) - filter_a, + gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, + gpu_se_filter_a: filter_a, + gpu_se_filter_1_a: filter_1_a, + gpu_se_ki_dt: pwr.se_ki * period_s, + gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, + unk_7c: f32!(65536.0), + gpu_se_kp: pwr.se_kp, + gpu_se_kp_1: pwr.se_kp_1, + + #[ver(V >= V13_3)] + unk_8c: 100, + #[ver(V < V13_3)] + unk_8c: 40, + + max_pstate_scaled_1: max_ps_scaled, + unk_9c: f32!(8000.0), + unk_a0: 1400, + gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant * period_ms, + gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 + * period_ms, + gpu_se_filter_time_constant_clks: U64((pwr.se_filter_time_constant + * clocks_per_period_coarse) + .into()), + gpu_se_filter_time_constant_1_clks: U64((pwr + .se_filter_time_constant_1 + * clocks_per_period_coarse) + .into()), + unk_c4: f32!(65536.0), + unk_114: f32!(65536.0), + unk_124: 40, + max_pstate_scaled_2: max_ps_scaled, + ..Zeroable::init_zeroed() + }) + }, + fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask_2: U64(cfg.fast_sensor_mask[1]), + unk_e24: cfg.da.unk_e24, + unk_e28: 1, + fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), + #[ver(G >= G14X)] + fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), + #[ver(V < V13_0B4)] + fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), + unk_163c: 1, + unk_3644: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + unk_3ce8: 1, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + for i in 0..self.dyncfg.pwr.perf_states.len() { + raw.sram_k[i] = self.cfg.sram_k; + } + + for (i, coef) in pwr.core_leak_coef.iter().enumerate() { + raw.core_leak_coef[i] = *coef; + } + + for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { + raw.sram_leak_coef[i] = *coef; + } + + #[ver(V >= V13_0B4)] + if let Some(csafr) = pwr.csafr.as_ref() { + for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { + raw.aux_leak_coef.cs_1[i] = *coef; + raw.aux_leak_coef.cs_2[i] = *coef; + } + + for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { + raw.aux_leak_coef.afr_1[i] = *coef; + raw.aux_leak_coef.afr_2[i] = *coef; + } + } + + for i in 0..self.dyncfg.id.num_clusters as usize { + if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { + (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); + (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); + } + if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { + (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); + (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); + } + } + + for (i, pz) in pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; + raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; + let filter_a = f32!(1.0) / pz.filter_tc.into(); + raw.power_zones[i].filter_a = filter_a; + raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; + #[ver(V >= V13_0B4)] + raw.power_zones[i].unk_10 = 1320000000; + } + + #[ver(V >= V13_0B4 && G >= G14X)] + for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { + raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; + } + + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataA::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_a.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Hwdata A size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_a.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { + if sla[i] != dyncfg.hw_data_a[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Hwdata A match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the HwDataB structure. This mostly contains GPU-related configuration. + fn hwdata_b(&mut self) -> Result> { + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + try_init!(raw::HwDataB::ver { + // Userspace VA map related + #[ver(V < V13_0B4)] + unk_0: U64(0x13_00000000), + unk_8: U64(0x14_00000000), + #[ver(V < V13_0B4)] + unk_10: U64(0x1_00000000), + unk_18: U64(0xffc00000), + // USC start + unk_20: U64(0), // U64(0x11_00000000), + unk_28: U64(0), // U64(0x11_00000000), + // Unknown page + //unk_30: U64(0x6f_ffff8000), + unk_30: U64(mmu::IOVA_UNK_PAGE), + timestamp_area_base: U64(gpu::IOVA_KERN_TIMESTAMP_RANGE.start), + // TODO: yuv matrices + chip_id: cfg.chip_id, + unk_454: cfg.db.unk_454, + unk_458: 0x1, + unk_460: 0x1, + unk_464: 0x1, + unk_468: 0x1, + unk_47c: 0x1, + unk_484: 0x1, + unk_48c: 0x1, + base_clock_khz: cfg.base_clock_hz / 1000, + power_sample_period: dyncfg.pwr.power_sample_period, + unk_49c: 0x1, + unk_4a0: 0x1, + unk_4a4: 0x1, + unk_4c0: 0x1f, + unk_4e0: U64(cfg.db.unk_4e0), + unk_4f0: 0x1, + unk_4f4: 0x1, + unk_504: 0x31, + unk_524: 0x1, // use_secure_cache_flush + unk_534: cfg.db.unk_534, + num_frags: dyncfg.id.num_frags * dyncfg.id.num_clusters, + unk_554: 0x1, + uat_ttb_base: U64(dyncfg.uat_ttb_base), + gpu_core_id: cfg.gpu_core as u32, + gpu_rev_id: dyncfg.id.gpu_rev_id as u32, + num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, + max_pstate: dyncfg.pwr.perf_states.len() as u32 - 1, + #[ver(V < V13_0B4)] + num_pstates: dyncfg.pwr.perf_states.len() as u32, + #[ver(V < V13_0B4)] + min_sram_volt: dyncfg.pwr.min_sram_microvolt / 1000, + #[ver(V < V13_0B4)] + unk_ab8: cfg.db.unk_ab8, + #[ver(V < V13_0B4)] + unk_abc: cfg.db.unk_abc, + #[ver(V < V13_0B4)] + unk_ac0: 0x1020, + + #[ver(V >= V13_0B4)] + unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), + #[ver(V < V13_0B4)] + unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), + unk_b10: 0x1, + timer_offset: U64(0), + unk_b24: 0x1, + unk_b28: 0x1, + unk_b2c: 0x1, + unk_b30: cfg.db.unk_b30, + #[ver(V >= V13_0B4)] + unk_b38_0: 1, + #[ver(V >= V13_0B4)] + unk_b38_4: 1, + unk_b38: Array::new([0xffffffff; 12]), + #[ver(V >= V13_0B4 && V < V13_3)] + unk_c3c: 0x19, + #[ver(V >= V13_3)] + unk_c3c: 0x1a, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + #[ver(V >= V13_3)] + for i in 0..16 { + raw.unk_arr_0[i] = i as u32; + } + + let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; + let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; + let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; + let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; + + for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { + raw.frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); + raw.voltages[i][j] = *mv; + raw.voltages_sram[i][j] = sram_mv; + } + for j in ps.volt_mv.len()..raw.voltages[i].len() { + raw.voltages[i][j] = raw.voltages[i][0]; + raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; + } + raw.sram_k[i] = self.cfg.sram_k; + raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; + raw.rel_boost_freqs[i] = if i > base_ps { + (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) + } else { + 0 + }; + } + + #[ver(V >= V13_0B4)] + if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { + let aux = &mut raw.aux_ps; + aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; + aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; + + for (i, ps) in csafr.perf_states_cs.iter().enumerate() { + aux.cs_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.cs_voltages[i][j] = *mv; + aux.cs_voltages_sram[i][j] = sram_mv; + } + } + + for (i, ps) in csafr.perf_states_afr.iter().enumerate() { + aux.afr_frequencies[i] = ps.freq_hz / 1000000; + for (j, mv) in ps.volt_mv.iter().enumerate() { + let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); + aux.afr_voltages[i][j] = *mv; + aux.afr_voltages_sram[i][j] = sram_mv; + } + } + } + + // Special case override for T602x + #[ver(G == G14X)] + if dyncfg.id.gpu_rev_id == hw::GpuRevisionID::B1 { + raw.gpu_rev_id = hw::GpuRevisionID::B0 as u32; + } + + if !dyncfg.hw_data_b.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::HwDataB::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_data_b.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Hwdata B size mismatch: {} {}", + sla.len(), + dyncfg.hw_data_b.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { + if sla[i] != dyncfg.hw_data_b[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Hwdata B match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the Globals structure, which contains global firmware config including more power + /// configuration data and globals used to exchange state between the firmware and driver. + fn globals(&mut self) -> Result> { + self.alloc + .private + .new_init(pin_init::init_zeroed(), |_inner, _ptr| { + let cfg = &self.cfg; + let dyncfg = &self.dyncfg; + let pwr = &dyncfg.pwr; + let period_ms = pwr.power_sample_period; + let period_s = F32::from(period_ms) / f32!(1000.0); + let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; + + let max_ps = pwr.perf_max_pstate; + let max_ps_scaled = 100 * max_ps; + + try_init!(raw::Globals::ver { + //ktrace_enable: 0xffffffff, + ktrace_enable: 0, + #[ver(V >= V13_2)] + unk_24_0: 3000, + unk_24: 0, + #[ver(V >= V13_0B4)] + debug: 0, + unk_28: 1, + #[ver(G >= G14X)] + unk_2c_0: 1, + #[ver(V >= V13_0B4 && G < G14X)] + unk_2c_0: 0, + unk_2c: 1, + unk_30: 0, + unk_34: 120, + // sub <- try_init!(raw::GlobalsSub::ver { + unk_54: cfg.global_unk_54, + unk_56: 40, + unk_58: 0xffff, + unk_5e: U32(1), + unk_66: U32(1), + // ..Zeroable::init_zeroed() + // }), + unk_8900: 1, + pending_submissions: AtomicU32::new(0), + max_power: pwr.max_power_mw, + max_pstate_scaled: max_ps_scaled, + max_pstate_scaled_2: max_ps_scaled, + max_pstate_scaled_3: max_ps_scaled, + power_zone_count: pwr.power_zones.len() as u32, + avg_power_filter_tc_periods: avg_power_filter_tc_periods, + avg_power_ki_dt: pwr.avg_power_ki_only * period_s, + avg_power_kp: pwr.avg_power_kp, + avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, + avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, + unk_89bc: cfg.da.unk_8cc, + fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, + unk_89c4: cfg.da.unk_87c, + fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, + fast_die0_kp: pwr.fast_die0_proportional_gain, + fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, + unk_89e0: 1, + max_power_2: pwr.max_power_mw, + ppm_kp: pwr.ppm_kp, + ppm_ki_dt: pwr.ppm_ki * period_s, + #[ver(V >= V13_0B4)] + unk_89f4_8: 1, + unk_89f4: 0, + hws1 <- Self::hw_shared1(cfg), + hws2 <- Self::hw_shared2(cfg, dyncfg), + hws3 <- Self::hw_shared3(cfg), + #[ver(V >= V13_0B4)] + idle_off_standby_timer: pwr.idle_off_standby_timer, + #[ver(V >= V13_0B4)] + unk_hws2_4: cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), + #[ver(V >= V13_0B4)] + unk_hws2_24: cfg.unk_hws2_24, + unk_900c: 1, + #[ver(V >= V13_0B4)] + unk_9010_0: 1, + #[ver(V >= V13_0B4)] + unk_903c: 1, + #[ver(V < V13_0B4)] + unk_903c: 0, + fault_control: *module_parameters::fault_control.value(), + do_init: 1, + progress_check_interval_3d: 40, + progress_check_interval_ta: 10, + progress_check_interval_cl: 250, + #[ver(V >= V13_0B4)] + unk_1102c_0: 1, + #[ver(V >= V13_0B4)] + unk_1102c_4: 1, + #[ver(V >= V13_0B4)] + unk_1102c_8: 100, + #[ver(V >= V13_0B4)] + unk_1102c_c: 1, + idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), + fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, + fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, + cl_context_switch_timeout_ms: 40, + #[ver(V >= V13_0B4)] + cl_kill_timeout_ms: 50, + #[ver(V >= V13_0B4)] + unk_11edc: 0, + #[ver(V >= V13_0B4)] + unk_11efc: 0, + ..Zeroable::init_zeroed() + }) + .chain(|raw| { + for (i, pz) in self.dyncfg.pwr.power_zones.iter().enumerate() { + raw.power_zones[i].target = pz.target; + raw.power_zones[i].target_off = pz.target - pz.target_offset; + raw.power_zones[i].filter_tc = pz.filter_tc; + } + + if let Some(tab) = self.cfg.global_tab.as_ref() { + for (i, x) in tab.iter().enumerate() { + raw.unk_118ec[i] = *x; + } + raw.unk_118e8 = 1; + } + + if !dyncfg.hw_globals.is_empty() { + unsafe { + let mut matches: bool = true; + let sla = core::slice::from_raw_parts( + raw as *const raw::Globals::ver as *const u8, + core::mem::size_of::(), + ); + if sla.len() != dyncfg.hw_globals.len() { + matches = false; + dev_err!( + self.dev.as_ref(), + "!!! Globals size mismatch: {} {}", + sla.len(), + dyncfg.hw_globals.len(), + ); + } + for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { + if sla[i] != dyncfg.hw_globals[i] { + matches = false; + dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); + break; + } + } + if matches { + dev_info!(self.dev.as_ref(), "!!! Globals match"); + } + } + } + + Ok(()) + }) + }) + } + + /// Create the RuntimePointers structure, which contains pointers to most of the other + /// structures including the ring buffer channels, statistics structures, and HwDataA/HwDataB. + fn runtime_pointers(&mut self) -> Result> { + let hwa = self.hwdata_a()?; + let hwb = self.hwdata_b()?; + + let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; + buffer_mgr_ctl.vmap()?.memset(0); + + GpuObject::new_init_prealloc( + self.alloc.private.alloc_object()?, + |_ptr| { + let alloc = &mut *self.alloc; + try_init!(RuntimePointers::ver { + stats <- { + let alloc = &mut *alloc; + try_init!(Stats::ver { + vtx: alloc.private.new_default::()?, + frag: alloc.private.new_init( + pin_init::init_zeroed::(), + |_inner, _ptr| { + try_init!(raw::GpuGlobalStatsFrag::ver { + total_cmds: 0, + unk_4: 0, + stats: Default::default(), + }) + } + )?, + comp: alloc.private.new_default::()?, + }) + }, + + hwdata_a: hwa, + unkptr_190: alloc.private.array_empty_tagged(0x80, b"I190")?, + unkptr_198: alloc.private.array_empty_tagged(0xc0, b"I198")?, + hwdata_b: hwb, + + unkptr_1b8: alloc.private.array_empty_tagged(0x1000, b"I1B8")?, + unkptr_1c0: alloc.private.array_empty_tagged(0x300, b"I1C0")?, + unkptr_1c8: alloc.private.array_empty_tagged(0x1000, b"I1C8")?, + + buffer_mgr_ctl, + buffer_mgr_ctl_low_mapping: None, + buffer_mgr_ctl_high_mapping: None, + }) + }, + |inner, _ptr| { + try_init!(raw::RuntimePointers::ver { + pipes: Default::default(), + device_control: Default::default(), + event: Default::default(), + fw_log: Default::default(), + ktrace: Default::default(), + stats: Default::default(), + + stats_vtx: inner.stats.vtx.gpu_pointer(), + stats_frag: inner.stats.frag.gpu_pointer(), + stats_comp: inner.stats.comp.gpu_pointer(), + + hwdata_a: inner.hwdata_a.gpu_pointer(), + unkptr_190: inner.unkptr_190.gpu_pointer(), + unkptr_198: inner.unkptr_198.gpu_pointer(), + hwdata_b: inner.hwdata_b.gpu_pointer(), + hwdata_b_2: inner.hwdata_b.gpu_pointer(), + + fwlog_buf: None, + + unkptr_1b8: inner.unkptr_1b8.gpu_pointer(), + + #[ver(G < G14X)] + unkptr_1c0: inner.unkptr_1c0.gpu_pointer(), + #[ver(G < G14X)] + unkptr_1c8: inner.unkptr_1c8.gpu_pointer(), + + buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), + buffer_mgr_ctl_fw_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_HIGH), + + __pad0: Default::default(), + unk_160: U64(0), + unk_168: U64(0), + unk_1d0: 0, + unk_1d4: 0, + unk_1d8: Default::default(), + + __pad1: Default::default(), + gpu_scratch: raw::RuntimeScratch::ver { + unk_6b38: 0xff, + ..Default::default() + }, + }) + }, + ) + } + + /// Create the FwStatus structure, which is used to coordinate the firmware halt state between + /// the firmware and the driver. + fn fw_status(&mut self) -> Result> { + self.alloc + .shared + .new_object(Default::default(), |_inner| Default::default()) + } + + /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. + fn uat_level_info( + cfg: &'static hw::HwConfig, + index_shift: usize, + num_entries: usize, + ) -> raw::UatLevelInfo { + raw::UatLevelInfo { + index_shift: index_shift as _, + unk_1: 14, + unk_2: 14, + unk_3: 8, + unk_4: 0x4000, + num_entries: num_entries as _, + unk_8: U64(1), + unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), + index_mask: U64(((num_entries - 1) << index_shift) as u64), + } + } + + /// Build the top-level InitData object. + #[inline(never)] + pub(crate) fn build(&mut self) -> Result>> { + let runtime_pointers = self.runtime_pointers()?; + let globals = self.globals()?; + let fw_status = self.fw_status()?; + let shared_ro = &mut self.alloc.shared_ro; + + let obj = self.alloc.private.new_init( + try_init!(InitData::ver { + unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, + runtime_pointers, + globals, + fw_status, + }), + |inner, _ptr| { + let cfg = &self.cfg; + try_init!(raw::InitData::ver { + #[ver(V == V13_5 && G != G14X)] + ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), + #[ver(V == V13_5 && G == G14X)] + ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), + unk_buf: inner.unk_buf.gpu_pointer(), + unk_8: 0, + unk_c: 0, + runtime_pointers: inner.runtime_pointers.gpu_pointer(), + globals: inner.globals.gpu_pointer(), + fw_status: inner.fw_status.gpu_pointer(), + uat_page_size: 0x4000, + uat_page_bits: 14, + uat_num_levels: 3, + uat_level_info: Array::new([ + Self::uat_level_info(cfg, 36, 8), + Self::uat_level_info(cfg, 25, 2048), + Self::uat_level_info(cfg, 14, 2048), + ]), + __pad0: Default::default(), + host_mapped_fw_allocations: 1, + unk_ac: 0, + unk_b0: 0, + unk_b4: 0, + unk_b8: 0, + }) + }, + )?; + Ok(KBox::new(obj, GFP_KERNEL)?) + } +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs new file mode 100644 index 00000000000000..60a64e23a161c5 --- /dev/null +++ b/drivers/gpu/drm/asahi/mem.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! ARM64 low level memory operations. +//! +//! This GPU uses CPU-side `tlbi` outer-shareable instructions to manage its TLBs. +//! Yes, really. Even though the VA address spaces are unrelated. +//! +//! Right now we pick our own ASIDs and don't coordinate with the CPU. This might result +//! in needless TLB shootdowns on the CPU side... TODO: fix this. + +use core::arch::asm; +use core::cmp::min; + +use crate::debug::*; +use crate::mmu; + +type Asid = u8; + +/// Invalidate the entire GPU TLB. +#[inline(always)] +pub(crate) fn tlbi_all() { + // SAFETY: tlbi is always safe by definition + unsafe { + asm!(".arch armv8.4-a", "tlbi vmalle1os",); + } +} + +/// Invalidate all TLB entries for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_asid(asid: Asid) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi aside1os, {x}", + x = in(reg) ((asid as u64) << 48) + ); + } +} + +/// Invalidate a single page for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_page(asid: Asid, va: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi vae1os, {x}", + x = in(reg) val + ); + } +} + +/// Invalidate a range of pages for a given ASID. +#[inline(always)] +pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + let start_pg = va >> mmu::UAT_PGBIT; + let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; + + let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & 0x1fffffffff); + let pages = end_pg - start_pg; + + // Guess? It's possible that the page count is in terms of 4K pages + // when the CPU is in 4K mode... + #[cfg(CONFIG_ARM64_4K_PAGES)] + let pages = 4 * pages; + + if pages == 1 { + tlbi_page(asid, va); + return; + } + + // Page count is always in units of 2 + let num = ((pages + 1) >> 1) as u64; + // base: 5 bits + // exp: 2 bits + // pages = (base + 1) << (5 * exp + 1) + // 0:00000 -> 2 pages = 2 << 0 + // 0:11111 -> 32 * 2 pages = 2 << 5 + // 1:00000 -> 1 * 32 * 2 pages = 2 << 5 + // 1:11111 -> 32 * 32 * 2 pages = 2 << 10 + // 2:00000 -> 1 * 32 * 32 * 2 pages = 2 << 10 + // 2:11111 -> 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:00000 -> 1 * 32 * 32 * 32 * 2 pages = 2 << 15 + // 3:11111 -> 32 * 32 * 32 * 32 * 2 pages = 2 << 20 + let exp = min(3, (64 - num.leading_zeros()) / 5); + let bits = 5 * exp; + let mut base = (num + (1 << bits) - 1) >> bits; + + val |= (exp as u64) << 44; + + while base > 32 { + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | (31 << 39) + ); + } + base -= 32; + } + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val | ((base - 1) << 39) + ); + } +} + +/// Issue a memory barrier (`dsb sy`). +#[inline(always)] +pub(crate) fn sync() { + // SAFETY: Barriers are always safe + unsafe { + asm!("dsb sy"); + } +} diff --git a/drivers/gpu/drm/asahi/microseq.rs b/drivers/gpu/drm/asahi/microseq.rs new file mode 100644 index 00000000000000..cbdb5de62e9218 --- /dev/null +++ b/drivers/gpu/drm/asahi/microseq.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU Micro operation sequence builder +//! +//! As part of a single job submisssion to the GPU, the GPU firmware interprets a sequence of +//! commands that we call a "microsequence". These are responsible for setting up the job execution, +//! timestamping the process, waiting for completion, tearing up any resources, and signaling +//! completion to the driver via the event stamp mechanism. +//! +//! Although the microsequences used by the macOS driver are usually quite uniform and simple, the +//! firmware actually implements enough operations to make this interpreter Turing-complete (!). +//! Most of those aren't implemented yet, since we don't need them, but they could come in handy in +//! the future to do strange things or work around firmware bugs... +//! +//! This module simply implements a collection of microsequence operations that can be appended to +//! and later concatenated into one buffer, ready for firmware execution. + +use crate::fw::microseq; +pub(crate) use crate::fw::microseq::*; +use crate::fw::types::*; +use kernel::prelude::*; + +/// MicroSequence object type, which is just an opaque byte array. +pub(crate) type MicroSequence = GpuArray; + +/// MicroSequence builder. +pub(crate) struct Builder { + ops: KVec, +} + +impl Builder { + /// Create a new Builder object + pub(crate) fn new() -> Builder { + Builder { ops: KVec::new() } + } + + /// Get the relative offset from the current pointer to a given target offset. + /// + /// Used for relative jumps. + pub(crate) fn offset_to(&self, target: i32) -> i32 { + target - self.ops.len() as i32 + } + + /// Add an operation to the end of the sequence. + pub(crate) fn add(&mut self, op: T) -> Result { + let off = self.ops.len(); + let p: *const T = &op; + let p: *const u8 = p as *const u8; + // SAFETY: Microseq operations always have no padding bytes, so it is safe to + // access them as a byte slice. + let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; + self.ops.extend_from_slice(s, GFP_KERNEL)?; + Ok(off as i32) + } + + /// Collect all submitted operations into a finalized GPU object. + pub(crate) fn build(self, alloc: &mut Allocator) -> Result { + let mut array = alloc.array_empty::(self.ops.len())?; + + array.as_mut_slice().clone_from_slice(self.ops.as_slice()); + Ok(array) + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs new file mode 100644 index 00000000000000..93692e27f2061f --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -0,0 +1,1602 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU UAT (MMU) management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the global MMU structures, including a shared handoff structure +//! that is used to coordinate VM management operations with the firmware, the TTBAT which points +//! to currently active GPU VM contexts, as well as the individual `Vm` operations to map and +//! unmap buffer objects into a single user or kernel address space. +//! +//! The actual page table management is in the `pt` module. + +use core::fmt::Debug; +use core::mem::size_of; +use core::num::NonZeroUsize; +use core::ops::Range; +use core::sync::atomic::{ + fence, + AtomicU32, + AtomicU64, + AtomicU8, + Ordering, // +}; + +use kernel::{ + addr::PhysicalAddr, + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + c_str, + device, + drm::{ + gem::shmem, + gpuvm, + mm, // + }, + error::Result, + io, + new_mutex, + prelude::*, + static_lock_class, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, Mutex, + }, + time::{ + delay::fsleep, + Delta, + Instant, + Monotonic, // + }, + types::ARef, // +}; + +use crate::debug::*; +use crate::module_parameters; +use crate::no_debug; +use crate::{ + driver, + fw, + gem, + hw, + mem, + pgtable, + slotalloc, + util::RangeExt, // +}; + +// KernelMapping protection types +pub(crate) use crate::pgtable::Prot; +pub(crate) use pgtable::prot::*; +pub(crate) use pgtable::{ + UatPageTable, + UAT_PGBIT, + UAT_PGMSK, + UAT_PGSZ, // +}; + +use pgtable::UAT_IAS; + +use pin_init; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; + +/// PPL magic number for the handoff region +const PPL_MAGIC: u64 = 0x4b1d000000000002; + +/// Number of supported context entries in the TTBAT +const UAT_NUM_CTX: usize = 64; +/// First context available for users +const UAT_USER_CTX_START: usize = 1; +/// Number of available user contexts +const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; + +/// Lower/user base VA +pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; +/// Lower/user top VA +pub(crate) const IOVA_USER_TOP: u64 = 1 << (UAT_IAS as u64); +/// Lower/user VA range +pub(crate) const IOVA_USER_RANGE: Range = IOVA_USER_BASE..IOVA_USER_TOP; + +/// Upper/kernel base VA +#[cfg(CONFIG_DEV_COREDUMP)] +const IOVA_TTBR1_BASE: u64 = 0xffffff8000000000; +/// Driver-managed kernel base VA +const IOVA_KERN_BASE: u64 = 0xffffffa000000000; +/// Driver-managed kernel top VA +const IOVA_KERN_TOP: u64 = 0xffffffb000000000; +/// Driver-managed kernel VA range +const IOVA_KERN_RANGE: Range = IOVA_KERN_BASE..IOVA_KERN_TOP; +/// Full kernel VA range +#[cfg(CONFIG_DEV_COREDUMP)] +const IOVA_KERN_FULL_RANGE: Range = IOVA_TTBR1_BASE..(!UAT_PGMSK as u64); + +const TTBR_VALID: u64 = 0x1; // BIT(0) +const TTBR_ASID_SHIFT: usize = 48; + +/// Address of a special dummy page? +//const IOVA_UNK_PAGE: u64 = 0x6f_ffff8000; +pub(crate) const IOVA_UNK_PAGE: u64 = IOVA_USER_TOP - 2 * UAT_PGSZ as u64; +/// User VA range excluding the unk page +pub(crate) const IOVA_USER_USABLE_RANGE: Range = IOVA_USER_BASE..IOVA_UNK_PAGE; + +/// A pre-allocated memory region for UAT management +struct UatRegion { + base: PhysicalAddr, + map: io::mem::Mem, +} + +/// SAFETY: It's safe to share UAT region records across threads. +unsafe impl Send for UatRegion {} +/// SAFETY: It's safe to share UAT region records across threads. +unsafe impl Sync for UatRegion {} + +/// Handoff region flush info structure +#[repr(C)] +struct FlushInfo { + state: AtomicU64, + addr: AtomicU64, + size: AtomicU64, +} + +/// UAT Handoff region layout +#[repr(C)] +struct Handoff { + magic_ap: AtomicU64, + magic_fw: AtomicU64, + + lock_ap: AtomicU8, + lock_fw: AtomicU8, + // Implicit padding: 2 bytes + turn: AtomicU32, + cur_slot: AtomicU32, + // Implicit padding: 4 bytes + flush: [FlushInfo; UAT_NUM_CTX + 1], + + unk2: AtomicU8, + // Implicit padding: 7 bytes + unk3: AtomicU64, +} + +const HANDOFF_SIZE: usize = size_of::(); + +/// One VM slot in the TTBAT +#[repr(C)] +struct SlotTTBS { + ttb0: AtomicU64, + ttb1: AtomicU64, +} + +const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); + +// We need at least page 0 (ttb0) +const PAGETABLES_SIZE: usize = UAT_PGSZ; + +/// Inner data for a Vm instance. This is reference-counted by the outer Vm object. +struct VmInner { + dev: driver::AsahiDevRef, + is_kernel: bool, + va_range: Range, + page_table: UatPageTable, + mm: mm::Allocator<(), KernelMappingInner>, + uat_inner: Arc, + binding: Arc>, + id: u64, +} + +/// Slot binding-related inner data for a Vm instance. +struct VmBinding { + active_users: usize, + binding: Option>, + bind_token: Option, + ttb: u64, +} + +/// Data associated with a VM <=> BO pairing +#[pin_data] +struct VmBo { + #[pin] + sgt: Mutex>>, +} + +impl gpuvm::DriverGpuVmBo for VmBo { + fn new() -> impl PinInit { + pin_init!(VmBo { + sgt <- new_mutex!(None, "VmBinding"), + }) + } +} + +#[derive(Default)] +struct StepContext { + new_va: Option>>>, + prev_va: Option>>>, + next_va: Option>>>, + vm_bo: Option>>, + prot: Prot, +} + +impl gpuvm::DriverGpuVm for VmInner { + type Driver = driver::AsahiDriver; + type GpuVmBo = VmBo; + type StepContext = StepContext; + + fn step_map( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpMap, + ctx: &mut Self::StepContext, + ) -> Result { + let mut iova = op.addr(); + let mut left = op.range() as usize; + let mut offset = op.offset() as usize; + + let bo = ctx.vm_bo.as_ref().expect("step_map with no BO"); + + let one_page = op.flags().contains(gpuvm::GpuVaFlags::REPEAT); + + let guard = bo.inner().sgt.lock(); + for range in guard.as_ref().expect("step_map with no SGT").iter() { + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if left == 0 { + break; + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + if len == 0 { + continue; + } + + assert!(offset == 0); + + if one_page { + len = left; + } else { + len = len.min(left); + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x} [OP={}]\n", + addr, + len, + iova, + one_page + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + ctx.prot, + one_page, + )?; + + left -= len; + iova += len as u64; + } + + let gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + + if op + .map_and_link_va( + self, + gpuva, + ctx.vm_bo.as_ref().expect("step_map with no BO"), + ) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "map_and_link_va failed: {:#x} [{:#x}] -> {:#x}\n", + op.offset(), + op.range(), + op.addr() + ); + return Err(EINVAL); + } + Ok(()) + } + fn step_unmap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpUnMap, + _ctx: &mut Self::StepContext, + ) -> Result { + let va = op.va().expect("step_unmap: missing VA"); + + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, va.addr() as usize, va.range() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.range(), + ); + mem::sync(); + } + + if op.unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + Ok(()) + } + fn step_remap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpReMap, + vm_bo: &gpuvm::GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result { + let va = op.unmap().va().expect("No previous VA"); + let orig_addr = va.addr(); + let orig_range = va.range(); + + // Only unmap the hole between prev/next, if they exist + let unmap_start = if let Some(op) = op.prev_map() { + op.addr() + op.range() + } else { + orig_addr + }; + + let unmap_end = if let Some(op) = op.next_map() { + op.addr() + } else { + orig_addr + orig_range + }; + + mod_dev_dbg!( + self.dev, + "MMU: unmap for remap: {:#x}..{:#x} (from {:#x}:{:#x})\n", + unmap_start, + unmap_end, + orig_addr, + orig_range + ); + + let unmap_range = unmap_end - unmap_start; + + self.page_table.unmap_pages(unmap_start..unmap_end)?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + unmap_start, + unmap_range, + ); + mem::sync(); + } + + if op.unmap().unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + + if let Some(prev_op) = op.prev_map() { + let prev_gpuva = ctx + .prev_va + .take() + .expect("Multiple step_remap calls with prev_op"); + if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); + return Err(EINVAL); + } + } + + if let Some(next_op) = op.next_map() { + let next_gpuva = ctx + .next_va + .take() + .expect("Multiple step_remap calls with next_op"); + if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); + return Err(EINVAL); + } + } + + Ok(()) + } +} + +impl VmInner { + /// Returns the slot index, if this VM is bound. + fn slot(&self) -> Option { + if self.is_kernel { + // The GFX ASC does not care about the ASID. Pick an arbitrary one. + // TODO: This needs to be a persistently reserved ASID once we integrate + // with the ARM64 kernel ASID machinery to avoid overlap. + Some(0) + } else { + // We don't check whether we lost the slot, which could cause unnecessary + // invalidations against another Vm. However, this situation should be very + // rare (e.g. a Vm lost its slot, which means 63 other Vms bound in the + // interim, and then it gets killed / drops its mappings without doing any + // final rendering). Anything doing active maps/unmaps is probably also + // rendering and therefore likely bound. + self.binding + .lock() + .bind_token + .as_ref() + .map(|token| token.last_slot() + UAT_USER_CTX_START as u32) + } + } + + /// Returns the translation table base for this Vm + fn ttb(&self) -> u64 { + self.page_table.ttb() + } + + /// Map an `mm::Node` representing an mapping in VA space. + fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let mut iova = node.start(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().sgt.lock(); + let sgt = guard.as_ref().ok_or(EINVAL)?; + let mut offset = node.offset; + let mut left = node.mapped_size; + + for range in sgt.iter() { + if left == 0 { + break; + } + + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if (offset | addr | len | iova as usize) & UAT_PGMSK != 0 { + dev_err!( + self.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + addr, + len, + iova + ); + return Err(EINVAL); + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + len = len.min(left); + + if len == 0 { + continue; + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x}\n", + addr, + len, + iova + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + prot, + false, + )?; + + iova += len as u64; + left -= len; + } + Ok(()) + } +} + +/// Shared reference to a virtual memory address space ([`Vm`]). +#[derive(Clone)] +pub(crate) struct Vm { + id: u64, + inner: ARef>, + dummy_obj: ARef, + binding: Arc>, +} +no_debug!(Vm); + +/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). +pub(crate) struct SlotInner(); + +impl slotalloc::SlotItem for SlotInner { + type Data = (); +} + +/// Represents a single user of a binding of a [`Vm`] to a slot. +/// +/// The number of users is counted, and the slot will be freed when it drops to 0. +#[derive(Debug)] +pub(crate) struct VmBind(Vm, u32); + +impl VmBind { + /// Returns the slot that this `Vm` is bound to. + pub(crate) fn slot(&self) -> u32 { + self.1 + } +} + +impl Drop for VmBind { + fn drop(&mut self) { + let mut binding = self.0.binding.lock(); + + assert_ne!(binding.active_users, 0); + binding.active_users -= 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + if binding.active_users == 0 { + binding.binding = None; + } + } +} + +impl Clone for VmBind { + fn clone(&self) -> VmBind { + let mut binding = self.0.binding.lock(); + + binding.active_users += 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + VmBind(self.0.clone(), self.1) + } +} + +/// Inner data required for an object mapping into a [`Vm`]. +pub(crate) struct KernelMappingInner { + // Drop order matters: + // - Drop the GpuVmBo first, which resv locks its BO and drops a GpuVm reference + // - Drop the GEM BO next, since BO free can take the resv lock itself + // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 + bo: Option>>, + _gem: Option>, + owner: ARef>, + uat_inner: Arc, + prot: Prot, + offset: usize, + mapped_size: usize, +} + +/// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. +pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); + +impl KernelMapping { + /// Returns the IOVA base of this mapping + pub(crate) fn iova(&self) -> u64 { + self.0.start() + } + + /// Returns the size of this mapping in bytes + pub(crate) fn size(&self) -> usize { + self.0.mapped_size + } + + /// Returns the IOVA base of this mapping + pub(crate) fn iova_range(&self) -> Range { + self.0.start()..(self.0.start() + self.0.mapped_size as u64) + } + + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the + /// coprocessor cache. This is required to safely unmap cached/private mappings. + fn remap_uncached_and_flush(&mut self) { + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("Failed to exec_lock in remap_uncached_and_flush"); + + mod_dev_dbg!( + owner.dev, + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + // Remap in-place as uncached. + // Do not try to unmap the guard page (-1) + let prot = self.0.prot.as_uncached(); + if owner + .page_table + .reprot_pages(self.iova_range(), prot) + .is_err() + { + dev_err!( + owner.dev.as_ref(), + "MMU: remap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + fence(Ordering::SeqCst); + + // If we don't have (and have never had) a VM slot, just return + let slot = match owner.slot() { + None => return, + Some(slot) => slot, + }; + + let flush_slot = if owner.is_kernel { + // If this is a kernel mapping, always flush on index 64 + UAT_NUM_CTX as u32 + } else { + // Otherwise, check if this slot is the active one, otherwise return + // Also check that we actually own this slot + let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.0.uat_inner.lock(); + uat_inner.handoff().lock(); + let cur_slot = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); + uat_inner.handoff().unlock(); + if cur_slot == Some(slot) && ttb_cur == ttb { + slot + } else { + return; + } + }; + + // FIXME: There is a race here, though it'll probably never happen in practice. + // In theory, it's possible for the ASC to finish using our slot, whatever command + // it was processing to complete, the slot to be lost to another context, and the ASC + // to begin using it again with a different page table, thus faulting when it gets a + // flush request here. In practice, the chance of this happening is probably vanishingly + // small, as all 62 other slots would have to be recycled or in use before that slot can + // be reused, and the ASC using user contexts at all is very rare. + + // Still, the locking around UAT/Handoff/TTBs should probably be redesigned to better + // model the interactions with the firmware and avoid these races. + // Possibly TTB changes should be tied to slot locks: + + // Flush: + // - Can early check handoff here (no need to lock). + // If user slot and it doesn't match the active ASC slot, + // we can elide the flush as the ASC guarantees it flushes + // TLBs/caches when it switches context. We just need a + // barrier to ensure ordering. + // - Lock TTB slot + // - If user ctx: + // - Lock handoff AP-side + // - Lock handoff dekker + // - Check TTB & handoff cur ctx + // - Perform flush if necessary + // - This implies taking the fwring lock + // + // TTB change: + // - lock TTB slot + // - lock handoff AP-side + // - lock handoff dekker + // change TTB + + // Lock this flush slot, and write the range to it + let flush = self.0.uat_inner.lock_flush(flush_slot); + let pages = self.size() >> UAT_PGBIT; + flush.begin_flush(self.iova(), self.size() as u64); + if pages >= 0x10000 { + dev_err!( + owner.dev.as_ref(), + "MMU: Flush too big ({:#x} pages))\n", + pages + ); + } + + let cmd = fw::channels::FwCtlMsg { + addr: fw::types::U64(self.iova()), + unk_8: 0, + slot: flush_slot, + page_count: pages as u16, + unk_12: 2, // ? + }; + + // Tell the firmware to do a cache flush + if let Err(e) = (*owner.dev).gpu.fwctl(cmd) { + dev_err!( + owner.dev.as_ref(), + "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", + self.iova(), + self.size(), + e + ); + } + + // Finish the flush + flush.end_flush(); + + // Slot is unlocked here + } +} +no_debug!(KernelMapping); + +impl Drop for KernelMapping { + fn drop(&mut self) { + // This is the main unmap function for UAT mappings. + // The sequence of operations here is finicky, due to the interaction + // between cached GFX ASC mappings and the page tables. These mappings + // always have to be flushed from the cache before being unmapped. + + // For uncached mappings, just unmapping and flushing the TLB is sufficient. + + // For cached mappings, this is the required sequence: + // 1. Remap it as uncached + // 2. Flush the TLB range + // 3. If kernel VA mapping OR user VA mapping and handoff.current_slot() == slot: + // a. Take a lock for this slot + // b. Write the flush range to the right context slot in handoff area + // c. Issue a cache invalidation request via FwCtl queue + // d. Poll for completion via queue + // e. Check for completion flag in the handoff area + // f. Drop the lock + // 4. Unmap + // 5. Flush the TLB range again + + if self.0.prot.is_cached_noncoherent() { + mod_pr_debug!( + "MMU: remap as uncached {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + self.remap_uncached_and_flush(); + } + + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("exec_lock failed in KernelMapping::drop"); + mod_dev_dbg!( + owner.dev, + "MMU: unmap {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + if owner.page_table.unmap_pages(self.iova_range()).is_err() { + dev_err!( + owner.dev.as_ref(), + "MMU: unmap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + if let Some(asid) = owner.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); + mod_dev_dbg!( + owner.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + self.iova(), + self.size() + ); + mem::sync(); + } + } +} + +/// Shared UAT global data structures +struct UatShared { + kernel_ttb1: u64, + map_kernel_to_user: bool, + handoff_rgn: UatRegion, + ttbs_rgn: UatRegion, +} + +impl UatShared { + /// Returns the handoff region area + fn handoff(&self) -> &Handoff { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() + } + + /// Returns the TTBAT area + fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { + // SAFETY: pointer is non-null per the type invariant + unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() + } +} + +// SAFETY: Nothing here is unsafe to send across threads. +unsafe impl Send for UatShared {} + +/// Inner data for the top-level UAT instance. +#[pin_data] +struct UatInner { + #[pin] + shared: Mutex, + #[pin] + handoff_flush: [Mutex; UAT_NUM_CTX + 1], +} + +impl UatInner { + /// Take the lock on the shared data and return the guard. + fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { + self.shared.lock() + } + + /// Take a lock on a handoff flush slot and return the guard. + fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { + self.handoff_flush[slot as usize].lock() + } +} + +/// Top-level UAT manager object +pub(crate) struct Uat { + dev: driver::AsahiDevRef, + cfg: &'static hw::HwConfig, + + inner: Arc, + slots: slotalloc::SlotAllocator, + + kernel_vm: Vm, + kernel_lower_vm: Vm, +} + +impl Handoff { + /// Lock the handoff region from firmware access + fn lock(&self) { + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + + while self.lock_fw.load(Ordering::Relaxed) != 0 { + if self.turn.load(Ordering::Relaxed) != 0 { + self.lock_ap.store(0, Ordering::Relaxed); + while self.turn.load(Ordering::Relaxed) != 0 {} + self.lock_ap.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + } + } + fence(Ordering::Acquire); + } + + /// Unlock the handoff region, allowing firmware access + fn unlock(&self) { + self.turn.store(1, Ordering::Relaxed); + self.lock_ap.store(0, Ordering::Release); + } + + /// Returns the current Vm slot mapped by the firmware for lower/unprivileged access, if any. + fn current_slot(&self) -> Option { + let slot = self.cur_slot.load(Ordering::Relaxed); + if slot == 0 || slot == u32::MAX { + None + } else { + Some(slot) + } + } + + /// Initialize the handoff region + fn init(&self) -> Result { + self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); + self.cur_slot.store(0, Ordering::Relaxed); + self.unk3.store(0, Ordering::Relaxed); + fence(Ordering::SeqCst); + + let start = Instant::::now(); + const TIMEOUT: Delta = Delta::from_millis(1000); + + self.lock(); + while start.elapsed() < TIMEOUT { + if self.magic_fw.load(Ordering::Relaxed) == PPL_MAGIC { + break; + } else { + self.unlock(); + fsleep(Delta::from_millis(10)); + self.lock(); + } + } + + if self.magic_fw.load(Ordering::Relaxed) != PPL_MAGIC { + self.unlock(); + pr_err!("Handoff: Failed to initialize (firmware not running?)\n"); + return Err(EIO); + } + + self.unlock(); + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Represents a single flush info slot in the handoff region. +/// +/// # Invariants +/// The pointer is valid and there is no aliasing HandoffFlush instance. +struct HandoffFlush(*const FlushInfo); + +// SAFETY: These pointers are safe to send across threads. +unsafe impl Send for HandoffFlush {} + +impl HandoffFlush { + /// Set up a flush operation for the coprocessor + fn begin_flush(&self, start: u64, size: u64) { + // SAFETY: Per the type invariant, this is safe + let flush = unsafe { self.0.as_ref().unwrap() }; + + let state = flush.state.load(Ordering::Relaxed); + if state != 0 { + pr_err!("Handoff: expected flush state 0, got {}\n", state); + } + flush.addr.store(start, Ordering::Relaxed); + flush.size.store(size, Ordering::Relaxed); + flush.state.store(1, Ordering::Relaxed); + } + + /// Complete a flush operation for the coprocessor + fn end_flush(&self) { + // SAFETY: Per the type invariant, this is safe + let flush = unsafe { self.0.as_ref().unwrap() }; + let state = flush.state.load(Ordering::Relaxed); + if state != 2 { + pr_err!("Handoff: expected flush state 2, got {}\n", state); + } + flush.state.store(0, Ordering::Relaxed); + } +} + +impl Vm { + /// Create a new virtual memory address space + fn new( + dev: &driver::AsahiDevice, + uat_inner: Arc, + kernel_range: Range, + cfg: &'static hw::HwConfig, + ttb: Option, + id: u64, + ) -> Result { + let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; + let is_kernel = ttb.is_some(); + + let page_table = if let Some(ttb) = ttb { + UatPageTable::new_with_ttb(ttb, IOVA_KERN_RANGE, cfg.uat_oas)? + } else { + UatPageTable::new(cfg.uat_oas)? + }; + + let (va_range, gpuvm_range) = if is_kernel { + (IOVA_KERN_RANGE, kernel_range.clone()) + } else { + (IOVA_USER_RANGE, IOVA_USER_USABLE_RANGE) + }; + + let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; + + let binding = Arc::pin_init( + new_mutex!( + VmBinding { + binding: None, + bind_token: None, + active_users: 0, + ttb: page_table.ttb(), + }, + "VmBinding", + ), + GFP_KERNEL, + )?; + + let binding_clone = binding.clone(); + Ok(Vm { + id, + dummy_obj: dummy_obj.gem.clone(), + inner: gpuvm::GpuVm::new( + c_str!("Asahi::GpuVm"), + // TODO: should we using DRM_GPUVM_RESV_PROTECTED as well? + drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + dev, + dummy_obj.gem.clone(), + gpuvm_range, + kernel_range, + init!(VmInner { + dev: dev.into(), + va_range, + is_kernel, + page_table, + mm, + uat_inner, + binding: binding_clone, + id, + }), + )?, + binding, + }) + } + + /// Get the translation table base for this Vm + fn ttb(&self) -> u64 { + self.binding.lock().ttb + } + + /// Map a GEM object (using its `SGTable`) into this Vm at a free address in a given range. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard: bool, + ) -> Result { + let size = object_range.range(); + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.insert_node_in_range( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: Some(vm_bo), + _gem: Some(gem.into()), + offset: object_range.start, + mapped_size: size, + }, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + alignment, + 0, + range.start, + range.end, + mm::InsertMode::Best, + )?; + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Map a GEM object into this Vm at a specific address. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_at( + &self, + addr: u64, + size: usize, + gem: ARef, + prot: Prot, + guard: bool, + ) -> Result { + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(&gem), false)?; + + let vm_bo = self.inner.obtain_bo(&gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: Some(vm_bo), + _gem: Some(gem.clone()), + offset: 0, + mapped_size: size, + }, + addr, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + 0, + )?; + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Map a range of a GEM object into this Vm using GPUVM. + #[allow(clippy::too_many_arguments)] + pub(crate) fn bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + // Mapping needs a complete context + let mut ctx = StepContext { + new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prot, + ..Default::default() + }; + + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), true)?; + + // Preallocate the page tables, to fail early if we ENOMEM + inner.page_table.alloc_pages(addr..(addr + size))?; + + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().sgt.lock(); + if vm_bo_guard.is_none() { + vm_bo_guard.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + ctx.vm_bo = Some(vm_bo); + + if (addr | size | offset) & (UAT_PGMSK as u64) != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: Map step {:#x} [{:#x}] -> {:#x} is not page-aligned\n", + offset, + size, + addr + ); + return Err(EINVAL); + } + + let (flags, gem_range) = if single_page { + (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) + } else { + (gpuvm::GpuVaFlags::NONE, 0u32) + }; + + mod_dev_dbg!( + inner.dev, + "MMU: sm_map: {:#x} [{:#x}] -> {:#x}\n", + offset, + size, + addr + ); + inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) + } + + /// Add a direct MMIO mapping to this Vm at a free address. + pub(crate) fn map_io( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + let mut inner = self.inner.exec_lock(None, false)?; + + if (iova as usize | phys | size) & UAT_PGMSK != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + + dev_info!( + inner.dev.as_ref(), + "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", + phys, + size, + iova + ); + + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }, + iova, + size as u64, + 0, + )?; + + let ret = inner.page_table.map_pages( + iova..(iova + size as u64), + phys as PhysicalAddr, + prot, + false, + ); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(node)) + } + + /// Unmap everything in an address range. + pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { + // Unmapping a range can only do a single split, so just preallocate + // the prev and next GpuVas + let mut ctx = StepContext { + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + ..Default::default() + }; + + let mut inner = self.inner.exec_lock(None, false)?; + + mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); + inner.sm_unmap(&mut ctx, iova, size) + } + + /// Drop mappings for a given bo. + pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + // Removing whole mappings only does unmaps, so no preallocated VAs + let mut ctx = Default::default(); + + let inner = self.inner.exec_lock(Some(gem), false)?; + + if let Some(bo) = self.inner.find_bo(gem) { + mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); + self.inner.bo_unmap(&mut ctx, &bo)?; + mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); + // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + core::mem::drop(inner); + core::mem::drop(bo); + } + + Ok(()) + } + + /// Returns the dummy GEM object used to hold the shared DMA reservation locks + pub(crate) fn get_resv_obj(&self) -> ARef { + self.dummy_obj.clone() + } + + /// Check whether an object is external to this GpuVm + pub(crate) fn is_extobj(&self, gem: &gem::Object) -> bool { + self.inner.is_extobj(gem) + } + + /// Check whether an object is external to this GpuVm + pub(crate) fn bo_deferred_cleanup(&self) { + self.inner.bo_deferred_cleanup() + } +} + +impl Drop for VmInner { + fn drop(&mut self) { + let mut binding = self.binding.lock(); + assert_eq!(binding.active_users, 0); + + mod_pr_debug!( + "VmInner::Drop [{}]: bind_token={:?}\n", + self.id, + binding.bind_token + ); + + // Make sure this VM is not mapped to a TTB if it was + if let Some(token) = binding.bind_token.take() { + let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; + let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.uat_inner.lock(); + uat_inner.handoff().lock(); + let handoff_cur = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); + let inval = ttb_cur == ttb; + if inval { + if handoff_cur == Some(idx as u32) { + pr_err!( + "VmInner::drop owning slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); + uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + } + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // In principle we dropped all the KernelMappings already, but we might as + // well play it safe and invalidate the whole ASID. + if inval { + mod_pr_debug!( + "VmInner::Drop [{}]: need inval for ASID {:#x}\n", + self.id, + idx + ); + mem::tlbi_asid(idx as u8); + mem::sync(); + } + } + } +} + +impl Uat { + /// Map a bootloader-preallocated memory region + fn map_region( + dev: &device::Device, + name: &CStr, + size: usize, + cached: bool, + ) -> Result { + let of_node = dev.of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(name)?; + let base = res.start(); + let res_size = res.size().try_into()?; + + if size > res_size { + dev_err!( + dev, + "Region {} is too small (expected {}, got {})\n", + name, + size, + res_size + ); + return Err(ENOMEM); + } + + let flags = if cached { + io::mem::MemFlag::WB + } else { + io::mem::MemFlag::WC + }; + + // SAFETY: The safety of this operation hinges on the correctness of + // much of this file and also the `pgtable` module, so it is difficult + // to prove in a single safety comment. Such is life with raw GPU + // page table management... + let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }.inspect_err(|_| { + dev_err!(dev, "Failed to remap {} mem resource\n", name); + })?; + + Ok(UatRegion { base, map }) + } + + /// Returns a reference to the global kernel (upper half) `Vm` + pub(crate) fn kernel_vm(&self) -> &Vm { + &self.kernel_vm + } + + /// Returns a reference to the local kernel (lower half) `Vm` + pub(crate) fn kernel_lower_vm(&self) -> &Vm { + &self.kernel_lower_vm + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_kernel_pages(&self) -> Result> { + let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; + inner.page_table.dump_pages(IOVA_KERN_FULL_RANGE) + } + + /// Returns the base physical address of the TTBAT region. + pub(crate) fn ttb_base(&self) -> u64 { + let inner = self.inner.lock(); + + inner.ttbs_rgn.base + } + + /// Binds a `Vm` to a slot, preferring the last used one. + pub(crate) fn bind(&self, vm: &Vm) -> Result { + let mut binding = vm.binding.lock(); + + if binding.binding.is_none() { + assert_eq!(binding.active_users, 0); + + let isolation = *module_parameters::robust_isolation.value() != 0; + + self.slots.set_limit(if isolation { + NonZeroUsize::new(1) + } else { + None + }); + + let slot = self.slots.get(binding.bind_token)?; + if slot.changed() { + mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); + let idx = (slot.slot() as usize) + UAT_USER_CTX_START; + let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.inner.lock(); + + let ttb1 = if uat_inner.map_kernel_to_user { + uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT + } else { + 0 + }; + + let ttbs = uat_inner.ttbs(); + uat_inner.handoff().lock(); + if uat_inner.handoff().current_slot() == Some(idx as u32) { + pr_err!( + "Vm::bind to slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + ttbs[idx].ttb0.store(ttb, Ordering::Release); + ttbs[idx].ttb1.store(ttb1, Ordering::Release); + uat_inner.handoff().unlock(); + core::mem::drop(uat_inner); + + // Make sure all TLB entries from the previous owner of this ASID are gone + mem::tlbi_asid(idx as u8); + mem::sync(); + } + + binding.bind_token = Some(slot.token()); + binding.binding = Some(slot); + } + + binding.active_users += 1; + + let slot = binding.binding.as_ref().unwrap().slot() + UAT_USER_CTX_START as u32; + mod_pr_debug!("MMU: slot {} active users {}\n", slot, binding.active_users); + Ok(VmBind(vm.clone(), slot)) + } + + /// Creates a new `Vm` linked to this UAT. + pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { + Vm::new( + &self.dev, + self.inner.clone(), + kernel_range, + self.cfg, + None, + id, + ) + } + + /// Creates the reference-counted inner data for a new `Uat` instance. + #[inline(never)] + fn make_inner(dev: &driver::AsahiDevice) -> Result> { + let handoff_rgn = Self::map_region(dev.as_ref(), c_str!("handoff"), HANDOFF_SIZE, true)?; + let ttbs_rgn = Self::map_region(dev.as_ref(), c_str!("ttbs"), SLOTS_SIZE, true)?; + + // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, + // and the region is at least large enough per the size specified above. + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + + dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); + + Arc::pin_init( + try_pin_init!(UatInner { + handoff_flush <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + }), + shared <- new_mutex!( + UatShared { + kernel_ttb1: 0, + map_kernel_to_user: false, + handoff_rgn, + ttbs_rgn, + }, + "uat_shared" + ), + }), + GFP_KERNEL, + ) + } + + /// Creates a new `Uat` instance given the relevant hardware config. + #[inline(never)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + cfg: &'static hw::HwConfig, + map_kernel_to_user: bool, + ) -> Result { + dev_info!(dev.as_ref(), "MMU: Initializing...\n"); + + let inner = Self::make_inner(dev)?; + + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; + let ttb1 = res.start(); + let ttb1size: usize = res.size().try_into()?; + + if ttb1size < PAGETABLES_SIZE { + dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); + return Err(ENOMEM); + } + + dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); + let kernel_lower_vm = Vm::new(dev, inner.clone(), IOVA_USER_RANGE, cfg, None, 1)?; + let kernel_vm = Vm::new(dev, inner.clone(), IOVA_KERN_RANGE, cfg, Some(ttb1), 0)?; + + dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); + + let ttb0 = kernel_lower_vm.ttb(); + + let uat = Self { + dev: dev.into(), + cfg, + kernel_vm, + kernel_lower_vm, + inner, + slots: slotalloc::SlotAllocator::new( + UAT_USER_CTX as u32, + (), + |_inner, _slot| Some(SlotInner()), + c_str!("Uat::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }; + + let mut inner = uat.inner.lock(); + + inner.map_kernel_to_user = map_kernel_to_user; + inner.kernel_ttb1 = ttb1; + + inner.handoff().init()?; + + dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); + + inner.handoff().lock(); + + let ttbs = inner.ttbs(); + + ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::SeqCst); + ttbs[0].ttb1.store(ttb1 | TTBR_VALID, Ordering::SeqCst); + + for ctx in &ttbs[1..] { + ctx.ttb0.store(0, Ordering::Relaxed); + ctx.ttb1.store(0, Ordering::Relaxed); + } + + inner.handoff().unlock(); + + core::mem::drop(inner); + + dev_info!(dev.as_ref(), "MMU: initialized\n"); + + Ok(uat) + } +} + +impl Drop for Uat { + fn drop(&mut self) { + // Make sure we flush the TLBs + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + } +} diff --git a/drivers/gpu/drm/asahi/object.rs b/drivers/gpu/drm/asahi/object.rs new file mode 100644 index 00000000000000..38a2268137effb --- /dev/null +++ b/drivers/gpu/drm/asahi/object.rs @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Asahi GPU object model +//! +//! The AGX GPU includes a coprocessor that uses a large number of shared memory structures to +//! communicate with the driver. These structures contain GPU VA pointers to each other, which are +//! directly dereferenced by the firmware and are expected to always be valid for the usage +//! lifetime of the containing struct (which is an implicit contract, not explicitly managed). +//! Any faults cause an unrecoverable firmware crash, requiring a full system reboot. +//! +//! In order to manage this complexity safely, we implement a GPU object model using Rust's type +//! system to enforce GPU object lifetime relationships. GPU objects represent an allocated piece +//! of memory of a given type, mapped to the GPU (and usually also the CPU). On the CPU side, +//! these objects are associated with a pure Rust structure that contains the objects it depends +//! on (or references to them). This allows us to map Rust lifetimes into the GPU object model +//! system. Then, GPU VA pointers also inherit those lifetimes, which means the Rust borrow checker +//! can ensure that all pointers are assigned an address that is guaranteed to outlive the GPU +//! object it points to. +//! +//! Since the firmware object model does have self-referencing pointers (and there is of course no +//! underlying revocability mechanism to make it safe), we must have an escape hatch. GPU pointers +//! can be weak pointers, which do not enforce lifetimes. In those cases, it is the user's +//! responsibility to ensure that lifetime requirements are met. +//! +//! In other words, the model is necessarily leaky and there is no way to fully map Rust safety to +//! GPU firmware object safety. The goal of the model is to make it easy to model the lifetimes of +//! GPU objects and have the compiler help in avoiding mistakes, rather than to guarantee safety +//! 100% of the time as would be the case for CPU-side Rust code. + +// TODO: There is a fundamental soundness issue with sharing memory with the GPU (that even affects +// C code too). Since the GPU is free to mutate that memory at any time, normal reference invariants +// cannot be enforced on the CPU side. For example, the compiler could perform an optimization that +// assumes that a given memory location does not change between two reads, and causes UB otherwise, +// and then the GPU could mutate that memory out from under the CPU. +// +// For cases where we *expect* this to happen, we use atomic types, which avoid this issue. However, +// doing so for every single field of every type is a non-starter. Right now, there seems to be no +// good solution for this that does not come with significant performance or ergonomics downsides. +// +// In *practice* we are almost always only writing GPU memory, and only reading from atomics, so the +// chances of this actually triggering UB (e.g. a security issue that can be triggered from the GPU +// side) due to a compiler optimization are very slim. +// +// Further discussion: https://github.com/rust-lang/unsafe-code-guidelines/issues/152 + +use kernel::{ + error::code::*, + prelude::*, + sync::Arc, // +}; + +use core::fmt; +use core::fmt::Debug; +use core::fmt::Formatter; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::num::NonZeroU64; +use core::ops::{ + Deref, + DerefMut, + Index, + IndexMut, // +}; +use core::{mem, ptr, slice}; + +use crate::alloc::Allocation; +use crate::debug::*; +use crate::fw::types::Zeroable; +use crate::mmu; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Object; + +/// A GPU-side strong pointer, which is a 64-bit non-zero VA with an associated lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuPointer<'a, T: ?Sized>(NonZeroU64, PhantomData<&'a T>); + +impl<'a, T: ?Sized> GpuPointer<'a, T> { + /// Logical OR the pointer with an arbitrary `u64`. This is used when GPU struct fields contain + /// misc flag fields in the upper bits. The lifetime is retained. This is GPU-unsafe in + /// principle, but we assert that only non-implemented address bits are touched, which is safe + /// for pointers used by the GPU (not by firmware). + pub(crate) fn or(&self, other: u64) -> GpuPointer<'a, T> { + // This will fail for kernel-half pointers, which should not be ORed. + assert_eq!(self.0.get() & other, 0); + // Assert that we only touch the high bits. + assert_eq!(other & 0xffffffffff, 0); + GpuPointer(self.0 | other, PhantomData) + } + + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_ptr` macro to get pointers to inner fields, hence we mark + /// it `unsafe` to discourage direct use. + /// + /// # Safety + /// Do not use directly, only via `inner_ptr`. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuPointer<'a, U> { + GpuPointer::<'a, U>( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } +} + +impl<'a, T> GpuPointer<'a, T> { + /// Create a GPU pointer from a KernelMapping and an offset. + /// TODO: Change all GPU pointers to point to the raw types so size_of here is GPU-sound. + pub(crate) fn from_mapping( + mapping: &'a Arc, + offset: usize, + ) -> Result> { + let addr = mapping.iova().checked_add(offset as u64).ok_or(EINVAL)?; + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > mapping.size() { + Err(ERANGE) + } else { + Ok(Self(addr.try_into().unwrap(), PhantomData)) + } + } +} + +impl<'a, T: ?Sized> Debug for GpuPointer<'a, T> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +impl<'a, T: ?Sized> From> for u64 { + fn from(value: GpuPointer<'a, T>) -> Self { + value.0.get() + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuPointer, keeping the +/// lifetime. +#[macro_export] +macro_rules! inner_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // This mirrors kernel::offset_of(), except we use type inference to avoid having to know + // the type of the pointer explicitly. + fn uninit_from(_: GpuPointer<'_, T>) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// A GPU-side weak pointer, which is a 64-bit non-zero VA with no lifetime. +/// +/// In rare cases these pointers are not aligned, so this is `packed(1)`. +#[repr(C, packed(1))] +pub(crate) struct GpuWeakPointer(NonZeroU64, PhantomData<*const T>); + +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Send for GpuWeakPointer {} +/// SAFETY: GPU weak pointers are always safe to share between threads. +unsafe impl Sync for GpuWeakPointer {} + +// Weak pointers can be copied/cloned regardless of their target type. +impl Copy for GpuWeakPointer {} + +impl Clone for GpuWeakPointer { + fn clone(&self) -> Self { + *self + } +} + +impl GpuWeakPointer { + /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and + /// should only be used via the `inner_weak_ptr` macro to get pointers to inner fields, hence we + /// mark it `unsafe` to discourage direct use. + /// + /// # Safety + /// Do not use directly, only via `inner_weak_ptr`. + // NOTE: The third argument is a type inference hack. + pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuWeakPointer { + GpuWeakPointer::( + NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), + PhantomData, + ) + } + + /// Upgrade a weak pointer into a strong pointer. This is not considered safe from the GPU + /// perspective. + /// + /// # Safety + /// The caller must ensure tht the data pointed to lives in the GPU at least as long as the + /// returned lifetime. + pub(crate) unsafe fn upgrade<'a>(&self) -> GpuPointer<'a, T> { + GpuPointer(self.0, PhantomData) + } +} + +impl From> for u64 { + fn from(value: GpuWeakPointer) -> Self { + value.0.get() + } +} + +impl Debug for GpuWeakPointer { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let val = self.0; + f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) + } +} + +/// Take a pointer to a sub-field within a structure pointed to by a GpuWeakPointer. +#[macro_export] +macro_rules! inner_weak_ptr { + ($gpuva:expr, $($f:tt)*) => ({ + // See inner_ptr() + fn uninit_from(_: GpuWeakPointer) -> core::mem::MaybeUninit> { + core::mem::MaybeUninit::uninit() + } + let tmp = uninit_from($gpuva); + let outer = tmp.as_ptr(); + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; + let inner = p as *const u8; + // SAFETY: The two pointers are within the same allocation block. + let off = unsafe { inner.offset_from(outer as *const u8) }; + // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer + // object. + unsafe { $gpuva.offset(off.try_into().unwrap(), p) } + }) +} + +/// Types that implement this trait represent a GPU structure from the CPU side. +/// +/// The `Raw` type represents the actual raw structure definition on the GPU side. +/// +/// Types implementing [`GpuStruct`] must have fields owning any objects (or strong references +/// to them) that GPU pointers in the `Raw` structure point to. This mechanism is used to enforce +/// lifetimes. +pub(crate) trait GpuStruct: 'static { + /// The type of the GPU-side structure definition representing the firmware struct layout. + type Raw<'a>; +} + +/// An instance of a GPU object in memory. +/// +/// # Invariants +/// `raw` must point to a valid mapping of the `T::Raw` type associated with the `alloc` allocation. +/// `gpu_ptr` must be the GPU address of the same object. +pub(crate) struct GpuObject> { + raw: *mut T::Raw<'static>, + alloc: U, + gpu_ptr: GpuWeakPointer, + inner: KBox, +} + +impl> GpuObject { + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that constructs the `T::Raw` type given a reference to the + /// `GpuStruct`. This is the mechanism used to enforce lifetimes. + pub(crate) fn new( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, + ) -> Result { + let size = mem::size_of::>(); + if size > 0x1000 { + dev_crit!( + alloc.device().as_ref(), + "Allocating {} of size {:#x}, with new, please use new_boxed!\n", + core::any::type_name::(), + size + ); + } + if alloc.size() < size { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; + let mut raw = callback(&inner); + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is + // identical to the type of `raw` other than the lifetime. + unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; + mem::forget(raw); + Ok(Self { + raw: p, + gpu_ptr, + alloc, + inner: KBox::new(inner, GFP_KERNEL)?, + }) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_boxed( + alloc: U, + inner: KBox, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + let raw = callback(&inner, unsafe { &mut *p })?; + if p as *mut T::Raw<'_> != raw as *mut _ { + dev_err!( + alloc.device().as_ref(), + "Allocation callback returned a mismatched reference ({})\n", + core::any::type_name::(), + ); + return Err(EINVAL); + } + Ok(Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner, + }) + } + + /// Create a new GpuObject given an allocator and the inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_inplace( + alloc: U, + inner: T, + callback: impl for<'a> FnOnce( + &'a T, + &'a mut MaybeUninit>, + ) -> Result<&'a mut T::Raw<'a>>, + ) -> Result { + GpuObject::::new_boxed(alloc, KBox::new(inner, GFP_KERNEL)?, callback) + } + + /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing + /// GpuStruct). + /// + /// The caller passes a closure that initializes the `T::Raw` type given a reference to the + /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() + /// macro to avoid constructing the whole `T::Raw` object on the stack. + pub(crate) fn new_init_prealloc<'a, I: Init, R: PinInit, F>, E, F>( + alloc: U, + inner_init: impl FnOnce(GpuWeakPointer) -> I, + raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, + ) -> Result + where + kernel::error::Error: core::convert::From, + kernel::error::Error: core::convert::From, + { + if alloc.size() < mem::size_of::>() { + return Err(ENOMEM); + } + let gpu_ptr = + GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); + mod_dev_dbg!( + alloc.device(), + "Allocating {} @ {:#x}\n", + core::any::type_name::(), + alloc.gpu_ptr() + ); + let inner = inner_init(gpu_ptr); + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; + let ret = Self { + raw: p as *mut u8 as *mut T::Raw<'static>, + gpu_ptr, + alloc, + inner: KBox::init(inner, GFP_KERNEL)?, + }; + let q = &*ret.inner as *const T; + // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. + unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; + Ok(ret) + } + + /// Returns the GPU VA of this object (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr.0 + } + + /// Returns a strong GPU pointer to this object, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, T> { + GpuPointer(self.gpu_ptr.0, PhantomData) + } + + /// Returns a weak GPU pointer to this object, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { + GpuWeakPointer(self.gpu_ptr.0, PhantomData) + } + + /// Perform a mutation to the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a mutable reference to the `GpuStruct` type. + pub(crate) fn with_mut( + &mut self, + callback: impl for<'a> FnOnce(&'a mut ::Raw<'a>, &'a mut T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } + } + + /// Access the inner `Raw` data given a user-supplied callback. + /// + /// The callback gets a reference to the `GpuStruct` type. + pub(crate) fn with( + &self, + callback: impl for<'a> FnOnce(&'a ::Raw<'a>, &'a T) -> RetVal, + ) -> RetVal { + // SAFETY: `self.raw` is valid per the type invariant, and the second half is just + // converting lifetimes. + unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } + } +} + +impl> Deref for GpuObject { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl> DerefMut for GpuObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl> Debug for GpuObject +where + ::Raw<'static>: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + // SAFETY: `self.raw` is valid per the type invariant. + .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) + .field("inner", &format_args!("{:#X?}", &self.inner)) + .field("alloc", &format_args!("{:?}", &self.alloc)) + .finish() + } +} + +impl> GpuObject +where + for<'a> ::Raw<'a>: Default + Zeroable, +{ + /// Create a new GpuObject with default data. `T` must implement `Default` and `T::Raw` must + /// implement `Zeroable`, since the GPU-side memory is initialized by zeroing. + pub(crate) fn new_default(alloc: U) -> Result { + GpuObject::::new_inplace(alloc, Default::default(), |_inner, raw| { + // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroable`. + Ok(unsafe { + ptr::write_bytes(raw, 0, 1); + (*raw).assume_init_mut() + }) + }) + } +} + +impl> Drop for GpuObject { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Send for GpuObject {} +// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send +unsafe impl> Sync for GpuObject {} + +/// Trait used to erase the type of a GpuObject, used when we need to keep a list of heterogenous +/// objects around. +pub(crate) trait OpaqueGpuObject: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; +} + +impl> OpaqueGpuObject for GpuObject { + fn gpu_va(&self) -> NonZeroU64 { + Self::gpu_va(self) + } +} + +/// An array of raw GPU objects that is only accessible to the GPU (no CPU-side mapping required). +/// +/// This must necessarily be uninitialized as far as the GPU is concerned, so it cannot be used +/// when initialization is required. +/// +/// # Invariants +/// +/// `alloc` is valid and at least as large as `len` times the size of one `T`. +/// `gpu_ptr` is valid and points to the allocation start. +pub(crate) struct GpuOnlyArray> { + len: usize, + alloc: U, + gpu_ptr: NonZeroU64, + _p: PhantomData, +} + +impl> GpuOnlyArray { + /// Allocate a new GPU-only array with the given length. + pub(crate) fn new(alloc: U, count: usize) -> Result> { + let bytes = count * mem::size_of::(); + let gpu_ptr = NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?; + if alloc.size() < bytes { + return Err(ENOMEM); + } + Ok(Self { + len: count, + alloc, + gpu_ptr, + _p: PhantomData, + }) + } + + /// Returns the GPU VA of this arraw (as a raw [`NonZeroU64`]) + pub(crate) fn gpu_va(&self) -> NonZeroU64 { + self.gpu_ptr + } + + /// Returns a strong GPU pointer to this array, with a lifetime. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, &'_ [T]> { + GpuPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a weak GPU pointer to this array, with no lifetime. + pub(crate) fn weak_pointer(&self) -> GpuWeakPointer<[T]> { + GpuWeakPointer(self.gpu_ptr, PhantomData) + } + + /// Returns a pointer to an offset within the array (as a subslice). + pub(crate) fn gpu_offset_pointer(&self, offset: usize) -> GpuPointer<'_, &'_ [T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /* Not used yet + /// Returns a weak pointer to an offset within the array (as a subslice). + pub(crate) fn weak_offset_pointer(&self, offset: usize) -> GpuWeakPointer<[T]> { + if offset > self.len { + panic!("Index {} out of bounds (len: {})", offset, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns a pointer to an element within the array. + pub(crate) fn gpu_item_pointer(&self, index: usize) -> GpuPointer<'_, &'_ T> { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + */ + + /// Returns a weak pointer to an element within the array. + pub(crate) fn weak_item_pointer(&self, index: usize) -> GpuWeakPointer { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + GpuWeakPointer( + NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), + PhantomData, + ) + } + + /// Returns the length of the array. + pub(crate) fn len(&self) -> usize { + self.len + } +} + +impl> Debug for GpuOnlyArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("len", &format_args!("{:#X?}", self.len())) + .finish() + } +} + +impl> Drop for GpuOnlyArray { + fn drop(&mut self) { + mod_dev_dbg!( + self.alloc.device(), + "Dropping {} @ {:?}\n", + core::any::type_name::(), + self.gpu_pointer() + ); + } +} + +/// An array of raw GPU objects that is also CPU-accessible. +/// +/// # Invariants +/// +/// `raw` is valid and points to the CPU-side view of the array (which must have one). +pub(crate) struct GpuArray> { + raw: *mut T, + array: GpuOnlyArray, +} + +impl> GpuArray { + /// Allocate a new GPU array, initializing each element to its default. + pub(crate) fn empty(alloc: U, count: usize) -> Result> { + let p = alloc.ptr().ok_or(EINVAL)?.as_ptr(); + let inner = GpuOnlyArray::new(alloc, count)?; + let mut pi = p; + for _i in 0..count { + // SAFETY: `pi` is valid per the Allocation type invariant, and GpuOnlyArray guarantees + // that it can never iterate beyond the buffer length. + unsafe { + pi.write(Default::default()); + pi = pi.add(1); + } + } + Ok(Self { + raw: p, + array: inner, + }) + } +} + +impl> GpuArray { + /// Get a slice view of the array contents. + pub(crate) fn as_slice(&self) -> &[T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts(self.raw, self.len) } + } + + /// Get a mutable slice view of the array contents. + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + // SAFETY: self.raw / self.len are valid per the type invariant + unsafe { slice::from_raw_parts_mut(self.raw, self.len) } + } +} + +impl> Deref for GpuArray { + type Target = GpuOnlyArray; + + fn deref(&self) -> &GpuOnlyArray { + &self.array + } +} + +impl> Index for GpuArray { + type Output = T; + + fn index(&self, index: usize) -> &T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &*(self.raw.add(index)) } + } +} + +impl> IndexMut for GpuArray { + fn index_mut(&mut self, index: usize) -> &mut T { + if index >= self.len { + panic!("Index {} out of bounds (len: {})", index, self.len); + } + // SAFETY: This is bounds checked above + unsafe { &mut *(self.raw.add(index)) } + } +} + +// SAFETY: GpuArray are Send as long as the contained type itself is Send +unsafe impl> Send for GpuArray {} +// SAFETY: GpuArray are Sync as long as the contained type itself is Sync +unsafe impl> Sync for GpuArray {} + +impl> Debug for GpuArray { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.debug_struct(core::any::type_name::()) + .field("array", &format_args!("{:#X?}", self.as_slice())) + .finish() + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs new file mode 100644 index 00000000000000..7c9b54c2242515 --- /dev/null +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! UAT Page Table management +//! +//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table +//! format. This module manages the actual page tables by allocating raw memory pages from +//! the kernel page allocator. + +use core::fmt::Debug; +use core::mem::size_of; +use core::ops::Range; +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +use kernel::{ + addr::PhysicalAddr, + error::Result, + page::Page, + prelude::*, // +}; +#[cfg(CONFIG_DEV_COREDUMP)] +use kernel::{ + types::Owned, + uapi::{ + PF_R, + PF_W, + PF_X, // + }, +}; + +use crate::debug::*; +use crate::util::align; + +const DEBUG_CLASS: DebugFlags = DebugFlags::PgTable; + +/// Number of bits in a page offset. +pub(crate) const UAT_PGBIT: usize = 14; +/// UAT page size. +pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; +/// UAT page offset mask. +pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; + +type Pte = AtomicU64; + +const PTE_BIT: usize = 3; // log2(sizeof(Pte)) +const PTE_SIZE: usize = 1 << PTE_BIT; + +/// Number of PTEs per page. +const UAT_NPTE: usize = UAT_PGSZ / size_of::(); + +/// Number of address bits to address a level +const UAT_LVBIT: usize = UAT_PGBIT - PTE_BIT; +/// Number of entries per level +const UAT_LVSZ: usize = UAT_NPTE; +/// Mask of level bits +const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; + +const UAT_LEVELS: usize = 3; + +/// UAT input address space +pub(crate) const UAT_IAS: usize = 39; +const UAT_IASMSK: u64 = (1u64 << UAT_IAS) - 1; + +const PTE_TYPE_BITS: u64 = 3; +const PTE_TYPE_LEAF_TABLE: u64 = 3; + +const UAT_NON_GLOBAL: u64 = 1 << 11; +const UAT_AP_SHIFT: u32 = 6; +const UAT_AP_BITS: u64 = 3 << UAT_AP_SHIFT; +const UAT_HIGH_BITS_SHIFT: u32 = 52; +const UAT_HIGH_BITS: u64 = 0xfff << UAT_HIGH_BITS_SHIFT; +const UAT_MEMATTR_SHIFT: u32 = 2; +const UAT_MEMATTR_BITS: u64 = 7 << UAT_MEMATTR_SHIFT; + +const UAT_PROT_BITS: u64 = UAT_AP_BITS | UAT_MEMATTR_BITS | UAT_HIGH_BITS; + +const UAT_AF: u64 = 1 << 10; + +const MEMATTR_CACHED: u8 = 0; +const MEMATTR_DEV: u8 = 1; +const MEMATTR_UNCACHED: u8 = 2; + +const AP_FW_GPU: u8 = 0; +const AP_FW: u8 = 1; +const AP_GPU: u8 = 2; + +const HIGH_BITS_PXN: u16 = 1 << 1; +const HIGH_BITS_UXN: u16 = 1 << 2; +const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) const PTE_ADDR_BITS: u64 = (!UAT_PGMSK as u64) & (!UAT_HIGH_BITS); + +#[derive(Debug, Copy, Clone)] +pub(crate) struct Prot { + memattr: u8, + ap: u8, + high_bits: u16, +} + +// Firmware + GPU access +const PROT_FW_GPU_NA: Prot = Prot::from_bits(AP_FW_GPU, 0, 0); +const _PROT_FW_GPU_RO: Prot = Prot::from_bits(AP_FW_GPU, 0, 1); +const _PROT_FW_GPU_WO: Prot = Prot::from_bits(AP_FW_GPU, 1, 0); +const PROT_FW_GPU_RW: Prot = Prot::from_bits(AP_FW_GPU, 1, 1); + +// Firmware only access +const PROT_FW_RO: Prot = Prot::from_bits(AP_FW, 0, 0); +const _PROT_FW_NA: Prot = Prot::from_bits(AP_FW, 0, 1); +const PROT_FW_RW: Prot = Prot::from_bits(AP_FW, 1, 0); +const PROT_FW_RW_GPU_RO: Prot = Prot::from_bits(AP_FW, 1, 1); + +// GPU only access +const PROT_GPU_RO: Prot = Prot::from_bits(AP_GPU, 0, 0); +const PROT_GPU_WO: Prot = Prot::from_bits(AP_GPU, 0, 1); +const PROT_GPU_RW: Prot = Prot::from_bits(AP_GPU, 1, 0); +const _PROT_GPU_NA: Prot = Prot::from_bits(AP_GPU, 1, 1); + +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RW: u32 = PF_R | PF_W; +#[cfg(CONFIG_DEV_COREDUMP)] +const PF_RX: u32 = PF_R | PF_X; + +// For crash dumps +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_FW: [[u32; 4]; 4] = [ + [0, 0, 0, PF_RW], + [0, PF_RW, 0, PF_RW], + [PF_RX, PF_RX, 0, PF_R], + [PF_RX, PF_RW, 0, PF_R], +]; +#[cfg(CONFIG_DEV_COREDUMP)] +const PROT_TO_PERMS_OS: [[u32; 4]; 4] = [ + [0, PF_R, PF_W, PF_RW], + [PF_R, 0, PF_RW, PF_RW], + [0, 0, 0, 0], + [0, 0, 0, 0], +]; + +pub(crate) mod prot { + pub(crate) use super::Prot; + use super::*; + + /// Firmware MMIO R/W + pub(crate) const PROT_FW_MMIO_RW: Prot = PROT_FW_RW.memattr(MEMATTR_DEV); + /// Firmware MMIO R/O + pub(crate) const PROT_FW_MMIO_RO: Prot = PROT_FW_RO.memattr(MEMATTR_DEV); + /// Firmware shared (uncached) RW + pub(crate) const PROT_FW_SHARED_RW: Prot = PROT_FW_RW.memattr(MEMATTR_UNCACHED); + /// Firmware shared (uncached) RO + pub(crate) const PROT_FW_SHARED_RO: Prot = PROT_FW_RO.memattr(MEMATTR_UNCACHED); + /// Firmware private (cached) RW + pub(crate) const PROT_FW_PRIV_RW: Prot = PROT_FW_RW.memattr(MEMATTR_CACHED); + /// Firmware/GPU shared (uncached) RW + pub(crate) const PROT_GPU_FW_SHARED_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_UNCACHED); + /// Firmware/GPU shared (private) RW + pub(crate) const PROT_GPU_FW_PRIV_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_CACHED); + /// Firmware-RW/GPU-RO shared (private) RW + pub(crate) const PROT_GPU_RO_FW_PRIV_RW: Prot = PROT_FW_RW_GPU_RO.memattr(MEMATTR_CACHED); + /// GPU shared/coherent RW + pub(crate) const PROT_GPU_SHARED_RW: Prot = PROT_GPU_RW.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent RO + pub(crate) const PROT_GPU_SHARED_RO: Prot = PROT_GPU_RO.memattr(MEMATTR_UNCACHED); + /// GPU shared/coherent WO + pub(crate) const PROT_GPU_SHARED_WO: Prot = PROT_GPU_WO.memattr(MEMATTR_UNCACHED); +} + +impl Prot { + const fn from_bits(ap: u8, uxn: u16, pxn: u16) -> Self { + assert!(uxn <= 1); + assert!(pxn <= 1); + assert!(ap <= 3); + + Prot { + high_bits: HIGH_BITS_GPU_ACCESS | (pxn * HIGH_BITS_PXN) | (uxn * HIGH_BITS_UXN), + memattr: 0, + ap, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn from_pte(pte: u64) -> Self { + Prot { + high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, + ap: ((pte & UAT_AP_BITS) >> UAT_AP_SHIFT) as u8, + memattr: ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8, + } + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) const fn elf_flags(&self) -> u32 { + let ap = (self.ap & 3) as usize; + let uxn = if self.high_bits & HIGH_BITS_UXN != 0 { + 1 + } else { + 0 + }; + let pxn = if self.high_bits & HIGH_BITS_PXN != 0 { + 1 + } else { + 0 + }; + let gpu = self.high_bits & HIGH_BITS_GPU_ACCESS != 0; + + // Format: + // [12 top bits of PTE] [12 bottom bits of PTE] [5 bits pad] [ELF RWX] + let mut perms = if gpu { + PROT_TO_PERMS_OS[ap][(uxn << 1) | pxn] + } else { + PROT_TO_PERMS_FW[ap][(uxn << 1) | pxn] + }; + + perms |= ((self.as_pte() >> 52) << 20) as u32; + perms |= ((self.as_pte() & 0xfff) << 8) as u32; + + perms + } + + const fn memattr(&self, memattr: u8) -> Self { + Self { memattr, ..*self } + } + + const fn as_pte(&self) -> u64 { + (self.ap as u64) << UAT_AP_SHIFT + | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT + | (self.memattr as u64) << UAT_MEMATTR_SHIFT + | UAT_AF + } + + pub(crate) const fn is_cached_noncoherent(&self) -> bool { + self.ap != AP_GPU && self.memattr == MEMATTR_CACHED + } + + pub(crate) const fn as_uncached(&self) -> Self { + self.memattr(MEMATTR_UNCACHED) + } +} + +impl Default for Prot { + fn default() -> Self { + PROT_FW_GPU_NA + } +} + +#[cfg(CONFIG_DEV_COREDUMP)] +pub(crate) struct DumpedPage { + pub(crate) iova: u64, + pub(crate) pte: u64, + pub(crate) data: Option>, +} + +pub(crate) struct UatPageTable { + ttb: PhysicalAddr, + ttb_owned: bool, + va_range: Range, + oas_mask: u64, +} + +impl UatPageTable { + pub(crate) fn new(oas: u32) -> Result { + mod_pr_debug!("UATPageTable::new: oas={}\n", oas); + let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let ttb = Page::into_phys(ttb_page); + Ok(UatPageTable { + ttb, + ttb_owned: true, + va_range: 0..(1u64 << UAT_IAS), + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn new_with_ttb(ttb: PhysicalAddr, va_range: Range, oas: u32) -> Result { + mod_pr_debug!( + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} oas={}\n", + ttb, + va_range, + oas + ); + if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + if unsafe { Page::borrow_phys(&ttb) }.is_none() { + pr_err!( + "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", + ttb + ); + return Err(EIO); + } + + Ok(UatPageTable { + ttb, + ttb_owned: false, + va_range, + oas_mask: (1u64 << oas) - 1, + }) + } + + pub(crate) fn ttb(&self) -> PhysicalAddr { + self.ttb + } + + fn with_pages( + &mut self, + iova_range: Range, + alloc: bool, + free: bool, + mut cb: F, + ) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + mod_pr_debug!( + "UATPageTable::with_pages: {:#x?} alloc={} free={}\n", + iova_range, + alloc, + free + ); + if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { + pr_err!( + "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", + iova_range + ); + return Err(EINVAL); + } + + if iova_range.is_empty() { + return Ok(()); + } + + let mut iova = iova_range.start & UAT_IASMSK; + let mut last_iova = iova; + // Handle the case where iova_range.end is just at the top boundary of the IAS + let end = ((iova_range.end - 1) & UAT_IASMSK) + 1; + + let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + pt_addr[UAT_LEVELS - 1] = Some(self.ttb); + + 'outer: while iova < end { + mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); + let addr_diff = last_iova ^ iova; + for level in (0..UAT_LEVELS - 1).rev() { + // If the iova has changed at this level or above, invalidate the physaddr + if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { + if let Some(phys) = pt_addr[level].take() { + if free { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); + } + } + } + last_iova = iova; + for level in (0..UAT_LEVELS - 1).rev() { + // Fetch the page table base address for this level + if pt_addr[level].is_none() { + let phys = pt_addr[level + 1].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", + level, + phys + ); + let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) + as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; + mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + upte.store(upte_val, Ordering::Relaxed); + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + })?; + mod_pr_debug!( + "UATPageTable::with_pages: level {} PT {:#x?}\n", + level, + pt_addr[level] + ); + } + // If we don't have a page table, skip this entire level + if pt_addr[level].is_none() { + let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); + let old = iova; + iova = align(iova + 1, block); + mod_pr_debug!( + "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", + block, + old, + iova + ); + continue 'outer; + } + } + + let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; + let max_count = UAT_NPTE - idx; + let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); + let phys = pt_addr[0].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", + phys, + idx, + count, + iova + ); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; + // SAFETY: We know this is a valid pointer to PTEs and the range is valid and + // checked by with_pointer_into_page(). + let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; + cb(iova, ptes)?; + Ok(()) + })?; + + let block = 1 << (UAT_PGBIT + UAT_LVBIT); + iova = align(iova + 1, block); + } + + if free { + for level in (0..UAT_LEVELS - 1).rev() { + if let Some(phys) = pt_addr[level] { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + } + } + + Ok(()) + } + + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, true, false, |_, _| Ok(())) + } + + fn pte_bits(&self) -> u64 { + if self.ttb_owned { + // Owned page tables are userspace, so non-global + PTE_TYPE_LEAF_TABLE | UAT_NON_GLOBAL + } else { + // The sole non-owned page table is kernelspace, so global + PTE_TYPE_LEAF_TABLE + } + } + + pub(crate) fn map_pages( + &mut self, + iova_range: Range, + mut phys: PhysicalAddr, + prot: Prot, + one_page: bool, + ) -> Result { + mod_pr_debug!( + "UATPageTable::map_pages: {:#x?} {:#x?} {:?}\n", + iova_range, + phys, + prot + ); + if phys & (UAT_PGMSK as PhysicalAddr) != 0 { + pr_err!("UATPageTable::map_pages: phys not aligned: {:#x?}\n", phys); + return Err(EINVAL); + } + + let pte_bits = self.pte_bits(); + + self.with_pages(iova_range, true, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev != 0 { + pr_err!( + "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + } + pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); + if !one_page { + phys += UAT_PGSZ as PhysicalAddr; + } + } + Ok(()) + }) + } + + pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { + mod_pr_debug!( + "UATPageTable::reprot_pages: {:#x?} {:?}\n", + iova_range, + prot + ); + self.with_pages(iova_range, true, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + let ptev = pte.load(Ordering::Relaxed); + if ptev & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + pr_err!( + "UATPageTable::reprot_pages: Page at IOVA {:#x} is unmapped (PTE: {:#x})\n", + iova + (idx * UAT_PGSZ) as u64, + ptev + ); + continue; + } + pte.store((ptev & !UAT_PROT_BITS) | prot.as_pte(), Ordering::Relaxed); + } + Ok(()) + }) + } + + pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { + mod_pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); + self.with_pages(iova_range, false, false, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { + pr_err!( + "UATPageTable::unmap_pages: Page at IOVA {:#x} already unmapped\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) + } + + #[cfg(CONFIG_DEV_COREDUMP)] + pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { + let mut pages = KVVec::new(); + let oas_mask = self.oas_mask; + let iova_base = self.va_range.start & !UAT_IASMSK; + self.with_pages(iova_range, false, false, |iova, ptes| { + let iova = iova | iova_base; + for (idx, ppte) in ptes.iter().enumerate() { + let pte = ppte.load(Ordering::Relaxed); + if (pte & PTE_TYPE_LEAF_TABLE) != PTE_TYPE_LEAF_TABLE { + continue; + } + let memattr = ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8; + + if !(memattr == MEMATTR_CACHED || memattr == MEMATTR_UNCACHED) { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + let phys = pte & oas_mask & (!UAT_PGMSK as u64); + // SAFETY: GPU pages are either firmware/preallocated pages + // (which the kernel isn't concerned with and are either in + // the page map or not, and if they aren't, borrow_phys() + // will fail), or GPU page table pages (which we own), + // or GEM buffer pages (which are locked while they are + // mapped in the page table), so they should be safe to + // borrow. + // + // This does trust the firmware not to have any weird + // mappings in its own internal page tables, but since + // those are managed by the uPPL which is privileged anyway, + // this trust does not actually extend any trust boundary. + let src_page = match unsafe { Page::borrow_phys(&phys) } { + Some(page) => page, + None => { + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: None, + }, + GFP_KERNEL, + )?; + continue; + } + }; + let dst_page = Page::alloc_page(GFP_KERNEL)?; + src_page.with_page_mapped(|psrc| -> Result { + // SAFETY: This could technically still have a data race with the firmware + // or other driver code (or even userspace with timestamp buffers), but while + // the Rust language technically says this is UB, in the real world, using + // atomic reads for this is guaranteed to never cause any harmful effects + // other than possibly reading torn/unreliable data. At least on ARM64 anyway. + // + // (Yes, I checked with Rust people about this. ~~ Lina) + // + let src_items = unsafe { + core::slice::from_raw_parts( + psrc as *const AtomicU64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + dst_page.with_page_mapped(|pdst| -> Result { + // SAFETY: We own the destination page, so it is safe to view its contents + // as a u64 slice. + let dst_items = unsafe { + core::slice::from_raw_parts_mut( + pdst as *mut u64, + UAT_PGSZ / core::mem::size_of::(), + ) + }; + for (si, di) in src_items.iter().zip(dst_items.iter_mut()) { + *di = si.load(Ordering::Relaxed); + } + Ok(()) + })?; + Ok(()) + })?; + pages.push( + DumpedPage { + iova: iova + (idx * UAT_PGSZ) as u64, + pte, + data: Some(dst_page), + }, + GFP_KERNEL, + )?; + } + Ok(()) + })?; + Ok(pages) + } +} + +impl Drop for UatPageTable { + fn drop(&mut self) { + mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); + if self + .with_pages(self.va_range.clone(), false, true, |iova, ptes| { + for (idx, pte) in ptes.iter().enumerate() { + if pte.load(Ordering::Relaxed) != 0 { + pr_err!( + "UATPageTable::drop: Leaked page at IOVA {:#x}\n", + iova + (idx * UAT_PGSZ) as u64 + ); + } + } + Ok(()) + }) + .is_err() + { + pr_err!("UATPageTable::drop failed to free page tables\n",); + } + if self.ttb_owned { + mod_pr_debug!("UATPageTable::drop: Free TTB {:#x}\n", self.ttb); + // SAFETY: If we own the ttb, it was allocated with Page::into_phys(). + unsafe { + Page::from_phys(self.ttb); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/queue/common.rs b/drivers/gpu/drm/asahi/queue/common.rs new file mode 100644 index 00000000000000..a68352828cfbc3 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/common.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common queue functionality. +//! +//! Shared helpers used by the submission logic for multiple command types. + +use crate::file; +use crate::fw::job::UserTimestamp; + +use kernel::prelude::*; +use kernel::uapi; +use kernel::xarray; + +pub(super) fn get_timestamp_object( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, +) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + + let guard = objects.lock(); + let object = guard + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + core::mem::drop(guard); + + #[allow(irrefutable_let_patterns)] + if let file::Object::TimestampBuffer(mapping) = object { + let offset = timestamp.offset; + if (offset.checked_add(8).ok_or(EINVAL)?) as usize > mapping.size() { + return Err(ERANGE); + } + Ok(Some(UserTimestamp { + mapping: mapping.clone(), + offset: offset as usize, + })) + } else { + Err(EINVAL) + } +} diff --git a/drivers/gpu/drm/asahi/queue/compute.rs b/drivers/gpu/drm/asahi/queue/compute.rs new file mode 100644 index 00000000000000..62afc561806703 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/compute.rs @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! Compute work queue. +//! +//! A compute queue consists of one underlying WorkQueue. +//! This module is in charge of creating all of the firmware structures required to submit compute +//! work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::{ + file, + fw, + gpu, + microseq, // +}; +use crate::{ + inner_ptr, + inner_weak_ptr, // +}; +use core::sync::atomic::Ordering; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::uapi; +use kernel::xarray; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; + +#[versions(AGX)] +impl super::QueueInner::ver { + /// Submit work to a compute queue. + pub(super) fn submit_compute( + &self, + job: &mut Job, + cmdbuf: &uapi::drm_asahi_cmd_compute, + attachments: µseq::Attachments, + objects: Pin<&xarray::XArray>>, + id: u64, + flush_stamps: bool, + ) -> Result { + let gpu = match (*self.dev) + .gpu + .as_any() + .downcast_ref::() + { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + mod_dev_dbg!(self.dev, "[Submission {}] Compute!\n", id); + + if cmdbuf.flags != 0 { + return Err(EINVAL); + } + + let mut user_timestamps: fw::job::UserTimestamps = Default::default(); + user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; + user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let notifier = self.notifier.clone(); + + let fence = job.fence.clone(); + let comp_job = job.get_comp()?; + let ev_comp = comp_job.event_info(); + + let preempt2_off = gpu.get_cfg().compute_preempt1_size; + let preempt3_off = preempt2_off + 8; + let preempt4_off = preempt3_off + 8; + let preempt5_off = preempt4_off + 8; + let preempt_size = preempt5_off + 8; + + let preempt_buf = self + .ualloc + .lock() + .array_empty_tagged(preempt_size, b"CPMT")?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Event #{} {:#x?} -> {:#x?}\n", + id, + ev_comp.slot, + ev_comp.value, + ev_comp.value.next(), + ); + + let timestamps = Arc::new( + kalloc.shared.new_default::()?, + GFP_KERNEL, + )?; + + let uuid = 0; + mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); + + // TODO: check + #[ver(V >= V13_0B4)] + let count = self.counter.fetch_add(1, Ordering::Relaxed); + + let comp = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + try_init!(fw::compute::RunCompute::ver { + preempt_buf: preempt_buf, + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); + + let start_comp = builder.add(microseq::StartCompute::ver { + header: microseq::op::StartCompute::HEADER, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x1, + event_generation: self.id as u32, + event_seq: U64(ev_comp.event_seq), + unk_38: 0x0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_44: 0x0, + uuid, + attachments: *attachments, + padding: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + counter: U64(count), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_comp.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, context_store_req), + uuid, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_comp); + builder.add(microseq::FinalizeCompute::ver { + header: microseq::op::FinalizeCompute::HEADER, + stats, + work_queue: ev_comp.info_ptr, + vm_slot: vm_bind.slot(), + #[ver(V < V13_0B4)] + unk_18: 0, + job_params2: inner_weak_ptr!(ptr, job_params2), + unk_24: 0, + uuid, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + unk_38: 0, + unk_3c: 0, + unk_40: 0, + unk_44: 0, + unk_48: 0, + unk_4c: 0, + unk_50: 0, + unk_54: 0, + unk_58: 0, + #[ver(G == G14 && V < V13_0B4)] + unk_5c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_64: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: inner_weak_ptr!(ptr, unk_flag), + #[ver(V >= V13_0B4)] + unk_79: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(&mut kalloc.private)? + }, + notifier, + vm_bind, + timestamps, + user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::compute::raw::RunCompute::ver { + tag: fw::workqueue::CommandType::RunCompute, + #[ver(V >= V13_0B4)] + counter: U64(count), + unk_4: 0, + vm_slot, + notifier: inner.notifier.gpu_pointer(), + unk_pointee: Default::default(), + #[ver(G < G14X)] + __pad0: Default::default(), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::compute::raw::JobParameters1 { + preempt_buf1: inner.preempt_buf.gpu_pointer(), + cdm_ctrl_stream_base: U64(cmdbuf.cdm_ctrl_stream_base), + // buf2-5 Only if internal program is used + preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), + preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), + preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), + preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), + usc_exec_base_cp: U64(self.usc_exec_base), + unk_38: U64(0x8c60), + helper_program: cmdbuf.helper.binary, // Internal program addr | 1 + unk_44: 0, + helper_arg: U64(cmdbuf.helper.data), // Only if internal program used + helper_cfg: cmdbuf.helper.cfg, // 0x40 if internal program used + unk_54: 0, + unk_58: 1, + unk_5c: 0, + iogpu_unk_40: 0, // 0x1c if internal program used + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1a510, inner.preempt_buf.gpu_pointer().into()); + r.add(0x1a420, cmdbuf.cdm_ctrl_stream_base); + // buf2-5 Only if internal program is used + r.add(0x1a4d0, inner.preempt_buf.gpu_offset_pointer(preempt2_off).into()); + r.add(0x1a4d8, inner.preempt_buf.gpu_offset_pointer(preempt3_off).into()); + r.add(0x1a4e0, inner.preempt_buf.gpu_offset_pointer(preempt4_off).into()); + r.add(0x1a4e8, inner.preempt_buf.gpu_offset_pointer(preempt5_off).into()); + r.add(0x10071, self.usc_exec_base); // USC_EXEC_BASE_CP + r.add(0x11841, cmdbuf.helper.binary.into()); + r.add(0x11849, cmdbuf.helper.data); + r.add(0x11f81, cmdbuf.helper.cfg.into()); + r.add(0x1a440, 0x24201); + r.add(0x12091, 0 /* iogpu_unk_40 */); + /* + r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? + r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? + */ + } + ), + __pad1: Default::default(), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + job_params2 <- try_init!(fw::compute::raw::JobParameters2::ver { + #[ver(V >= V13_0B4)] + unk_0_0: 0, + unk_0: Default::default(), + preempt_buf1: inner.preempt_buf.gpu_pointer(), + cdm_ctrl_stream_end: U64(cmdbuf.cdm_ctrl_stream_end), + unk_34: Default::default(), + #[ver(G < G14X)] + unk_g14x: 0, + #[ver(G >= G14X)] + unk_g14x: 0x24201, + unk_58: 0, + #[ver(V < V13_0B4)] + unk_5c: 0, + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + sync_grow: 0x0, // check! + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffff, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: 0, + stamp: ev_comp.stamp_pointer, + fw_stamp: ev_comp.fw_stamp_pointer, + stamp_value: ev_comp.value.next(), + stamp_slot: ev_comp.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid, + event_seq: ev_comp.event_seq as u32, + }), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_2d1: Default::default(), + unk_2d4: 0, + unk_2d8: 0, + #[ver(V >= V13_0B4)] + context_store_req: U64(0), + #[ver(V >= V13_0B4)] + context_store_compl: U64(0), + #[ver(V >= V13_0B4)] + unk_2e9: Default::default(), + #[ver(V >= V13_0B4)] + unk_flag: U32(0), + #[ver(V >= V13_0B4)] + unk_pad: Default::default(), + }) + }, + )?; + + core::mem::drop(alloc); + + fence.add_command(); + comp_job.add_cb(comp, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + + fence.command_complete(); + })?; + + comp_job.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs new file mode 100644 index 00000000000000..85da1d539d1bc5 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -0,0 +1,935 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submission queue management +//! +//! This module implements the userspace view of submission queues and the logic to map userspace +//! submissions to firmware queues. + +use kernel::dma_fence::*; +use kernel::prelude::*; +use kernel::{ + c_str, + dma_fence, + drm::sched, + macros::versions, + sync::{ + Arc, + LockClassKey, + Mutex, // + }, + uapi, + xarray, // +}; + +use crate::alloc::Allocator; +use crate::debug::*; +use crate::driver::{AsahiDevRef, AsahiDevice}; +use crate::file::MAX_COMMANDS_PER_SUBMISSION; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::inner_weak_ptr; +use crate::microseq; +use crate::module_parameters; +use crate::util::{ + AnyBitPattern, + Reader, // +}; +use crate::{ + alloc, + buffer, + channel, + event, + file, + fw, + gpu, + mmu, + workqueue, // +}; + +use core::sync::atomic::{ + AtomicU64, + Ordering, // +}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Queue; + +const WQ_SIZE: u32 = 0x500; + +mod common; +mod compute; +mod render; + +/// Trait implemented by all versioned queues. +pub(crate) trait Queue: Send + Sync { + fn submit( + &mut self, + id: u64, + syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result; +} + +#[versions(AGX)] +struct SubQueue { + wq: Arc, +} + +#[versions(AGX)] +impl SubQueue::ver { + fn new_job(&mut self, fence: dma_fence::Fence) -> SubQueueJob::ver { + SubQueueJob::ver { + wq: self.wq.clone(), + fence: Some(fence), + job: None, + } + } +} + +#[versions(AGX)] +struct SubQueueJob { + wq: Arc, + job: Option, + fence: Option, +} + +#[versions(AGX)] +impl SubQueueJob::ver { + fn get(&mut self) -> Result<&mut workqueue::Job::ver> { + if self.job.is_none() { + mod_pr_debug!("SubQueueJob: Creating {:?} job\n", self.wq.pipe_type()); + self.job + .replace(self.wq.new_job(self.fence.take().unwrap())?); + } + Ok(self.job.as_mut().expect("expected a Job")) + } + + fn commit(&mut self) -> Result { + match self.job.as_mut() { + Some(job) => job.commit(), + None => Ok(()), + } + } + + fn can_submit(&self) -> Option { + self.job.as_ref().and_then(|job| job.can_submit()) + } +} + +#[versions(AGX)] +pub(crate) struct Queue { + dev: AsahiDevRef, + _sched: sched::Scheduler, + entity: sched::Entity, + vm: mmu::Vm, + q_vtx: Option, + q_frag: Option, + q_comp: Option, + fence_ctx: FenceContexts, + inner: QueueInner::ver, +} + +#[versions(AGX)] +pub(crate) struct QueueInner { + dev: AsahiDevRef, + ualloc: Arc>, + buffer: buffer::Buffer::ver, + gpu_context: Arc, + notifier_list: Arc>, + notifier: Arc>, + usc_exec_base: u64, + id: u64, + #[ver(V >= V13_0B4)] + counter: AtomicU64, +} + +#[versions(AGX)] +#[derive(Default)] +pub(crate) struct JobFence { + id: u64, + pending: AtomicU64, +} + +#[versions(AGX)] +impl JobFence::ver { + fn add_command(self: &FenceObject) { + self.pending.fetch_add(1, Ordering::Relaxed); + } + + fn command_complete(self: &FenceObject) { + let remain = self.pending.fetch_sub(1, Ordering::Relaxed) - 1; + mod_pr_debug!( + "JobFence[{}]: Command complete (remain: {})\n", + self.id, + remain + ); + if remain == 0 { + mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); + self.signal(); + } + } +} + +#[versions(AGX)] +#[vtable] +impl dma_fence::FenceOps for JobFence::ver { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("queue") + } +} + +#[versions(AGX)] +pub(crate) struct QueueJob { + dev: AsahiDevRef, + vm_bind: mmu::VmBind, + op_guard: Option, + sj_vtx: Option, + sj_frag: Option, + sj_comp: Option, + fence: UserFence, + notifier: Arc>, + notification_count: u32, + did_run: bool, + id: u64, +} + +#[versions(AGX)] +impl QueueJob::ver { + fn get_vtx(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_vtx + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No vertex queue\n"); + EINVAL + })? + .get() + } + fn get_frag(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_frag + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No fragment queue\n"); + EINVAL + })? + .get() + } + fn get_comp(&mut self) -> Result<&mut workqueue::Job::ver> { + self.sj_comp + .as_mut() + .ok_or_else(|| { + cls_pr_debug!(Errors, "No compute queue\n"); + EINVAL + })? + .get() + } + + fn commit(&mut self) -> Result { + mod_dev_dbg!(self.dev, "QueueJob {}: Committing\n", self.id); + + self.sj_vtx.as_mut().map(|a| a.commit()).unwrap_or(Ok(()))?; + self.sj_frag + .as_mut() + .map(|a| a.commit()) + .unwrap_or(Ok(()))?; + self.sj_comp.as_mut().map(|a| a.commit()).unwrap_or(Ok(())) + } +} + +#[versions(AGX)] +impl sched::JobImpl for QueueJob::ver { + fn prepare(job: &mut sched::Job) -> Option { + mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); + + if let Some(sj) = job.sj_vtx.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to vertex queue full\n", + job.id + ); + return Some(fence); + } + } + if let Some(sj) = job.sj_frag.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to fragment queue full\n", + job.id + ); + return Some(fence); + } + } + if let Some(sj) = job.sj_comp.as_ref() { + if let Some(fence) = sj.can_submit() { + mod_dev_dbg!( + job.dev, + "QueueJob {}: Blocking due to compute queue full\n", + job.id + ); + return Some(fence); + } + } + None + } + + #[allow(unused_assignments)] + fn run(job: &mut sched::Job) -> Result> { + mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); + + // We can only increase the notifier threshold here, now that we are + // actually running the job. We cannot increase it while queueing the + // job without introducing subtle race conditions. Suppose we did, as + // early versions of drm/asahi did: + // + // 1. When processing the ioctl submit, a job is queued to drm_sched. + // Incorrectly, the notifier threshold is increased, gating firmware + // events. + // 2. When DRM schedules an event, the hardware is kicked. + // 3. When the number of processed jobs equals the threshold, the + // firmware signals the complete event to the kernel + // 4. When the kernel gets a complete event, we signal the out-syncs. + // + // Does that work? There are a few scenarios. + // + // 1. There is nothing else ioctl submitted before the job completes. + // The job is scheduled, completes, and signals immediately. + // Everything works. + // 2. There is nontrivial sync across different queues. Since each queue + // has a separate own notifier threshold, submitting one does not + // block scheduling of the other. Everything works the way you'd + // expect. drm/sched handles the wait/signal ordering. + // 3. Two ioctls are submitted back-to-back. The first signals a fence + // that the second waits on. Due to the notifier threshold increment, + // the first job's completion event is deferred. But in good + // conditions, drm/sched will schedule the second submit anyway + // because it kills the pointless intra-queue sync. Then both + // commands execute and are signalled together. + // 4. Two ioctls are submitted back-to-back as above, but conditions are + // bad. Reporting completion of the first job is still masked by the + // notifier threshold, but the intra-queue fences are not optimized + // out in drm/sched... drm/sched doesn't schedule the second job + // until the first is signalled, but the first isn't signalled until + // the second is completed, but the second can't complete until it's + // scheduled. We hang! + // + // In good conditions, everything works properly and/or we win the race + // to mask the issue. So the issue here is challenging to hit. + // Nevertheless, we do need to get it right. + // + // The intention with drm/sched is that jobs that are not yet scheduled + // are "invisible" to the firmware. Incrementing the notifier threshold + // earlier than this violates that which leads to circles like the + // above. Deferring the increment to submit solves the race. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + + let gpu = match (*job.dev) + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(job.dev.as_ref(), "GpuManager mismatched with QueueJob!\n"); + return Err(EIO); + } + }; + + if job.op_guard.is_none() { + job.op_guard = Some(gpu.start_op()?); + } + + // First submit all the commands for each queue. This can fail. + + let mut frag_job = None; + let mut frag_sub = None; + if let Some(sj) = job.sj_frag.as_mut() { + frag_job = sj.job.take(); + if let Some(wqjob) = frag_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit fragment\n", job.id); + frag_sub = Some(wqjob.submit()?); + } + } + + let mut vtx_job = None; + let mut vtx_sub = None; + if let Some(sj) = job.sj_vtx.as_mut() { + vtx_job = sj.job.take(); + if let Some(wqjob) = vtx_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit vertex\n", job.id); + vtx_sub = Some(wqjob.submit()?); + } + } + + let mut comp_job = None; + let mut comp_sub = None; + if let Some(sj) = job.sj_comp.as_mut() { + comp_job = sj.job.take(); + if let Some(wqjob) = comp_job.as_mut() { + mod_dev_dbg!(job.dev, "QueueJob {}: Submit compute\n", job.id); + comp_sub = Some(wqjob.submit()?); + } + } + + // Now we fully commit to running the job + mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); + frag_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Run vertex\n", job.id); + vtx_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Run compute\n", job.id); + comp_sub.map(|a| gpu.run_job(a)).transpose()?; + + mod_dev_dbg!(job.dev, "QueueJob {}: Drop compute job\n", job.id); + core::mem::drop(comp_job); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop vertex job\n", job.id); + core::mem::drop(vtx_job); + mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); + core::mem::drop(frag_job); + + job.did_run = true; + + Ok(Some(Fence::from_fence(&job.fence))) + } + + fn timed_out(job: &mut sched::Job) -> sched::Status { + // FIXME: Handle timeouts properly + dev_err!( + job.dev.as_ref(), + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", + job.id, job.did_run + ); + sched::Status::NoDevice + } + + fn cancel(job: &mut sched::Job) { + dev_info!( + job.dev.as_ref(), + "QueueJob {}: Job canceled on DRM scheduler teardown\n", + job.id + ); + } +} + +#[versions(AGX)] +impl Drop for QueueJob::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "QueueJob {}: Dropping\n", self.id); + } +} + +static QUEUE_NAME: &CStr = c_str!("asahi_fence"); +static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +#[versions(AGX)] +impl Queue::ver { + /// Create a new user queue. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &AsahiDevice, + vm: mmu::Vm, + alloc: &mut gpu::KernelAllocators, + ualloc: Arc>, + ualloc_priv: Arc>, + event_manager: Arc, + mgr: &buffer::BufferManager::ver, + id: u64, + priority: u32, + usc_exec_base: u64, + ) -> Result { + mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); + + // Must be shared, no cache management on this one! + let mut notifier_list = alloc.shared.new_default::()?; + + let self_ptr = notifier_list.weak_pointer(); + notifier_list.with_mut(|raw, _inner| { + raw.list_head.next = Some(inner_weak_ptr!(self_ptr, list_head)); + }); + + let threshold = alloc.shared.new_default::()?; + + let notifier: Arc> = Arc::new( + alloc.private.new_init( + /*try_*/ init!(fw::event::Notifier::ver { threshold }), + |inner, _p| { + try_init!(fw::event::raw::Notifier::ver { + threshold: inner.threshold.gpu_pointer(), + generation: AtomicU32::new(id as u32), + cur_count: AtomicU32::new(0), + unk_10: AtomicU32::new(0x50), + state: Default::default() + }) + }, + )?, + GFP_KERNEL, + )?; + + // Priorities are handled by the AGX scheduler, there is no meaning within a + // per-queue scheduler. Use a single run queue wth Kernel priority. + let sched = + sched::Scheduler::new(dev.as_ref(), 1, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; + let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; + + let buffer = + buffer::Buffer::ver::new(&*(*dev).gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; + + let mut ret = Queue::ver { + dev: dev.into(), + _sched: sched, + entity, + vm, + q_vtx: None, + q_frag: None, + q_comp: None, + fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, + inner: QueueInner::ver { + dev: dev.into(), + ualloc, + gpu_context: Arc::new( + workqueue::GpuContext::new(dev, alloc, buffer.any_ref())?, + GFP_KERNEL, + )?, + + buffer, + notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, + notifier, + usc_exec_base, + id, + #[ver(V >= V13_0B4)] + counter: AtomicU64::new(0), + }, + }; + + // Rendering structures + let tvb_blocks = *module_parameters::initial_tvb_size.value(); + + ret.inner.buffer.ensure_blocks(tvb_blocks)?; + + ret.q_vtx = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager.clone(), + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Vertex, + id, + priority, + WQ_SIZE, + )?, + }); + + ret.q_frag = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager.clone(), + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Fragment, + id, + priority, + WQ_SIZE, + )?, + }); + + // Compute structures + ret.q_comp = Some(SubQueue::ver { + wq: workqueue::WorkQueue::ver::new( + dev, + alloc, + event_manager, + ret.inner.gpu_context.clone(), + ret.inner.notifier_list.clone(), + channel::PipeType::Compute, + id, + priority, + WQ_SIZE, + )?, + }); + + mod_dev_dbg!(dev, "[Queue {}] Queue created\n", id); + Ok(ret) + } +} + +const SQ_RENDER: usize = 0; +const SQ_COMPUTE: usize = 1; +const SQ_COUNT: usize = 2; + +// SAFETY: All bit patterns are valid by construction. +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} +unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_compute {} +unsafe impl AnyBitPattern for uapi::drm_asahi_attachment {} + +fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { + const STRIDE: usize = core::mem::size_of::(); + let count = size / STRIDE; + + if count > microseq::MAX_ATTACHMENTS { + return Err(EINVAL); + } + + let mut attachments: microseq::Attachments = Default::default(); + attachments.count = count as u32; + + for i in 0..count { + let att: uapi::drm_asahi_attachment = reader.read()?; + + if att.flags != 0 || att.pad != 0 { + return Err(EINVAL); + } + + // Some kind of power-of-2 exponent related to attachment size, in + // bounds [1, 6]? We don't know what this is exactly yet. + let unk_e = 1; + + let cache_lines = (att.size + 127) >> 7; + attachments.list[i as usize] = microseq::Attachment { + address: U64(att.pointer), + size: cache_lines.try_into()?, + unk_c: 0x17, + unk_e: unk_e as u16, + }; + } + + Ok(attachments) +} + +#[versions(AGX)] +impl Queue for Queue::ver { + fn submit( + &mut self, + id: u64, + mut syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let gpu = match (*self.dev) + .gpu + .clone() + .arc_as_any() + .downcast::() + { + Ok(gpu) => gpu, + Err(_) => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with JobImpl!\n"); + return Err(EIO); + } + }; + + mod_dev_dbg!(self.dev, "[Submission {}] Submit job\n", id); + + if gpu.is_crashed() { + dev_err!( + self.dev.as_ref(), + "[Submission {}] GPU is crashed, cannot submit\n", + id + ); + return Err(ENODEV); + } + + let op_guard = if in_sync_count > 0 { + Some(gpu.start_op()?) + } else { + None + }; + + let mut events: [KVec>; SQ_COUNT] = + Default::default(); + + events[SQ_RENDER].push( + self.q_frag.as_ref().and_then(|a| a.wq.event_info()), + GFP_KERNEL, + )?; + events[SQ_COMPUTE].push( + self.q_comp.as_ref().and_then(|a| a.wq.event_info()), + GFP_KERNEL, + )?; + + let vm_bind = gpu.bind_vm(&self.vm)?; + let vm_slot = vm_bind.slot(); + + mod_dev_dbg!(self.dev, "[Submission {}] Creating job\n", id); + + // FIXME: I think this can violate the fence seqno ordering contract. + // If we have e.g. a render submission with no barriers and then a compute submission + // with no barriers, it's possible for the compute submission to complete first, and + // therefore its fence. Maybe we should have separate fence contexts for render + // and compute, and then do a ? (Vert+frag should be fine since there is no vert + // without frag, and frag always serializes.) + let fence: UserFence = self + .fence_ctx + .new_fence::( + 0, + JobFence::ver { + id, + pending: Default::default(), + }, + )? + .into(); + + let mut cmdbuf = Reader::new(cmdbuf_raw); + + // First, parse the headers to determine the number of compute/render + // commands. This will be used to determine when to flush stamps. + // + // We also use it to determine how many notifications the job will + // generate. We could calculate that in the second pass since we don't + // need until much later, but it's convenient to gather everything at + // the same time. + let mut nr_commands = 0; + let mut last_compute = 0; + let mut last_render = 0; + let mut nr_render = 0; + let mut nr_compute = 0; + + while !cmdbuf.is_empty() { + let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; + cmdbuf.skip(header.size as usize); + nr_commands += 1; + + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + last_compute = nr_commands; + nr_render += 1; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + last_render = nr_commands; + nr_compute += 1; + } + _ => {} + } + } + + let mut job = self.entity.new_job( + 1, + QueueJob::ver { + dev: self.dev.clone(), + vm_bind, + op_guard, + sj_vtx: self + .q_vtx + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + sj_frag: self + .q_frag + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + sj_comp: self + .q_comp + .as_mut() + .map(|a| a.new_job(Fence::from_fence(&fence))), + fence, + notifier: self.inner.notifier.clone(), + + // Each render command generates 2 notifications: 1 for the + // vertex part, 1 for the fragment part. Each compute command + // generates 1 notification. Sum up to calculate the total + // notification count for the job. + notification_count: (2 * nr_render) + nr_compute, + + did_run: false, + id, + }, + )?; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Adding {} in_syncs\n", + id, + in_sync_count + ); + for sync in syncs.drain(0..in_sync_count) { + if let Some(fence) = sync.fence { + job.add_dependency(fence)?; + } + } + + // Validate the number of hardware commands, ignoring software commands + let nr_hw_commands = nr_render + nr_compute; + if nr_hw_commands == 0 || nr_hw_commands > MAX_COMMANDS_PER_SUBMISSION { + cls_pr_debug!( + Errors, + "submit: Command count {} out of valid range [1, {}]\n", + nr_hw_commands, + MAX_COMMANDS_PER_SUBMISSION - 1 + ); + return Err(EINVAL); + } + + cmdbuf.rewind(); + + let mut command_index = 0; + let mut vertex_attachments: microseq::Attachments = Default::default(); + let mut fragment_attachments: microseq::Attachments = Default::default(); + let mut compute_attachments: microseq::Attachments = Default::default(); + + // Parse the full command buffer submitting as we go + while !cmdbuf.is_empty() { + let header: uapi::drm_asahi_cmd_header = cmdbuf.read()?; + let header_size = header.size as usize; + + // Pre-increment command index to match last_compute/last_render + command_index += 1; + + for (queue_idx, index) in [header.vdm_barrier, header.cdm_barrier].iter().enumerate() { + if *index == uapi::DRM_ASAHI_BARRIER_NONE as u16 { + continue; + } + if let Some(event) = events[queue_idx].get(*index as usize).ok_or_else(|| { + cls_pr_debug!(Errors, "Invalid barrier #{}: {}\n", queue_idx, index); + EINVAL + })? { + let mut alloc = gpu.alloc(); + let queue_job = match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => job.get_vtx()?, + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => job.get_comp()?, + _ => return Err(EINVAL), + }; + mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); + let barrier = alloc.private.new_init( + pin_init::zeroed::(), + |_inner, _p| { + let queue_job = &queue_job; + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: event.fw_stamp_pointer, + wait_value: event.value, + wait_slot: event.slot, + stamp_self: queue_job.event_info().value.next(), + uuid: 0xffffbbbb, + external_barrier: 0, + internal_barrier_type: 1, + padding: Default::default(), + }) + }, + )?; + mod_dev_dbg!(self.dev, "[Submission {}] Add Explicit Barrier\n", id); + queue_job.add(barrier, vm_slot)?; + } else { + assert!(*index == 0); + } + } + + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { + let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(header_size)?; + + self.inner.submit_render( + &mut job, + &render, + &vertex_attachments, + &fragment_attachments, + objects, + id, + command_index == last_render, + )?; + events[SQ_RENDER].push( + Some( + job.sj_frag + .as_ref() + .expect("No frag queue?") + .job + .as_ref() + .expect("No frag job?") + .event_info(), + ), + GFP_KERNEL, + )?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(header_size)?; + + self.inner.submit_compute( + &mut job, + &compute, + &compute_attachments, + objects, + id, + command_index == last_compute, + )?; + events[SQ_COMPUTE].push( + Some( + job.sj_comp + .as_ref() + .expect("No comp queue?") + .job + .as_ref() + .expect("No comp job?") + .event_info(), + ), + GFP_KERNEL, + )?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS => { + vertex_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS => { + fragment_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + compute_attachments = build_attachments(&mut cmdbuf, header_size)?; + } + _ => { + cls_pr_debug!(Errors, "Unknown command type {}\n", header.cmd_type); + return Err(EINVAL); + } + } + } + + mod_dev_dbg!( + self.dev, + "Queue {}: Committing job {}\n", + self.inner.id, + job.id + ); + job.commit()?; + + mod_dev_dbg!(self.dev, "Queue {}: Arming job {}\n", self.inner.id, job.id); + let mut job = job.arm(); + let out_fence = job.fences().finished(); + mod_dev_dbg!( + self.dev, + "Queue {}: Pushing job {}\n", + self.inner.id, + job.id + ); + job.push(); + + mod_dev_dbg!( + self.dev, + "Queue {}: Adding {} out_syncs\n", + self.inner.id, + syncs.len() + ); + for mut sync in syncs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj + .add_point(chain, &out_fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(&out_fence)); + } + } + + Ok(()) + } +} + +#[versions(AGX)] +impl Drop for Queue::ver { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.inner.id); + } +} diff --git a/drivers/gpu/drm/asahi/queue/render.rs b/drivers/gpu/drm/asahi/queue/render.rs new file mode 100644 index 00000000000000..32273b2975b505 --- /dev/null +++ b/drivers/gpu/drm/asahi/queue/render.rs @@ -0,0 +1,1400 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![allow(clippy::unusual_byte_groupings)] + +//! Render work queue. +//! +//! A render queue consists of two underlying WorkQueues, one for vertex and one for fragment work. +//! This module is in charge of creating all of the firmware structures required to submit 3D +//! rendering work to the GPU, based on the userspace command buffer. + +use super::common; +use crate::alloc::Allocator; +use crate::debug::*; +use crate::fw::types::*; +use crate::gpu::GpuManager; +use crate::util::*; +use crate::{ + buffer, + file, + fw, + gpu, + microseq, // +}; +use crate::{ + inner_ptr, + inner_weak_ptr, // +}; +use core::sync::atomic::Ordering; +use kernel::dma_fence::RawDmaFence; +use kernel::drm::sched::Job; +use kernel::prelude::*; +use kernel::sync::Arc; +use kernel::uapi; +use kernel::xarray; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Render; + +/// Tiling/Vertex control bit to disable using more than one GPU cluster. This results in decreased +/// throughput but also less latency, which is probably desirable for light vertex loads where the +/// overhead of clustering/merging would exceed the time it takes to just run the job on one +/// cluster. +const TILECTL_DISABLE_CLUSTERING: u32 = 1u32 << 0; + +#[versions(AGX)] +impl super::QueueInner::ver { + /// Get the appropriate tiling parameters for a given userspace command buffer. + fn get_tiling_params( + cmdbuf: &uapi::drm_asahi_cmd_render, + num_clusters: u32, + ) -> Result { + let width: u32 = cmdbuf.width_px as u32; + let height: u32 = cmdbuf.height_px as u32; + let layers: u32 = cmdbuf.layers as u32; + + if layers == 0 || layers > 2048 { + cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); + return Err(EINVAL); + } + + // This is overflow safe: all these calculations are done in u32. + // At 64Kx64K max dimensions above, this is 2**32 pixels max. + // In terms of tiles that are always larger than one pixel, + // this can never overflow. Note that real actual dimensions + // are limited to 16K * 16K below anyway. + // + // Once we multiply by the layer count, then we need to check + // for overflow or use u64. + + let tile_width = 32u32; + let tile_height = 32u32; + + let utile_width = cmdbuf.utile_width_px as u32; + let utile_height = cmdbuf.utile_height_px as u32; + + match (utile_width, utile_height) { + (32, 32) | (32, 16) | (16, 16) => (), + _ => { + cls_pr_debug!( + Errors, + "uTile size invalid ({} x {})\n", + utile_width, + utile_height + ); + return Err(EINVAL); + } + }; + + let utiles_per_tile_x = tile_width / utile_width; + let utiles_per_tile_y = tile_height / utile_height; + + let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; + + let tiles_x = width.div_ceil(tile_width); + let tiles_y = height.div_ceil(tile_height); + let tiles = tiles_x * tiles_y; + + let mtiles_x = 4u32; + let mtiles_y = 4u32; + let mtiles = mtiles_x * mtiles_y; + + let tiles_per_mtile_x = align(tiles_x.div_ceil(mtiles_x), 4); + let tiles_per_mtile_y = align(tiles_y.div_ceil(mtiles_y), 4); + let tiles_per_mtile = tiles_per_mtile_x * tiles_per_mtile_y; + + let mtile_x1 = tiles_per_mtile_x; + let mtile_x2 = 2 * tiles_per_mtile_x; + let mtile_x3 = 3 * tiles_per_mtile_x; + + let mtile_y1 = tiles_per_mtile_y; + let mtile_y2 = 2 * tiles_per_mtile_y; + let mtile_y3 = 3 * tiles_per_mtile_y; + + let rgn_entry_size = 5; + // Macrotile stride in 32-bit words + let rgn_size = align(rgn_entry_size * tiles_per_mtile * utiles_per_tile, 4) / 4; + let tilemap_size = (4 * rgn_size * mtiles) as usize * layers as usize; + + let tpc_entry_size = 8; + // TPC stride in 32-bit words + let tpc_mtile_stride = tpc_entry_size * utiles_per_tile * tiles_per_mtile / 4; + let tpc_size = + (4 * tpc_mtile_stride * mtiles) as usize * layers as usize * num_clusters as usize; + + // No idea where this comes from, but it fits what macOS does... + // GUESS: Number of 32K heap blocks to fit a 5-byte region header/pointer per tile? + // That would make a ton of sense... + let meta1_layer_stride = if num_clusters > 1 { + (align(tiles_x, 2) * align(tiles_y, 4) * utiles_per_tile).div_ceil(0x1980) + } else { + 0 + }; + + let mut min_tvb_blocks = align((tiles_x * tiles_y).div_ceil(128), 8); + + if num_clusters > 1 { + min_tvb_blocks = min_tvb_blocks.max(7 + 2 * layers); + } + + Ok(buffer::TileInfo { + tiles_x, + tiles_y, + tiles, + utile_width, + utile_height, + //mtiles_x, + //mtiles_y, + tiles_per_mtile_x, + tiles_per_mtile_y, + //tiles_per_mtile, + utiles_per_mtile_x: tiles_per_mtile_x * utiles_per_tile_x, + utiles_per_mtile_y: tiles_per_mtile_y * utiles_per_tile_y, + //utiles_per_mtile: tiles_per_mtile * utiles_per_tile, + tilemap_size, + tpc_size, + meta1_layer_stride, + #[ver(G < G14X)] + meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), + #[ver(G >= G14X)] + meta1_blocks: meta1_layer_stride, + layermeta_size: if layers > 1 { 0x100 } else { 0 }, + min_tvb_blocks: min_tvb_blocks as usize, + params: fw::vertex::raw::TilingParameters { + rgn_size, + unk_4: 0x88, + ppp_ctrl: cmdbuf.ppp_ctrl, + x_max: (width - 1) as u16, + y_max: (height - 1) as u16, + te_screen: ((tiles_y - 1) << 12) | (tiles_x - 1), + te_mtile1: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), + te_mtile2: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), + tiles_per_mtile, + tpc_stride: tpc_mtile_stride, + unk_24: 0x100, + unk_28: if layers > 1 { + 0xe000 | (layers - 1) + } else { + 0x8000 + }, + helper_cfg: cmdbuf.vertex_helper.cfg, + __pad: Default::default(), + }, + }) + } + + /// Submit work to a render queue. + pub(super) fn submit_render( + &self, + job: &mut Job, + cmdbuf: &uapi::drm_asahi_cmd_render, + vertex_attachments: µseq::Attachments, + fragment_attachments: µseq::Attachments, + objects: Pin<&xarray::XArray>>, + id: u64, + flush_stamps: bool, + ) -> Result { + mod_dev_dbg!(self.dev, "[Submission {}] Render!\n", id); + + if cmdbuf.flags + & !(uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING + | uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT) as u32 + != 0 + { + cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); + return Err(EINVAL); + } + + if cmdbuf.width_px == 0 + || cmdbuf.height_px == 0 + || cmdbuf.width_px > 16384 + || cmdbuf.height_px > 16384 + { + cls_pr_debug!( + Errors, + "Invalid dimensions ({}x{})\n", + cmdbuf.width_px, + cmdbuf.height_px + ); + return Err(EINVAL); + } + + let mut vtx_user_timestamps: fw::job::UserTimestamps = Default::default(); + let mut frg_user_timestamps: fw::job::UserTimestamps = Default::default(); + + vtx_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_vtx.start)?; + vtx_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_vtx.end)?; + frg_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_frag.start)?; + frg_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_frag.end)?; + + let gpu = match (*self.dev) + .gpu + .as_any() + .downcast_ref::() + { + Some(gpu) => gpu, + None => { + dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); + return Err(EIO); + } + }; + + let nclusters = gpu.get_dyncfg().id.num_clusters; + + // Can be set to false to disable clustering (for simpler jobs), but then the + // core masks below should be adjusted to cover a single rolling cluster. + let mut clustering = nclusters > 1; + + if debug_enabled(debug::DebugFlags::DisableClustering) + || cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING as u32 + != 0 + { + clustering = false; + } + + #[ver(G != G14)] + let tiling_control = { + let render_cfg = gpu.get_cfg().render; + let mut tiling_control = render_cfg.tiling_control; + + if !clustering { + tiling_control |= TILECTL_DISABLE_CLUSTERING; + } + tiling_control + }; + + let mut alloc = gpu.alloc(); + let kalloc = &mut *alloc; + + // This sequence number increases per new client/VM? assigned to some slot, + // but it's unclear *which* slot... + let slot_client_seq: u8 = (self.id & 0xff) as u8; + + let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; + + let buffer = &self.buffer; + let notifier = self.notifier.clone(); + + let tvb_autogrown = buffer.auto_grow()?; + if tvb_autogrown { + let new_size = buffer.block_count() as usize; + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to overflows\n", + id, + new_size * buffer::BLOCK_SIZE, + new_size, + ); + } + + let tvb_grown = buffer.ensure_blocks(tile_info.min_tvb_blocks)?; + if tvb_grown { + cls_dev_dbg!( + TVBStats, + &self.dev, + "[Submission {}] TVB grew to {} bytes ({} blocks) due to dimensions ({}x{})\n", + id, + tile_info.min_tvb_blocks * buffer::BLOCK_SIZE, + tile_info.min_tvb_blocks, + cmdbuf.width_px, + cmdbuf.height_px + ); + } + + let scene = Arc::new(buffer.new_scene(kalloc, &tile_info)?, GFP_KERNEL)?; + + let vm_bind = job.vm_bind.clone(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] VM slot = {}\n", + id, + vm_bind.slot() + ); + + let ev_vtx = job.get_vtx()?.event_info(); + let ev_frag = job.get_frag()?.event_info(); + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert event #{} -> {:#x?}\n", + id, + ev_vtx.slot, + ev_vtx.value.next(), + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag event #{} -> {:#x?}\n", + id, + ev_frag.slot, + ev_frag.value.next(), + ); + + let uuid_3d = 0; + let uuid_ta = 0; + + mod_dev_dbg!( + self.dev, + "[Submission {}] Vert UUID = {:#x?}\n", + id, + uuid_ta + ); + mod_dev_dbg!( + self.dev, + "[Submission {}] Frag UUID = {:#x?}\n", + id, + uuid_3d + ); + + let fence = job.fence.clone(); + let frag_job = job.get_frag()?; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Barrier\n", id); + let barrier = kalloc.private.new_init( + pin_init::zeroed::(), + |_inner, _p| { + try_init!(fw::workqueue::raw::Barrier { + tag: fw::workqueue::CommandType::Barrier, + wait_stamp: ev_vtx.fw_stamp_pointer, + wait_value: ev_vtx.value.next(), + wait_slot: ev_vtx.slot, + stamp_self: ev_frag.value.next(), + uuid: uuid_3d, + external_barrier: 0, + internal_barrier_type: 0, + padding: Default::default(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Barrier\n", id); + frag_job.add(barrier, vm_bind.slot())?; + + let timestamps = Arc::new( + kalloc.shared.new_default::()?, + GFP_KERNEL, + )?; + + let unk1 = false; + + let mut tile_config: u64 = 0; + if !unk1 { + tile_config |= 0x280; + } + if cmdbuf.layers > 1 { + tile_config |= 1; + } + if cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 + != 0 + { + tile_config |= 0x10000; + } + + let samples_log2 = match cmdbuf.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => { + cls_pr_debug!(Errors, "Invalid sample count {}\n", cmdbuf.samples); + return Err(EINVAL); + } + }; + + let utile_config = ((tile_info.utile_width / 16) << 12) + | ((tile_info.utile_height / 16) << 14) + | samples_log2; + + // Calculate the number of 2KiB blocks to allocate per utile. This is + // just a bit of dimensional analysis. + let pixels_per_utile: u32 = + (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); + let samples_per_utile: u32 = pixels_per_utile << samples_log2; + let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; + let block_size_bytes: u32 = 2048; + let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); + + #[ver(G >= G14X)] + let frg_tilecfg = 0x0000000_00036011 + | (((tile_info.tiles_x - 1) as u64) << 44) + | (((tile_info.tiles_y - 1) as u64) << 53) + | (if unk1 { 0 } else { 0x20_00000000 }) + | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) + | ((utile_config as u64 & 0xf000) << 28); + + // TODO: check + #[ver(V >= V13_0B4)] + let count_frag = self.counter.fetch_add(2, Ordering::Relaxed); + #[ver(V >= V13_0B4)] + let count_vtx = count_frag + 1; + + // Unknowns handling + + #[ver(G >= G14)] + let g14_unk = 0x4040404; + #[ver(G < G14)] + let g14_unk = 0; + #[ver(G < G14X)] + let frg_unk_140 = 0x8c60; + let frg_unk_158 = 0x1c; + #[ver(G >= G14)] + let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; + #[ver(G < G14)] + let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; + let reload_zlsctrl = cmdbuf.zls_ctrl; + let iogpu_unk54 = 0x3a0012006b0003; + let iogpu_unk56 = 1; + #[ver(G < G14)] + let tiling_control_2 = 0; + #[ver(G >= G14X)] + let tiling_control_2 = 4; + #[ver(G >= G14X)] + let vtx_unk_f0 = 0x1c; + #[ver(G < G14)] + let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); + let vtx_unk_118 = 0x1c; + + // DRM_ASAHI_RENDER_DBIAS_IS_INT chosen to match hardware bit. + let isp_ctl = 0xc000u32 + | (cmdbuf.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT as u32); + + // Always allow preemption at the UAPI level + let no_preemption = false; + + mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); + let frag = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let scene = scene.clone(); + let notifier = notifier.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::fragment::RunFragment::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), + stats + ); + + let start_frag = builder.add(microseq::StartFragment::ver { + header: microseq::op::StartFragment::HEADER, + #[ver(G < G14X)] + job_params2: Some(inner_weak_ptr!(ptr, job_params2)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + job_params2: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + scene: scene.gpu_pointer(), + stats, + busy_flag: inner_weak_ptr!(ptr, busy_flag), + tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_50: 0x1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + sync_grow: 0, + event_seq: U64(ev_frag.event_seq), + unk_68: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + #[ver(V >= V13_3)] + unk_7c_0: U64(0), + unk_7c: 0, + unk_80: 0, + unk_84: unk1.into(), + uuid: uuid_3d, + attachments: *fragment_attachments, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + })?; + + if frg_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if frg_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_frag.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_3d, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_frag); + builder.add(microseq::FinalizeFragment::ver { + header: microseq::op::FinalizeFragment::HEADER, + uuid: uuid_3d, + unk_8: 0, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + unk_18: 0, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + unk_2c: U64(1), + stats, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + busy_flag: inner_weak_ptr!(ptr, busy_flag), + work_queue: ev_frag.info_ptr, + work_item: ptr, + vm_slot: vm_bind.slot(), + unk_60: 0, + unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), + #[ver(V >= V13_3)] + unk_6c_0: U64(0), + unk_6c: U64(0), + unk_74: U64(0), + unk_7c: U64(0), + unk_84: U64(0), + unk_8c: U64(0), + #[ver(G == G14 && V < V13_0B4)] + unk_8c_g14: U64(0), + restart_branch_offset: off, + has_attachments: (fragment_attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_9c: Default::default(), + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + + builder.build(private)? + }, + notifier, + scene, + vm_bind, + aux_fb: self.ualloc.lock().array_empty_tagged(0x8000, b"AXFB")?, + timestamps, + user_timestamps: frg_user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { + isp_ctl: isp_ctl, + unk2: 0, + width: cmdbuf.width_px as u32, + height: cmdbuf.height_px as u32, + #[ver(V >= V13_0B4)] + unk3: U64(0x100000), + }; + + try_init!(fw::fragment::raw::RunFragment::ver { + tag: fw::workqueue::CommandType::RunFragment, + #[ver(V >= V13_0B4)] + counter: U64(count_frag), + vm_slot, + unk_8: 0, + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + notifier: inner.notifier.gpu_pointer(), + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + samples: cmdbuf.samples as u32, + tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, + tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, + unk_50: U64(0), + unk_58: U64(0), + isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), + isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), + unk_68: U64(0), + tile_count: U64(tile_info.tiles as u64), + #[ver(G < G14X)] + job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { + utile_config, + unk_4: 0, + bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), + address: U64(cmdbuf.bg.usc as u64), + }, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), + isp_scissor_base: U64(cmdbuf.isp_scissor_base), + isp_dbias_base: U64(cmdbuf.isp_dbias_base), + isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), + aux_fb_info, + isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + zls_ctrl: U64(cmdbuf.zls_ctrl), + #[ver(G >= G14)] + unk_58_g14_0: U64(g14_unk), + #[ver(G >= G14)] + unk_58_g14_8: U64(0), + z_load: U64(cmdbuf.depth.base), + z_store: U64(cmdbuf.depth.base), + s_load: U64(cmdbuf.stencil.base), + s_store: U64(cmdbuf.stencil.base), + #[ver(G >= G14)] + unk_68_g14_0: Default::default(), + z_load_stride: U64(cmdbuf.depth.stride as u64), + z_store_stride: U64(cmdbuf.depth.stride as u64), + s_load_stride: U64(cmdbuf.stencil.stride as u64), + s_store_stride: U64(cmdbuf.stencil.stride as u64), + z_load_comp: U64(cmdbuf.depth.comp_base), + z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store_comp: U64(cmdbuf.depth.comp_base), + z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + s_load_comp: U64(cmdbuf.stencil.comp_base), + s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store_comp: U64(cmdbuf.stencil.comp_base), + s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + tvb_layermeta: inner.scene.tvb_layermeta_pointer(), + mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), + tile_config: U64(tile_config), + aux_fb: inner.aux_fb.gpu_pointer(), + unk_108: Default::default(), + usc_exec_base_isp: U64(self.usc_exec_base), + unk_140: U64(frg_unk_140), + helper_program: cmdbuf.fragment_helper.binary, + unk_14c: 0, + helper_arg: U64(cmdbuf.fragment_helper.data), + unk_158: U64(frg_unk_158), + unk_160: U64(0), + __pad: Default::default(), + #[ver(V < V13_0B4)] + __pad1: Default::default(), + }), + #[ver(G < G14X)] + job_params2 <- try_init!(fw::fragment::raw::JobParameters2 { + eot_rsrc_spec: cmdbuf.eot.rsrc_spec, + eot_usc: cmdbuf.eot.usc, + unk_8: 0x0, + unk_c: 0x0, + isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), + isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), + unk_18: U64(0x0), + utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, + utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, + unk_24: 0x0, + tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), + tib_blocks: blocks_per_utile, + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + // TODO: does this flag need to be exposed to userspace? + isp_bgobjvals: load_bgobjvals as u32, + unk_38: 0x0, + unk_3c: 0x1, + helper_cfg: cmdbuf.fragment_helper.cfg, + __pad: Default::default(), + }), + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x1739, 1); + r.add(0x10009, utile_config.into()); + r.add(0x15379, cmdbuf.eot.rsrc_spec.into()); + r.add(0x15381, cmdbuf.eot.usc.into()); + r.add(0x15369, cmdbuf.bg.rsrc_spec.into()); + r.add(0x15371, cmdbuf.bg.usc.into()); + r.add(0x15131, cmdbuf.isp_merge_upper_x.into()); + r.add(0x15139, cmdbuf.isp_merge_upper_y.into()); + r.add(0x100a1, 0); + r.add(0x15069, 0); + r.add(0x15071, 0); // pointer + r.add(0x16058, 0); + r.add(0x10019, cmdbuf.ppp_multisamplectl); + let isp_mtile_size = (tile_info.utiles_per_mtile_y + | (tile_info.utiles_per_mtile_x << 16)) + .into(); + r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE + r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE + r.add( + 0x100d9, + (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)).into(), + ); // TE_SCREEN + r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); + r.add(0x15109, cmdbuf.isp_scissor_base); // ISP_SCISSOR_BASE + r.add(0x15101, cmdbuf.isp_dbias_base); // ISP_DBIAS_BASE + r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 + r.add( + 0x15211, + ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, + ); // aux_fb_info.{width, heigh + r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 + r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c + r.add(0x15321, cmdbuf.isp_zls_pixels.into()); // ISP_ZLS_PIXELS + r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH + r.add(0x15309, load_bgobjvals); // ISP_BGOBJVALS + r.add(0x15311, cmdbuf.isp_oclqry_base); // ISP_OCLQRY_BASE + r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL + r.add(0x15349, g14_unk); // s2.unk_58_g14_0 + r.add(0x15351, 0); // s2.unk_58_g14_8 + r.add(0x15329, cmdbuf.depth.base); // ISP_ZLOAD_BASE + r.add(0x15331, cmdbuf.depth.base); // ISP_ZSTORE_BASE + r.add(0x15339, cmdbuf.stencil.base); // ISP_STENCIL_LOAD_BASE + r.add(0x15341, cmdbuf.stencil.base); // ISP_STENCIL_STORE_BASE + r.add(0x15231, 0); + r.add(0x15221, 0); + r.add(0x15239, 0); + r.add(0x15229, 0); + r.add(0x15401, cmdbuf.depth.stride as u64); // load + r.add(0x15421, cmdbuf.depth.stride as u64); // store + r.add(0x15409, cmdbuf.stencil.stride as u64); // load + r.add(0x15429, cmdbuf.stencil.stride as u64); + r.add(0x153c1, cmdbuf.depth.comp_base); // load + r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load + r.add(0x153c9, cmdbuf.depth.comp_base); // store + r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store + r.add(0x153d1, cmdbuf.stencil.comp_base); // load + r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load + r.add(0x153d9, cmdbuf.stencil.comp_base); // store + r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store + r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); + r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? + r.add(0x10039, tile_config); // tile_config ISP_CTL? + r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN + r.add(0x11821, cmdbuf.fragment_helper.binary.into()); + r.add(0x11829, cmdbuf.fragment_helper.data); + r.add(0x11f79, cmdbuf.fragment_helper.cfg.into()); + r.add(0x15359, 0); + r.add(0x10069, self.usc_exec_base); // frag; USC_EXEC_BASE_ISP + r.add(0x16020, 0); + r.add(0x16461, inner.aux_fb.gpu_pointer().into()); + r.add(0x16090, inner.aux_fb.gpu_pointer().into()); + r.add(0x120a1, frg_unk_158); + r.add(0x160a8, 0); + r.add(0x16068, frg_tilecfg); + r.add(0x160b8, 0x0); + /* + r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? + r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c838, 1); // ? + r.add(0x1ca28, 0x1502960f00); // ?? + r.add(0x1731, 0x1); // ?? + */ + } + ), + job_params3 <- try_init!(fw::fragment::raw::JobParameters3::ver { + isp_dbias_base: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.isp_dbias_base), + unk_padding: U64(0), + }, + isp_scissor_base: fw::fragment::raw::ArrayAddr { + ptr: U64(cmdbuf.isp_scissor_base), + unk_padding: U64(0), + }, + isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), + unk_118: U64(0x0), + unk_120: Default::default(), + unk_partial_bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), + address: U64(cmdbuf.partial_bg.usc as u64), + }, + unk_258: U64(0), + unk_260: U64(0), + unk_268: U64(0), + unk_270: U64(0), + partial_bg: fw::fragment::raw::BackgroundProgram { + rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), + address: U64(cmdbuf.partial_bg.usc as u64), + }, + zls_ctrl: U64(reload_zlsctrl), + unk_290: U64(g14_unk), + z_load: U64(cmdbuf.depth.base), + z_partial_stride: U64(cmdbuf.depth.stride as u64), + z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), + z_store: U64(cmdbuf.depth.base), + z_partial: U64(cmdbuf.depth.base), + z_partial_comp: U64(cmdbuf.depth.comp_base), + s_load: U64(cmdbuf.stencil.base), + s_partial_stride: U64(cmdbuf.stencil.stride as u64), + s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), + s_store: U64(cmdbuf.stencil.base), + s_partial: U64(cmdbuf.stencil.base), + s_partial_comp: U64(cmdbuf.stencil.comp_base), + unk_2f8: Default::default(), + tib_blocks: blocks_per_utile, + unk_30c: 0x0, + aux_fb_info, + tile_config: U64(tile_config), + unk_328_padding: Default::default(), + unk_partial_eot: fw::fragment::raw::EotProgram::new( + cmdbuf.partial_eot.rsrc_spec, + cmdbuf.partial_eot.usc + ), + partial_eot: fw::fragment::raw::EotProgram::new( + cmdbuf.partial_eot.rsrc_spec, + cmdbuf.partial_eot.usc + ), + isp_bgobjdepth: cmdbuf.isp_bgobjdepth, + isp_bgobjvals: cmdbuf.isp_bgobjvals, + sample_size: cmdbuf.sample_size_B as u32, + unk_37c: 0x0, + unk_380: U64(0x0), + unk_388: U64(0x0), + #[ver(V >= V13_0B4)] + unk_390_0: U64(0x0), + isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), + }), + unk_758_flag: 0, + unk_75c_flag: 0, + unk_buf: Default::default(), + busy_flag: 0, + tvb_overflow_count: 0, + unk_878: 0, + encoder_params <- try_init!(fw::job::raw::EncoderParams { + // Maybe set when reloading z/s? + unk_8: 0, + sync_grow: 0, + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffffu32, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + process_empty_tiles: (cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES as u32 + != 0) as u32, + // TODO: needs to be investigated + no_clear_pipeline_textures: 1, + // TODO: needs to be investigated + msaa_zs: 0, + unk_pointee: 0, + #[ver(V >= V13_3)] + unk_v13_3: 0, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: no_preemption as u8, + stamp: ev_frag.stamp_pointer, + fw_stamp: ev_frag.fw_stamp_pointer, + stamp_value: ev_frag.value.next(), + stamp_slot: ev_frag.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_3d, + event_seq: ev_frag.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + #[ver(G < G14X)] + unk_buf_10: U64(1), + #[ver(G >= G14X)] + unk_buf_10: U64(0), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_925: Default::default(), + unk_928: 0, + unk_92c: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_92d_8: Default::default(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); + fence.add_command(); + + frag_job.add_cb(frag, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()); + } + + fence.command_complete(); + })?; + + let fence = job.fence.clone(); + let vtx_job = job.get_vtx()?; + + if scene.rebind() || tvb_grown || tvb_autogrown { + mod_dev_dbg!(self.dev, "[Submission {}] Create Bind Buffer\n", id); + let bind_buffer = kalloc.private.new_init( + { + let scene = scene.clone(); + try_init!(fw::buffer::InitBuffer::ver { scene }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + try_init!(fw::buffer::raw::InitBuffer::ver { + tag: fw::workqueue::CommandType::InitBuffer, + vm_slot, + buffer_slot: inner.scene.slot(), + unk_c: 0, + block_count: buffer.block_count(), + buffer: inner.scene.buffer_pointer(), + stamp_value: ev_vtx.value.next(), + }) + }, + )?; + + mod_dev_dbg!(self.dev, "[Submission {}] Add Bind Buffer\n", id); + vtx_job.add(bind_buffer, vm_bind.slot())?; + } + + mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); + let vtx = GpuObject::new_init_prealloc( + kalloc.gpu_ro.alloc_object()?, + |ptr: GpuWeakPointer| { + let scene = scene.clone(); + let vm_bind = vm_bind.clone(); + let timestamps = timestamps.clone(); + let private = &mut kalloc.private; + try_init!(fw::vertex::RunVertex::ver { + micro_seq: { + let mut builder = microseq::Builder::new(); + + let stats = inner_weak_ptr!( + gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), + stats + ); + + let start_vtx = builder.add(microseq::StartVertex::ver { + header: microseq::op::StartVertex::HEADER, + #[ver(G < G14X)] + tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), + #[ver(G < G14X)] + job_params1: Some(inner_weak_ptr!(ptr, job_params1)), + #[ver(G >= G14X)] + tiling_params: None, + #[ver(G >= G14X)] + job_params1: None, + #[ver(G >= G14X)] + registers: inner_weak_ptr!(ptr, registers), + buffer: scene.weak_buffer_pointer(), + scene: scene.weak_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_38: 1, // fixed + event_generation: self.id as u32, + buffer_slot: scene.slot(), + unk_44: 0, + event_seq: U64(ev_vtx.event_seq), + unk_50: 0, + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), + unk_64: 0x0, // fixed + unk_68: unk1.into(), + uuid: uuid_ta, + attachments: *vertex_attachments, + padding: 0, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + #[ver(V >= V13_0B4)] + notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), + #[ver(V < V13_0B4)] + unk_178: 0x0, // padding? + #[ver(V >= V13_0B4)] + unk_178: (!clustering) as u32, + })?; + + if vtx_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(true), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + #[ver(G < G14X)] + builder.add(microseq::WaitForIdle { + header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), + })?; + #[ver(G >= G14X)] + builder.add(microseq::WaitForIdle2 { + header: microseq::op::WaitForIdle2::HEADER, + })?; + + if vtx_user_timestamps.any() { + builder.add(microseq::Timestamp::ver { + header: microseq::op::Timestamp::new(false), + command_time: inner_weak_ptr!(ptr, command_time), + ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), + update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), + work_queue: ev_vtx.info_ptr, + user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), + #[ver(V >= V13_0B4)] + unk_ts: inner_weak_ptr!(ptr, unk_ts), + uuid: uuid_ta, + unk_30_padding: 0, + })?; + } + + let off = builder.offset_to(start_vtx); + builder.add(microseq::FinalizeVertex::ver { + header: microseq::op::FinalizeVertex::HEADER, + scene: scene.weak_pointer(), + buffer: scene.weak_buffer_pointer(), + stats, + work_queue: ev_vtx.info_ptr, + vm_slot: vm_bind.slot(), + unk_28: 0x0, // fixed + unk_pointer: inner_weak_ptr!(ptr, unk_pointee), + unk_34: 0x0, // fixed + uuid: uuid_ta, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + unk_48: U64(0x0), // fixed + unk_50: 0x0, // fixed + unk_54: 0x0, // fixed + unk_58: U64(0x0), // fixed + unk_60: 0x0, // fixed + unk_64: 0x0, // fixed + unk_68: 0x0, // fixed + #[ver(G >= G14 && V < V13_0B4)] + unk_68_g14: U64(0), + restart_branch_offset: off, + has_attachments: (vertex_attachments.count > 0) as u32, + #[ver(V >= V13_0B4)] + unk_74: Default::default(), // Ventura + })?; + + builder.add(microseq::RetireStamp { + header: microseq::op::RetireStamp::HEADER, + })?; + builder.build(private)? + }, + notifier, + scene, + vm_bind, + timestamps, + user_timestamps: vtx_user_timestamps, + }) + }, + |inner, _ptr| { + let vm_slot = vm_bind.slot(); + #[ver(G < G14)] + let core_masks = gpu.core_masks_packed(); + + try_init!(fw::vertex::raw::RunVertex::ver { + tag: fw::workqueue::CommandType::RunVertex, + #[ver(V >= V13_0B4)] + counter: U64(count_vtx), + vm_slot, + unk_8: 0, + notifier: inner.notifier.gpu_pointer(), + buffer_slot: inner.scene.slot(), + unk_1c: 0, + buffer: inner.scene.buffer_pointer(), + scene: inner.scene.gpu_pointer(), + unk_buffer_buf: inner.scene.kernel_buffer_pointer(), + unk_34: 0, + #[ver(G < G14X)] + job_params1 <- try_init!(fw::vertex::raw::JobParameters1::ver { + unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 + unk_8: f32!(1e-20), // fixed + unk_c: f32!(1e-20), // fixed + tvb_tilemap: inner.scene.tvb_tilemap_pointer(), + #[ver(G < G14)] + tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), + tpc: inner.scene.tpc_pointer(), + tvb_heapmeta: inner.scene.tvb_heapmeta_pointer().or(0x8000_0000_0000_0000), + iogpu_unk_54: U64(iogpu_unk54), // fixed + iogpu_unk_56: U64(iogpu_unk56), // fixed + #[ver(G < G14)] + tvb_cluster_meta1: inner + .scene + .meta_1_pointer() + .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), + utile_config, + unk_4c: 0, + ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed + tvb_layermeta: inner.scene.tvb_layermeta_pointer(), + #[ver(G < G14)] + tvb_cluster_layermeta: inner.scene.tvb_cluster_layermeta_pointer(), + #[ver(G < G14)] + core_mask: Array::new([ + *core_masks.first().unwrap_or(&0), + *core_masks.get(1).unwrap_or(&0), + ]), + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + preempt_buf2: inner.scene.preempt_buf_2_pointer(), + unk_80: U64(0x1), // fixed + preempt_buf3: inner.scene.preempt_buf_3_pointer().or(0x4_0000_0000_0000), // check + vdm_ctrl_stream_base: U64(cmdbuf.vdm_ctrl_stream_base), + #[ver(G < G14)] + tvb_cluster_meta2: inner.scene.meta_2_pointer(), + #[ver(G < G14)] + tvb_cluster_meta3: inner.scene.meta_3_pointer(), + #[ver(G < G14)] + tiling_control, + #[ver(G < G14)] + unk_ac: tiling_control_2 as u32, // fixed + unk_b0: Default::default(), // fixed + usc_exec_base_ta: U64(self.usc_exec_base), + #[ver(G < G14)] + tvb_cluster_meta4: inner + .scene + .meta_4_pointer() + .map(|x| x.or(0x3000_0000_0000_0000)), + #[ver(G < G14)] + unk_f0: U64(vtx_unk_f0), + unk_f8: U64(0x8c60), // fixed + helper_program: cmdbuf.vertex_helper.binary, + unk_104: 0, + helper_arg: U64(cmdbuf.vertex_helper.data), + unk_110: Default::default(), // fixed + unk_118: vtx_unk_118 as u32, // fixed + __pad: Default::default(), + }), + #[ver(G < G14X)] + tiling_params: tile_info.params, + #[ver(G >= G14X)] + registers: fw::job::raw::RegisterArray::new( + inner_weak_ptr!(_ptr, registers.registers), + |r| { + r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 + r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); + r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); + + let cl_tilemaps_ptr = inner + .scene + .cluster_tilemaps_pointer() + .map_or(0, |a| a.into()); + r.add(0x1c041, cl_tilemaps_ptr); + r.add(0x1c9d0, cl_tilemaps_ptr); + r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR + + let tvb_heapmeta_ptr = inner + .scene + .tvb_heapmeta_pointer() + .or(0x8000_0000_0000_0000) + .into(); + r.add(0x1c031, tvb_heapmeta_ptr); + r.add(0x1c9c0, tvb_heapmeta_ptr); + r.add(0x1c051, iogpu_unk54); // iogpu_unk_54/55 + r.add(0x1c061, iogpu_unk56); // iogpu_unk_56 + r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config + r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL + r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); + r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); + r.add(0x1c958, 1); // s2.unk_80 + r.add( + 0x1c950, + inner + .scene + .preempt_buf_3_pointer() + .or(0x4_0000_0000_0000) + .into(), + ); + r.add(0x1c930, 0); // VCE related addr, lsb to enable + r.add(0x1c880, cmdbuf.vdm_ctrl_stream_base); // VDM_CTRL_STREAM_BASE + r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. + r.add( + 0x1c948, + inner.scene.meta_2_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta2 + r.add( + 0x1c888, + inner.scene.meta_3_pointer().map_or(0, |a| a.into()), + ); // tvb_cluster_meta3 + r.add(0x1c890, tiling_control.into()); // tvb_tiling_control + r.add(0x1c918, tiling_control_2); + r.add(0x1c079, inner.scene.tvb_layermeta_pointer().into()); + r.add(0x1c9d8, inner.scene.tvb_layermeta_pointer().into()); + let cl_layermeta_pointer = + inner.scene.tvb_cluster_layermeta_pointer().map_or(0, |a| a.into()); + r.add(0x1c089, cl_layermeta_pointer); + r.add(0x1c9e0, cl_layermeta_pointer); + let cl_meta_4_pointer = + inner.scene.meta_4_pointer().map_or(0, |a| a.into()); + r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 + r.add(0x1c9a8, vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? + r.add( + 0x1c920, + inner.scene.meta_1_pointer().map_or(0, |a| a.into()), + ); // ??? | meta1_blocks? + r.add(0x10151, 0); + r.add(0x1c199, 0); + r.add(0x1c1a1, 0); + r.add(0x1c1a9, 0); // 0x10151 bit 1 enables + r.add(0x1c1b1, 0); + r.add(0x1c1b9, 0); + r.add(0x10061, self.usc_exec_base); // USC_EXEC_BASE_TA + r.add(0x11801, cmdbuf.vertex_helper.binary.into()); + r.add(0x11809, cmdbuf.vertex_helper.data); + r.add(0x11f71, cmdbuf.vertex_helper.cfg.into()); + r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG + r.add(0x1c850, tile_info.params.rgn_size.into()); + r.add(0x10131, tile_info.params.unk_4.into()); + r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL + r.add( + 0x10129, + tile_info.params.x_max as u64 + | ((tile_info.params.y_max as u64) << 16), + ); // PPP_SCREEN + r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN + r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 + r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 + r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE + r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC + r.add(0x10171, tile_info.params.unk_24.into()); + r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX + r.add(0x12099, vtx_unk_118); + r.add(0x1c9e8, (tile_info.params.unk_28 & 0x4fff).into()); + /* + r.add(0x10209, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c9f0, 0x100); // Some kind of counter?? Does this matter? + r.add(0x1c830, 1); // ? + r.add(0x1ca30, 0x1502960e60); // ? + r.add(0x16c39, 0x1502960e60); // ? + r.add(0x1c910, 0xa0000b011d); // ? + r.add(0x1c8e0, 0xff); // cluster mask + r.add(0x1c8e8, 0); // ? + */ + } + ), + tpc: inner.scene.tpc_pointer(), + tpc_size: U64(tile_info.tpc_size as u64), + microsequence: inner.micro_seq.gpu_pointer(), + microsequence_size: inner.micro_seq.len() as u32, + fragment_stamp_slot: ev_frag.slot, + fragment_stamp_value: ev_frag.value.next(), + unk_pointee: 0, + unk_pad: 0, + job_params2 <- try_init!(fw::vertex::raw::JobParameters2 { + unk_480: Default::default(), // fixed + unk_498: U64(0x0), // fixed + unk_4a0: 0x0, // fixed + preempt_buf1: inner.scene.preempt_buf_1_pointer(), + unk_4ac: 0x0, // fixed + unk_4b0: U64(0x0), // fixed + unk_4b8: 0x0, // fixed + unk_4bc: U64(0x0), // fixed + unk_4c4_padding: Default::default(), + unk_50c: 0x0, // fixed + unk_510: U64(0x0), // fixed + unk_518: U64(0x0), // fixed + unk_520: U64(0x0), // fixed + }), + encoder_params <- try_init!(fw::job::raw::EncoderParams { + unk_8: 0x0, // fixed + sync_grow: 0x0, // fixed + unk_10: 0x0, // fixed + encoder_id: 0, + unk_18: 0x0, // fixed + unk_mask: 0xffffffffu32, + sampler_array: U64(cmdbuf.sampler_heap), + sampler_count: cmdbuf.sampler_count as u32, + sampler_max: (cmdbuf.sampler_count as u32) + 1, + }), + unk_55c: 0, + unk_560: 0, + sync_grow: 0, + unk_568: 0, + uses_scratch: (cmdbuf.flags + & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH as u32 + != 0) as u32, + meta <- try_init!(fw::job::raw::JobMeta { + unk_0: 0, + unk_2: 0, + no_preemption: no_preemption as u8, + stamp: ev_vtx.stamp_pointer, + fw_stamp: ev_vtx.fw_stamp_pointer, + stamp_value: ev_vtx.value.next(), + stamp_slot: ev_vtx.slot, + evctl_index: 0, // fixed + flush_stamps: flush_stamps as u32, + uuid: uuid_ta, + event_seq: ev_vtx.event_seq as u32, + }), + unk_after_meta: unk1.into(), + unk_buf_0: U64(0), + unk_buf_8: U64(0), + unk_buf_10: U64(0), + command_time: U64(0), + timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { + start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), + end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), + }), + user_timestamp_pointers: inner.user_timestamps.pointers()?, + client_sequence: slot_client_seq, + pad_5d5: Default::default(), + unk_5d8: 0, + unk_5dc: 0, + #[ver(V >= V13_0B4)] + unk_ts: U64(0), + #[ver(V >= V13_0B4)] + unk_5dd_8: Default::default(), + }) + }, + )?; + + core::mem::drop(alloc); + + mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); + fence.add_command(); + vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { + if let Some(err) = error { + fence.set_error(err.into()) + } + + fence.command_complete(); + })?; + + mod_dev_dbg!(self.dev, "[Submission {}] Increment counters\n", id); + + // TODO: handle rollbacks, move to job submit? + buffer.increment(); + + job.get_vtx()?.next_seq(); + job.get_frag()?.next_seq(); + + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs new file mode 100644 index 00000000000000..6ebbaa56f48c81 --- /dev/null +++ b/drivers/gpu/drm/asahi/regs.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU MMIO register abstraction +//! +//! Since the vast majority of the interactions with the GPU are brokered through the firmware, +//! there is very little need to interact directly with GPU MMIO register. This module abstracts +//! the few operations that require that, mainly reading the MMU fault status, reading GPU ID +//! information, and starting the GPU firmware coprocessor. + +use crate::hw; +use kernel::{ + c_str, + device::Core, + devres::Devres, + io::mem::IoMem, + platform, + prelude::*, + types::ARef, // +}; + +/// Size of the ASC control MMIO region. +pub(crate) const ASC_CTL_SIZE: usize = 0x4000; + +/// Size of the SGX MMIO region. +pub(crate) const SGX_SIZE: usize = 0x1000000; + +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; // BIT(4) + +const FAULT_INFO: usize = 0x17030; + +const ID_VERSION: usize = 0xd04000; +const ID_UNK08: usize = 0xd04008; +const ID_COUNTS_1: usize = 0xd04010; +const ID_COUNTS_2: usize = 0xd04014; +const ID_UNK18: usize = 0xd04018; +const ID_CLUSTERS: usize = 0xd0401c; + +const CORE_MASK_0: usize = 0xd01500; +const CORE_MASK_1: usize = 0xd01514; + +const CORE_MASKS_G14X: usize = 0xe01500; +const FAULT_INFO_G14X: usize = 0xd8c0; +const FAULT_ADDR_G14X: usize = 0xd8c8; + +/// Enum representing the unit that caused an MMU fault. +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultUnit { + /// Decompress / pixel fetch + DCMP(u8), + /// USC L1 Cache (device loads/stores) + UL1C(u8), + /// Compress / pixel store + CMP(u8), + GSL1(u8), + IAP(u8), + VCE(u8), + /// Tiling Engine + TE(u8), + RAS(u8), + /// Vertex Data Master + VDM(u8), + PPP(u8), + /// ISP Parameter Fetch + IPF(u8), + IPF_CPF(u8), + VF(u8), + VF_CPF(u8), + /// Depth/Stencil load/store + ZLS(u8), + + /// Parameter Management + dPM, + /// Compute Data Master + dCDM_KS(u8), + dIPP, + dIPP_CS, + // Vertex Data Master + dVDM_CSD, + dVDM_SSD, + dVDM_ILF, + dVDM_ILD, + dRDE(u8), + FC, + GSL2, + + /// Graphics L2 Cache Control? + GL2CC_META(u8), + GL2CC_MB, + + /// Parameter Management + gPM_SP(u8), + /// Vertex Data Master - CSD + gVDM_CSD_SP(u8), + gVDM_SSD_SP(u8), + gVDM_ILF_SP(u8), + gVDM_TFP_SP(u8), + gVDM_MMB_SP(u8), + /// Compute Data Master + gCDM_CS_KS0_SP(u8), + gCDM_CS_KS1_SP(u8), + gCDM_CS_KS2_SP(u8), + gCDM_KS0_SP(u8), + gCDM_KS1_SP(u8), + gCDM_KS2_SP(u8), + gIPP_SP(u8), + gIPP_CS_SP(u8), + gRDE0_SP(u8), + gRDE1_SP(u8), + + gCDM_CS, + gCDM_ID, + gCDM_CSR, + gCDM_CSW, + gCDM_CTXR, + gCDM_CTXW, + gIPP, + gIPP_CS, + gKSM_RCE, + + Unknown(u8), +} + +/// Reason for an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum FaultReason { + Unmapped, + AfFault, + WriteOnly, + ReadOnly, + NoAccess, + Unknown(u8), +} + +/// Collection of information about an MMU fault. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) struct FaultInfo { + pub(crate) address: u64, + pub(crate) sideband: u8, + pub(crate) vm_slot: u32, + pub(crate) unit_code: u8, + pub(crate) unit: FaultUnit, + pub(crate) level: u8, + pub(crate) unk_5: u8, + pub(crate) read: bool, + pub(crate) reason: FaultReason, +} + +/// Device resources for this GPU instance. +pub(crate) struct Resources { + dev: ARef, + sgx: Pin>>>, +} + +impl Resources { + /// Map the required resources given our platform device. + pub(crate) fn new(pdev: &platform::Device) -> Result { + let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let sgx_iomem = KBox::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; + + Ok(Resources { + // SAFETY: This device does DMA via the UAT IOMMU. + dev: pdev.into(), + sgx: sgx_iomem, + }) + } + + fn sgx_read32(&self) -> u32 { + if let Some(sgx) = self.sgx.try_access() { + sgx.read32_relaxed(OFF) + } else { + 0 + } + } + + /* Not yet used + fn sgx_write32(&self, val: u32) { + if let Some(sgx) = self.sgx.try_access() { + sgx.write32_relaxed(val, OFF) + } + } + */ + + fn sgx_read64(&self) -> u64 { + if let Some(sgx) = self.sgx.try_access() { + sgx.read64_relaxed(OFF) + } else { + 0 + } + } + + /* Not yet used + fn sgx_write64(&self, val: u64) { + if let Some(sgx) = self.sgx.try_access() { + sgx.write64_relaxed(val, OFF) + } + } + */ + + /// Initialize the MMIO registers for the GPU. + pub(crate) fn init_mmio(&self) -> Result { + // Nothing to do for now... + + Ok(()) + } + + /// Start the ASC coprocessor CPU. + pub(crate) fn start_cpu(pdev: &platform::Device) -> Result { + let asc_req = pdev.io_request_by_name(c_str!("asc")).ok_or(EINVAL)?; + let asc_iomem = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + let res = asc_iomem.access(pdev.as_ref())?; + + let val = res.read32_relaxed(CPU_CONTROL); + res.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + + /// Get the GPU identification info from registers. + /// + /// See [`hw::GpuIdConfig`] for the result. + pub(crate) fn get_gpu_id(&self) -> Result { + let id_version = self.sgx_read32::(); + let id_unk08 = self.sgx_read32::(); + let id_counts_1 = self.sgx_read32::(); + let id_counts_2 = self.sgx_read32::(); + let id_unk18 = self.sgx_read32::(); + let id_clusters = self.sgx_read32::(); + + dev_info!( + self.dev.as_ref(), + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + let gpu_gen = (id_version >> 24) & 0xff; + + let mut core_mask_regs = KVec::new(); + + let num_clusters = match gpu_gen { + 4 | 5 => { + // G13 | G14G + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + (id_clusters >> 12) & 0xff + } + 6 => { + // G14X + core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), GFP_KERNEL)?; + core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), GFP_KERNEL)?; + // Clusters per die * num dies + ((id_counts_1 >> 8) & 0xff) * ((id_counts_1 >> 16) & 0xf) + } + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }; + + let mut core_masks_packed = KVec::new(); + core_masks_packed.extend_from_slice(&core_mask_regs, GFP_KERNEL)?; + + dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_masks_packed); + + let num_cores = id_counts_1 & 0xff; + + if num_cores > 32 { + dev_err!( + self.dev.as_ref(), + "Too many cores per cluster ({} > 32)\n", + num_cores + ); + return Err(ENODEV); + } + + if num_cores * num_clusters > (core_mask_regs.len() * 32) as u32 { + dev_err!( + self.dev.as_ref(), + "Too many total cores ({} x {} > {})\n", + num_clusters, + num_cores, + core_mask_regs.len() * 32 + ); + return Err(ENODEV); + } + + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] >>= num_cores; + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + self.dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }; + + Ok(hw::GpuIdConfig { + gpu_gen: match (id_version >> 24) & 0xff { + 4 => hw::GpuGen::G13, + 5 => hw::GpuGen::G14, + 6 => hw::GpuGen::G14, // G14X has a separate ID + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); + return Err(ENODEV); + } + }, + gpu_variant: match (id_version >> 16) & 0xff { + 1 => hw::GpuVariant::P, // Guess + 2 => hw::GpuVariant::G, + 3 => hw::GpuVariant::S, + 4 => { + if num_clusters > 4 { + hw::GpuVariant::D + } else { + hw::GpuVariant::C + } + } + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU variant {}\n", a); + return Err(ENODEV); + } + }, + gpu_rev, + gpu_rev_id, + num_clusters, + num_cores, + num_frags: num_cores, // Used to be id_counts_1[15:8] but does not work for G14X + num_gps: (id_counts_2 >> 16) & 0xff, + total_active_cores, + core_masks, + core_masks_packed, + }) + } + + /// Get the fault information from the MMU status register, if one occurred. + pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { + let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; + + let fault_info = if g14x { + self.sgx_read64::() + } else { + self.sgx_read64::() + }; + + if fault_info & 1 == 0 { + return None; + } + + let fault_addr = if g14x { + self.sgx_read64::() + } else { + fault_info >> 30 + }; + + let unit_code = ((fault_info >> 9) & 0xff) as u8; + let unit = match unit_code { + 0x00..=0x9f => match unit_code & 0xf { + 0x0 => FaultUnit::DCMP(unit_code >> 4), + 0x1 => FaultUnit::UL1C(unit_code >> 4), + 0x2 => FaultUnit::CMP(unit_code >> 4), + 0x3 => FaultUnit::GSL1(unit_code >> 4), + 0x4 => FaultUnit::IAP(unit_code >> 4), + 0x5 => FaultUnit::VCE(unit_code >> 4), + 0x6 => FaultUnit::TE(unit_code >> 4), + 0x7 => FaultUnit::RAS(unit_code >> 4), + 0x8 => FaultUnit::VDM(unit_code >> 4), + 0x9 => FaultUnit::PPP(unit_code >> 4), + 0xa => FaultUnit::IPF(unit_code >> 4), + 0xb => FaultUnit::IPF_CPF(unit_code >> 4), + 0xc => FaultUnit::VF(unit_code >> 4), + 0xd => FaultUnit::VF_CPF(unit_code >> 4), + 0xe => FaultUnit::ZLS(unit_code >> 4), + _ => FaultUnit::Unknown(unit_code), + }, + 0xa1 => FaultUnit::dPM, + 0xa2 => FaultUnit::dCDM_KS(0), + 0xa3 => FaultUnit::dCDM_KS(1), + 0xa4 => FaultUnit::dCDM_KS(2), + 0xa5 => FaultUnit::dIPP, + 0xa6 => FaultUnit::dIPP_CS, + 0xa7 => FaultUnit::dVDM_CSD, + 0xa8 => FaultUnit::dVDM_SSD, + 0xa9 => FaultUnit::dVDM_ILF, + 0xaa => FaultUnit::dVDM_ILD, + 0xab => FaultUnit::dRDE(0), + 0xac => FaultUnit::dRDE(1), + 0xad => FaultUnit::FC, + 0xae => FaultUnit::GSL2, + 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), + 0xb8 => FaultUnit::GL2CC_MB, + 0xd0..=0xdf if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gCDM_CS, + 0x1 => FaultUnit::gCDM_ID, + 0x2 => FaultUnit::gCDM_CSR, + 0x3 => FaultUnit::gCDM_CSW, + 0x4 => FaultUnit::gCDM_CTXR, + 0x5 => FaultUnit::gCDM_CTXW, + 0x6 => FaultUnit::gIPP, + 0x7 => FaultUnit::gIPP_CS, + 0x8 => FaultUnit::gKSM_RCE, + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + 0xe0..=0xff if !g14x => match unit_code & 0xf { + 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), + 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), + 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), + 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), + 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), + 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), + 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), + 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), + 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), + 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), + 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), + 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), + 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), + 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), + 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), + 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), + _ => FaultUnit::Unknown(unit_code), + }, + _ => FaultUnit::Unknown(unit_code), + }; + + let reason = match (fault_info >> 1) & 0x7 { + 0 => FaultReason::Unmapped, + 1 => FaultReason::AfFault, + 2 => FaultReason::WriteOnly, + 3 => FaultReason::ReadOnly, + 4 => FaultReason::NoAccess, + a => FaultReason::Unknown(a as u8), + }; + + Some(FaultInfo { + address: fault_addr << 6, + sideband: ((fault_info >> 23) & 0x7f) as u8, + vm_slot: ((fault_info >> 17) & 0x3f) as u32, + unit_code, + unit, + level: ((fault_info >> 7) & 3) as u8, + unk_5: ((fault_info >> 5) & 3) as u8, + read: (fault_info & (1 << 4)) != 0, + reason, + }) + } +} diff --git a/drivers/gpu/drm/asahi/slotalloc.rs b/drivers/gpu/drm/asahi/slotalloc.rs new file mode 100644 index 00000000000000..fde7470fe57791 --- /dev/null +++ b/drivers/gpu/drm/asahi/slotalloc.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Generic slot allocator +//! +//! This is a simple allocator to manage fixed-size pools of GPU resources that are transiently +//! required during command execution. Each item resides in a "slot" at a given index. Users borrow +//! and return free items from the available pool. +//! +//! Allocations are "sticky", and return a token that callers can use to request the same slot +//! again later. This allows slots to be lazily invalidated, so that multiple uses by the same user +//! avoid any actual cleanup work. +//! +//! The allocation policy is currently a simple LRU mechanism, doing a full linear scan over the +//! slots when no token was previously provided. This is probably good enough, since in the absence +//! of serious system contention most allocation requests will be immediately fulfilled from the +//! previous slot without doing an LRU scan. + +use core::num::NonZeroUsize; +use core::ops::{ + Deref, + DerefMut, // +}; +use kernel::{ + error::{ + code::*, + Result, // + }, + prelude::*, + str::CStr, + sync::{ + Arc, + CondVar, + LockClassKey, + Mutex, // + }, +}; + +/// Trait representing a single item within a slot. +pub(crate) trait SlotItem { + /// Arbitrary user data associated with the SlotAllocator. + type Data; + + /// Called eagerly when this item is released back into the available pool. + fn release(&mut self, _data: &mut Self::Data, _slot: u32) {} +} + +/// Trivial implementation for users which do not require any slot data nor any allocator data. +impl SlotItem for () { + type Data = (); +} + +/// Represents a current or previous allocation of an item from a slot. Users keep `SlotToken`s +/// around across allocations to request that, if possible, the same slot be reused. +#[derive(Copy, Clone, Debug)] +pub(crate) struct SlotToken { + time: u64, + slot: u32, +} + +impl SlotToken { + /// Returns the slot index that this token represents a past assignment to. + pub(crate) fn last_slot(&self) -> u32 { + self.slot + } +} + +/// A guard representing active ownership of a slot. +pub(crate) struct Guard { + item: Option, + changed: bool, + token: SlotToken, + alloc: Arc>, +} + +impl Guard { + /// Returns the active slot owned by this `Guard`. + pub(crate) fn slot(&self) -> u32 { + self.token.slot + } + + /// Returns `true` if the slot changed since the last allocation (or no `SlotToken` was + /// provided), or `false` if the previously allocated slot was successfully re-acquired with + /// no other users in the interim. + pub(crate) fn changed(&self) -> bool { + self.changed + } + + /// Returns a `SlotToken` that can be used to re-request the same slot at a later time, after + /// this `Guard` is dropped. + pub(crate) fn token(&self) -> SlotToken { + self.token + } +} + +impl Deref for Guard { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.item.as_ref().expect("SlotItem Guard lost our item!") + } +} + +impl DerefMut for Guard { + fn deref_mut(&mut self) -> &mut Self::Target { + self.item.as_mut().expect("SlotItem Guard lost our item!") + } +} + +/// A slot item that is currently free. +struct Entry { + item: T, + get_time: u64, + drop_time: u64, +} + +/// Inner data for the `SlotAllocator`, protected by a `Mutex`. +struct SlotAllocatorInner { + data: T::Data, + slots: KVec>>, + get_count: u64, + drop_count: u64, + slot_limit: usize, +} + +/// A single slot allocator instance. +#[pin_data] +struct SlotAllocatorOuter { + #[pin] + inner: Mutex>, + #[pin] + cond: CondVar, +} + +/// A shared reference to a slot allocator instance. +pub(crate) struct SlotAllocator(Arc>); + +impl SlotAllocator { + /// Creates a new `SlotAllocator`, with a fixed number of slots and arbitrary associated data. + /// + /// The caller provides a constructor callback which takes a reference to the `T::Data` and + /// creates a single slot. This is called during construction to create all the initial + /// items, which then live the lifetime of the `SlotAllocator`. + pub(crate) fn new( + num_slots: u32, + mut data: T::Data, + mut constructor: impl FnMut(&mut T::Data, u32) -> Option, + name: &'static CStr, + lock_key1: Pin<&'static LockClassKey>, + lock_key2: Pin<&'static LockClassKey>, + ) -> Result> { + let mut slots = KVec::with_capacity(num_slots as usize, GFP_KERNEL)?; + + for i in 0..num_slots { + slots + .push( + constructor(&mut data, i).map(|item| Entry { + item, + get_time: 0, + drop_time: 0, + }), + GFP_KERNEL, + ) + .expect("try_push() failed after reservation"); + } + + let inner = SlotAllocatorInner { + data, + slots, + get_count: 0, + drop_count: 0, + slot_limit: usize::MAX, + }; + + let alloc = Arc::pin_init( + pin_init!(SlotAllocatorOuter { + // SAFETY: `mutex_init!` is called below. + inner <- Mutex::new(inner, name, lock_key1), + // SAFETY: `condvar_init!` is called below. + cond <- CondVar::new(name, lock_key2), + }), + GFP_KERNEL, + )?; + + Ok(SlotAllocator(alloc)) + } + + /// Calls a callback on the inner data associated with this allocator, taking the lock. + pub(crate) fn with_inner(&self, cb: impl FnOnce(&mut T::Data) -> RetVal) -> RetVal { + let mut inner = self.0.inner.lock(); + cb(&mut inner.data) + } + + /// Set the slot limit for this allocator. New bindings will not use slots above + /// this threshold. + pub(crate) fn set_limit(&self, limit: Option) { + let mut inner = self.0.inner.lock(); + inner.slot_limit = limit.unwrap_or(NonZeroUsize::MAX).get(); + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + pub(crate) fn get(&self, token: Option) -> Result> { + self.get_inner(token, |_a, _b| Ok(())) + } + + /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. + /// + /// Blocks if no slots are free. + /// + /// This version allows the caller to pass in a callback that gets a mutable reference to the + /// user data for the allocator and the freshly acquired slot, which is called before the + /// allocator lock is released. This can be used to perform bookkeeping associated with + /// specific slots (such as tracking their current owner). + pub(crate) fn get_inner( + &self, + token: Option, + cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, + ) -> Result> { + let mut inner = self.0.inner.lock(); + + if let Some(token) = token { + if (token.slot as usize) < inner.slot_limit { + let slot = &mut inner.slots[token.slot as usize]; + if slot.is_some() { + let count = slot.as_ref().unwrap().get_time; + if count == token.time { + let mut guard = Guard { + item: Some(slot.take().unwrap().item), + token, + changed: false, + alloc: self.0.clone(), + }; + cb(&mut inner.data, &mut guard)?; + return Ok(guard); + } + } + } + } + + let mut first = true; + let slot = loop { + let mut oldest_time = u64::MAX; + let mut oldest_slot = 0u32; + + for (i, slot) in inner.slots.iter().enumerate() { + if i >= inner.slot_limit { + break; + } + if let Some(slot) = slot.as_ref() { + if slot.drop_time < oldest_time { + oldest_slot = i as u32; + oldest_time = slot.drop_time; + } + } + } + + if oldest_time == u64::MAX { + if first && inner.slot_limit == usize::MAX { + pr_warn!( + "{}: out of slots, blocking\n", + core::any::type_name::() + ); + } + first = false; + if self.0.cond.wait_interruptible(&mut inner) { + return Err(ERESTARTSYS); + } + } else { + break oldest_slot; + } + }; + + inner.get_count += 1; + + let item = inner.slots[slot as usize] + .take() + .expect("Someone stole our slot?") + .item; + + let mut guard = Guard { + item: Some(item), + changed: true, + token: SlotToken { + time: inner.get_count, + slot, + }, + alloc: self.0.clone(), + }; + + cb(&mut inner.data, &mut guard)?; + Ok(guard) + } +} + +impl Clone for SlotAllocator { + fn clone(&self) -> Self { + SlotAllocator(self.0.clone()) + } +} + +impl Drop for Guard { + fn drop(&mut self) { + let mut inner = self.alloc.inner.lock(); + if inner.slots[self.token.slot as usize].is_some() { + pr_crit!( + "{}: tried to return an item into a full slot ({})\n", + core::any::type_name::(), + self.token.slot + ); + } else { + inner.drop_count += 1; + let mut item = self.item.take().expect("Guard lost its item"); + item.release(&mut inner.data, self.token.slot); + inner.slots[self.token.slot as usize] = Some(Entry { + item, + get_time: self.token.time, + drop_time: inner.drop_count, + }); + self.alloc.cond.notify_one(); + } + } +} diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs new file mode 100644 index 00000000000000..1a41d8f16d4432 --- /dev/null +++ b/drivers/gpu/drm/asahi/util.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Miscellaneous utility functions + +use core::ops::{ + Add, + BitAnd, + Div, + Not, + Sub, // +}; +use kernel::prelude::*; + +/// Aligns an integer type to a power of two. +pub(crate) fn align(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Add + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + (a + b - one) & !(b - one) +} + +/// Aligns an integer type down to a power of two. +pub(crate) fn align_down(a: T, b: T) -> T +where + T: Copy + + Default + + BitAnd + + Not + + Sub + + Div + + core::cmp::PartialEq, +{ + let def: T = Default::default(); + #[allow(clippy::eq_op)] + let one: T = !def / !def; + + assert!((b & (b - one)) == def); + + a & !(b - one) +} + +pub(crate) trait RangeExt { + fn overlaps(&self, other: Self) -> bool; + fn is_superset(&self, other: Self) -> bool; + // fn len(&self) -> usize; + fn range(&self) -> T; +} + +impl + Default + Copy + Sub> RangeExt for core::ops::Range +where + usize: core::convert::TryFrom, + >::Error: core::fmt::Debug, +{ + fn overlaps(&self, other: Self) -> bool { + !(self.is_empty() || other.is_empty() || self.end <= other.start || other.end <= self.start) + } + fn is_superset(&self, other: Self) -> bool { + !self.is_empty() + && (other.is_empty() || (other.start >= self.start && other.end <= self.end)) + } + fn range(&self) -> T { + if self.is_empty() { + Default::default() + } else { + self.end - self.start + } + } + // fn len(&self) -> usize { + // self.range().try_into().unwrap() + // } +} + +pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { + let mut n = in_n; + let mut m = in_m; + + while n != 0 { + let remainder = m % n; + m = n; + n = remainder; + } + + m +} + +pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} + +pub(crate) struct Reader<'a> { + buffer: &'a [u8], + offset: usize, +} + +impl<'a> Reader<'a> { + pub(crate) fn new(buffer: &'a [u8]) -> Self { + Reader { buffer, offset: 0 } + } + + pub(crate) fn read_up_to(&mut self, max_size: usize) -> Result { + let mut obj: T = Default::default(); + let size: usize = core::mem::size_of::().min(max_size); + let range = self.offset..self.offset + size; + let src = self.buffer.get(range).ok_or(EINVAL)?; + + // SAFETY: The output pointer is valid, and the size does not exceed + // the type size, and all bit patterns are valid. + let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; + + dst.copy_from_slice(src); + self.offset += size; + Ok(obj) + } + + pub(crate) fn read(&mut self) -> Result { + self.read_up_to(!0) + } + + pub(crate) fn is_empty(&self) -> bool { + self.offset >= self.buffer.len() + } + + pub(crate) fn skip(&mut self, size: usize) { + self.offset += size + } + + pub(crate) fn rewind(&mut self) { + self.offset = 0 + } +} diff --git a/drivers/gpu/drm/asahi/vm/mod.rs b/drivers/gpu/drm/asahi/vm/mod.rs new file mode 100644 index 00000000000000..63cf8a76cd5ce4 --- /dev/null +++ b/drivers/gpu/drm/asahi/vm/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Virtual address space management + +mod range; diff --git a/drivers/gpu/drm/asahi/vm/range.rs b/drivers/gpu/drm/asahi/vm/range.rs new file mode 100644 index 00000000000000..727d29a5cccf02 --- /dev/null +++ b/drivers/gpu/drm/asahi/vm/range.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +// Copied from tyr's mmu/vm/range.rs + +//! Range allocator. +//! +//! This module allows you to search for unused ranges to store GEM objects. + +use kernel::alloc::Flags; +use kernel::maple_tree::MapleTreeAlloc; +use kernel::prelude::*; +use kernel::sync::Arc; + +use core::ops::Range; + +/// The actual storage for the ranges. +/// +/// All ranges must fit within the `range` field. +/// +/// The implementation is different on 32-bit and 64-bit cpus. On 64-bit, the 64-bit addresses are +/// stored directly in the maple tree, but on 32-bit, the maple tree stores the ranges translated +/// in the range zero until `range.end-range.start`. This is done because the maple tree uses +/// unsigned long as its address type, which is too small to store the 64-bit address directly on +/// 32-bit machines. +#[pin_data] +struct RangeAllocInner { + #[pin] + maple: MapleTreeAlloc<()>, + range: Range, +} + +/// This object allows you to allocate ranges on the inner maple tree. +pub(crate) struct RangeAlloc { + inner: Arc, +} + +/// Represents a live range in the maple tree. +/// +/// The destructor removes the range from the maple tree, allowing others to allocate it in the +/// future. +pub(crate) struct LiveRange { + inner: Arc, + offset: u64, + size: usize, +} + +impl RangeAlloc { + pub(crate) fn new(start: u64, end: u64, gfp: Flags) -> Result { + if end < start { + return Err(EINVAL); + } + + let inner = Arc::pin_init( + try_pin_init!(RangeAllocInner { + maple <- MapleTreeAlloc::new(), + range: start..end, + }), + gfp, + )?; + + Ok(RangeAlloc { inner }) + } + + pub(crate) fn allocate(&self, size: usize, gfp: Flags) -> Result { + let maple_start = self.inner.range.start as usize; + let maple_end = self.inner.range.end as usize; + + let offset = self + .inner + .maple + .alloc_range(size, (), maple_start..maple_end, gfp)?; + + Ok(LiveRange { + inner: self.inner.clone(), + offset: offset as u64, + size, + }) + } + + pub(crate) fn insert(&self, start: u64, end: u64, gfp: Flags) -> Result { + if end < start { + return Err(EINVAL); + } + if start < self.inner.range.start { + return Err(EINVAL); + } + if end > self.inner.range.end { + return Err(EINVAL); + } + + self.inner + .maple + .insert_range(start as usize..end as usize, (), gfp)?; + + Ok(LiveRange { + inner: self.inner.clone(), + offset: start, + size: (end - start) as usize, + }) + } +} + +impl LiveRange { + pub(crate) fn size(&self) -> usize { + self.size + } + + pub(crate) fn start(&self) -> u64 { + self.offset + } + + pub(crate) fn end(&self) -> u64 { + self.offset + self.size as u64 + } + + pub(crate) fn range(&self) -> Range { + self.start()..self.end() + } +} + +impl Drop for LiveRange { + fn drop(&mut self) { + self.inner.maple.erase(self.offset as usize); + } +} diff --git a/drivers/gpu/drm/asahi/workqueue.rs b/drivers/gpu/drm/asahi/workqueue.rs new file mode 100644 index 00000000000000..55b6058d50224f --- /dev/null +++ b/drivers/gpu/drm/asahi/workqueue.rs @@ -0,0 +1,1015 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! GPU command execution queues +//! +//! The AGX GPU firmware schedules GPU work commands out of work queues, which are ring buffers of +//! pointers to work commands. There can be an arbitrary number of work queues. Work queues have an +//! associated type (vertex, fragment, or compute) and may only contain generic commands or commands +//! specific to that type. +//! +//! This module manages queueing work commands into a work queue and submitting them for execution +//! by the firmware. An active work queue needs an event to signal completion of its work, which is +//! owned by what we call a batch. This event then notifies the work queue when work is completed, +//! and that triggers freeing of all resources associated with that work. An idle work queue gives +//! up its associated event. + +use crate::debug::*; +use crate::fw::channels::{ + ChannelErrorType, + PipeType, // +}; +use crate::fw::types::*; +use crate::fw::workqueue::*; +use crate::no_debug; +use crate::object::OpaqueGpuObject; +use crate::{ + channel, + driver, + event, + fw, + gpu, + regs, // +}; +use core::any::Any; +use core::num::NonZeroU64; +use core::sync::atomic::Ordering; +use kernel::{ + dma_fence, + error::code::*, + new_mutex, + prelude::*, + sync::{ + lock::{ + mutex::MutexBackend, + Guard, // + }, + Arc, + Mutex, // + }, + workqueue::{ + self, + impl_has_work, + new_work, + Work, + WorkItem, // + }, // +}; + +pub(crate) trait OpaqueCommandObject: OpaqueGpuObject {} + +impl OpaqueCommandObject for GpuObject where T: Command {} + +const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; + +const MAX_JOB_SLOTS: u32 = 127; + +/// An enum of possible errors that might cause a piece of work to fail execution. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum WorkError { + /// GPU timeout (command execution took too long). + Timeout, + /// GPU MMU fault (invalid access). + Fault(regs::FaultInfo), + /// Work failed due to an error caused by other concurrent GPU work. + Killed, + /// Channel error + ChannelError(ChannelErrorType), + /// The GPU crashed. + NoDevice, + /// Unknown reason. + Unknown, +} + +impl From for kernel::error::Error { + fn from(err: WorkError) -> Self { + match err { + WorkError::Timeout => ETIMEDOUT, + // Not EFAULT because that's for userspace faults + WorkError::Fault(_) => EIO, + WorkError::Unknown => ENODATA, + WorkError::Killed => ECANCELED, + WorkError::NoDevice => ENODEV, + WorkError::ChannelError(_) => EIO, + } + } +} + +/// A GPU context tracking structure, which must be explicitly invalidated when dropped. +pub(crate) struct GpuContext { + dev: driver::AsahiDevRef, + data: Option>>, +} +no_debug!(GpuContext); + +impl GpuContext { + /// Allocate a new GPU context. + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + buffer: Arc, + ) -> Result { + Ok(GpuContext { + dev: dev.into(), + data: Some(KBox::new( + alloc.shared.new_object( + fw::workqueue::GpuContextData { _buffer: buffer }, + |_inner| Default::default(), + )?, + GFP_KERNEL, + )?), + }) + } + + /// Returns the GPU pointer to the inner GPU context data structure. + pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { + self.data.as_ref().unwrap().gpu_pointer() + } +} + +impl Drop for GpuContext { + fn drop(&mut self) { + mod_dev_dbg!(self.dev, "GpuContext: Freeing GPU context\n"); + let data = self.data.take().unwrap(); + (*self.dev).gpu.free_context(data); + } +} + +struct SubmittedWork +where + O: OpaqueCommandObject, + C: FnOnce(Option) + Send + Sync + 'static, +{ + object: O, + value: EventValue, + error: Option, + wptr: u32, + vm_slot: u32, + callback: Option, + fence: dma_fence::Fence, +} + +pub(crate) trait GenSubmittedWork: Send + Sync { + fn gpu_va(&self) -> NonZeroU64; + fn value(&self) -> event::EventValue; + fn wptr(&self) -> u32; + fn set_wptr(&mut self, wptr: u32); + fn mark_error(&mut self, error: WorkError); + fn complete(&mut self); + fn get_fence(&self) -> dma_fence::Fence; +} + +#[pin_data] +struct SubmittedWorkContainer { + #[pin] + work: Work, + inner: KBox, +} + +impl_has_work! { + impl HasWork for SubmittedWorkContainer { self.work } +} + +impl WorkItem for SubmittedWorkContainer { + type Pointer = Pin>; + + fn run(this: Pin>) { + mod_pr_debug!("WorkQueue: Freeing command @ {:?}\n", this.inner.gpu_va()); + } +} + +impl SubmittedWorkContainer { + fn inner_mut(self: Pin<&mut Self>) -> &mut KBox { + // SAFETY: inner does not require structural pinning. + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl) + Send + Sync> GenSubmittedWork + for SubmittedWork +{ + fn gpu_va(&self) -> NonZeroU64 { + self.object.gpu_va() + } + + fn value(&self) -> event::EventValue { + self.value + } + + fn wptr(&self) -> u32 { + self.wptr + } + + fn set_wptr(&mut self, wptr: u32) { + self.wptr = wptr; + } + + fn complete(&mut self) { + if let Some(cb) = self.callback.take() { + cb(self.error); + } + } + + fn mark_error(&mut self, error: WorkError) { + mod_pr_debug!("WorkQueue: Command at value {:#x?} failed\n", self.value); + self.error = Some(match error { + WorkError::Fault(info) if info.vm_slot != self.vm_slot => WorkError::Killed, + err => err, + }); + } + + fn get_fence(&self) -> dma_fence::Fence { + self.fence.clone() + } +} + +/// Inner data for managing a single work queue. +#[versions(AGX)] +struct WorkQueueInner { + dev: driver::AsahiDevRef, + event_manager: Arc, + info: GpuObject, + new: bool, + pipe_type: PipeType, + size: u32, + wptr: u32, + pending: KVec>>, + last_token: Option, + pending_jobs: usize, + last_submitted: Option, + last_completed: Option, + event: Option<(event::Event, event::EventValue)>, + priority: u32, + commit_seq: u64, + submit_seq: u64, + event_seq: u64, +} + +/// An instance of a work queue. +#[versions(AGX)] +#[pin_data] +pub(crate) struct WorkQueue { + info_pointer: GpuWeakPointer, + #[pin] + inner: Mutex, +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the GPU done pointer, representing how many work items have been completed by the + /// GPU. + fn doneptr(&self) -> u32 { + self.info + .state + .with(|raw, _inner| raw.gpu_doneptr.load(Ordering::Acquire)) + } +} + +#[versions(AGX)] +#[derive(Copy, Clone)] +pub(crate) struct QueueEventInfo { + pub(crate) stamp_pointer: GpuWeakPointer, + pub(crate) fw_stamp_pointer: GpuWeakPointer, + pub(crate) slot: u32, + pub(crate) value: event::EventValue, + pub(crate) cmd_seq: u64, + pub(crate) event_seq: u64, + pub(crate) info_ptr: GpuWeakPointer, +} + +#[versions(AGX)] +pub(crate) struct Job { + wq: Arc, + event_info: QueueEventInfo::ver, + start_value: EventValue, + pending: KVec>>, + committed: bool, + submitted: bool, + event_count: usize, + fence: dma_fence::Fence, +} + +#[versions(AGX)] +pub(crate) struct JobSubmission<'a> { + inner: Option>, + wptr: u32, + event_count: usize, + command_count: usize, +} + +#[versions(AGX)] +impl Job::ver { + pub(crate) fn event_info(&self) -> QueueEventInfo::ver { + let mut info = self.event_info; + info.cmd_seq += self.pending.len() as u64; + info.event_seq += self.event_count as u64; + + info + } + + pub(crate) fn next_seq(&mut self) { + self.event_count += 1; + self.event_info.value.increment(); + } + + pub(crate) fn add( + &mut self, + command: O, + vm_slot: u32, + ) -> Result { + self.add_cb(command, vm_slot, |_| {}) + } + + pub(crate) fn add_cb( + &mut self, + command: O, + vm_slot: u32, + callback: impl FnOnce(Option) + Sync + Send + 'static, + ) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to mutate committed Job\n"); + return Err(EINVAL); + } + + let fence = self.fence.clone(); + let value = self.event_info.value.next(); + + self.pending.push( + KBox::try_pin_init( + try_pin_init!(SubmittedWorkContainer { + work <- new_work!("SubmittedWorkWrapper::work"), + inner: KBox::new(SubmittedWork::<_, _> { + object: command, + value, + error: None, + callback: Some(callback), + wptr: 0, + vm_slot, + fence, + }, GFP_KERNEL)? + }), + GFP_KERNEL, + )?, + GFP_KERNEL, + )?; + + Ok(()) + } + + pub(crate) fn commit(&mut self) -> Result { + if self.committed { + pr_err!("WorkQueue: Tried to commit committed Job\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::commit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + let ev = inner.event.as_mut().expect("WorkQueue: Job lost its event"); + + if ev.1 != self.start_value { + pr_err!( + "WorkQueue: Job::commit() out of order (event slot {} {:?} != {:?}\n", + ev.0.slot(), + ev.1, + self.start_value + ); + return Err(EINVAL); + } + + ev.1 = self.event_info.value; + inner.commit_seq += self.pending.len() as u64; + inner.event_seq += self.event_count as u64; + self.committed = true; + + Ok(()) + } + + pub(crate) fn can_submit(&self) -> Option { + let inner = self.wq.inner.lock(); + if inner.free_slots() > self.event_count && inner.free_space() > self.pending.len() { + None + } else if let Some(work) = inner.pending.first() { + Some(work.inner.get_fence()) + } else { + pr_err!( + "WorkQueue: Cannot submit, but queue is empty? {} > {}, {} > {} (pend={} ls={:#x?} lc={:#x?}) ev={:#x?} cur={:#x?} slot {:?}\n", + inner.free_slots(), + self.event_count, + inner.free_space(), + self.pending.len(), + inner.pending.len(), + inner.last_submitted, + inner.last_completed, + inner.event.as_ref().map(|a| a.1), + inner.event.as_ref().map(|a| a.0.current()), + inner.event.as_ref().map(|a| a.0.slot()), + ); + None + } + } + + pub(crate) fn submit(&mut self) -> Result> { + if !self.committed { + pr_err!("WorkQueue: Tried to submit uncommitted Job\n"); + return Err(EINVAL); + } + + if self.submitted { + pr_err!("WorkQueue: Tried to submit Job twice\n"); + return Err(EINVAL); + } + + if self.pending.is_empty() { + pr_err!("WorkQueue: Job::submit() with no commands\n"); + return Err(EINVAL); + } + + let mut inner = self.wq.inner.lock(); + + if inner.submit_seq != self.event_info.cmd_seq { + pr_err!( + "WorkQueue: Job::submit() out of order (submit_seq {} != {})\n", + inner.submit_seq, + self.event_info.cmd_seq + ); + return Err(EINVAL); + } + + if inner.commit_seq < (self.event_info.cmd_seq + self.pending.len() as u64) { + pr_err!( + "WorkQueue: Job::submit() out of order (commit_seq {} != {})\n", + inner.commit_seq, + (self.event_info.cmd_seq + self.pending.len() as u64) + ); + return Err(EINVAL); + } + + let mut wptr = inner.wptr; + let command_count = self.pending.len(); + + if inner.free_space() <= command_count { + pr_err!("WorkQueue: Job does not fit in ring buffer\n"); + return Err(EBUSY); + } + + inner.pending.reserve(command_count, GFP_KERNEL)?; + + inner.last_submitted = Some(self.event_info.value); + mod_dev_dbg!( + inner.dev, + "WorkQueue: submitting {} cmds at {:#x?}, lc {:#x?}, cur {:#x?}, pending {}, events {}\n", + self.pending.len(), + inner.last_submitted, + inner.last_completed, + inner.event.as_ref().map(|a| a.0.current()), + inner.pending.len(), + self.event_count, + ); + + for mut command in self.pending.drain(..) { + command.as_mut().inner_mut().set_wptr(wptr); + + let next_wptr = (wptr + 1) % inner.size; + assert!(inner.doneptr() != next_wptr); + inner.info.ring[wptr as usize] = command.inner.gpu_va().get(); + wptr = next_wptr; + + // Cannot fail, since we did a reserve(1) above + inner + .pending + .push(command, GFP_KERNEL) + .expect("push() failed after reserve()"); + } + + self.submitted = true; + + Ok(JobSubmission::ver { + inner: Some(inner), + wptr, + command_count, + event_count: self.event_count, + }) + } +} + +#[versions(AGX)] +impl<'a> JobSubmission::ver<'a> { + pub(crate) fn run(mut self, channel: &mut channel::PipeChannel::ver) { + let command_count = self.command_count; + let mut inner = self.inner.take().expect("No inner?"); + let wptr = self.wptr; + core::mem::forget(self); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_wptr.store(wptr, Ordering::Release)); + + inner.wptr = wptr; + + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + + let event_slot = event.0.slot(); + + let msg = fw::channels::RunWorkQueueMsg::ver { + pipe_type: inner.pipe_type, + work_queue: Some(inner.info.weak_pointer()), + wptr: inner.wptr, + event_slot, + is_new: inner.new, + __pad: Default::default(), + }; + channel.send(&msg); + inner.new = false; + + inner.submit_seq += command_count as u64; + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.as_ref().expect("No inner?").pipe_type + } + + pub(crate) fn priority(&self) -> u32 { + self.inner.as_ref().expect("No inner?").priority + } +} + +#[versions(AGX)] +impl Drop for Job::ver { + fn drop(&mut self) { + mod_pr_debug!("WorkQueue: Dropping Job\n"); + let mut inner = self.wq.inner.lock(); + + if !self.committed { + pr_info!( + "WorkQueue: Dropping uncommitted job with {} events\n", + self.event_count + ); + } + + if self.committed && !self.submitted { + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("Job lost its event"); + pr_info!( + "WorkQueue({:?}): Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.pending.len() + ); + event.1.sub(self.event_count as u32); + inner.commit_seq -= self.pending.len() as u64; + inner.event_seq -= self.event_count as u64; + } + + inner.pending_jobs -= 1; + + if inner.pending.is_empty() && inner.pending_jobs == 0 { + mod_pr_debug!("WorkQueue({:?}): Dropping event\n", inner.pipe_type); + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + mod_pr_debug!("WorkQueue({:?}): Dropped Job\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl<'a> Drop for JobSubmission::ver<'a> { + fn drop(&mut self) { + let inner = self.inner.as_mut().expect("No inner?"); + mod_pr_debug!("WorkQueue({:?}): Dropping JobSubmission\n", inner.pipe_type); + + let new_len = inner.pending.len() - self.command_count; + inner.pending.truncate(new_len); + + let pipe_type = inner.pipe_type; + let event = inner.event.as_mut().expect("JobSubmission lost its event"); + pr_info!( + "WorkQueue({:?}): JobSubmission: Roll back {} events (slot {} val {:#x?}) and {} commands\n", + pipe_type, + self.event_count, + event.0.slot(), + event.1, + self.command_count + ); + event.1.sub(self.event_count as u32); + let val = event.1; + inner.commit_seq -= self.command_count as u64; + inner.event_seq -= self.event_count as u64; + inner.last_submitted = Some(val); + mod_pr_debug!("WorkQueue({:?}): Dropped JobSubmission\n", inner.pipe_type); + } +} + +#[versions(AGX)] +impl WorkQueueInner::ver { + /// Return the number of free entries in the workqueue + pub(crate) fn free_space(&self) -> usize { + self.size as usize - self.pending.len() - 1 + } + + pub(crate) fn free_slots(&self) -> usize { + let busy_slots = if let Some(ls) = self.last_submitted { + let lc = self + .last_completed + .expect("last_submitted but not completed?"); + ls.delta(&lc) + } else { + 0 + }; + + ((MAX_JOB_SLOTS as i32) - busy_slots).max(0) as usize + } +} + +#[versions(AGX)] +impl WorkQueue::ver { + /// Create a new WorkQueue of a given type and priority. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + alloc: &mut gpu::KernelAllocators, + event_manager: Arc, + gpu_context: Arc, + notifier_list: Arc>, + pipe_type: PipeType, + id: u64, + priority: u32, + size: u32, + ) -> Result> { + let gpu_buf = alloc.private.array_empty_tagged(0x2c18, b"GPBF")?; + let mut state = alloc.shared.new_default::()?; + let ring = alloc.shared.array_empty(size as usize)?; + let mut prio = *raw::PRIORITY.get(priority as usize).ok_or(EINVAL)?; + + if pipe_type == PipeType::Compute && !debug_enabled(DebugFlags::Debug0) { + // Hack to disable compute preemption until we fix it + prio.0 = 0; + prio.5 = 1; + } + + let inner = WorkQueueInner::ver { + dev: dev.into(), + event_manager, + // Use shared (coherent) state with verbose faults so we can dump state correctly + info: if debug_enabled(DebugFlags::VerboseFaults) { + &mut alloc.shared + } else { + &mut alloc.private + } + .new_init( + try_init!(QueueInfo::ver { + state: { + state.with_mut(|raw, _inner| { + raw.rb_size = size; + }); + state + }, + ring, + gpu_buf, + notifier_list: notifier_list, + gpu_context: gpu_context, + }), + |inner, _p| { + try_init!(raw::QueueInfo::ver { + state: inner.state.gpu_pointer(), + ring: inner.ring.gpu_pointer(), + notifier_list: inner.notifier_list.gpu_pointer(), + gpu_buf: inner.gpu_buf.gpu_pointer(), + gpu_rptr1: Default::default(), + gpu_rptr2: Default::default(), + gpu_rptr3: Default::default(), + event_id: AtomicI32::new(-1), + priority: prio, + unk_4c: -1, + uuid: id as u32, + unk_54: -1, + unk_58: Default::default(), + busy: Default::default(), + __pad: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_84_0: 0, + unk_84_state: Default::default(), + error_count: Default::default(), + unk_8c: 0, + unk_90: 0, + unk_94: 0, + pending: Default::default(), + unk_9c: 0, + gpu_context: inner.gpu_context.gpu_pointer(), + unk_a8: Default::default(), + #[ver(V >= V13_2 && G < G14X)] + unk_b0: 0, + }) + }, + )?, + new: true, + pipe_type, + size, + wptr: 0, + pending: KVec::new(), + last_token: None, + event: None, + priority, + pending_jobs: 0, + commit_seq: 0, + submit_seq: 0, + event_seq: 0, + last_completed: None, + last_submitted: None, + }; + + let info_pointer = inner.info.weak_pointer(); + + Arc::pin_init( + pin_init!(Self { + info_pointer, + inner <- match pipe_type { + PipeType::Vertex => new_mutex!(inner, "WorkQueue::inner (Vertex)"), + PipeType::Fragment => new_mutex!(inner, "WorkQueue::inner (Fragment)"), + PipeType::Compute => new_mutex!(inner, "WorkQueue::inner (Compute)"), + }, + }), + GFP_KERNEL, + ) + } + + pub(crate) fn event_info(&self) -> Option { + let inner = self.inner.lock(); + + inner.event.as_ref().map(|ev| QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, + info_ptr: self.info_pointer, + }) + } + + pub(crate) fn new_job(self: &Arc, fence: dma_fence::Fence) -> Result { + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue({:?}): Grabbing event\n", inner.pipe_type); + let event = inner.event_manager.get(inner.last_token, self.clone())?; + let cur = event.current(); + inner.last_token = Some(event.token()); + mod_pr_debug!( + "WorkQueue({:?}): Grabbed event slot {}: {:#x?}\n", + inner.pipe_type, + event.slot(), + cur + ); + inner.event = Some((event, cur)); + inner.last_submitted = Some(cur); + inner.last_completed = Some(cur); + } + + inner.pending_jobs += 1; + + let ev = &inner.event.as_ref().unwrap(); + + mod_pr_debug!( + "WorkQueue({:?}): New job at value {:#x?} slot {}\n", + inner.pipe_type, + ev.1, + ev.0.slot() + ); + Ok(Job::ver { + wq: self.clone(), + event_info: QueueEventInfo::ver { + stamp_pointer: ev.0.stamp_pointer(), + fw_stamp_pointer: ev.0.fw_stamp_pointer(), + slot: ev.0.slot(), + value: ev.1, + cmd_seq: inner.commit_seq, + event_seq: inner.event_seq, + info_ptr: self.info_pointer, + }, + start_value: ev.1, + pending: KVec::new(), + event_count: 0, + committed: false, + submitted: false, + fence, + }) + } + + pub(crate) fn pipe_type(&self) -> PipeType { + self.inner.lock().pipe_type + } + + pub(crate) fn dump_info(&self) { + pr_info!("WorkQueue @ {:?}:", self.info_pointer); + self.inner.lock().info.with(|raw, _inner| { + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr1.load(Ordering::Relaxed)); + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr2.load(Ordering::Relaxed)); + pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr3.load(Ordering::Relaxed)); + pr_info!(" Event ID: {:#x}", raw.event_id.load(Ordering::Relaxed)); + pr_info!(" Busy: {:#x}", raw.busy.load(Ordering::Relaxed)); + pr_info!(" Unk 84: {:#x}", raw.unk_84_state.load(Ordering::Relaxed)); + pr_info!( + " Error count: {:#x}", + raw.error_count.load(Ordering::Relaxed) + ); + pr_info!(" Pending: {:#x}", raw.pending.load(Ordering::Relaxed)); + }); + } + + pub(crate) fn info_pointer(&self) -> GpuWeakPointer { + self.info_pointer + } +} + +/// Trait used to erase the version-specific type of WorkQueues, to avoid leaking +/// version-specificity into the event module. +pub(crate) trait WorkQueue { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + + fn signal(&self) -> bool; + fn mark_error(&self, value: event::EventValue, error: WorkError); + fn fail_all(&self, error: WorkError); +} + +#[versions(AGX)] +impl WorkQueue for WorkQueue::ver { + fn as_any(&self) -> &dyn Any { + self + } + + /// Signal a workqueue that some work was completed. + /// + /// This will check the event stamp value to find out exactly how many commands were processed. + fn signal(&self) -> bool { + let mut inner = self.inner.lock(); + let event = inner.event.as_ref(); + let value = match event { + None => { + mod_pr_debug!("WorkQueue: signal() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: signal() called with no event and pending jobs.\n"); + } + return true; + } + Some(event) => event.0.current(), + }; + + if let Some(lc) = inner.last_completed { + if value < lc { + pr_err!( + "WorkQueue: event rolled back? cur {:#x?}, lc {:#x?}, ls {:#x?}", + value, + inner.last_completed, + inner.last_submitted + ); + } + } else { + pr_crit!("WorkQueue: signal() called with no last_completed.\n"); + } + inner.last_completed = Some(value); + + mod_pr_debug!( + "WorkQueue({:?}): Signaling event {:?} value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + let mut completed_commands: usize = 0; + + for cmd in inner.pending.iter() { + if cmd.inner.value() <= value { + mod_pr_debug!( + "WorkQueue({:?}): Command at value {:#x?} complete\n", + inner.pipe_type, + cmd.inner.value() + ); + completed_commands += 1; + } else { + break; + } + } + + if completed_commands == 0 { + return inner.pending.is_empty(); + } + + let last_wptr = inner.pending[completed_commands - 1].inner.wptr(); + let pipe_type = inner.pipe_type; + + for mut cmd in inner.pending.drain(..completed_commands) { + mod_pr_debug!( + "WorkQueue({:?}): Queueing command @ {:?} for cleanup\n", + pipe_type, + cmd.inner.gpu_va() + ); + cmd.as_mut().inner_mut().complete(); + workqueue::system().enqueue(cmd); + } + + mod_pr_debug!( + "WorkQueue({:?}): Completed {} commands, left pending {}, ls {:#x?}, lc {:#x?}\n", + inner.pipe_type, + completed_commands, + inner.pending.len(), + inner.last_submitted, + inner.last_completed, + ); + + inner + .info + .state + .with(|raw, _inner| raw.cpu_freeptr.store(last_wptr, Ordering::Release)); + + let empty = inner.pending.is_empty(); + if empty && inner.pending_jobs == 0 { + inner.event = None; + inner.last_submitted = None; + inner.last_completed = None; + } + + empty + } + + /// Mark this queue's work up to a certain stamp value as having failed. + fn mark_error(&self, value: event::EventValue, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue: signal_fault() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: signal_fault() called with no event and pending jobs.\n"); + } + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Signaling fault for event {:?} at value {:#x?}\n", + inner.pipe_type, + inner.last_token, + value + ); + + for cmd in inner.pending.iter_mut() { + if cmd.inner.value() <= value { + cmd.as_mut().inner_mut().mark_error(error); + } else { + break; + } + } + } + + /// Mark all of this queue's work as having failed, and complete it. + fn fail_all(&self, error: WorkError) { + // If anything is marked completed, we can consider it successful + // at this point, even if we didn't get the signal event yet. + self.signal(); + + let mut inner = self.inner.lock(); + + if inner.event.is_none() { + mod_pr_debug!("WorkQueue: fail_all() called but no event?\n"); + + if inner.pending_jobs > 0 || !inner.pending.is_empty() { + pr_crit!("WorkQueue: fail_all() called with no event and pending jobs.\n"); + } + return; + } + + mod_pr_debug!( + "WorkQueue({:?}): Failing all jobs {:?}\n", + inner.pipe_type, + error + ); + + let mut cmds = KVec::new(); + + core::mem::swap(&mut inner.pending, &mut cmds); + + if inner.pending_jobs == 0 { + inner.event = None; + } + + core::mem::drop(inner); + + for mut cmd in cmds { + cmd.as_mut().inner_mut().mark_error(error); + cmd.as_mut().inner_mut().complete(); + } + } +} diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c index 9392c226ff5b11..c7cd0234d16861 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c @@ -740,7 +740,7 @@ static void cdns_mhdp_fw_cb(const struct firmware *fw, void *context) bridge_attached = mhdp->bridge_attached; spin_unlock(&mhdp->start_lock); if (bridge_attached) { - if (mhdp->connector.dev) + if (mhdp->connector_ptr) drm_kms_helper_hotplug_event(mhdp->bridge.dev); else drm_bridge_hpd_notify(&mhdp->bridge, cdns_mhdp_detect(mhdp)); @@ -1636,6 +1636,7 @@ static int cdns_mhdp_connector_init(struct cdns_mhdp_device *mhdp) return ret; } + mhdp->connector_ptr = conn; drm_connector_helper_add(conn, &cdns_mhdp_conn_helper_funcs); ret = drm_display_info_set_bus_formats(&conn->display_info, @@ -1915,17 +1916,25 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge, struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge); struct cdns_mhdp_bridge_state *mhdp_state; struct drm_crtc_state *crtc_state; - struct drm_connector *connector; struct drm_connector_state *conn_state; struct drm_bridge_state *new_state; const struct drm_display_mode *mode; u32 resp; - int ret; + int ret = 0; dev_dbg(mhdp->dev, "bridge enable\n"); mutex_lock(&mhdp->link_mutex); + mhdp->connector_ptr = drm_atomic_get_new_connector_for_encoder(state, + bridge->encoder); + if (WARN_ON(!mhdp->connector_ptr)) + goto out; + + conn_state = drm_atomic_get_new_connector_state(state, mhdp->connector_ptr); + if (WARN_ON(!conn_state)) + goto out; + if (mhdp->plugged && !mhdp->link_up) { ret = cdns_mhdp_link_up(mhdp); if (ret < 0) @@ -1945,15 +1954,6 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge, cdns_mhdp_reg_write(mhdp, CDNS_DPTX_CAR, resp | CDNS_VIF_CLK_EN | CDNS_VIF_CLK_RSTN); - connector = drm_atomic_get_new_connector_for_encoder(state, - bridge->encoder); - if (WARN_ON(!connector)) - goto out; - - conn_state = drm_atomic_get_new_connector_state(state, connector); - if (WARN_ON(!conn_state)) - goto out; - if (mhdp->hdcp_supported && mhdp->hw_state == MHDP_HW_READY && conn_state->content_protection == @@ -2030,6 +2030,7 @@ static void cdns_mhdp_atomic_disable(struct drm_bridge *bridge, if (mhdp->info && mhdp->info->ops && mhdp->info->ops->disable) mhdp->info->ops->disable(mhdp); + mhdp->connector_ptr = NULL; mutex_unlock(&mhdp->link_mutex); } @@ -2122,6 +2123,10 @@ static int cdns_mhdp_atomic_check(struct drm_bridge *bridge, { struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge); const struct drm_display_mode *mode = &crtc_state->adjusted_mode; + struct drm_connector_state *old_state, *new_state; + struct drm_atomic_state *state = crtc_state->state; + struct drm_connector *conn = mhdp->connector_ptr; + u64 old_cp, new_cp; mutex_lock(&mhdp->link_mutex); @@ -2141,6 +2146,25 @@ static int cdns_mhdp_atomic_check(struct drm_bridge *bridge, if (mhdp->info) bridge_state->input_bus_cfg.flags = *mhdp->info->input_bus_flags; + if (conn && mhdp->hdcp_supported) { + old_state = drm_atomic_get_old_connector_state(state, conn); + new_state = drm_atomic_get_new_connector_state(state, conn); + old_cp = old_state->content_protection; + new_cp = new_state->content_protection; + + if (old_state->hdcp_content_type != new_state->hdcp_content_type && + new_cp != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { + new_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; + crtc_state = drm_atomic_get_new_crtc_state(state, new_state->crtc); + crtc_state->mode_changed = true; + } + + if (!new_state->crtc) { + if (old_cp == DRM_MODE_CONTENT_PROTECTION_ENABLED) + new_state->content_protection = DRM_MODE_CONTENT_PROTECTION_DESIRED; + } + } + mutex_unlock(&mhdp->link_mutex); return 0; } @@ -2161,6 +2185,25 @@ static const struct drm_edid *cdns_mhdp_bridge_edid_read(struct drm_bridge *brid return cdns_mhdp_edid_read(mhdp, connector); } +static enum drm_mode_status +cdns_mhdp_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) +{ + struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge); + + mutex_lock(&mhdp->link_mutex); + + if (!cdns_mhdp_bandwidth_ok(mhdp, mode, mhdp->link.num_lanes, + mhdp->link.rate)) { + mutex_unlock(&mhdp->link_mutex); + return MODE_CLOCK_HIGH; + } + + mutex_unlock(&mhdp->link_mutex); + return MODE_OK; +} + static const struct drm_bridge_funcs cdns_mhdp_bridge_funcs = { .atomic_enable = cdns_mhdp_atomic_enable, .atomic_disable = cdns_mhdp_atomic_disable, @@ -2175,6 +2218,7 @@ static const struct drm_bridge_funcs cdns_mhdp_bridge_funcs = { .edid_read = cdns_mhdp_bridge_edid_read, .hpd_enable = cdns_mhdp_bridge_hpd_enable, .hpd_disable = cdns_mhdp_bridge_hpd_disable, + .mode_valid = cdns_mhdp_bridge_mode_valid, }; static bool cdns_mhdp_detect_hpd(struct cdns_mhdp_device *mhdp, bool *hpd_pulse) @@ -2296,7 +2340,7 @@ static void cdns_mhdp_modeset_retry_fn(struct work_struct *work) mhdp = container_of(work, typeof(*mhdp), modeset_retry_work); - conn = &mhdp->connector; + conn = mhdp->connector_ptr; /* Grab the locks before changing connector property */ mutex_lock(&conn->dev->mode_config.mutex); @@ -2373,7 +2417,7 @@ static void cdns_mhdp_hpd_work(struct work_struct *work) int ret; ret = cdns_mhdp_update_link_status(mhdp); - if (mhdp->connector.dev) { + if (mhdp->connector_ptr) { if (ret < 0) schedule_work(&mhdp->modeset_retry_work); else diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h index bad2fc0c730660..a76775c7689568 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.h @@ -376,6 +376,7 @@ struct cdns_mhdp_device { struct mutex link_mutex; struct drm_connector connector; + struct drm_connector *connector_ptr; struct drm_bridge bridge; struct cdns_mhdp_link link; diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c index 42248f179b69de..21a7d2fb266e49 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-hdcp.c @@ -394,7 +394,7 @@ static int _cdns_mhdp_hdcp_disable(struct cdns_mhdp_device *mhdp) int ret; dev_dbg(mhdp->dev, "[%s:%d] HDCP is being disabled...\n", - mhdp->connector.name, mhdp->connector.base.id); + mhdp->connector_ptr->name, mhdp->connector_ptr->base.id); ret = cdns_mhdp_hdcp_set_config(mhdp, 0, false); @@ -436,6 +436,10 @@ static int cdns_mhdp_hdcp_check_link(struct cdns_mhdp_device *mhdp) int ret = 0; mutex_lock(&mhdp->hdcp.mutex); + + if (!mhdp->connector_ptr) + goto out; + if (mhdp->hdcp.value == DRM_MODE_CONTENT_PROTECTION_UNDESIRED) goto out; @@ -445,7 +449,7 @@ static int cdns_mhdp_hdcp_check_link(struct cdns_mhdp_device *mhdp) dev_err(mhdp->dev, "[%s:%d] HDCP link failed, retrying authentication\n", - mhdp->connector.name, mhdp->connector.base.id); + mhdp->connector_ptr->name, mhdp->connector_ptr->base.id); ret = _cdns_mhdp_hdcp_disable(mhdp); if (ret) { @@ -487,13 +491,19 @@ static void cdns_mhdp_hdcp_prop_work(struct work_struct *work) struct cdns_mhdp_device *mhdp = container_of(hdcp, struct cdns_mhdp_device, hdcp); - struct drm_device *dev = mhdp->connector.dev; + struct drm_device *dev = NULL; struct drm_connector_state *state; + if (mhdp->connector_ptr) + dev = mhdp->connector_ptr->dev; + + if (!dev) + return; + drm_modeset_lock(&dev->mode_config.connection_mutex, NULL); mutex_lock(&mhdp->hdcp.mutex); if (mhdp->hdcp.value != DRM_MODE_CONTENT_PROTECTION_UNDESIRED) { - state = mhdp->connector.state; + state = mhdp->connector_ptr->state; state->content_protection = mhdp->hdcp.value; } mutex_unlock(&mhdp->hdcp.mutex); diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c index 441fd32dc91c7a..d64e328bf542f3 100644 --- a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c +++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c @@ -222,52 +222,58 @@ static const struct drm_bridge_funcs imx8qxp_pxl2dpi_bridge_funcs = { imx8qxp_pxl2dpi_bridge_atomic_get_output_bus_fmts, }; -static struct device_node * +static int imx8qxp_pxl2dpi_get_available_ep_from_port(struct imx8qxp_pxl2dpi *p2d, - u32 port_id) + u32 port_id, + struct device_node **ep) { - struct device_node *port, *ep; + struct device_node *port; + int ret = 0; int ep_cnt; + *ep = NULL; + port = of_graph_get_port_by_id(p2d->dev->of_node, port_id); if (!port) { DRM_DEV_ERROR(p2d->dev, "failed to get port@%u\n", port_id); - return ERR_PTR(-ENODEV); + return -ENODEV; } ep_cnt = of_get_available_child_count(port); if (ep_cnt == 0) { DRM_DEV_ERROR(p2d->dev, "no available endpoints of port@%u\n", port_id); - ep = ERR_PTR(-ENODEV); + ret = -ENODEV; goto out; } else if (ep_cnt > 1) { DRM_DEV_ERROR(p2d->dev, "invalid available endpoints of port@%u\n", port_id); - ep = ERR_PTR(-EINVAL); + ret = -EINVAL; goto out; } - ep = of_get_next_available_child(port, NULL); - if (!ep) { + *ep = of_get_next_available_child(port, NULL); + if (!*ep) { DRM_DEV_ERROR(p2d->dev, "failed to get available endpoint of port@%u\n", port_id); - ep = ERR_PTR(-ENODEV); + ret = -ENODEV; goto out; } out: of_node_put(port); - return ep; + return ret; } static int imx8qxp_pxl2dpi_find_next_bridge(struct imx8qxp_pxl2dpi *p2d) { - struct device_node *ep __free(device_node) = - imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 1); - if (IS_ERR(ep)) - return PTR_ERR(ep); + struct device_node *ep __free(device_node) = NULL; + int ret; + + ret = imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 1, &ep); + if (ret) + return ret; struct device_node *remote __free(device_node) = of_graph_get_remote_port_parent(ep); if (!remote || !of_device_is_available(remote)) { @@ -291,9 +297,9 @@ static int imx8qxp_pxl2dpi_set_pixel_link_sel(struct imx8qxp_pxl2dpi *p2d) struct of_endpoint endpoint; int ret; - ep = imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 0); - if (IS_ERR(ep)) - return PTR_ERR(ep); + ret = imx8qxp_pxl2dpi_get_available_ep_from_port(p2d, 0, &ep); + if (ret) + return ret; ret = of_graph_parse_endpoint(ep, &endpoint); if (ret) { diff --git a/drivers/gpu/drm/bridge/tda998x_drv.c b/drivers/gpu/drm/bridge/tda998x_drv.c index e636459d91857a..f90b08869267a6 100644 --- a/drivers/gpu/drm/bridge/tda998x_drv.c +++ b/drivers/gpu/drm/bridge/tda998x_drv.c @@ -1697,7 +1697,7 @@ static const struct drm_bridge_funcs tda998x_bridge_funcs = { static int tda998x_get_audio_ports(struct tda998x_priv *priv, struct device_node *np) { - const u32 *port_data; + const __be32 *port_data; u32 size; int i; diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index f2cd2e25f00980..ec7534227f66d4 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -1569,6 +1569,7 @@ drm_atomic_add_affected_planes(struct drm_atomic_state *state, const struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc); struct drm_plane *plane; + int ret; WARN_ON(!drm_atomic_get_new_crtc_state(state, crtc)); @@ -1582,6 +1583,12 @@ drm_atomic_add_affected_planes(struct drm_atomic_state *state, if (IS_ERR(plane_state)) return PTR_ERR(plane_state); + + if (plane_state->color_pipeline) { + ret = drm_atomic_add_affected_colorops(state, plane); + if (ret) + return ret; + } } return 0; } diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c index c598b99673fc15..e7db4e4ea700fe 100644 --- a/drivers/gpu/drm/drm_color_mgmt.c +++ b/drivers/gpu/drm/drm_color_mgmt.c @@ -831,7 +831,7 @@ static void fill_palette_332(struct drm_crtc *crtc, u16 r, u16 g, u16 b, } /** - * drm_crtc_fill_palette_332 - Programs a default palette for R332-like formats + * drm_crtc_fill_palette_332 - Programs a default palette for RGB332-like formats * @crtc: The displaying CRTC * @set_palette: Callback for programming the hardware gamma LUT * diff --git a/drivers/gpu/drm/drm_colorop.c b/drivers/gpu/drm/drm_colorop.c index 398cc81ae5880f..27139862b12086 100644 --- a/drivers/gpu/drm/drm_colorop.c +++ b/drivers/gpu/drm/drm_colorop.c @@ -169,12 +169,8 @@ void drm_colorop_cleanup(struct drm_colorop *colorop) list_del(&colorop->head); config->num_colorop--; - if (colorop->state && colorop->state->data) { - drm_property_blob_put(colorop->state->data); - colorop->state->data = NULL; - } - - kfree(colorop->state); + if (colorop->state) + drm_colorop_atomic_destroy_state(colorop, colorop->state); } EXPORT_SYMBOL(drm_colorop_cleanup); @@ -441,8 +437,6 @@ static void __drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colo if (state->data) drm_property_blob_get(state->data); - - state->bypass = true; } struct drm_colorop_state * @@ -460,9 +454,23 @@ drm_atomic_helper_colorop_duplicate_state(struct drm_colorop *colorop) return state; } +/** + * __drm_atomic_helper_colorop_destroy_state - release colorop state + * @state: colorop state object to release + * + * Releases all resources stored in the colorop state without actually freeing + * the memory of the colorop state. This is useful for drivers that subclass the + * colorop state. + */ +static void __drm_atomic_helper_colorop_destroy_state(struct drm_colorop_state *state) +{ + drm_property_blob_put(state->data); +} + void drm_colorop_atomic_destroy_state(struct drm_colorop *colorop, struct drm_colorop_state *state) { + __drm_atomic_helper_colorop_destroy_state(state); kfree(state); } @@ -513,7 +521,9 @@ static void __drm_colorop_reset(struct drm_colorop *colorop, void drm_colorop_reset(struct drm_colorop *colorop) { - kfree(colorop->state); + if (colorop->state) + drm_colorop_atomic_destroy_state(colorop, colorop->state); + colorop->state = kzalloc_obj(*colorop->state); if (colorop->state) diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 05803169bed571..16bfbfb0af1615 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -1641,8 +1641,10 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper) drm_client_modeset_probe(&fb_helper->client, width, height); info = drm_fb_helper_alloc_info(fb_helper); - if (IS_ERR(info)) + if (IS_ERR(info)) { + mutex_unlock(&fb_helper->lock); return PTR_ERR(info); + } ret = drm_fb_helper_single_fb_probe(fb_helper); if (ret < 0) { diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 891c3bff5ae009..2b152e3103c325 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1001,7 +1001,7 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_gem_change_handle *args = data; - struct drm_gem_object *obj; + struct drm_gem_object *obj, *idrobj; int handle, ret; if (!drm_core_check_feature(dev, DRIVER_GEM)) @@ -1024,12 +1024,30 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, mutex_lock(&file_priv->prime.lock); spin_lock(&file_priv->table_lock); + + /* When create_tail allocs an obj idr, it needs to first alloc as NULL, + * then later replace with the correct object. This is not necessary + * here, because the only operations that could race are drm_prime + * bookkeeping, and we hold the prime lock. + */ ret = idr_alloc(&file_priv->object_idr, obj, handle, handle + 1, GFP_NOWAIT); - spin_unlock(&file_priv->table_lock); - if (ret < 0) - goto out_unlock; + if (ret < 0) { + spin_unlock(&file_priv->table_lock); + goto out_unlock; + } + + idrobj = idr_replace(&file_priv->object_idr, NULL, handle); + if (idrobj != obj) { + idr_replace(&file_priv->object_idr, idrobj, handle); + idr_remove(&file_priv->object_idr, args->new_handle); + spin_unlock(&file_priv->table_lock); + ret = -ENOENT; + goto out_unlock; + } + + spin_unlock(&file_priv->table_lock); if (obj->dma_buf) { ret = drm_prime_add_buf_handle(&file_priv->prime, obj->dma_buf, @@ -1048,7 +1066,9 @@ int drm_gem_change_handle_ioctl(struct drm_device *dev, void *data, spin_lock(&file_priv->table_lock); idr_remove(&file_priv->object_idr, args->handle); + idrobj = idr_replace(&file_priv->object_idr, obj, handle); spin_unlock(&file_priv->table_lock); + WARN_ON(idrobj != NULL); out_unlock: mutex_unlock(&file_priv->prime.lock); diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c index 9166c353f131cb..88808e972cc156 100644 --- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c +++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c @@ -172,8 +172,8 @@ int drm_gem_fb_init_with_funcs(struct drm_device *dev, } for (i = 0; i < info->num_planes; i++) { - unsigned int width = mode_cmd->width / (i ? info->hsub : 1); - unsigned int height = mode_cmd->height / (i ? info->vsub : 1); + unsigned int width = drm_format_info_plane_width(info, mode_cmd->width, i); + unsigned int height = drm_format_info_plane_height(info, mode_cmd->height, i); unsigned int min_size; objs[i] = drm_gem_object_lookup(file, mode_cmd->handles[i]); diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 9ef9e52c054739..35dd07297dd084 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -819,7 +819,7 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, if (!(pfns[i] & HMM_PFN_VALID)) { state = DRM_GPUSVM_SCAN_UNPOPULATED; - goto err_free; + break; } page = hmm_pfn_to_page(pfns[i]); @@ -856,9 +856,9 @@ enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); } -err_free: drm_gpusvm_notifier_unlock(range->gpusvm); +err_free: kvfree(pfns); return state; } @@ -1495,7 +1495,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, } zdd = page->zone_device_data; if (pagemap != page_pgmap(page)) { - if (i > 0) { + if (pagemap) { err = -EOPNOTSUPP; goto err_unmap; } @@ -1572,6 +1572,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, return 0; err_unmap: + svm_pages->flags.has_dma_mapping = true; __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped); drm_gpusvm_notifier_unlock(gpusvm); err_free: diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 44acfe4120d243..d262562f496469 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -1862,7 +1862,7 @@ drm_gpuvm_bo_obtain_locked(struct drm_gpuvm *gpuvm, * and this method allocates. Immediate mode drivers should use * drm_gpuvm_bo_obtain_prealloc() instead. */ - drm_WARN_ON(gpuvm->drm, drm_gpuvm_immediate_mode(gpuvm)); + // drm_WARN_ON(gpuvm->drm, drm_gpuvm_immediate_mode(gpuvm)); vm_bo = drm_gpuvm_bo_find(gpuvm, obj); if (vm_bo) @@ -2362,6 +2362,7 @@ op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset; + op.map.flags = req->map.flags; return fn->sm_step_map(&op, priv); } @@ -2400,6 +2401,97 @@ op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, return fn->sm_step_unmap(&op, priv); } +static bool can_merge_flags(struct drm_gpuvm *gpuvm, enum drm_gpuva_flags a, + enum drm_gpuva_flags b) +{ + if (gpuvm->ops->sm_can_merge_flags) + return gpuvm->ops->sm_can_merge_flags(a, b); + return a == b; +} + +static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map *a, + const struct drm_gpuva_op_map *b) +{ + /* Only GEM-based mappings can be merged, and they must point to + * the same GEM object. + */ + if (a->gem.obj != b->gem.obj || !a->gem.obj) + return false; + + if (can_merge_flags(gpuvm, a->flags, b->flags)) + return false; + + /* Order VAs for the rest of the checks. */ + if (a->va.addr > b->va.addr) + swap(a, b); + + /* We assume the caller already checked that VAs overlap or are + * contiguous. + */ + if (drm_WARN_ON(gpuvm->drm, b->va.addr > a->va.addr + a->va.range)) + return false; + + if (a->flags & DRM_GPUVA_REPEAT) { + u64 va_diff = b->va.addr - a->va.addr; + + /* If this is a repeated mapping, both the GEM range + * and offset must match. + */ + if (a->gem.range != b->gem.range || + a->gem.offset != b->gem.offset) + return false; + + /* The difference between the VA addresses must be a + * multiple of the repeated range, otherwise there's + * a shift. + */ + if (do_div(va_diff, a->gem.range)) + return false; + + return true; + } + + /* We intentionally ignore u64 underflows because all we care about + * here is whether the VA diff matches the GEM offset diff. + */ + return b->va.addr - a->va.addr == b->gem.offset - a->gem.offset; +} + +static bool can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva *a, + const struct drm_gpuva_op_map *b) +{ + struct drm_gpuva_op_map tmp = { + .va.addr = a->va.addr, + .va.range = a->va.range, + .gem.offset = a->gem.offset, + .gem.obj = a->gem.obj, + .flags = a->flags, + }; + + return __can_merge(gpuvm, &tmp, b); +} + +static int validate_map_request(struct drm_gpuvm *gpuvm, + const struct drm_gpuva_op_map *req) +{ + if (unlikely(!drm_gpuvm_range_valid(gpuvm, req->va.addr, req->va.range))) + return -EINVAL; + + if (req->flags & DRM_GPUVA_REPEAT) { + u64 va_range = req->va.range; + + /* For a repeated mapping, GEM range must be > 0 + * and a multiple of the VA range. + */ + if (unlikely(!req->gem.range || + va_range < req->gem.range || + do_div(va_range, req->gem.range))) + return -EINVAL; + } + + return 0; +} + static int __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, const struct drm_gpuvm_ops *ops, void *priv, @@ -2415,7 +2507,8 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, u64 req_end = req_addr + req_range; int ret; - if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) + ret = validate_map_request(gpuvm, &req->map); + if (unlikely(ret)) return -EINVAL; drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { @@ -2424,7 +2517,7 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, u64 addr = va->va.addr; u64 range = va->va.range; u64 end = addr + range; - bool merge = !!va->gem.obj; + bool merge = can_merge(gpuvm, va, &req->map); if (madvise && obj) continue; @@ -2452,7 +2545,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = range - req_range, .gem.obj = obj, - .gem.offset = offset + req_range, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_range), + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va, @@ -2473,7 +2569,9 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = addr, .va.range = ls_range, .gem.obj = obj, + .gem.range = va->gem.range, .gem.offset = offset, + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va }; @@ -2515,8 +2613,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, - .gem.offset = offset + ls_range + - req_range, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : ls_range + req_range), + .flags = va->flags, }; ret = op_remap_cb(ops, priv, &p, &n, &u); @@ -2553,7 +2653,10 @@ __drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, .va.addr = req_end, .va.range = end - req_end, .gem.obj = obj, - .gem.offset = offset + req_end - addr, + .gem.range = va->gem.range, + .gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr), + .flags = va->flags, }; struct drm_gpuva_op_unmap u = { .va = va, @@ -2604,7 +2707,9 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, prev.va.addr = addr; prev.va.range = req_addr - addr; prev.gem.obj = obj; + prev.gem.range = va->gem.range; prev.gem.offset = offset; + prev.flags = va->flags; prev_split = true; } @@ -2613,7 +2718,10 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; - next.gem.offset = offset + (req_end - addr); + prev.gem.range = va->gem.range; + next.gem.offset = offset + + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr); + next.flags = va->flags; next_split = true; } @@ -3156,6 +3264,55 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, } EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); +/** + * drm_gpuvm_bo_unmap() - unmaps a GEM + * @vm_bo: the &drm_gpuvm_bo abstraction + * + * This function calls the unmap callback for every GPUVA attached to a GEM. + * + * It is the callers responsibility to protect the GEMs GPUVA list against + * concurrent access using the GEMs dma_resv lock. + * + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ +int +drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *vm_bo, void *priv) +{ + struct drm_gpuva_ops *ops; + struct drm_gpuva_op *op; + int ret; + + if (unlikely(!vm_bo->vm)) + return -EINVAL; + + const struct drm_gpuvm_ops *vm_ops = vm_bo->vm->ops; + + if (unlikely(!(vm_ops && vm_ops->sm_step_unmap))) + return -EINVAL; + + if (drm_gpuvm_immediate_mode(vm_bo->vm)) { + guard(mutex)(&vm_bo->obj->gpuva.lock); + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + } else { + ops = drm_gpuvm_bo_unmap_ops_create(vm_bo); + } + if (IS_ERR(ops)) + return PTR_ERR(ops); + + drm_gpuva_for_each_op(op, ops) { + drm_WARN_ON(vm_bo->vm->drm, op->op != DRM_GPUVA_OP_UNMAP); + + ret = op_unmap_cb(vm_ops, priv, op->unmap.va, false, false); + if (ret) + goto cleanup; + } + +cleanup: + drm_gpuva_ops_free(vm_bo->vm, ops); + return ret; +} +EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap); + /** * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM * @vm_bo: the &drm_gpuvm_bo abstraction diff --git a/drivers/gpu/drm/exynos/exynos_drm_mic.c b/drivers/gpu/drm/exynos/exynos_drm_mic.c index 29a8366513fa70..e68c954ec3e614 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_mic.c +++ b/drivers/gpu/drm/exynos/exynos_drm_mic.c @@ -423,7 +423,9 @@ static int exynos_mic_probe(struct platform_device *pdev) mic->bridge.of_node = dev->of_node; - drm_bridge_add(&mic->bridge); + ret = devm_drm_bridge_add(dev, &mic->bridge); + if (ret) + goto err; pm_runtime_enable(dev); @@ -443,12 +445,8 @@ static int exynos_mic_probe(struct platform_device *pdev) static void exynos_mic_remove(struct platform_device *pdev) { - struct exynos_mic *mic = platform_get_drvdata(pdev); - component_del(&pdev->dev, &exynos_mic_component_ops); pm_runtime_disable(&pdev->dev); - - drm_bridge_remove(&mic->bridge); } static const struct of_device_id exynos_mic_of_match[] = { diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c index 58d7e191fd56f8..403d21cbb3a230 100644 --- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c +++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c @@ -580,6 +580,7 @@ static int oaktrail_hdmi_get_modes(struct drm_connector *connector) } else { edid = (struct edid *)raw_edid; /* FIXME ? edid = drm_get_edid(connector, i2c_adap); */ + i2c_put_adapter(i2c_adap); } if (edid) { diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c index 884d324f004408..e194d0cce0671a 100644 --- a/drivers/gpu/drm/gma500/oaktrail_lvds.c +++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c @@ -293,7 +293,7 @@ void oaktrail_lvds_init(struct drm_device *dev, { struct gma_encoder *gma_encoder; struct gma_connector *gma_connector; - struct gma_i2c_chan *ddc_bus; + struct gma_i2c_chan *ddc_bus = NULL; struct drm_connector *connector; struct drm_encoder *encoder; struct drm_psb_private *dev_priv = to_drm_psb_private(dev); @@ -367,6 +367,8 @@ void oaktrail_lvds_init(struct drm_device *dev, if (edid == NULL && dev_priv->lpc_gpio_base) { ddc_bus = oaktrail_lvds_i2c_init(dev); if (!IS_ERR(ddc_bus)) { + if (i2c_adap) + i2c_put_adapter(i2c_adap); i2c_adap = &ddc_bus->base; edid = drm_get_edid(connector, i2c_adap); } @@ -421,7 +423,10 @@ void oaktrail_lvds_init(struct drm_device *dev, err_unlock: mutex_unlock(&dev->mode_config.mutex); - gma_i2c_destroy(to_gma_i2c_chan(connector->ddc)); + if (!IS_ERR_OR_NULL(ddc_bus)) + gma_i2c_destroy(ddc_bus); + else if (i2c_adap) + i2c_put_adapter(i2c_adap); drm_encoder_cleanup(encoder); err_connector_cleanup: drm_connector_cleanup(connector); diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 696edf40b24361..2906dc6e630ecc 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -2981,8 +2981,13 @@ static void intel_dp_compute_vsc_colorimetry(const struct intel_crtc_state *crtc drm_WARN_ON(display->drm, vsc->bpc == 6 && vsc->pixelformat != DP_PIXELFORMAT_RGB); - /* all YCbCr are always limited range */ - vsc->dynamic_range = DP_DYNAMIC_RANGE_CTA; + /* All YCbCr formats are always limited range. */ + if (vsc->pixelformat == DP_PIXELFORMAT_RGB) + vsc->dynamic_range = crtc_state->limited_color_range ? + DP_DYNAMIC_RANGE_CTA : DP_DYNAMIC_RANGE_VESA; + else + vsc->dynamic_range = DP_DYNAMIC_RANGE_CTA; + vsc->content_type = DP_CONTENT_TYPE_NOT_DEFINED; } diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 097e18c1adb279..8a7075c4a2480f 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -2970,7 +2970,7 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state, return ret; do { - bool cursor_in_su_area; + bool cursor_in_su_area = false; /* * Adjust su area to cover cursor fully as necessary diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c index f5a6fae815d1df..fa51f976ea725d 100644 --- a/drivers/gpu/drm/i915/display/skl_watermark.c +++ b/drivers/gpu/drm/i915/display/skl_watermark.c @@ -3980,8 +3980,8 @@ void intel_wm_state_verify(struct intel_atomic_state *state, } /* DDB */ - hw_ddb_entry = &hw->ddb[PLANE_CURSOR]; - sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR]; + hw_ddb_entry = &hw->ddb[plane->id]; + sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[plane->id]; if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) { drm_err(display->drm, diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 41b5036dc53858..64ef68e6ac625b 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -132,7 +132,8 @@ void __i915_request_reset(struct i915_request *rq, bool guilty) rcu_read_lock(); /* protect the GEM context */ if (guilty) { i915_request_set_error_once(rq, -EIO); - __i915_request_skip(rq); + if (!i915_request_signaled(rq)) + __i915_request_skip(rq); banned = mark_guilty(rq); } else { i915_request_set_error_once(rq, -EAGAIN); diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 160733619a4ace..3cfa1344b1b2ac 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -520,7 +520,7 @@ ggtt_write(struct io_mapping *mapping, /* We can use the cpu mem copy function because this is X86. */ vaddr = io_mapping_map_atomic_wc(mapping, base); - unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset, + unwritten = copy_from_user_inatomic_nontemporal((void __force *)vaddr + offset, user_data, length); io_mapping_unmap_atomic(vaddr); if (unwritten) { diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c index e154cb35f604dc..6193811ef7bebb 100644 --- a/drivers/gpu/drm/imagination/pvr_fw_trace.c +++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c @@ -558,6 +558,6 @@ pvr_fw_trace_debugfs_init(struct pvr_device *pvr_dev, struct dentry *dir) &pvr_fw_trace_fops); } - debugfs_create_file("trace_mask", 0600, dir, fw_trace, + debugfs_create_file("trace_mask", 0600, dir, pvr_dev, &pvr_fw_trace_mask_fops); } diff --git a/drivers/gpu/drm/imagination/pvr_rogue_fwif.h b/drivers/gpu/drm/imagination/pvr_rogue_fwif.h index 172886be4c820a..5d590c4c25663e 100644 --- a/drivers/gpu/drm/imagination/pvr_rogue_fwif.h +++ b/drivers/gpu/drm/imagination/pvr_rogue_fwif.h @@ -1347,8 +1347,12 @@ struct rogue_fwif_fwccb_cmd_freelists_reconstruction_data { struct rogue_fwif_fwccb_cmd_context_reset_data { /* Context affected by the reset */ u32 server_common_context_id; - /* Reason for reset */ - enum rogue_context_reset_reason reset_reason; + /* + * Reason for reset + * The valid values for reset_reason are the ones from + * enum rogue_context_reset_reason + */ + u32 reset_reason; /* Data Master affected by the reset */ u32 dm; /* Job ref running at the time of reset */ diff --git a/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h b/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h index 6c09c15bf9bd8d..f95acd5a1f8e80 100644 --- a/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h +++ b/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h @@ -249,7 +249,11 @@ enum rogue_context_reset_reason { }; struct rogue_context_reset_reason_data { - enum rogue_context_reset_reason reset_reason; + /* + * The valid values for reset_reason are the ones from + * enum rogue_context_reset_reason + */ + u32 reset_reason; u32 reset_ext_job_ref; }; diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c index 4ce772bc3cb3a1..a356f0b764cb79 100644 --- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c +++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c @@ -110,8 +110,7 @@ imx_pd_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge, output_fmt = imxpd->bus_format ? : MEDIA_BUS_FMT_RGB888_1X24; /* Now make sure the requested output format is supported. */ - if ((imxpd->bus_format && imxpd->bus_format != output_fmt) || - !imx_pd_format_supported(output_fmt)) { + if (!imx_pd_format_supported(output_fmt)) { *num_input_fmts = 0; return NULL; } @@ -121,7 +120,17 @@ imx_pd_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge, if (!input_fmts) return NULL; - input_fmts[0] = output_fmt; + /* + * Prefer bus format set via legacy "interface-pix-fmt" DT property + * over panel bus format. This is necessary to retain support for + * DTs which configure the IPUv3 parallel output as 24bit, but + * connect 18bit DPI panels to it with hardware swizzling. + */ + if (imxpd->bus_format && imxpd->bus_format != output_fmt) + input_fmts[0] = imxpd->bus_format; + else + input_fmts[0] = output_fmt; + return input_fmts; } diff --git a/drivers/gpu/drm/loongson/lsdc_drv.c b/drivers/gpu/drm/loongson/lsdc_drv.c index abf5bf68eec210..4b97750897dcf2 100644 --- a/drivers/gpu/drm/loongson/lsdc_drv.c +++ b/drivers/gpu/drm/loongson/lsdc_drv.c @@ -292,7 +292,7 @@ static int lsdc_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) vga_client_register(pdev, lsdc_vga_set_decode); - drm_kms_helper_poll_init(ddev); + drmm_kms_helper_poll_init(ddev); if (loongson_vblank) { ret = drm_vblank_init(ddev, descp->num_of_crtc); diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c index 8464d89e37f316..e6ab731f8e9a3a 100644 --- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c @@ -604,11 +604,9 @@ static int a4xx_pm_suspend(struct msm_gpu *gpu) { return 0; } -static int a4xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) +static u64 a4xx_get_timestamp(struct msm_gpu *gpu) { - *value = gpu_read64(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_LO); - - return 0; + return gpu_read64(gpu, REG_A4XX_RBBM_PERFCTR_CP_0_LO); } static u64 a4xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate) diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index ef9fd6171af717..e44302251de560 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -1435,11 +1435,9 @@ static int a5xx_pm_suspend(struct msm_gpu *gpu) return 0; } -static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) +static u64 a5xx_get_timestamp(struct msm_gpu *gpu) { - *value = gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO); - - return 0; + return gpu_read64(gpu, REG_A5XX_RBBM_ALWAYSON_COUNTER_LO); } struct a5xx_crashdumper { diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 9662201cd2e9d0..1b44b9e21ad868 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -91,10 +92,10 @@ bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu) } /* Check to see if the GX rail is still powered */ -bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu) +bool a6xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu) { - struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); - struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; u32 val; /* This can be called from gpu state code so make sure GMU is valid */ @@ -117,6 +118,40 @@ bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu) A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF)); } +bool a7xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu) +{ + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + u32 val; + + /* This can be called from gpu state code so make sure GMU is valid */ + if (!gmu->initialized) + return false; + + val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS); + + return !(val & + (A7XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF | + A7XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF)); +} + +bool a8xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu) +{ + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + u32 val; + + /* This can be called from gpu state code so make sure GMU is valid */ + if (!gmu->initialized) + return false; + + val = gmu_read(gmu, REG_A8XX_GMU_PWR_CLK_STATUS); + + return !(val & + (A8XX_GMU_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF | + A8XX_GMU_PWR_CLK_STATUS_GX_HM_CLK_OFF)); +} + void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp, bool suspended) { @@ -240,7 +275,7 @@ static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu) if (val == local) { if (gmu->idle_level != GMU_IDLE_STATE_IFPC || - !a6xx_gmu_gx_is_on(gmu)) + !adreno_gpu->funcs->gx_is_on(adreno_gpu)) return true; } @@ -1157,6 +1192,65 @@ static void a6xx_gmu_set_initial_bw(struct msm_gpu *gpu, struct a6xx_gmu *gmu) dev_pm_opp_put(gpu_opp); } +static int a6xx_gmu_secure_init(struct a6xx_gpu *a6xx_gpu) +{ + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + u32 fuse_val; + int ret; + + if (test_bit(GMU_STATUS_SECURE_INIT, &gmu->status)) + return 0; + + if (adreno_is_a750(adreno_gpu) || adreno_is_a8xx(adreno_gpu)) { + /* + * Assume that if qcom scm isn't available, that whatever + * replacement allows writing the fuse register ourselves. + * Users of alternative firmware need to make sure this + * register is writeable or indicate that it's not somehow. + * Print a warning because if you mess this up you're about to + * crash horribly. + */ + if (!qcom_scm_is_available()) { + dev_warn_once(gpu->dev->dev, + "SCM is not available, poking fuse register\n"); + a6xx_llc_write(a6xx_gpu, REG_A7XX_CX_MISC_SW_FUSE_VALUE, + A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING | + A7XX_CX_MISC_SW_FUSE_VALUE_FASTBLEND | + A7XX_CX_MISC_SW_FUSE_VALUE_LPAC); + adreno_gpu->has_ray_tracing = true; + goto done; + } + + ret = qcom_scm_gpu_init_regs(QCOM_SCM_GPU_ALWAYS_EN_REQ | + QCOM_SCM_GPU_TSENSE_EN_REQ); + if (ret) { + dev_warn_once(gpu->dev->dev, + "SCM call failed\n"); + return ret; + } + + /* + * On A7XX_GEN3 and newer, raytracing may be disabled by the + * firmware, find out whether that's the case. The scm call + * above sets the fuse register. + */ + fuse_val = a6xx_llc_read(a6xx_gpu, + REG_A7XX_CX_MISC_SW_FUSE_VALUE); + adreno_gpu->has_ray_tracing = + !!(fuse_val & A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING); + } else if (adreno_is_a740(adreno_gpu)) { + /* Raytracing is always enabled on a740 */ + adreno_gpu->has_ray_tracing = true; + } + +done: + set_bit(GMU_STATUS_SECURE_INIT, &gmu->status); + return 0; +} + + int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) { struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; @@ -1185,11 +1279,12 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) clk_set_rate(gmu->hub_clk, adreno_is_a740_family(adreno_gpu) ? 200000000 : 150000000); ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); - if (ret) { - pm_runtime_put(gmu->gxpd); - pm_runtime_put(gmu->dev); - return ret; - } + if (ret) + goto rpm_put; + + ret = a6xx_gmu_secure_init(a6xx_gpu); + if (ret) + goto disable_clk; /* Read the slice info on A8x GPUs */ a8xx_gpu_get_slice_info(gpu); @@ -1219,11 +1314,11 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) ret = a6xx_gmu_fw_start(gmu, status); if (ret) - goto out; + goto disable_irq; ret = a6xx_hfi_start(gmu, status); if (ret) - goto out; + goto disable_irq; /* * Turn on the GMU firmware fault interrupt after we know the boot @@ -1236,19 +1331,16 @@ int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) /* Set the GPU to the current freq */ a6xx_gmu_set_initial_freq(gpu, gmu); - if (refcount_read(&gpu->sysprof_active) > 1) { - ret = a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET); - if (!ret) - set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status); - } -out: - /* On failure, shut down the GMU to leave it in a good state */ - if (ret) { - disable_irq(gmu->gmu_irq); - a6xx_rpmh_stop(gmu); - pm_runtime_put(gmu->gxpd); - pm_runtime_put(gmu->dev); - } + return 0; + +disable_irq: + disable_irq(gmu->gmu_irq); + a6xx_rpmh_stop(gmu); +disable_clk: + clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks); +rpm_put: + pm_runtime_put(gmu->gxpd); + pm_runtime_put(gmu->dev); return ret; } diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index 2af074c8e8cfa7..0cd8ae1b4f5c82 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -10,6 +10,7 @@ #include #include #include "msm_drv.h" +#include "adreno_gpu.h" #include "a6xx_hfi.h" struct a6xx_gmu_bo { @@ -129,6 +130,8 @@ struct a6xx_gmu { #define GMU_STATUS_PDC_SLEEP 1 /* To track Perfcounter OOB set status */ #define GMU_STATUS_OOB_PERF_SET 2 +/* To track whether secure world init was done */ +#define GMU_STATUS_SECURE_INIT 3 unsigned long status; }; @@ -231,7 +234,9 @@ void a6xx_hfi_stop(struct a6xx_gmu *gmu); int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu); int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, u32 perf_index, u32 bw_index); -bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu); +bool a6xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu); +bool a7xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu); +bool a8xx_gmu_gx_is_on(struct adreno_gpu *adreno_gpu); bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu); void a6xx_sptprac_disable(struct a6xx_gmu *gmu); int a6xx_sptprac_enable(struct a6xx_gmu *gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c index d6dfe6337bc342..0e8a48ca816dd7 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -10,14 +10,15 @@ #include #include -#include #include #include #define GPU_PAS_ID 13 -static u64 read_gmu_ao_counter(struct a6xx_gpu *a6xx_gpu) +static u64 a6xx_gmu_get_timestamp(struct msm_gpu *gpu) { + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); u64 count_hi, count_lo, temp; do { @@ -345,7 +346,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) * GPU registers so we need to add 0x1a800 to the register value on A630 * to get the right value from PM4. */ - get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER, + get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_CONTEXT, rbmemptr_stats(ring, index, alwayson_start)); /* Invalidate CCU depth and color */ @@ -386,7 +387,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0), rbmemptr_stats(ring, index, cpcycles_end)); - get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER, + get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_CONTEXT, rbmemptr_stats(ring, index, alwayson_end)); /* Write the fence to the scratch register */ @@ -404,7 +405,7 @@ static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); OUT_RING(ring, submit->seqno); - trace_msm_gpu_submit_flush(submit, read_gmu_ao_counter(a6xx_gpu)); + trace_msm_gpu_submit_flush(submit, adreno_gpu->funcs->get_timestamp(gpu)); a6xx_flush(gpu, ring); } @@ -455,7 +456,7 @@ static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); struct msm_ringbuffer *ring = submit->ring; - u32 rbbm_perfctr_cp0, cp_always_on_counter; + u32 rbbm_perfctr_cp0, cp_always_on_context; unsigned int i, ibs = 0; adreno_check_and_reenable_stall(adreno_gpu); @@ -478,14 +479,14 @@ static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) if (adreno_is_a8xx(adreno_gpu)) { rbbm_perfctr_cp0 = REG_A8XX_RBBM_PERFCTR_CP(0); - cp_always_on_counter = REG_A8XX_CP_ALWAYS_ON_COUNTER; + cp_always_on_context = REG_A8XX_CP_ALWAYS_ON_CONTEXT; } else { rbbm_perfctr_cp0 = REG_A7XX_RBBM_PERFCTR_CP(0); - cp_always_on_counter = REG_A6XX_CP_ALWAYS_ON_COUNTER; + cp_always_on_context = REG_A6XX_CP_ALWAYS_ON_CONTEXT; } get_stats_counter(ring, rbbm_perfctr_cp0, rbmemptr_stats(ring, index, cpcycles_start)); - get_stats_counter(ring, cp_always_on_counter, rbmemptr_stats(ring, index, alwayson_start)); + get_stats_counter(ring, cp_always_on_context, rbmemptr_stats(ring, index, alwayson_start)); OUT_PKT7(ring, CP_THREAD_CONTROL, 1); OUT_RING(ring, CP_SET_THREAD_BOTH); @@ -533,7 +534,7 @@ static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) } get_stats_counter(ring, rbbm_perfctr_cp0, rbmemptr_stats(ring, index, cpcycles_end)); - get_stats_counter(ring, cp_always_on_counter, rbmemptr_stats(ring, index, alwayson_end)); + get_stats_counter(ring, cp_always_on_context, rbmemptr_stats(ring, index, alwayson_end)); /* Write the fence to the scratch register */ if (adreno_is_a8xx(adreno_gpu)) { @@ -614,7 +615,7 @@ static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) } - trace_msm_gpu_submit_flush(submit, read_gmu_ao_counter(a6xx_gpu)); + trace_msm_gpu_submit_flush(submit, adreno_gpu->funcs->get_timestamp(gpu)); a6xx_flush(gpu, ring); @@ -1603,6 +1604,12 @@ static int hw_init(struct msm_gpu *gpu) a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); } + if (!ret && (refcount_read(&gpu->sysprof_active) > 1)) { + ret = a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET); + if (!ret) + set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status); + } + return ret; } @@ -1635,7 +1642,7 @@ static void a6xx_recover(struct msm_gpu *gpu) adreno_dump_info(gpu); - if (a6xx_gmu_gx_is_on(&a6xx_gpu->gmu)) { + if (adreno_gpu->funcs->gx_is_on(adreno_gpu)) { /* Sometimes crashstate capture is skipped, so SQE should be halted here again */ gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3); @@ -2152,56 +2159,6 @@ static void a6xx_llc_slices_init(struct platform_device *pdev, a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL); } -static int a7xx_cx_mem_init(struct a6xx_gpu *a6xx_gpu) -{ - struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; - struct msm_gpu *gpu = &adreno_gpu->base; - u32 fuse_val; - int ret; - - if (adreno_is_a750(adreno_gpu) || adreno_is_a8xx(adreno_gpu)) { - /* - * Assume that if qcom scm isn't available, that whatever - * replacement allows writing the fuse register ourselves. - * Users of alternative firmware need to make sure this - * register is writeable or indicate that it's not somehow. - * Print a warning because if you mess this up you're about to - * crash horribly. - */ - if (!qcom_scm_is_available()) { - dev_warn_once(gpu->dev->dev, - "SCM is not available, poking fuse register\n"); - a6xx_llc_write(a6xx_gpu, REG_A7XX_CX_MISC_SW_FUSE_VALUE, - A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING | - A7XX_CX_MISC_SW_FUSE_VALUE_FASTBLEND | - A7XX_CX_MISC_SW_FUSE_VALUE_LPAC); - adreno_gpu->has_ray_tracing = true; - return 0; - } - - ret = qcom_scm_gpu_init_regs(QCOM_SCM_GPU_ALWAYS_EN_REQ | - QCOM_SCM_GPU_TSENSE_EN_REQ); - if (ret) - return ret; - - /* - * On A7XX_GEN3 and newer, raytracing may be disabled by the - * firmware, find out whether that's the case. The scm call - * above sets the fuse register. - */ - fuse_val = a6xx_llc_read(a6xx_gpu, - REG_A7XX_CX_MISC_SW_FUSE_VALUE); - adreno_gpu->has_ray_tracing = - !!(fuse_val & A7XX_CX_MISC_SW_FUSE_VALUE_RAYTRACING); - } else if (adreno_is_a740(adreno_gpu)) { - /* Raytracing is always enabled on a740 */ - adreno_gpu->has_ray_tracing = true; - } - - return 0; -} - - #define GBIF_CLIENT_HALT_MASK BIT(0) #define GBIF_ARB_HALT_MASK BIT(1) #define VBIF_XIN_HALT_CTRL0_MASK GENMASK(3, 0) @@ -2414,20 +2371,9 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu) return 0; } -static int a6xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value) -{ - struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); - struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); - - *value = read_gmu_ao_counter(a6xx_gpu); - - return 0; -} - -static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) +static u64 a6xx_get_timestamp(struct msm_gpu *gpu) { - *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER); - return 0; + return gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER); } static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu) @@ -2656,6 +2602,7 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) gpu = &adreno_gpu->base; mutex_init(&a6xx_gpu->gmu.lock); + spin_lock_init(&a6xx_gpu->aperture_lock); adreno_gpu->registers = NULL; @@ -2708,14 +2655,6 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) return ERR_PTR(ret); } - if (adreno_is_a7xx(adreno_gpu) || adreno_is_a8xx(adreno_gpu)) { - ret = a7xx_cx_mem_init(a6xx_gpu); - if (ret) { - a6xx_destroy(&(a6xx_gpu->base.base)); - return ERR_PTR(ret); - } - } - adreno_gpu->uche_trap_base = 0x1fffffffff000ull; msm_mmu_set_fault_handler(to_msm_vm(gpu->vm)->mmu, gpu, @@ -2765,6 +2704,7 @@ const struct adreno_gpu_funcs a6xx_gpu_funcs = { .get_timestamp = a6xx_gmu_get_timestamp, .bus_halt = a6xx_bus_clear_pending_transactions, .mmu_fault_handler = a6xx_fault_handler, + .gx_is_on = a6xx_gmu_gx_is_on, }; const struct adreno_gpu_funcs a6xx_gmuwrapper_funcs = { @@ -2797,6 +2737,7 @@ const struct adreno_gpu_funcs a6xx_gmuwrapper_funcs = { .get_timestamp = a6xx_get_timestamp, .bus_halt = a6xx_bus_clear_pending_transactions, .mmu_fault_handler = a6xx_fault_handler, + .gx_is_on = a6xx_gmu_gx_is_on, }; const struct adreno_gpu_funcs a7xx_gpu_funcs = { @@ -2831,6 +2772,7 @@ const struct adreno_gpu_funcs a7xx_gpu_funcs = { .get_timestamp = a6xx_gmu_get_timestamp, .bus_halt = a6xx_bus_clear_pending_transactions, .mmu_fault_handler = a6xx_fault_handler, + .gx_is_on = a7xx_gmu_gx_is_on, }; const struct adreno_gpu_funcs a8xx_gpu_funcs = { @@ -2858,4 +2800,5 @@ const struct adreno_gpu_funcs a8xx_gpu_funcs = { .get_timestamp = a8xx_gmu_get_timestamp, .bus_halt = a8xx_bus_clear_pending_transactions, .mmu_fault_handler = a8xx_fault_handler, + .gx_is_on = a8xx_gmu_gx_is_on, }; diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h index 4eaa0471124606..a4434a6a56dd84 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h @@ -320,7 +320,7 @@ int a6xx_zap_shader_init(struct msm_gpu *gpu); void a8xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off); int a8xx_fault_handler(void *arg, unsigned long iova, int flags, void *data); void a8xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring); -int a8xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value); +u64 a8xx_gmu_get_timestamp(struct msm_gpu *gpu); u64 a8xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate); int a8xx_gpu_feature_probe(struct msm_gpu *gpu); void a8xx_gpu_get_slice_info(struct msm_gpu *gpu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 2d56fe0a65b7e7..791623ddb67c94 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -361,7 +361,7 @@ static void a6xx_get_debugbus_blocks(struct msm_gpu *gpu, sizeof(*a6xx_state->debugbus)); if (a6xx_state->debugbus) { - int i; + int i, j; for (i = 0; i < ARRAY_SIZE(a6xx_debugbus_blocks); i++) a6xx_get_debugbus_block(gpu, @@ -369,8 +369,6 @@ static void a6xx_get_debugbus_blocks(struct msm_gpu *gpu, &a6xx_debugbus_blocks[i], &a6xx_state->debugbus[i]); - a6xx_state->nr_debugbus = ARRAY_SIZE(a6xx_debugbus_blocks); - /* * GBIF has same debugbus as of other GPU blocks, fall back to * default path if GPU uses GBIF, also GBIF uses exactly same @@ -381,17 +379,19 @@ static void a6xx_get_debugbus_blocks(struct msm_gpu *gpu, &a6xx_gbif_debugbus_block, &a6xx_state->debugbus[i]); - a6xx_state->nr_debugbus += 1; + i++; } if (adreno_is_a650_family(to_adreno_gpu(gpu))) { - for (i = 0; i < ARRAY_SIZE(a650_debugbus_blocks); i++) + for (j = 0; j < ARRAY_SIZE(a650_debugbus_blocks); i++, j++) a6xx_get_debugbus_block(gpu, a6xx_state, - &a650_debugbus_blocks[i], + &a650_debugbus_blocks[j], &a6xx_state->debugbus[i]); } + + a6xx_state->nr_debugbus = i; } } @@ -1013,7 +1013,7 @@ static void a6xx_get_crashdumper_hlsq_registers(struct msm_gpu *gpu, u64 out = dumper->iova + A6XX_CD_DATA_OFFSET; int i, regcount = 0; - in += CRASHDUMP_WRITE(in, REG_A6XX_HLSQ_DBG_READ_SEL, regs->val1); + in += CRASHDUMP_WRITE(in, REG_A6XX_HLSQ_DBG_READ_SEL, (regs->val1 & 0xff) << 8); for (i = 0; i < regs->count; i += 2) { u32 count = RANGE(regs->registers, i); @@ -1251,7 +1251,7 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu, _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gpucc_reg, &a6xx_state->gmu_registers[2], false); - if (!a6xx_gmu_gx_is_on(&a6xx_gpu->gmu)) + if (!adreno_gpu->funcs->gx_is_on(adreno_gpu)) return; /* Set the fence to ALLOW mode so we can access the registers */ @@ -1607,7 +1607,7 @@ struct msm_gpu_state *a6xx_gpu_state_get(struct msm_gpu *gpu) } /* If GX isn't on the rest of the data isn't going to be accessible */ - if (!a6xx_gmu_gx_is_on(&a6xx_gpu->gmu)) + if (!adreno_gpu->funcs->gx_is_on(adreno_gpu)) return &a6xx_state->base; /* Halt SQE first */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c index 53cfdf4e6c3468..4f5dbf46132ba7 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c @@ -34,7 +34,7 @@ static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu, struct a6xx_hfi_queue_header *header = queue->header; u32 i, hdr, index = header->read_index; - if (header->read_index == header->write_index) { + if (header->read_index == READ_ONCE(header->write_index)) { header->rx_request = 1; return 0; } @@ -62,7 +62,10 @@ static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu, if (!gmu->legacy) index = ALIGN(index, 4) % header->size; - header->read_index = index; + /* Ensure all memory operations are complete before updating the read index */ + dma_mb(); + + WRITE_ONCE(header->read_index, index); return HFI_HEADER_SIZE(hdr); } @@ -74,7 +77,7 @@ static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu, spin_lock(&queue->lock); - space = CIRC_SPACE(header->write_index, header->read_index, + space = CIRC_SPACE(header->write_index, READ_ONCE(header->read_index), header->size); if (space < dwords) { header->dropped++; @@ -95,7 +98,10 @@ static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu, queue->data[index] = 0xfafafafa; } - header->write_index = index; + /* Ensure all memory operations are complete before updating the write index */ + dma_mb(); + + WRITE_ONCE(header->write_index, index); spin_unlock(&queue->lock); gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01); diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c index b1887e0cf6983e..fafeac62aebf5e 100644 --- a/drivers/gpu/drm/msm/adreno/a8xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu.c @@ -711,6 +711,12 @@ static int hw_init(struct msm_gpu *gpu) */ a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + if (!ret && (refcount_read(&gpu->sysprof_active) > 1)) { + ret = a6xx_gmu_set_oob(gmu, GMU_OOB_PERFCOUNTER_SET); + if (!ret) + set_bit(GMU_STATUS_OOB_PERF_SET, &gmu->status); + } + return ret; } @@ -1174,23 +1180,19 @@ void a8xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_ gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0); } -int a8xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value) +u64 a8xx_gmu_get_timestamp(struct msm_gpu *gpu) { struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + u64 count_hi, count_lo, temp; - mutex_lock(&a6xx_gpu->gmu.lock); - - /* Force the GPU power on so we can read this register */ - a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); - - *value = gpu_read64(gpu, REG_A8XX_CP_ALWAYS_ON_COUNTER); - - a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET); - - mutex_unlock(&a6xx_gpu->gmu.lock); + do { + count_hi = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_H); + count_lo = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_L); + temp = gmu_read(&a6xx_gpu->gmu, REG_A8XX_GMU_ALWAYS_ON_COUNTER_H); + } while (unlikely(count_hi != temp)); - return 0; + return (count_hi << 32) | count_lo; } u64 a8xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate) diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index d5fe6f6f0decc4..785e99fb5bd5d4 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -391,13 +391,11 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx, return 0; case MSM_PARAM_TIMESTAMP: if (adreno_gpu->funcs->get_timestamp) { - int ret; - pm_runtime_get_sync(&gpu->pdev->dev); - ret = adreno_gpu->funcs->get_timestamp(gpu, value); + *value = adreno_gpu->funcs->get_timestamp(gpu); pm_runtime_put_autosuspend(&gpu->pdev->dev); - return ret; + return 0; } return -EINVAL; case MSM_PARAM_PRIORITIES: diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index 1d0145f8b3ecbd..29097e6b425350 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -79,9 +79,10 @@ struct adreno_gpu; struct adreno_gpu_funcs { struct msm_gpu_funcs base; struct msm_gpu *(*init)(struct drm_device *dev); - int (*get_timestamp)(struct msm_gpu *gpu, uint64_t *value); + u64 (*get_timestamp)(struct msm_gpu *gpu); void (*bus_halt)(struct adreno_gpu *adreno_gpu, bool gx_off); int (*mmu_fault_handler)(void *arg, unsigned long iova, int flags, void *data); + bool (*gx_is_on)(struct adreno_gpu *adreno_gpu); }; struct adreno_reglist { diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_1_16_msm8953.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_1_16_msm8953.h index b44d02b48418f7..2162ff917b0f85 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_1_16_msm8953.h +++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_1_16_msm8953.h @@ -121,13 +121,6 @@ static const struct dpu_dspp_cfg msm8953_dspp[] = { static const struct dpu_intf_cfg msm8953_intf[] = { { - .name = "intf_0", .id = INTF_0, - .base = 0x6a000, .len = 0x268, - .type = INTF_NONE, - .prog_fetch_lines_worst_case = 14, - .intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 24), - .intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 25), - }, { .name = "intf_1", .id = INTF_1, .base = 0x6a800, .len = 0x268, .type = INTF_DSI, diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c index 0f4921b1a89226..cbb7caa194c1e5 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c @@ -1410,7 +1410,8 @@ static struct msm_display_topology dpu_crtc_get_topology( topology.num_lm = 2; else if (topology.num_dsc == 2) topology.num_lm = 2; - else if (dpu_kms->catalog->caps->has_3d_merge) + else if (dpu_kms->catalog->caps->has_3d_merge && + topology.num_dsc == 0) topology.num_lm = (mode->hdisplay > MAX_HDISPLAY_SPLIT) ? 2 : 1; else topology.num_lm = 1; diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c index 61d7e65469b3ad..014b2c504eda67 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c @@ -1461,8 +1461,6 @@ static int __maybe_unused dpu_runtime_suspend(struct device *dev) struct msm_drm_private *priv = platform_get_drvdata(pdev); struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms); - /* Drop the performance state vote */ - dev_pm_opp_set_rate(dev, 0); clk_bulk_disable_unprepare(dpu_kms->num_clocks, dpu_kms->clocks); for (i = 0; i < dpu_kms->num_paths; i++) diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 476848bf8cd16b..d2124d6254855b 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -210,6 +210,7 @@ static const struct of_device_id msm_dp_dt_match[] = { { .compatible = "qcom,x1e80100-dp", .data = &msm_dp_desc_x1e80100 }, {} }; +MODULE_DEVICE_TABLE(of, msm_dp_dt_match); static struct msm_dp_display_private *dev_get_dp_display_private(struct device *dev) { diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c index d8bb40ef820e2b..3c9f01ed627133 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.c +++ b/drivers/gpu/drm/msm/dsi/dsi.c @@ -198,6 +198,7 @@ static const struct of_device_id dt_match[] = { { .compatible = "qcom,dsi-ctrl-6g-qcm2290" }, {} }; +MODULE_DEVICE_TABLE(of, dt_match); static const struct dev_pm_ops dsi_pm_ops = { SET_RUNTIME_PM_OPS(msm_dsi_runtime_suspend, msm_dsi_runtime_resume, NULL) diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c index bd3c51c350e731..da3fe68244952f 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c @@ -317,10 +317,10 @@ static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = { &msm8996_dsi_cfg, &msm_dsi_6g_host_ops}, {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V1_4_2, &msm8976_dsi_cfg, &msm_dsi_6g_host_ops}, + {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_0_0, + &msm8998_dsi_cfg, &msm_dsi_6g_v2_host_ops}, {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_1_0, &sdm660_dsi_cfg, &msm_dsi_6g_v2_host_ops}, - {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_2_0, - &msm8998_dsi_cfg, &msm_dsi_6g_v2_host_ops}, {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_2_1, &sdm845_dsi_cfg, &msm_dsi_6g_v2_host_ops}, {MSM_DSI_VER_MAJOR_6G, MSM_DSI_6G_VER_MINOR_V2_3_0, diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h index 5dc812028bd545..ccf06679608e03 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h +++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h @@ -19,8 +19,8 @@ #define MSM_DSI_6G_VER_MINOR_V1_3_1 0x10030001 #define MSM_DSI_6G_VER_MINOR_V1_4_1 0x10040001 #define MSM_DSI_6G_VER_MINOR_V1_4_2 0x10040002 +#define MSM_DSI_6G_VER_MINOR_V2_0_0 0x20000000 #define MSM_DSI_6G_VER_MINOR_V2_1_0 0x20010000 -#define MSM_DSI_6G_VER_MINOR_V2_2_0 0x20000000 #define MSM_DSI_6G_VER_MINOR_V2_2_1 0x20020001 #define MSM_DSI_6G_VER_MINOR_V2_3_0 0x20030000 #define MSM_DSI_6G_VER_MINOR_V2_3_1 0x20030001 diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index db6da99375a185..1c0841a1c10133 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -569,6 +569,7 @@ void dsi_link_clk_disable_v2(struct msm_dsi_host *msm_host) * dsi_adjust_pclk_for_compression() - Adjust the pclk rate for compression case * @mode: The selected mode for the DSI output * @dsc: DRM DSC configuration for this DSI output + * @is_bonded_dsi: True if two DSI controllers are bonded * * Adjust the pclk rate by calculating a new hdisplay proportional to * the compression ratio such that: @@ -1033,8 +1034,9 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) /* * DPU sends 3 bytes per pclk cycle to DSI. If widebus is * enabled, MDP always sends out 48-bit compressed data per - * pclk and on average, DSI consumes an amount of compressed - * data equivalent to the uncompressed pixel depth per pclk. + * pclk and on average, for video mode, DSI consumes only an + * amount of compressed data equivalent to the uncompressed + * pixel depth per pclk. * * Calculate the number of pclks needed to transmit one line of * the compressed data. @@ -1046,10 +1048,14 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi) * unused anyway. */ h_total -= hdisplay; - if (wide_bus_enabled) - bits_per_pclk = mipi_dsi_pixel_format_to_bpp(msm_host->format); - else + if (wide_bus_enabled) { + if (msm_host->mode_flags & MIPI_DSI_MODE_VIDEO) + bits_per_pclk = dsc->bits_per_component * 3; + else + bits_per_pclk = 48; + } else { bits_per_pclk = 24; + } hdisplay = DIV_ROUND_UP(msm_dsc_get_bytes_per_line(msm_host->dsc) * 8, bits_per_pclk); diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c index 7937266de1d28f..c59375aaae1974 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c @@ -582,6 +582,7 @@ static const struct of_device_id dsi_phy_dt_match[] = { #endif {} }; +MODULE_DEVICE_TABLE(of, dsi_phy_dt_match); /* * Currently, we only support one SoC for each PHY type. When we have multiple diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c index 5afac09c0d3347..d5ef5089c9e9c4 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi.c @@ -441,6 +441,7 @@ static const struct of_device_id msm_hdmi_dt_match[] = { { .compatible = "qcom,hdmi-tx-8660", .data = &hdmi_tx_8960_config }, {} }; +MODULE_DEVICE_TABLE(of, msm_hdmi_dt_match); static struct platform_driver msm_hdmi_driver = { .probe = msm_hdmi_dev_probe, diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c index 98cd490e7ab0f3..9b7012692ece99 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c @@ -153,7 +153,7 @@ static int msm_hdmi_bridge_write_avi_infoframe(struct drm_bridge *bridge, for (i = 0; i < ARRAY_SIZE(buf); i++) hdmi_write(hdmi, REG_HDMI_AVI_INFO(i), buf[i]); - val = hdmi_read(hdmi, REG_HDMI_INFOFRAME_CTRL1); + val = hdmi_read(hdmi, REG_HDMI_INFOFRAME_CTRL0); val |= HDMI_INFOFRAME_CTRL0_AVI_SEND | HDMI_INFOFRAME_CTRL0_AVI_CONT; hdmi_write(hdmi, REG_HDMI_INFOFRAME_CTRL0, val); @@ -193,7 +193,7 @@ static int msm_hdmi_bridge_write_audio_infoframe(struct drm_bridge *bridge, buffer[9] << 16 | buffer[10] << 24); - val = hdmi_read(hdmi, REG_HDMI_INFOFRAME_CTRL1); + val = hdmi_read(hdmi, REG_HDMI_INFOFRAME_CTRL0); val |= HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SEND | HDMI_INFOFRAME_CTRL0_AUDIO_INFO_CONT | HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SOURCE | diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_phy.c b/drivers/gpu/drm/msm/hdmi/hdmi_phy.c index 667573f1db7c6b..f726555bb68105 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi_phy.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi_phy.c @@ -204,6 +204,7 @@ static const struct of_device_id msm_hdmi_phy_dt_match[] = { .data = &msm_hdmi_phy_8998_cfg }, {} }; +MODULE_DEVICE_TABLE(of, msm_hdmi_phy_dt_match); static struct platform_driver msm_hdmi_phy_platform_driver = { .probe = msm_hdmi_phy_probe, diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index e5ab1e28851dfe..195f40e331e5a8 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -536,6 +536,11 @@ static int msm_ioctl_gem_info_get_metadata(struct drm_gem_object *obj, len = msm_obj->metadata_size; buf = kmemdup(msm_obj->metadata, len, GFP_KERNEL); + if (!buf) { + msm_gem_unlock(obj); + return -ENOMEM; + } + msm_gem_unlock(obj); if (*metadata_size < len) { @@ -548,7 +553,7 @@ static int msm_ioctl_gem_info_get_metadata(struct drm_gem_object *obj, kfree(buf); - return 0; + return ret; } static int msm_ioctl_gem_info(struct drm_device *dev, void *data, diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c index b3fdb83202ab84..9b681e144c07aa 100644 --- a/drivers/gpu/drm/msm/msm_fb.c +++ b/drivers/gpu/drm/msm/msm_fb.c @@ -219,7 +219,12 @@ static struct drm_framebuffer *msm_framebuffer_init(struct drm_device *dev, + mode_cmd->offsets[i]; if (bos[i]->size < min_size) { - ret = -EINVAL; + ret = UERR(EINVAL, dev, "plane %d too small", i); + goto fail; + } + + if (to_msm_bo(bos[i])->flags & MSM_BO_NO_SHARE) { + ret = UERR(EINVAL, dev, "Cannot map _NO_SHARE to kms vm"); goto fail; } diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index b27abaa13926de..2cb3ab04f12502 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -507,8 +507,11 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj) */ void msm_gem_unpin_active(struct drm_gem_object *obj) { + struct msm_drm_private *priv = obj->dev->dev_private; struct msm_gem_object *msm_obj = to_msm_bo(obj); + GEM_WARN_ON(!mutex_is_locked(&priv->lru.lock)); + msm_obj->pin_count--; GEM_WARN_ON(msm_obj->pin_count < 0); update_lru_active(obj); diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 1039e3c0a47bff..31fa51a44f86e3 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -26,9 +26,8 @@ static bool can_swap(void) static bool can_block(struct shrink_control *sc) { - if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM)) - return false; - return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM); + return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) || + (current_is_kswapd() && (sc->gfp_mask & __GFP_KSWAPD_RECLAIM)); } static unsigned long diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index adf88cf8f41aa9..9e3632019bc92f 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -696,6 +696,7 @@ static struct dma_fence * msm_vma_job_run(struct drm_sched_job *_job) { struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job); + struct msm_drm_private *priv = job->vm->drm->dev_private; struct msm_gem_vm *vm = to_msm_vm(job->vm); struct drm_gem_object *obj; int ret = vm->unusable ? -EINVAL : 0; @@ -738,12 +739,14 @@ msm_vma_job_run(struct drm_sched_job *_job) if (ret) msm_gem_vm_unusable(job->vm); + mutex_lock(&priv->lru.lock); + job_foreach_bo (obj, job) { - msm_gem_lock(obj); - msm_gem_unpin_locked(obj); - msm_gem_unlock(obj); + msm_gem_unpin_active(obj); } + mutex_unlock(&priv->lru.lock); + /* VM_BIND ops are synchronous, so no fence to wait on: */ return NULL; } @@ -1242,7 +1245,7 @@ vm_bind_job_lock_objects(struct msm_vm_bind_job *job, struct drm_exec *exec) case MSM_VM_BIND_OP_UNMAP: ret = drm_gpuvm_sm_unmap_exec_lock(job->vm, exec, op->iova, - op->obj_offset); + op->range); break; case MSM_VM_BIND_OP_MAP: case MSM_VM_BIND_OP_MAP_NULL: { diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 84d6c7f50c8df4..d178bb9b813ae6 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -546,32 +546,30 @@ static void recover_worker(struct kthread_work *work) msm_update_fence(ring->fctx, fence); } - if (msm_gpu_active(gpu)) { - /* retire completed submits, plus the one that hung: */ - retire_submits(gpu); + /* retire completed submits, plus the one that hung: */ + retire_submits(gpu); - gpu->funcs->recover(gpu); + gpu->funcs->recover(gpu); - /* - * Replay all remaining submits starting with highest priority - * ring - */ - for (i = 0; i < gpu->nr_rings; i++) { - struct msm_ringbuffer *ring = gpu->rb[i]; - unsigned long flags; + /* + * Replay all remaining submits starting with highest priority + * ring + */ + for (i = 0; i < gpu->nr_rings; i++) { + struct msm_ringbuffer *ring = gpu->rb[i]; + unsigned long flags; - spin_lock_irqsave(&ring->submit_lock, flags); - list_for_each_entry(submit, &ring->submits, node) { - /* - * If the submit uses an unusable vm make sure - * we don't actually run it - */ - if (to_msm_vm(submit->vm)->unusable) - submit->nr_cmds = 0; - gpu->funcs->submit(gpu, submit); - } - spin_unlock_irqrestore(&ring->submit_lock, flags); + spin_lock_irqsave(&ring->submit_lock, flags); + list_for_each_entry(submit, &ring->submits, node) { + /* + * If the submit uses an unusable vm make sure + * we don't actually run it + */ + if (to_msm_vm(submit->vm)->unusable) + submit->nr_cmds = 0; + gpu->funcs->submit(gpu, submit); } + spin_unlock_irqrestore(&ring->submit_lock, flags); } pm_runtime_put(&gpu->pdev->dev); diff --git a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml index c4e00b1263cda6..33404eb18fd023 100644 --- a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml +++ b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml @@ -141,8 +141,10 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - + + + + diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 915f73279302d0..0c23398dd4f1d8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -874,7 +874,7 @@ static int nouveau_drm_probe(struct pci_dev *pdev, /* Remove conflicting drivers (vesafb, efifb etc). */ ret = aperture_remove_conflicting_pci_devices(pdev, driver_pci.name); if (ret) - return ret; + goto fail_nvkm; pci_set_master(pdev); diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 82621ede42e1ec..20dba02d6175bc 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -686,7 +686,7 @@ nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, } nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; - if (unlikely(r->reloc_bo_offset + 4 > + if (unlikely((u64)r->reloc_bo_offset + 4 > nvbo->bo.base.size)) { NV_PRINTK(err, cli, "reloc outside of bo\n"); ret = -EINVAL; diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs index b1af0a099551d1..1cf5b83dbbab5b 100644 --- a/drivers/gpu/drm/nova/driver.rs +++ b/drivers/gpu/drm/nova/driver.rs @@ -71,6 +71,8 @@ impl drm::Driver for NovaDriver { const INFO: drm::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM; + kernel::declare_drm_ioctls! { (NOVA_GETPARAM, drm_nova_getparam, ioctl::RENDER_ALLOW, File::get_param), (NOVA_GEM_CREATE, drm_nova_gem_create, ioctl::AUTH | ioctl::RENDER_ALLOW, File::gem_create), diff --git a/drivers/gpu/drm/nova/file.rs b/drivers/gpu/drm/nova/file.rs index a3b7bd36792c1b..a8288bc014c250 100644 --- a/drivers/gpu/drm/nova/file.rs +++ b/drivers/gpu/drm/nova/file.rs @@ -4,6 +4,7 @@ use crate::driver::{NovaDevice, NovaDriver}; use crate::gem::NovaObject; use kernel::{ alloc::flags::*, + bindings, drm::{self, gem::BaseObject}, pci, prelude::*, @@ -18,6 +19,10 @@ impl drm::file::DriverFile for File { fn open(_dev: &NovaDevice) -> Result>> { Ok(KBox::new(Self, GFP_KERNEL)?.into()) } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } } impl File { diff --git a/drivers/gpu/drm/nova/gem.rs b/drivers/gpu/drm/nova/gem.rs index 6ccfa5da57617f..1cc48eda8d8889 100644 --- a/drivers/gpu/drm/nova/gem.rs +++ b/drivers/gpu/drm/nova/gem.rs @@ -17,10 +17,12 @@ use crate::{ #[pin_data] pub(crate) struct NovaObject {} +#[vtable] impl gem::DriverObject for NovaObject { type Driver = NovaDriver; + type Args = (); - fn new(_dev: &NovaDevice, _size: usize) -> impl PinInit { + fn new(_dev: &NovaDevice, _size: usize, _args: Self::Args) -> impl PinInit { try_pin_init!(NovaObject {}) } } @@ -33,7 +35,7 @@ impl NovaObject { } let aligned_size = page::page_align(size).ok_or(EINVAL)?; - gem::Object::new(dev, aligned_size) + gem::Object::new(dev, aligned_size, ()) } /// Look up a GEM object handle for a `File` and return an `ObjectRef` for it. diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig index 307152ad77591a..79264f7bbd0e20 100644 --- a/drivers/gpu/drm/panel/Kconfig +++ b/drivers/gpu/drm/panel/Kconfig @@ -280,6 +280,7 @@ config DRM_PANEL_ILITEK_ILI9882T depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE + select DRM_DISPLAY_DSC_HELPER help Say Y if you want to enable support for panels based on the Ilitek ILI9882t controller. diff --git a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c index d5fe105bdbdde5..658ce64c71eb2b 100644 --- a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c +++ b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c @@ -1324,6 +1324,8 @@ static int boe_panel_disable(struct drm_panel *panel) mipi_dsi_dcs_set_display_off_multi(&ctx); mipi_dsi_dcs_enter_sleep_mode_multi(&ctx); + boe->dsi->mode_flags |= MIPI_DSI_MODE_LPM; + mipi_dsi_msleep(&ctx, 150); return ctx.accum_err; diff --git a/drivers/gpu/drm/panel/panel-himax-hx83102.c b/drivers/gpu/drm/panel/panel-himax-hx83102.c index 1d3bb5dca5594c..5dfec2a958f7be 100644 --- a/drivers/gpu/drm/panel/panel-himax-hx83102.c +++ b/drivers/gpu/drm/panel/panel-himax-hx83102.c @@ -850,6 +850,8 @@ static int hx83102_disable(struct drm_panel *panel) mipi_dsi_dcs_set_display_off_multi(&dsi_ctx); mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx); + dsi->mode_flags |= MIPI_DSI_MODE_LPM; + mipi_dsi_msleep(&dsi_ctx, 150); return dsi_ctx.accum_err; diff --git a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c index 36abfa2e65e962..dd1eaba23ad3c6 100644 --- a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c +++ b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c @@ -201,6 +201,7 @@ static int sharp_nt_panel_add(struct sharp_nt_panel *sharp_nt) drm_panel_init(&sharp_nt->base, &sharp_nt->dsi->dev, &sharp_nt_panel_funcs, DRM_MODE_CONNECTOR_DSI); + sharp_nt->base.prepare_prev_first = true; ret = drm_panel_of_backlight(&sharp_nt->base); if (ret) diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 91ab280869bacf..a242ae284db5a4 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -1295,7 +1295,7 @@ static const struct panel_desc auo_g190ean01 = { .height = 301, }, .delay = { - .prepare = 50, + .prepare = 30, .enable = 200, .disable = 110, .unprepare = 1000, diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index 711f5101aa04ce..074c0995ddc26c 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -390,6 +390,8 @@ panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, true, timeout); if (!ret) ret = timeout ? -ETIMEDOUT : -EBUSY; + else if (ret > 0) + ret = 0; drm_gem_object_put(gem_obj); diff --git a/drivers/gpu/drm/qxl/qxl_ioctl.c b/drivers/gpu/drm/qxl/qxl_ioctl.c index 4ee2b5acf2e08d..591b026ceff9ce 100644 --- a/drivers/gpu/drm/qxl/qxl_ioctl.c +++ b/drivers/gpu/drm/qxl/qxl_ioctl.c @@ -183,7 +183,7 @@ static int qxl_process_single_command(struct qxl_device *qdev, /* TODO copy slow path code from i915 */ fb_cmd = qxl_bo_kmap_atomic_page(qdev, cmd_bo, (release->release_offset & PAGE_MASK)); - unwritten = __copy_from_user_inatomic_nocache + unwritten = copy_from_user_inatomic_nontemporal (fb_cmd + sizeof(union qxl_release_info) + (release->release_offset & ~PAGE_MASK), u64_to_user_ptr(cmd->command), cmd->command_size); diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c index 22321eb95b7d5d..703848fac18933 100644 --- a/drivers/gpu/drm/radeon/ci_dpm.c +++ b/drivers/gpu/drm/radeon/ci_dpm.c @@ -2461,7 +2461,8 @@ static void ci_register_patching_mc_arb(struct radeon_device *rdev, if (patch && ((rdev->pdev->device == 0x67B0) || - (rdev->pdev->device == 0x67B1))) { + (rdev->pdev->device == 0x67B1)) && + (rdev->pdev->revision == 0)) { if ((memory_clock > 100000) && (memory_clock <= 125000)) { tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff; *dram_timimg2 &= ~0x00ff0000; @@ -3304,7 +3305,8 @@ static int ci_populate_all_memory_levels(struct radeon_device *rdev) pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1; if ((dpm_table->mclk_table.count >= 2) && - ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1))) { + ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1)) && + (rdev->pdev->revision == 0)) { pi->smc_state_table.MemoryLevel[1].MinVddc = pi->smc_state_table.MemoryLevel[0].MinVddc; pi->smc_state_table.MemoryLevel[1].MinVddcPhases = @@ -4493,7 +4495,8 @@ static int ci_register_patching_mc_seq(struct radeon_device *rdev, if (patch && ((rdev->pdev->device == 0x67B0) || - (rdev->pdev->device == 0x67B1))) { + (rdev->pdev->device == 0x67B1)) && + (rdev->pdev->revision == 0)) { for (i = 0; i < table->last; i++) { if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE) return -EINVAL; diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index fe174a4857be7f..e1920f3f920aee 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -420,7 +420,12 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity, /* * Fence is from the same scheduler, only need to wait for - * it to be scheduled + * it to be scheduled. + * + * Note: s_fence->sched could have been freed and reallocated + * as another scheduler. This false positive case is okay, as if + * the old scheduler was freed all of its jobs must have + * signaled their completion fences. */ fence = dma_fence_get(&s_fence->scheduled); dma_fence_put(entity->dependency); diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c index 9391d6f0dc01d7..d05ab041a581a6 100644 --- a/drivers/gpu/drm/scheduler/sched_fence.c +++ b/drivers/gpu/drm/scheduler/sched_fence.c @@ -92,7 +92,7 @@ static const char *drm_sched_fence_get_driver_name(struct dma_fence *fence) static const char *drm_sched_fence_get_timeline_name(struct dma_fence *f) { struct drm_sched_fence *fence = to_drm_sched_fence(f); - return (const char *)fence->sched->name; + return (const char *)fence->sched_name; } static void drm_sched_fence_free_rcu(struct rcu_head *rcu) @@ -228,6 +228,8 @@ void drm_sched_fence_init(struct drm_sched_fence *fence, unsigned seq; fence->sched = entity->rq->sched; + strscpy(fence->sched_name, entity->rq->sched->name, + sizeof(fence->sched_name)); seq = atomic_inc_return(&entity->fence_seq); dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled, &fence->lock, entity->fence_context, seq); diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c index b7397827889c94..360a88ca8f0c5a 100644 --- a/drivers/gpu/drm/sti/sti_hda.c +++ b/drivers/gpu/drm/sti/sti_hda.c @@ -741,6 +741,7 @@ static int sti_hda_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct sti_hda *hda; struct resource *res; + int ret; DRM_INFO("%s\n", __func__); @@ -779,7 +780,9 @@ static int sti_hda_probe(struct platform_device *pdev) return PTR_ERR(hda->clk_hddac); } - drm_bridge_add(&hda->bridge); + ret = devm_drm_bridge_add(dev, &hda->bridge); + if (ret) + return ret; platform_set_drvdata(pdev, hda); @@ -788,10 +791,7 @@ static int sti_hda_probe(struct platform_device *pdev) static void sti_hda_remove(struct platform_device *pdev) { - struct sti_hda *hda = platform_get_drvdata(pdev); - component_del(&pdev->dev, &sti_hda_ops); - drm_bridge_remove(&hda->bridge); } static const struct of_device_id hda_of_match[] = { diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c index 40405a52a073af..e989f75c09b7da 100644 --- a/drivers/gpu/drm/sun4i/sun4i_backend.c +++ b/drivers/gpu/drm/sun4i/sun4i_backend.c @@ -491,6 +491,9 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine, drm_for_each_plane_mask(plane, drm, crtc_state->plane_mask) { struct drm_plane_state *plane_state = drm_atomic_get_plane_state(state, plane); + if (IS_ERR(plane_state)) + return PTR_ERR(plane_state); + struct sun4i_layer_state *layer_state = state_to_sun4i_layer_state(plane_state); struct drm_framebuffer *fb = plane_state->fb; @@ -878,7 +881,8 @@ static int sun4i_backend_bind(struct device *dev, struct device *master, &sun4i_backend_regmap_config); if (IS_ERR(backend->engine.regs)) { dev_err(dev, "Couldn't create the backend regmap\n"); - return PTR_ERR(backend->engine.regs); + ret = PTR_ERR(backend->engine.regs); + goto err_disable_ram_clk; } list_add_tail(&backend->engine.list, &drv->engine_list); diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c index ce9c155bfad7fc..02acc7cbdb979d 100644 --- a/drivers/gpu/drm/sun4i/sun8i_mixer.c +++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c @@ -321,7 +321,7 @@ static struct drm_plane **sun8i_layers_init(struct drm_device *drm, unsigned int phy_index; int i; - planes = devm_kcalloc(drm->dev, plane_cnt, sizeof(*planes), GFP_KERNEL); + planes = devm_kcalloc(drm->dev, plane_cnt + 1, sizeof(*planes), GFP_KERNEL); if (!planes) return ERR_PTR(-ENOMEM); diff --git a/drivers/gpu/drm/sysfb/ofdrm.c b/drivers/gpu/drm/sysfb/ofdrm.c index d38ba70f4e0d37..247cf13c80a055 100644 --- a/drivers/gpu/drm/sysfb/ofdrm.c +++ b/drivers/gpu/drm/sysfb/ofdrm.c @@ -350,6 +350,7 @@ static void ofdrm_pci_release(void *data) struct pci_dev *pcidev = data; pci_disable_device(pcidev); + pci_dev_put(pcidev); } static int ofdrm_device_init_pci(struct ofdrm_device *odev) @@ -375,6 +376,7 @@ static int ofdrm_device_init_pci(struct ofdrm_device *odev) if (ret) { drm_err(dev, "pci_enable_device(%s) failed: %d\n", dev_name(&pcidev->dev), ret); + pci_dev_put(pcidev); return ret; } ret = devm_add_action_or_reset(&pdev->dev, ofdrm_pci_release, pcidev); diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c index 7a95d2dacd9d27..2ab6d96b4ec9e7 100644 --- a/drivers/gpu/drm/sysfb/simpledrm.c +++ b/drivers/gpu/drm/sysfb/simpledrm.c @@ -841,6 +841,12 @@ static int simpledrm_probe(struct platform_device *pdev) struct drm_device *dev; int ret; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to set dma mask\n"); + sdev = simpledrm_device_create(&simpledrm_driver, pdev); if (IS_ERR(sdev)) return PTR_ERR(sdev); diff --git a/drivers/gpu/drm/tiny/appletbdrm.c b/drivers/gpu/drm/tiny/appletbdrm.c index 3bae91d7eefed7..278bb23fe4c8d0 100644 --- a/drivers/gpu/drm/tiny/appletbdrm.c +++ b/drivers/gpu/drm/tiny/appletbdrm.c @@ -353,7 +353,7 @@ static int appletbdrm_primary_plane_helper_atomic_check(struct drm_plane *plane, frames_size + sizeof(struct appletbdrm_fb_request_footer), 16); - appletbdrm_state->request = kzalloc(request_size, GFP_KERNEL); + appletbdrm_state->request = kvzalloc(request_size, GFP_KERNEL); if (!appletbdrm_state->request) return -ENOMEM; @@ -543,7 +543,7 @@ static void appletbdrm_primary_plane_destroy_state(struct drm_plane *plane, { struct appletbdrm_plane_state *appletbdrm_state = to_appletbdrm_plane_state(state); - kfree(appletbdrm_state->request); + kvfree(appletbdrm_state->request); kfree(appletbdrm_state->response); __drm_gem_destroy_shadow_plane_state(&appletbdrm_state->base); diff --git a/drivers/gpu/drm/tiny/arcpgu.c b/drivers/gpu/drm/tiny/arcpgu.c index 50588849748213..c93d61ac0bb755 100644 --- a/drivers/gpu/drm/tiny/arcpgu.c +++ b/drivers/gpu/drm/tiny/arcpgu.c @@ -250,7 +250,8 @@ DEFINE_DRM_GEM_DMA_FOPS(arcpgu_drm_ops); static int arcpgu_load(struct arcpgu_drm_private *arcpgu) { struct platform_device *pdev = to_platform_device(arcpgu->drm.dev); - struct device_node *encoder_node = NULL, *endpoint_node = NULL; + struct device_node *encoder_node __free(device_node) = NULL; + struct device_node *endpoint_node = NULL; struct drm_connector *connector = NULL; struct drm_device *drm = &arcpgu->drm; int ret; diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 0765d69423d2f4..08d07ffbb1674a 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -740,7 +740,7 @@ static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL); if (ret) { - if (ret != -ENOSPC && ret != -EAGAIN) { + if (ret != -ENOSPC) { dmem_cgroup_pool_state_put(limit_pool); return ret; } @@ -1178,17 +1178,13 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) bdev->funcs->swap_notify(bo); if (ttm_tt_is_populated(tt)) { - spin_lock(&bdev->lru_lock); - ttm_resource_del_bulk_move(bo->resource, bo); - spin_unlock(&bdev->lru_lock); - ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags); - - spin_lock(&bdev->lru_lock); - if (ret) - ttm_resource_add_bulk_move(bo->resource, bo); - ttm_resource_move_to_lru_tail(bo->resource); - spin_unlock(&bdev->lru_lock); + if (!ret) { + spin_lock(&bdev->lru_lock); + ttm_resource_del_bulk_move_unevictable(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bdev->lru_lock); + } } out: diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index f83b7d5ec6c6d0..3e3c201a022267 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -1112,19 +1112,14 @@ long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, if (lret < 0) return lret; - if (bo->bulk_move) { - spin_lock(&bdev->lru_lock); - ttm_resource_del_bulk_move(bo->resource, bo); - spin_unlock(&bdev->lru_lock); - } - lret = ttm_tt_backup(bdev, bo->ttm, (struct ttm_backup_flags) {.purge = flags.purge, .writeback = flags.writeback}); - if (lret <= 0 && bo->bulk_move) { + if (lret > 0) { spin_lock(&bdev->lru_lock); - ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_del_bulk_move_unevictable(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); spin_unlock(&bdev->lru_lock); } diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 192fca24f37e44..bfd9c68fcd9c1f 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -292,6 +292,19 @@ void ttm_resource_del_bulk_move(struct ttm_resource *res, ttm_lru_bulk_move_del(bo->bulk_move, res); } +/* + * Remove a resource from its bulk_move, bypassing the unevictable check. + * Use only when the resource is known to still be tracked in the range despite + * the BO having just become unevictable; asserts that this is the case. + */ +void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res, + struct ttm_buffer_object *bo) +{ + WARN_ON_ONCE(!ttm_resource_unevictable(res, bo)); + if (bo->bulk_move) + ttm_lru_bulk_move_del(bo->bulk_move, res); +} + /* Move a resource to the LRU or bulk tail */ void ttm_resource_move_to_lru_tail(struct ttm_resource *res) { @@ -385,8 +398,11 @@ int ttm_resource_alloc(struct ttm_buffer_object *bo, if (man->cg) { ret = dmem_cgroup_try_charge(man->cg, bo->base.size, &pool, ret_limit_pool); - if (ret) + if (ret) { + if (ret == -EAGAIN) + ret = -ENOSPC; return ret; + } } ret = man->func->alloc(man, bo, place, res_ptr); diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index beeffe36b6cb06..d42db50f7c14d1 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -179,6 +179,8 @@ impl drm::Driver for TyrDriver { const INFO: drm::DriverInfo = INFO; + const FEATURES: u32 = drm::driver::FEAT_GEM; + kernel::declare_drm_ioctls! { (PANTHOR_DEV_QUERY, drm_panthor_dev_query, ioctl::RENDER_ALLOW, File::dev_query), } diff --git a/drivers/gpu/drm/tyr/file.rs b/drivers/gpu/drm/tyr/file.rs index 0ef432947b73d5..8fc26e03a9034b 100644 --- a/drivers/gpu/drm/tyr/file.rs +++ b/drivers/gpu/drm/tyr/file.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 or MIT +use kernel::bindings; use kernel::drm; use kernel::prelude::*; use kernel::uaccess::UserSlice; @@ -20,6 +21,10 @@ impl drm::file::DriverFile for File { fn open(_dev: &drm::Device) -> Result>> { KBox::try_pin_init(try_pin_init!(Self {}), GFP_KERNEL) } + + fn as_raw(&self) -> *mut bindings::drm_file { + todo!() + } } impl File { diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs index 1273bf89dbd5d7..83493904a13f5e 100644 --- a/drivers/gpu/drm/tyr/gem.rs +++ b/drivers/gpu/drm/tyr/gem.rs @@ -9,10 +9,12 @@ use kernel::prelude::*; #[pin_data] pub(crate) struct TyrObject {} +#[vtable] impl gem::DriverObject for TyrObject { type Driver = TyrDriver; + type Args = (); - fn new(_dev: &TyrDevice, _size: usize) -> impl PinInit { + fn new(_dev: &TyrDevice, _size: usize, _args: ()) -> impl PinInit { try_pin_init!(TyrObject {}) } } diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c index 08a0e9480d706c..17950fe3a0ec64 100644 --- a/drivers/gpu/drm/udl/udl_main.c +++ b/drivers/gpu/drm/udl/udl_main.c @@ -285,13 +285,12 @@ static struct urb *udl_get_urb_locked(struct udl_device *udl, long timeout) return unode->urb; } -#define GET_URB_TIMEOUT HZ struct urb *udl_get_urb(struct udl_device *udl) { struct urb *urb; spin_lock_irq(&udl->urbs.lock); - urb = udl_get_urb_locked(udl, GET_URB_TIMEOUT); + urb = udl_get_urb_locked(udl, HZ * 2); spin_unlock_irq(&udl->urbs.lock); return urb; } diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c index 231e829bd709a2..1ca073a4ecb250 100644 --- a/drivers/gpu/drm/udl/udl_modeset.c +++ b/drivers/gpu/drm/udl/udl_modeset.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -342,8 +343,10 @@ static void udl_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atom return; urb = udl_get_urb(udl); - if (!urb) + if (!urb) { + drm_err_ratelimited(dev, "get urb failed when enabling crtc\n"); goto out; + } buf = (char *)urb->transfer_buffer; buf = udl_vidreg_lock(buf); diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index dd60acdf52c2b2..86e05fcf6cf656 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -131,7 +131,7 @@ v3d_open(struct drm_device *dev, struct drm_file *file) struct v3d_dev *v3d = to_v3d_dev(dev); struct v3d_file_priv *v3d_priv; struct drm_gpu_scheduler *sched; - int i; + int i, ret; v3d_priv = kzalloc_obj(*v3d_priv); if (!v3d_priv) @@ -141,9 +141,11 @@ v3d_open(struct drm_device *dev, struct drm_file *file) for (i = 0; i < V3D_MAX_QUEUES; i++) { sched = &v3d->queue[i].sched; - drm_sched_entity_init(&v3d_priv->sched_entity[i], - DRM_SCHED_PRIORITY_NORMAL, &sched, - 1, NULL); + ret = drm_sched_entity_init(&v3d_priv->sched_entity[i], + DRM_SCHED_PRIORITY_NORMAL, &sched, + 1, NULL); + if (ret) + goto err_sched; memset(&v3d_priv->stats[i], 0, sizeof(v3d_priv->stats[i])); seqcount_init(&v3d_priv->stats[i].lock); @@ -153,6 +155,12 @@ v3d_open(struct drm_device *dev, struct drm_file *file) file->driver_priv = v3d_priv; return 0; + +err_sched: + for (i--; i >= 0; i--) + drm_sched_entity_destroy(&v3d_priv->sched_entity[i]); + kfree(v3d_priv); + return ret; } static void diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 18f2bf1fe89fac..fc74351efad5ce 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -393,6 +393,11 @@ v3d_get_multisync_submit_deps(struct drm_file *file_priv, if (multisync.pad) return -EINVAL; + if (!multisync.in_sync_count && !multisync.out_sync_count) { + drm_dbg(&v3d->drm, "Empty multisync extension\n"); + return -EINVAL; + } + ret = v3d_get_multisync_post_deps(file_priv, se, multisync.out_sync_count, multisync.out_syncs); if (ret) diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index fda214b5a466eb..2b0ba80ab4b0e7 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -2449,17 +2449,23 @@ static int vc4_hdmi_hotplug_init(struct vc4_hdmi *vc4_hdmi) int ret; if (vc4_hdmi->variant->external_irq_controller) { - unsigned int hpd_con = platform_get_irq_byname(pdev, "hpd-connected"); - unsigned int hpd_rm = platform_get_irq_byname(pdev, "hpd-removed"); + int hpd = platform_get_irq_byname(pdev, "hpd-connected"); - ret = devm_request_threaded_irq(&pdev->dev, hpd_con, + if (hpd < 0) + return hpd; + + ret = devm_request_threaded_irq(&pdev->dev, hpd, NULL, vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT, "vc4 hdmi hpd connected", vc4_hdmi); if (ret) return ret; - ret = devm_request_threaded_irq(&pdev->dev, hpd_rm, + hpd = platform_get_irq_byname(pdev, "hpd-removed"); + if (hpd < 0) + return hpd; + + ret = devm_request_threaded_irq(&pdev->dev, hpd, NULL, vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT, "vc4 hdmi hpd disconnected", vc4_hdmi); diff --git a/drivers/gpu/drm/virtio/virtgpu_prime.c b/drivers/gpu/drm/virtio/virtgpu_prime.c index 8adcf5c15d456f..05756ed4f1dac9 100644 --- a/drivers/gpu/drm/virtio/virtgpu_prime.c +++ b/drivers/gpu/drm/virtio/virtgpu_prime.c @@ -310,7 +310,7 @@ struct drm_gem_object *virtgpu_gem_prime_import(struct drm_device *dev, } } - if (!vgdev->has_resource_blob || vgdev->has_virgl_3d) + if (!vgdev->has_resource_blob) return drm_gem_prime_import(dev, buf); bo = kzalloc_obj(*bo); diff --git a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c index 29c72aa4b0d2d7..33494b86205d2e 100644 --- a/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c +++ b/drivers/gpu/drm/xe/display/xe_hdcp_gsc.c @@ -37,9 +37,17 @@ static bool intel_hdcp_gsc_check_status(struct drm_device *drm) struct xe_device *xe = to_xe_device(drm); struct xe_tile *tile = xe_device_get_root_tile(xe); struct xe_gt *gt = tile->media_gt; - struct xe_gsc *gsc = >->uc.gsc; + struct xe_gsc *gsc; + + if (!gt) { + drm_dbg_kms(&xe->drm, + "not checking GSC status for HDCP2.x: media GT not present or disabled\n"); + return false; + } + + gsc = >->uc.gsc; - if (!gsc || !xe_uc_fw_is_available(&gsc->fw)) { + if (!xe_uc_fw_is_available(&gsc->fw)) { drm_dbg_kms(&xe->drm, "GSC Components not ready for HDCP2.x\n"); return false; diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 29fffc81f2401b..8a182611ddad24 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -2154,8 +2154,10 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, } /* XE_BO_FLAG_GGTTx requires XE_BO_FLAG_GGTT also be set */ - if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT)) + if ((flags & XE_BO_FLAG_GGTT_ALL) && !(flags & XE_BO_FLAG_GGTT)) { + xe_bo_free(bo); return ERR_PTR(-EINVAL); + } if (flags & (XE_BO_FLAG_VRAM_MASK | XE_BO_FLAG_STOLEN) && !(flags & XE_BO_FLAG_IGNORE_MIN_PAGE_SIZE) && @@ -2174,8 +2176,10 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, alignment = SZ_4K >> PAGE_SHIFT; } - if (type == ttm_bo_type_device && aligned_size != size) + if (type == ttm_bo_type_device && aligned_size != size) { + xe_bo_free(bo); return ERR_PTR(-EINVAL); + } if (!bo) { bo = xe_bo_alloc(); diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index 7c74a31d448602..f9e5fa3bd2108a 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -239,8 +239,7 @@ struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) } static struct drm_gem_object * -xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, - struct dma_buf *dma_buf) +xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) { struct dma_resv *resv = dma_buf->resv; struct xe_device *xe = to_xe_device(dev); @@ -261,7 +260,7 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, if (ret) break; - bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size, + bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, 0, /* Will require 1way or 2way for vm_bind */ ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); drm_exec_retry_on_contention(&exec); @@ -312,7 +311,6 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, const struct dma_buf_attach_ops *attach_ops; struct dma_buf_attachment *attach; struct drm_gem_object *obj; - struct xe_bo *bo; if (dma_buf->ops == &xe_dmabuf_ops) { obj = dma_buf->priv; @@ -328,13 +326,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, } /* - * Don't publish the bo until we have a valid attachment, and a - * valid attachment needs the bo address. So pre-create a bo before - * creating the attachment and publish. + * This needs to happen before the attach, since it will create a new + * attachment for this, and add it to the list of attachments, at which + * point it is globally visible, and at any point the export side can + * call into on invalidate_mappings callback, which require a working + * object. */ - bo = xe_bo_alloc(); - if (IS_ERR(bo)) - return ERR_CAST(bo); + obj = xe_dma_buf_create_obj(dev, dma_buf); + if (IS_ERR(obj)) + return obj; attach_ops = &xe_dma_buf_attach_ops; #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) @@ -342,26 +342,15 @@ struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, attach_ops = test->attach_ops; #endif - attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base); + attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, obj); if (IS_ERR(attach)) { - obj = ERR_CAST(attach); - goto out_err; + xe_bo_put(gem_to_xe_bo(obj)); + return ERR_CAST(attach); } - /* Errors here will take care of freeing the bo. */ - obj = xe_dma_buf_init_obj(dev, bo, dma_buf); - if (IS_ERR(obj)) - return obj; - - get_dma_buf(dma_buf); obj->import_attach = attach; return obj; - -out_err: - xe_bo_free(bo); - - return obj; } #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 39723928a01999..7da14854f688e7 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -869,14 +869,14 @@ static int xe_eu_stall_stream_close(struct inode *inode, struct file *file) struct xe_eu_stall_data_stream *stream = file->private_data; struct xe_gt *gt = stream->gt; - drm_dev_put(>->tile->xe->drm); - mutex_lock(>->eu_stall->stream_lock); xe_eu_stall_disable_locked(stream); xe_eu_stall_data_buf_destroy(stream); xe_eu_stall_stream_free(stream); mutex_unlock(>->eu_stall->stream_lock); + drm_dev_put(>->tile->xe->drm); + return 0; } diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 8ecdf949f9e4c8..a49919da0eee68 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -1227,7 +1227,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (q->vm && q->hwe->hw_engine_group) { err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); if (err) - goto put_exec_queue; + goto kill_exec_queue; } } @@ -1236,12 +1236,15 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, /* user id alloc must always be last in ioctl to prevent UAF */ err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); if (err) - goto kill_exec_queue; + goto del_hw_engine_group; args->exec_queue_id = id; return 0; +del_hw_engine_group: + if (q->vm && q->hwe && q->hwe->hw_engine_group) + xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); kill_exec_queue: xe_exec_queue_kill(q); delete_queue_group: @@ -1574,7 +1577,7 @@ void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q, void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q, unsigned int type) { - xe_assert(q->vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || + xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT); dma_fence_put(q->tlb_inval[type].last_fence); diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c index e5c234f3d795ee..0d13e357fb43c5 100644 --- a/drivers/gpu/drm/xe/xe_gsc.c +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -166,7 +166,7 @@ static int query_compatibility_version(struct xe_gsc *gsc) &rd_offset); if (err) { xe_gt_err(gt, "HuC: invalid GSC reply for version query (err=%d)\n", err); - return err; + goto out_bo; } compat->major = version_query_rd(xe, &bo->vmap, rd_offset, proj_major); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index fc4f99d467635e..82412c8dfd37da 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -260,24 +260,12 @@ static void guc_submit_sw_fini(struct drm_device *drm, void *arg) } static void guc_submit_fini(void *arg) -{ - struct xe_guc *guc = arg; - - /* Forcefully kill any remaining exec queues */ - xe_guc_ct_stop(&guc->ct); - guc_submit_reset_prepare(guc); - xe_guc_softreset(guc); - xe_guc_submit_stop(guc); - xe_uc_fw_sanitize(&guc->fw); - xe_guc_submit_pause_abort(guc); -} - -static void guc_submit_wedged_fini(void *arg) { struct xe_guc *guc = arg; struct xe_exec_queue *q; unsigned long index; + /* Drop any wedged queue refs */ mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { if (exec_queue_wedged(q)) { @@ -287,6 +275,14 @@ static void guc_submit_wedged_fini(void *arg) } } mutex_unlock(&guc->submission_state.lock); + + /* Forcefully kill any remaining exec queues */ + xe_guc_ct_stop(&guc->ct); + guc_submit_reset_prepare(guc); + xe_guc_softreset(guc); + xe_guc_submit_stop(guc); + xe_uc_fw_sanitize(&guc->fw); + xe_guc_submit_pause_abort(guc); } static const struct xe_exec_queue_ops guc_exec_queue_ops; @@ -1272,10 +1268,8 @@ static void disable_scheduling_deregister(struct xe_guc *guc, void xe_guc_submit_wedge(struct xe_guc *guc) { struct xe_device *xe = guc_to_xe(guc); - struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; unsigned long index; - int err; xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); @@ -1286,15 +1280,7 @@ void xe_guc_submit_wedge(struct xe_guc *guc) if (!guc->submission_state.initialized) return; - if (xe->wedged.mode == 2) { - err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, - guc_submit_wedged_fini, guc); - if (err) { - xe_gt_err(gt, "Failed to register clean-up on wedged.mode=2; " - "Although device is wedged.\n"); - return; - } - + if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { mutex_lock(&guc->submission_state.lock); xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) if (xe_exec_queue_get_unless_zero(q)) diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 7b70cc01fdb387..fc38cdcc37714c 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1202,7 +1202,7 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, if (xe_gt_WARN_ON(lrc->gt, max_len < 3)) return -ENOSPC; - *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); + *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1); *cmd++ = CS_DEBUG_MODE2(0).addr; *cmd++ = _MASKED_BIT_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index 1d36c09681aaa8..fc4b6f835d4b32 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -218,7 +218,7 @@ void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, } range_start = reg & REG_GENMASK(25, range_bit); - range_end = range_start | REG_GENMASK(range_bit, 0); + range_end = range_start | REG_GENMASK(range_bit - 1, 0); switch (val & RING_FORCE_TO_NONPRIV_ACCESS_MASK) { case RING_FORCE_TO_NONPRIV_ACCESS_RW: diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c index b4086129a3648c..421224f89f573c 100644 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -357,6 +357,45 @@ static void xe_madvise_details_fini(struct xe_madvise_details *details) drm_pagemap_put(details->dpagemap); } +static bool check_pat_args_are_sane(struct xe_device *xe, + struct xe_vmas_in_madvise_range *madvise_range, + u16 pat_index) +{ + u16 coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + int i; + + /* + * Using coh_none with CPU cached buffers is not allowed on iGPU. + * On iGPU the GPU shares the LLC with the CPU, so with coh_none + * the GPU bypasses CPU caches and reads directly from DRAM, + * potentially seeing stale sensitive data from previously freed + * pages. On dGPU this restriction does not apply, because the + * platform does not provide a non-coherent system memory access + * path that would violate the DMA coherency contract. + */ + if (coh_mode != XE_COH_NONE || IS_DGFX(xe)) + return true; + + for (i = 0; i < madvise_range->num_vmas; i++) { + struct xe_vma *vma = madvise_range->vmas[i]; + struct xe_bo *bo = xe_vma_bo(vma); + + if (bo) { + /* BO with WB caching + COH_NONE is not allowed */ + if (XE_IOCTL_DBG(xe, bo->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) + return false; + /* Imported dma-buf without caching info, assume cached */ + if (XE_IOCTL_DBG(xe, !bo->cpu_caching)) + return false; + } else if (XE_IOCTL_DBG(xe, xe_vma_is_cpu_addr_mirror(vma) || + xe_vma_is_userptr(vma))) + /* System memory (userptr/SVM) is always CPU cached */ + return false; + } + + return true; +} + static bool check_bo_args_are_sane(struct xe_vm *vm, struct xe_vma **vmas, int num_vmas, u32 atomic_val) { @@ -454,6 +493,14 @@ int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *fil if (err || !madvise_range.num_vmas) goto madv_fini; + if (args->type == DRM_XE_MEM_RANGE_ATTR_PAT) { + if (!check_pat_args_are_sane(xe, &madvise_range, + args->pat_index.val)) { + err = -EINVAL; + goto free_vmas; + } + } + if (madvise_range.has_bo_vmas) { if (args->type == DRM_XE_MEM_RANGE_ATTR_ATOMIC) { if (!check_bo_args_are_sane(vm, madvise_range.vmas, diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c index d7e309ad9abaf5..9ddd21a21dcefd 100644 --- a/drivers/gpu/drm/xe/xe_wa.c +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -708,6 +708,14 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION(1200)), XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, DISABLE_TDC_LOAD_BALANCING_CALC)) }, + { XE_RTP_NAME("14019877138"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) + }, + { XE_RTP_NAME("14019386621"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)), + XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE)) + }, /* DG1 */ @@ -744,10 +752,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(PLATFORM(DG2)), XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE)) }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(PLATFORM(DG2)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, /* PVC */ @@ -765,21 +769,9 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274)), XE_RTP_ACTIONS(SET(CACHE_MODE_1, MSAA_OPTIMIZATION_REDUC_DISABLE)) }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1270, 1274), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, /* Xe2_LPG */ - { XE_RTP_NAME("14019386621"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE)) - }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14019988906"), XE_RTP_RULES(GRAPHICS_VERSION(2004), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) @@ -813,14 +805,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = { }, /* Xe2_HPG */ - { XE_RTP_NAME("15010599737"), - XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(CHICKEN_RASTER_1, DIS_SF_ROUND_NEAREST_EVEN)) - }, - { XE_RTP_NAME("14019386621"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(VF_SCRATCHPAD, XE2_VFG_TED_CREDIT_INTERFACE_DISABLE)) - }, + { XE_RTP_NAME("14020756599"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(WM_CHICKEN3, HIZ_PLANE_COMPRESSION_DIS)) @@ -829,10 +814,6 @@ static const struct xe_rtp_entry_sr lrc_was[] = { XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FLSH_IGNORES_PSD)) }, - { XE_RTP_NAME("14019877138"), - XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(SET(XEHP_PSS_CHICKEN, FD_END_COLLECT)) - }, { XE_RTP_NAME("14021490052"), XE_RTP_RULES(GRAPHICS_VERSION(2001), ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(SET(FF_MODE, diff --git a/drivers/gpu/nova-core/bitfield.rs b/drivers/gpu/nova-core/bitfield.rs index 16e143658c511d..02efdcf78d8944 100644 --- a/drivers/gpu/nova-core/bitfield.rs +++ b/drivers/gpu/nova-core/bitfield.rs @@ -314,12 +314,11 @@ macro_rules! bitfield { /// Returns a value for the bitfield where all fields are set to their default value. impl ::core::default::Default for $name { fn default() -> Self { - #[allow(unused_mut)] - let mut value = Self(Default::default()); + let value = Self(Default::default()); ::kernel::macros::paste!( $( - value.[](Default::default()); + let value = value.[](Default::default()); )* ); diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 5a4cc047bcfc9f..e39885c0d5ca5d 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -70,7 +70,7 @@ impl pci::Driver for NovaCore { fn probe(pdev: &pci::Device, _info: &Self::IdInfo) -> impl PinInit { pin_init::pin_init_scope(move || { - dev_dbg!(pdev.as_ref(), "Probe Nova Core GPU driver.\n"); + dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); pdev.enable_device_mem()?; pdev.set_master(); diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 37bfee1d094926..808c17e981d192 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -2,12 +2,13 @@ //! Falcon microprocessor base support -use core::ops::Deref; - use hal::FalconHal; use kernel::{ - device, + device::{ + self, + Device, // + }, dma::{ DmaAddress, DmaMask, // @@ -15,9 +16,7 @@ use kernel::{ io::poll::read_poll_timeout, prelude::*, sync::aref::ARef, - time::{ - Delta, // - }, + time::Delta, }; use crate::{ @@ -327,9 +326,10 @@ pub(crate) trait FalconEngine: const ID: Self; } -/// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM). +/// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM) +/// using DMA. #[derive(Debug, Clone)] -pub(crate) struct FalconLoadTarget { +pub(crate) struct FalconDmaLoadTarget { /// Offset from the start of the source object to copy from. pub(crate) src_start: u32, /// Offset from the start of the destination memory to copy into. @@ -349,17 +349,20 @@ pub(crate) struct FalconBromParams { pub(crate) ucode_id: u8, } -/// Trait for providing load parameters of falcon firmwares. -pub(crate) trait FalconLoadParams { +/// Trait implemented by falcon firmwares that can be loaded using DMA. +pub(crate) trait FalconDmaLoadable { + /// Returns the firmware data as a slice of bytes. + fn as_slice(&self) -> &[u8]; + /// Returns the load parameters for Secure `IMEM`. - fn imem_sec_load_params(&self) -> FalconLoadTarget; + fn imem_sec_load_params(&self) -> FalconDmaLoadTarget; /// Returns the load parameters for Non-Secure `IMEM`, /// used only on Turing and GA100. - fn imem_ns_load_params(&self) -> Option; + fn imem_ns_load_params(&self) -> Option; /// Returns the load parameters for `DMEM`. - fn dmem_load_params(&self) -> FalconLoadTarget; + fn dmem_load_params(&self) -> FalconDmaLoadTarget; /// Returns the parameters to write into the BROM registers. fn brom_params(&self) -> FalconBromParams; @@ -370,9 +373,8 @@ pub(crate) trait FalconLoadParams { /// Trait for a falcon firmware. /// -/// A falcon firmware can be loaded on a given engine, and is presented in the form of a DMA -/// object. -pub(crate) trait FalconFirmware: FalconLoadParams + Deref { +/// A falcon firmware can be loaded on a given engine. +pub(crate) trait FalconFirmware: FalconDmaLoadable { /// Engine on which this firmware is to be loaded. type Target: FalconEngine; } @@ -415,12 +417,12 @@ impl Falcon { /// `target_mem`. /// /// `sec` is set if the loaded firmware is expected to run in secure mode. - fn dma_wr>( + fn dma_wr( &self, bar: &Bar0, - fw: &F, + dma_obj: &DmaObject, target_mem: FalconMem, - load_offsets: FalconLoadTarget, + load_offsets: FalconDmaLoadTarget, ) -> Result { const DMA_LEN: u32 = 256; @@ -430,11 +432,11 @@ impl Falcon { // For DMEM we can fold the start offset into the DMA handle. let (src_start, dma_start) = match target_mem { FalconMem::ImemSecure | FalconMem::ImemNonSecure => { - (load_offsets.src_start, fw.dma_handle()) + (load_offsets.src_start, dma_obj.dma_handle()) } FalconMem::Dmem => ( 0, - fw.dma_handle_with_offset(load_offsets.src_start.into_safe_cast())?, + dma_obj.dma_handle_with_offset(load_offsets.src_start.into_safe_cast())?, ), }; if dma_start % DmaAddress::from(DMA_LEN) > 0 { @@ -466,7 +468,7 @@ impl Falcon { dev_err!(self.dev, "DMA transfer length overflow\n"); return Err(EOVERFLOW); } - Some(upper_bound) if usize::from_safe_cast(upper_bound) > fw.size() => { + Some(upper_bound) if usize::from_safe_cast(upper_bound) > dma_obj.size() => { dev_err!(self.dev, "DMA transfer goes beyond range of DMA object\n"); return Err(EINVAL); } @@ -515,7 +517,12 @@ impl Falcon { } /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. - fn dma_load>(&self, bar: &Bar0, fw: &F) -> Result { + fn dma_load>( + &self, + dev: &Device, + bar: &Bar0, + fw: &F, + ) -> Result { // The Non-Secure section only exists on firmware used by Turing and GA100, and // those platforms do not use DMA. if fw.imem_ns_load_params().is_some() { @@ -523,14 +530,22 @@ impl Falcon { return Err(EINVAL); } + // Create DMA object with firmware content as the source of the DMA engine. + let dma_obj = DmaObject::from_data(dev, fw.as_slice())?; + self.dma_reset(bar); regs::NV_PFALCON_FBIF_TRANSCFG::update(bar, &E::ID, 0, |v| { v.set_target(FalconFbifTarget::CoherentSysmem) .set_mem_type(FalconFbifMemType::Physical) }); - self.dma_wr(bar, fw, FalconMem::ImemSecure, fw.imem_sec_load_params())?; - self.dma_wr(bar, fw, FalconMem::Dmem, fw.dmem_load_params())?; + self.dma_wr( + bar, + &dma_obj, + FalconMem::ImemSecure, + fw.imem_sec_load_params(), + )?; + self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; self.hal.program_brom(self, bar, &fw.brom_params())?; @@ -641,9 +656,14 @@ impl Falcon { } // Load a firmware image into Falcon memory - pub(crate) fn load>(&self, bar: &Bar0, fw: &F) -> Result { + pub(crate) fn load>( + &self, + dev: &Device, + bar: &Bar0, + fw: &F, + ) -> Result { match self.hal.load_method() { - LoadMethod::Dma => self.dma_load(bar, fw), + LoadMethod::Dma => self.dma_load(dev, bar, fw), LoadMethod::Pio => Err(ENOTSUPP), } } diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 68779540aa2842..6a58118648652c 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -15,10 +15,9 @@ use kernel::{ }; use crate::{ - dma::DmaObject, falcon::{ - FalconFirmware, - FalconLoadTarget, // + FalconDmaLoadTarget, + FalconFirmware, // }, gpu, num::{ @@ -64,7 +63,8 @@ pub(crate) struct FalconUCodeDescV2 { pub(crate) interface_offset: u32, /// Base address at which to load the code segment into 'IMEM'. pub(crate) imem_phys_base: u32, - /// Size in bytes of the code to copy into 'IMEM'. + /// Size in bytes of the code to copy into 'IMEM' (includes both secure and non-secure + /// segments). pub(crate) imem_load_size: u32, /// Virtual 'IMEM' address (i.e. 'tag') at which the code should start. pub(crate) imem_virt_base: u32, @@ -171,9 +171,9 @@ pub(crate) trait FalconUCodeDescriptor { ((hdr & HDR_SIZE_MASK) >> HDR_SIZE_SHIFT).into_safe_cast() } - fn imem_sec_load_params(&self) -> FalconLoadTarget; - fn imem_ns_load_params(&self) -> Option; - fn dmem_load_params(&self) -> FalconLoadTarget; + fn imem_sec_load_params(&self) -> FalconDmaLoadTarget; + fn imem_ns_load_params(&self) -> Option; + fn dmem_load_params(&self) -> FalconDmaLoadTarget; } impl FalconUCodeDescriptor for FalconUCodeDescV2 { @@ -205,24 +205,31 @@ impl FalconUCodeDescriptor for FalconUCodeDescV2 { 0 } - fn imem_sec_load_params(&self) -> FalconLoadTarget { - FalconLoadTarget { - src_start: 0, - dst_start: self.imem_sec_base, + fn imem_sec_load_params(&self) -> FalconDmaLoadTarget { + // `imem_sec_base` is the *virtual* start address of the secure IMEM segment, so subtract + // `imem_virt_base` to get its physical offset. + let imem_sec_start = self.imem_sec_base.saturating_sub(self.imem_virt_base); + + FalconDmaLoadTarget { + src_start: imem_sec_start, + dst_start: self.imem_phys_base.saturating_add(imem_sec_start), len: self.imem_sec_size, } } - fn imem_ns_load_params(&self) -> Option { - Some(FalconLoadTarget { + fn imem_ns_load_params(&self) -> Option { + Some(FalconDmaLoadTarget { + // Non-secure code always starts at offset 0. src_start: 0, dst_start: self.imem_phys_base, - len: self.imem_load_size.checked_sub(self.imem_sec_size)?, + // `imem_load_size` includes the size of the secure segment, so subtract it to + // get the correct amount of data to copy. + len: self.imem_load_size.saturating_sub(self.imem_sec_size), }) } - fn dmem_load_params(&self) -> FalconLoadTarget { - FalconLoadTarget { + fn dmem_load_params(&self) -> FalconDmaLoadTarget { + FalconDmaLoadTarget { src_start: self.dmem_offset, dst_start: self.dmem_phys_base, len: self.dmem_load_size, @@ -259,21 +266,21 @@ impl FalconUCodeDescriptor for FalconUCodeDescV3 { self.signature_versions } - fn imem_sec_load_params(&self) -> FalconLoadTarget { - FalconLoadTarget { + fn imem_sec_load_params(&self) -> FalconDmaLoadTarget { + FalconDmaLoadTarget { src_start: 0, dst_start: self.imem_phys_base, len: self.imem_load_size, } } - fn imem_ns_load_params(&self) -> Option { + fn imem_ns_load_params(&self) -> Option { // Not used on V3 platforms None } - fn dmem_load_params(&self) -> FalconLoadTarget { - FalconLoadTarget { + fn dmem_load_params(&self) -> FalconDmaLoadTarget { + FalconDmaLoadTarget { src_start: self.imem_load_size, dst_start: self.dmem_phys_base, len: self.dmem_load_size, @@ -292,7 +299,7 @@ impl SignedState for Unsigned {} struct Signed; impl SignedState for Signed {} -/// A [`DmaObject`] containing a specific microcode ready to be loaded into a falcon. +/// Microcode to be loaded into a specific falcon. /// /// This is module-local and meant for sub-modules to use internally. /// @@ -300,34 +307,35 @@ impl SignedState for Signed {} /// before it can be loaded (with an exception for development hardware). The /// [`Self::patch_signature`] and [`Self::no_patch_signature`] methods are used to transition the /// firmware to its [`Signed`] state. -struct FirmwareDmaObject(DmaObject, PhantomData<(F, S)>); +// TODO: Consider replacing this with a coherent memory object once `CoherentAllocation` supports +// temporary CPU-exclusive access to the object without unsafe methods. +struct FirmwareObject(KVVec, PhantomData<(F, S)>); /// Trait for signatures to be patched directly into a given firmware. /// /// This is module-local and meant for sub-modules to use internally. trait FirmwareSignature: AsRef<[u8]> {} -impl FirmwareDmaObject { - /// Patches the firmware at offset `sig_base_img` with `signature`. +impl FirmwareObject { + /// Patches the firmware at offset `signature_start` with `signature`. fn patch_signature>( mut self, signature: &S, - sig_base_img: usize, - ) -> Result> { + signature_start: usize, + ) -> Result> { let signature_bytes = signature.as_ref(); - if sig_base_img + signature_bytes.len() > self.0.size() { - return Err(EINVAL); - } - - // SAFETY: We are the only user of this object, so there cannot be any race. - let dst = unsafe { self.0.start_ptr_mut().add(sig_base_img) }; + let signature_end = signature_start + .checked_add(signature_bytes.len()) + .ok_or(EOVERFLOW)?; + let dst = self + .0 + .get_mut(signature_start..signature_end) + .ok_or(EINVAL)?; - // SAFETY: `signature` and `dst` are valid, properly aligned, and do not overlap. - unsafe { - core::ptr::copy_nonoverlapping(signature_bytes.as_ptr(), dst, signature_bytes.len()) - }; + // PANIC: `dst` and `signature_bytes` have the same length. + dst.copy_from_slice(signature_bytes); - Ok(FirmwareDmaObject(self.0, PhantomData)) + Ok(FirmwareObject(self.0, PhantomData)) } /// Mark the firmware as signed without patching it. @@ -335,8 +343,8 @@ impl FirmwareDmaObject { /// This method is used to explicitly confirm that we do not need to sign the firmware, while /// allowing us to continue as if it was. This is typically only needed for development /// hardware. - fn no_patch_signature(self) -> FirmwareDmaObject { - FirmwareDmaObject(self.0, PhantomData) + fn no_patch_signature(self) -> FirmwareObject { + FirmwareObject(self.0, PhantomData) } } diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index 86556cee8e67b8..1a6b2a7e179062 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -4,10 +4,7 @@ //! running on [`Sec2`], that is used on Turing/Ampere to load the GSP firmware into the GSP falcon //! (and optionally unload it through a separate firmware image). -use core::{ - marker::PhantomData, - ops::Deref, // -}; +use core::marker::PhantomData; use kernel::{ device, @@ -16,19 +13,18 @@ use kernel::{ }; use crate::{ - dma::DmaObject, driver::Bar0, falcon::{ sec2::Sec2, Falcon, FalconBromParams, - FalconFirmware, - FalconLoadParams, - FalconLoadTarget, // + FalconDmaLoadTarget, + FalconDmaLoadable, + FalconFirmware, // }, firmware::{ BinFirmware, - FirmwareDmaObject, + FirmwareObject, FirmwareSignature, Signed, Unsigned, // @@ -252,21 +248,24 @@ impl<'a> FirmwareSignature for BooterSignature<'a> {} /// The `Booter` loader firmware, responsible for loading the GSP. pub(crate) struct BooterFirmware { // Load parameters for Secure `IMEM` falcon memory. - imem_sec_load_target: FalconLoadTarget, + imem_sec_load_target: FalconDmaLoadTarget, // Load parameters for Non-Secure `IMEM` falcon memory, // used only on Turing and GA100 - imem_ns_load_target: Option, + imem_ns_load_target: Option, // Load parameters for `DMEM` falcon memory. - dmem_load_target: FalconLoadTarget, + dmem_load_target: FalconDmaLoadTarget, // BROM falcon parameters. brom_params: FalconBromParams, // Device-mapped firmware image. - ucode: FirmwareDmaObject, + ucode: FirmwareObject, } -impl FirmwareDmaObject { - fn new_booter(dev: &device::Device, data: &[u8]) -> Result { - DmaObject::from_data(dev, data).map(|ucode| Self(ucode, PhantomData)) +impl FirmwareObject { + fn new_booter(data: &[u8]) -> Result { + let mut ucode = KVVec::new(); + ucode.extend_from_slice(data, GFP_KERNEL)?; + + Ok(Self(ucode, PhantomData)) } } @@ -320,7 +319,7 @@ impl BooterFirmware { let ucode = bin_fw .data() .ok_or(EINVAL) - .and_then(|data| FirmwareDmaObject::::new_booter(dev, data))?; + .and_then(FirmwareObject::::new_booter)?; let ucode_signed = { let mut signatures = hs_fw.signatures_iter()?.peekable(); @@ -363,7 +362,7 @@ impl BooterFirmware { let (imem_sec_dst_start, imem_ns_load_target) = if chipset <= Chipset::GA100 { ( app0.offset, - Some(FalconLoadTarget { + Some(FalconDmaLoadTarget { src_start: 0, dst_start: load_hdr.os_code_offset, len: load_hdr.os_code_size, @@ -374,13 +373,13 @@ impl BooterFirmware { }; Ok(Self { - imem_sec_load_target: FalconLoadTarget { + imem_sec_load_target: FalconDmaLoadTarget { src_start: app0.offset, dst_start: imem_sec_dst_start, len: app0.len, }, imem_ns_load_target, - dmem_load_target: FalconLoadTarget { + dmem_load_target: FalconDmaLoadTarget { src_start: load_hdr.os_data_offset, dst_start: 0, len: load_hdr.os_data_size, @@ -391,16 +390,20 @@ impl BooterFirmware { } } -impl FalconLoadParams for BooterFirmware { - fn imem_sec_load_params(&self) -> FalconLoadTarget { +impl FalconDmaLoadable for BooterFirmware { + fn as_slice(&self) -> &[u8] { + self.ucode.0.as_slice() + } + + fn imem_sec_load_params(&self) -> FalconDmaLoadTarget { self.imem_sec_load_target.clone() } - fn imem_ns_load_params(&self) -> Option { + fn imem_ns_load_params(&self) -> Option { self.imem_ns_load_target.clone() } - fn dmem_load_params(&self) -> FalconLoadTarget { + fn dmem_load_params(&self) -> FalconDmaLoadTarget { self.dmem_load_target.clone() } @@ -417,14 +420,6 @@ impl FalconLoadParams for BooterFirmware { } } -impl Deref for BooterFirmware { - type Target = DmaObject; - - fn deref(&self) -> &Self::Target { - &self.ucode.0 - } -} - impl FalconFirmware for BooterFirmware { type Target = Sec2; } diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index bfb7b06b13d150..7ac5cfeb594d4b 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -10,10 +10,7 @@ //! - The command to be run, as this firmware can perform several tasks ; //! - The ucode signature, so the GSP falcon can run FWSEC in HS mode. -use core::{ - marker::PhantomData, - ops::Deref, // -}; +use core::marker::PhantomData; use kernel::{ device::{ @@ -28,27 +25,23 @@ use kernel::{ }; use crate::{ - dma::DmaObject, driver::Bar0, falcon::{ gsp::Gsp, Falcon, FalconBromParams, - FalconFirmware, - FalconLoadParams, - FalconLoadTarget, // + FalconDmaLoadTarget, + FalconDmaLoadable, + FalconFirmware, // }, firmware::{ FalconUCodeDesc, - FirmwareDmaObject, + FirmwareObject, FirmwareSignature, Signed, Unsigned, // }, - num::{ - FromSafeCast, - IntoSafeCast, // - }, + num::FromSafeCast, vbios::Vbios, }; @@ -177,61 +170,30 @@ impl AsRef<[u8]> for Bcrt30Rsa3kSignature { impl FirmwareSignature for Bcrt30Rsa3kSignature {} -/// Reinterpret the area starting from `offset` in `fw` as an instance of `T` (which must implement -/// [`FromBytes`]) and return a reference to it. -/// -/// # Safety -/// -/// * Callers must ensure that the device does not read/write to/from memory while the returned -/// reference is live. -/// * Callers must ensure that this call does not race with a write to the same region while -/// the returned reference is live. -unsafe fn transmute(fw: &DmaObject, offset: usize) -> Result<&T> { - // SAFETY: The safety requirements of the function guarantee the device won't read - // or write to memory while the reference is alive and that this call won't race - // with writes to the same memory region. - T::from_bytes(unsafe { fw.as_slice(offset, size_of::())? }).ok_or(EINVAL) -} - -/// Reinterpret the area starting from `offset` in `fw` as a mutable instance of `T` (which must -/// implement [`FromBytes`]) and return a reference to it. -/// -/// # Safety -/// -/// * Callers must ensure that the device does not read/write to/from memory while the returned -/// slice is live. -/// * Callers must ensure that this call does not race with a read or write to the same region -/// while the returned slice is live. -unsafe fn transmute_mut( - fw: &mut DmaObject, - offset: usize, -) -> Result<&mut T> { - // SAFETY: The safety requirements of the function guarantee the device won't read - // or write to memory while the reference is alive and that this call won't race - // with writes or reads to the same memory region. - T::from_bytes_mut(unsafe { fw.as_slice_mut(offset, size_of::())? }).ok_or(EINVAL) -} - /// The FWSEC microcode, extracted from the BIOS and to be run on the GSP falcon. /// /// It is responsible for e.g. carving out the WPR2 region as the first step of the GSP bootflow. pub(crate) struct FwsecFirmware { /// Descriptor of the firmware. desc: FalconUCodeDesc, - /// GPU-accessible DMA object containing the firmware. - ucode: FirmwareDmaObject, + /// Object containing the firmware binary. + ucode: FirmwareObject, } -impl FalconLoadParams for FwsecFirmware { - fn imem_sec_load_params(&self) -> FalconLoadTarget { +impl FalconDmaLoadable for FwsecFirmware { + fn as_slice(&self) -> &[u8] { + self.ucode.0.as_slice() + } + + fn imem_sec_load_params(&self) -> FalconDmaLoadTarget { self.desc.imem_sec_load_params() } - fn imem_ns_load_params(&self) -> Option { + fn imem_ns_load_params(&self) -> Option { self.desc.imem_ns_load_params() } - fn dmem_load_params(&self) -> FalconLoadTarget { + fn dmem_load_params(&self) -> FalconDmaLoadTarget { self.desc.dmem_load_params() } @@ -248,27 +210,27 @@ impl FalconLoadParams for FwsecFirmware { } } -impl Deref for FwsecFirmware { - type Target = DmaObject; - - fn deref(&self) -> &Self::Target { - &self.ucode.0 - } -} - impl FalconFirmware for FwsecFirmware { type Target = Gsp; } -impl FirmwareDmaObject { - fn new_fwsec(dev: &Device, bios: &Vbios, cmd: FwsecCommand) -> Result { +impl FirmwareObject { + fn new_fwsec(bios: &Vbios, cmd: FwsecCommand) -> Result { let desc = bios.fwsec_image().header()?; - let ucode = bios.fwsec_image().ucode(&desc)?; - let mut dma_object = DmaObject::from_data(dev, ucode)?; + let mut ucode = KVVec::new(); + ucode.extend_from_slice(bios.fwsec_image().ucode(&desc)?, GFP_KERNEL)?; + + let hdr_offset = desc + .imem_load_size() + .checked_add(desc.interface_offset()) + .map(usize::from_safe_cast) + .ok_or(EINVAL)?; - let hdr_offset = usize::from_safe_cast(desc.imem_load_size() + desc.interface_offset()); - // SAFETY: we have exclusive access to `dma_object`. - let hdr: &FalconAppifHdrV1 = unsafe { transmute(&dma_object, hdr_offset) }?; + let hdr = ucode + .get(hdr_offset..) + .and_then(FalconAppifHdrV1::from_bytes_prefix) + .ok_or(EINVAL)? + .0; if hdr.version != 1 { return Err(EINVAL); @@ -276,26 +238,34 @@ impl FirmwareDmaObject { // Find the DMEM mapper section in the firmware. for i in 0..usize::from(hdr.entry_count) { - // SAFETY: we have exclusive access to `dma_object`. - let app: &FalconAppifV1 = unsafe { - transmute( - &dma_object, - hdr_offset + usize::from(hdr.header_size) + i * usize::from(hdr.entry_size), - ) - }?; + // CALC: hdr_offset + header_size + i * entry_size. + let entry_offset = hdr_offset + .checked_add(usize::from(hdr.header_size)) + .and_then(|o| o.checked_add(i.checked_mul(usize::from(hdr.entry_size))?)) + .ok_or(EINVAL)?; + + let app = ucode + .get(entry_offset..) + .and_then(FalconAppifV1::from_bytes_prefix) + .ok_or(EINVAL)? + .0; if app.id != NVFW_FALCON_APPIF_ID_DMEMMAPPER { continue; } let dmem_base = app.dmem_base; - // SAFETY: we have exclusive access to `dma_object`. - let dmem_mapper: &mut FalconAppifDmemmapperV3 = unsafe { - transmute_mut( - &mut dma_object, - (desc.imem_load_size() + dmem_base).into_safe_cast(), - ) - }?; + let dmem_mapper_offset = desc + .imem_load_size() + .checked_add(dmem_base) + .map(usize::from_safe_cast) + .ok_or(EINVAL)?; + + let dmem_mapper = ucode + .get_mut(dmem_mapper_offset..) + .and_then(FalconAppifDmemmapperV3::from_bytes_mut_prefix) + .ok_or(EINVAL)? + .0; dmem_mapper.init_cmd = match cmd { FwsecCommand::Frts { .. } => NVFW_FALCON_APPIF_DMEMMAPPER_CMD_FRTS, @@ -303,13 +273,17 @@ impl FirmwareDmaObject { }; let cmd_in_buffer_offset = dmem_mapper.cmd_in_buffer_offset; - // SAFETY: we have exclusive access to `dma_object`. - let frts_cmd: &mut FrtsCmd = unsafe { - transmute_mut( - &mut dma_object, - (desc.imem_load_size() + cmd_in_buffer_offset).into_safe_cast(), - ) - }?; + let frts_cmd_offset = desc + .imem_load_size() + .checked_add(cmd_in_buffer_offset) + .map(usize::from_safe_cast) + .ok_or(EINVAL)?; + + let frts_cmd = ucode + .get_mut(frts_cmd_offset..) + .and_then(FrtsCmd::from_bytes_mut_prefix) + .ok_or(EINVAL)? + .0; frts_cmd.read_vbios = ReadVbios { ver: 1, @@ -333,7 +307,7 @@ impl FirmwareDmaObject { } // Return early as we found and patched the DMEMMAPPER region. - return Ok(Self(dma_object, PhantomData)); + return Ok(Self(ucode, PhantomData)); } Err(ENOTSUPP) @@ -350,13 +324,16 @@ impl FwsecFirmware { bios: &Vbios, cmd: FwsecCommand, ) -> Result { - let ucode_dma = FirmwareDmaObject::::new_fwsec(dev, bios, cmd)?; + let ucode_dma = FirmwareObject::::new_fwsec(bios, cmd)?; // Patch signature if needed. let desc = bios.fwsec_image().header()?; let ucode_signed = if desc.signature_count() != 0 { - let sig_base_img = - usize::from_safe_cast(desc.imem_load_size() + desc.pkc_data_offset()); + let sig_base_img = desc + .imem_load_size() + .checked_add(desc.pkc_data_offset()) + .map(usize::from_safe_cast) + .ok_or(EINVAL)?; let desc_sig_versions = u32::from(desc.signature_versions()); let reg_fuse_version = falcon.signature_reg_fuse_version(bar, desc.engine_id_mask(), desc.ucode_id())?; @@ -419,7 +396,7 @@ impl FwsecFirmware { .reset(bar) .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?; falcon - .load(bar, self) + .load(dev, bar, self) .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; let (mbox0, _) = falcon .boot(bar, Some(0), None) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 9b042ef1a30868..60c85fffaeafd9 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -262,13 +262,13 @@ impl Gpu { ) -> impl PinInit + 'a { try_pin_init!(Self { spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { - dev_info!(pdev.as_ref(),"NVIDIA ({})\n", spec); + dev_info!(pdev,"NVIDIA ({})\n", spec); })?, // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. _: { gfw::wait_gfw_boot_completion(bar) - .inspect_err(|_| dev_err!(pdev.as_ref(), "GFW boot did not complete\n"))?; + .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; }, sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 94833f7996e8aa..99e7a0b7e3107d 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -170,39 +170,25 @@ impl super::Gsp { Some(libos_handle as u32), Some((libos_handle >> 32) as u32), )?; - dev_dbg!( - pdev.as_ref(), - "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", - mbox0, - mbox1 - ); + dev_dbg!(pdev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); dev_dbg!( - pdev.as_ref(), + pdev, "Using SEC2 to load and run the booter_load firmware...\n" ); sec2_falcon.reset(bar)?; - sec2_falcon.load(bar, &booter_loader)?; + sec2_falcon.load(dev, bar, &booter_loader)?; let wpr_handle = wpr_meta.dma_handle(); let (mbox0, mbox1) = sec2_falcon.boot( bar, Some(wpr_handle as u32), Some((wpr_handle >> 32) as u32), )?; - dev_dbg!( - pdev.as_ref(), - "SEC2 MBOX0: {:#x}, MBOX1{:#x}\n", - mbox0, - mbox1 - ); + dev_dbg!(pdev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); if mbox0 != 0 { - dev_err!( - pdev.as_ref(), - "Booter-load failed with error {:#x}\n", - mbox0 - ); + dev_err!(pdev, "Booter-load failed with error {:#x}\n", mbox0); return Err(ENODEV); } @@ -216,11 +202,7 @@ impl super::Gsp { Delta::from_secs(5), )?; - dev_dbg!( - pdev.as_ref(), - "RISC-V active? {}\n", - gsp_falcon.is_riscv_active(bar), - ); + dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),); // Create and run the GSP sequencer. let seq_params = GspSequencerParams { @@ -239,8 +221,8 @@ impl super::Gsp { // Obtain and display basic GPU information. let info = commands::get_gsp_info(&mut self.cmdq, bar)?; match info.gpu_name() { - Ok(name) => dev_info!(pdev.as_ref(), "GPU name: {}\n", name), - Err(e) => dev_warn!(pdev.as_ref(), "GPU name unavailable: {:?}\n", e), + Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), + Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), } Ok(()) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 03a4f359984988..5da153c71800a7 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -237,21 +237,27 @@ impl DmaGspMem { // PANIC: per the invariant of `cpu_write_ptr`, `tx` is `<= MSGQ_NUM_PAGES`. let (before_tx, after_tx) = gsp_mem.cpuq.msgq.data.split_at_mut(tx); - if rx <= tx { - // The area from `tx` up to the end of the ring, and from the beginning of the ring up - // to `rx`, minus one unit, belongs to the driver. - if rx == 0 { - let last = after_tx.len() - 1; - (&mut after_tx[..last], &mut before_tx[0..0]) - } else { - (after_tx, &mut before_tx[..rx]) - } + // The area starting at `tx` and ending at `rx - 2` modulo MSGQ_NUM_PAGES, inclusive, + // belongs to the driver for writing. + + if rx == 0 { + // Since `rx` is zero, leave an empty slot at end of the buffer. + let last = after_tx.len() - 1; + (&mut after_tx[..last], &mut []) + } else if rx <= tx { + // The area is discontiguous and we leave an empty slot before `rx`. + // PANIC: + // - The index `rx - 1` is non-negative because `rx != 0` in this branch. + // - The index does not exceed `before_tx.len()` (which equals `tx`) because + // `rx <= tx` in this branch. + (after_tx, &mut before_tx[..(rx - 1)]) } else { - // The area from `tx` to `rx`, minus one unit, belongs to the driver. - // - // PANIC: per the invariants of `cpu_write_ptr` and `gsp_read_ptr`, `rx` and `tx` are - // `<= MSGQ_NUM_PAGES`, and the test above ensured that `rx > tx`. - (after_tx.split_at_mut(rx - tx).0, &mut before_tx[0..0]) + // The area is contiguous and we leave an empty slot before `rx`. + // PANIC: + // - The index `rx - tx - 1` is non-negative because `rx > tx` in this branch. + // - The index does not exceed `after_tx.len()` (which is `MSGQ_NUM_PAGES - tx`) + // because `rx < MSGQ_NUM_PAGES` by the `gsp_read_ptr` invariant. + (&mut after_tx[..(rx - tx - 1)], &mut []) } } @@ -273,8 +279,8 @@ impl DmaGspMem { let (before_rx, after_rx) = gsp_mem.gspq.msgq.data.split_at(rx); match tx.cmp(&rx) { - cmp::Ordering::Equal => (&after_rx[0..0], &after_rx[0..0]), - cmp::Ordering::Greater => (&after_rx[..tx], &before_rx[0..0]), + cmp::Ordering::Equal => (&[], &[]), + cmp::Ordering::Greater => (&after_rx[..tx], &[]), cmp::Ordering::Less => (after_rx, &before_rx[..tx]), } } diff --git a/drivers/greybus/gb-beagleplay.c b/drivers/greybus/gb-beagleplay.c index 87186f891a6acb..e28d1e9ec95732 100644 --- a/drivers/greybus/gb-beagleplay.c +++ b/drivers/greybus/gb-beagleplay.c @@ -242,30 +242,26 @@ static void hdlc_write(struct gb_beagleplay *bg) } /** - * hdlc_append() - Queue HDLC data for sending. + * hdlc_append() - Queue a single HDLC byte for sending. * @bg: beagleplay greybus driver * @value: hdlc byte to transmit * - * Assumes that producer lock as been acquired. + * Caller must hold tx_producer_lock and must have ensured sufficient + * space in the circular buffer before calling (see hdlc_tx_frames()). */ static void hdlc_append(struct gb_beagleplay *bg, u8 value) { - int tail, head = bg->tx_circ_buf.head; + int head = bg->tx_circ_buf.head; + int tail = READ_ONCE(bg->tx_circ_buf.tail); - while (true) { - tail = READ_ONCE(bg->tx_circ_buf.tail); - - if (CIRC_SPACE(head, tail, TX_CIRC_BUF_SIZE) >= 1) { - bg->tx_circ_buf.buf[head] = value; + lockdep_assert_held(&bg->tx_producer_lock); + if (WARN_ON_ONCE(CIRC_SPACE(head, tail, TX_CIRC_BUF_SIZE) < 1)) + return; - /* Finish producing HDLC byte */ - smp_store_release(&bg->tx_circ_buf.head, - (head + 1) & (TX_CIRC_BUF_SIZE - 1)); - return; - } - dev_warn(&bg->sd->dev, "Tx circ buf full"); - usleep_range(3000, 5000); - } + bg->tx_circ_buf.buf[head] = value; + /* Ensure buffer write is visible before advancing head. */ + smp_store_release(&bg->tx_circ_buf.head, + (head + 1) & (TX_CIRC_BUF_SIZE - 1)); } static void hdlc_append_escaped(struct gb_beagleplay *bg, u8 value) @@ -313,13 +309,90 @@ static void hdlc_transmit(struct work_struct *work) spin_unlock_bh(&bg->tx_consumer_lock); } +/** + * hdlc_encoded_length() - Calculate worst-case encoded length of an HDLC frame. + * @payloads: array of payload buffers + * @count: number of payloads + * + * Returns the maximum number of bytes needed in the circular buffer. + */ +static size_t hdlc_encoded_length(const struct hdlc_payload payloads[], + size_t count) +{ + size_t i, payload_len = 0; + + for (i = 0; i < count; i++) + payload_len += payloads[i].len; + + /* + * Worst case: every data byte needs escaping (doubles in size). + * data bytes = address(1) + control(1) + payload + crc(2) + * framing = opening flag(1) + closing flag(1) + */ + return 2 + (1 + 1 + payload_len + 2) * 2; +} + +#define HDLC_TX_BUF_WAIT_RETRIES 500 +#define HDLC_TX_BUF_WAIT_US_MIN 3000 +#define HDLC_TX_BUF_WAIT_US_MAX 5000 + +/** + * hdlc_tx_frames() - Encode and queue an HDLC frame for transmission. + * @bg: beagleplay greybus driver + * @address: HDLC address field + * @control: HDLC control field + * @payloads: array of payload buffers + * @count: number of payloads + * + * Sleeps outside the spinlock until enough circular-buffer space is + * available, then verifies space under the lock and writes the entire + * frame atomically. Either a complete frame is enqueued or nothing is + * written, avoiding both sleeping in atomic context and partial frames. + */ static void hdlc_tx_frames(struct gb_beagleplay *bg, u8 address, u8 control, const struct hdlc_payload payloads[], size_t count) { + size_t needed = hdlc_encoded_length(payloads, count); + int retries = HDLC_TX_BUF_WAIT_RETRIES; size_t i; + int head, tail; + + /* Wait outside the lock for sufficient buffer space. */ + while (retries--) { + /* Pairs with smp_store_release() in hdlc_append(). */ + head = smp_load_acquire(&bg->tx_circ_buf.head); + tail = READ_ONCE(bg->tx_circ_buf.tail); + + if (CIRC_SPACE(head, tail, TX_CIRC_BUF_SIZE) >= needed) + break; + + /* Kick the consumer and sleep — no lock held. */ + schedule_work(&bg->tx_work); + usleep_range(HDLC_TX_BUF_WAIT_US_MIN, HDLC_TX_BUF_WAIT_US_MAX); + } + + if (retries < 0) { + dev_warn_ratelimited(&bg->sd->dev, + "Tx circ buf full, dropping frame\n"); + return; + } spin_lock(&bg->tx_producer_lock); + /* + * Re-check under the lock. Should not fail since + * tx_producer_lock serialises all producers and the + * consumer only frees space, but guard against it. + */ + head = bg->tx_circ_buf.head; + tail = READ_ONCE(bg->tx_circ_buf.tail); + if (unlikely(CIRC_SPACE(head, tail, TX_CIRC_BUF_SIZE) < needed)) { + spin_unlock(&bg->tx_producer_lock); + dev_warn_ratelimited(&bg->sd->dev, + "Tx circ buf space lost, dropping frame\n"); + return; + } + hdlc_append_tx_frame(bg); hdlc_append_tx_u8(bg, address); hdlc_append_tx_u8(bg, control); @@ -535,6 +608,13 @@ static size_t cc1352_bootloader_rx(struct gb_beagleplay *bg, const u8 *data, int ret; size_t off = 0; + if (count > sizeof(bg->rx_buffer) - bg->rx_buffer_len) { + dev_warn(&bg->sd->dev, + "dropping oversized bootloader receive chunk"); + bg->rx_buffer_len = 0; + return count; + } + memcpy(bg->rx_buffer + bg->rx_buffer_len, data, count); bg->rx_buffer_len += count; diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index c1d9f7c6a5f234..8d7b34803073a2 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -732,7 +732,8 @@ config HID_MAGICMOUSE Support for the Apple Magic Mouse/Trackpad multi-touch. Say Y here if you want support for the multi-touch features of the - Apple Wireless "Magic" Mouse and the Apple Wireless "Magic" Trackpad. + Apple Wireless "Magic" Mouse, the Apple Wireless "Magic" Trackpad and + force touch Trackpads in Macbooks starting from 2015. config HID_MALTRON tristate "Maltron L90 keyboard" @@ -1455,4 +1456,8 @@ endif # HID source "drivers/hid/usbhid/Kconfig" +source "drivers/hid/spi-hid/Kconfig" + +source "drivers/hid/dockchannel-hid/Kconfig" + endif # HID_SUPPORT diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index e01838239ae64c..947942cec9a78f 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -174,6 +174,12 @@ obj-$(CONFIG_INTEL_ISH_HID) += intel-ish-hid/ obj-$(CONFIG_AMD_SFH_HID) += amd-sfh-hid/ +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid/ + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid/ + obj-$(CONFIG_SURFACE_HID_CORE) += surface-hid/ obj-$(CONFIG_INTEL_THC_HID) += intel-thc-hid/ diff --git a/drivers/hid/bpf/hid_bpf_dispatch.c b/drivers/hid/bpf/hid_bpf_dispatch.c index 50c7b45c59e3fb..d0130658091b02 100644 --- a/drivers/hid/bpf/hid_bpf_dispatch.c +++ b/drivers/hid/bpf/hid_bpf_dispatch.c @@ -24,7 +24,8 @@ EXPORT_SYMBOL(hid_ops); u8 * dispatch_hid_bpf_device_event(struct hid_device *hdev, enum hid_report_type type, u8 *data, - u32 *size, int interrupt, u64 source, bool from_bpf) + size_t *buf_size, u32 *size, int interrupt, u64 source, + bool from_bpf) { struct hid_bpf_ctx_kern ctx_kern = { .ctx = { @@ -74,6 +75,7 @@ dispatch_hid_bpf_device_event(struct hid_device *hdev, enum hid_report_type type *size = ret; } + *buf_size = ctx_kern.ctx.allocated_size; return ctx_kern.data; } EXPORT_SYMBOL_GPL(dispatch_hid_bpf_device_event); @@ -505,7 +507,7 @@ __hid_bpf_input_report(struct hid_bpf_ctx *ctx, enum hid_report_type type, u8 *b if (ret) return ret; - return hid_ops->hid_input_report(ctx->hid, type, buf, size, 0, (u64)(long)ctx, true, + return hid_ops->hid_input_report(ctx->hid, type, buf, size, size, 0, (u64)(long)ctx, true, lock_already_taken); } diff --git a/drivers/hid/dockchannel-hid/Kconfig b/drivers/hid/dockchannel-hid/Kconfig new file mode 100644 index 00000000000000..254961ad15e19c --- /dev/null +++ b/drivers/hid/dockchannel-hid/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +menu "DockChannel HID support" + depends on APPLE_DOCKCHANNEL + +config HID_DOCKCHANNEL + tristate "HID over DockChannel transport layer for Apple Silicon SoCs" + depends on APPLE_DOCKCHANNEL && INPUT && OF && HID + help + Say Y here if you use an M2 or later Apple Silicon based laptop. + The keyboard and touchpad are HID based devices connected via the + proprietary DockChannel interface. + +endmenu diff --git a/drivers/hid/dockchannel-hid/Makefile b/drivers/hid/dockchannel-hid/Makefile new file mode 100644 index 00000000000000..7dba766b047fcc --- /dev/null +++ b/drivers/hid/dockchannel-hid/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# +# Makefile for DockChannel HID transport drivers +# + +obj-$(CONFIG_HID_DOCKCHANNEL) += dockchannel-hid.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c new file mode 100644 index 00000000000000..9cab638b30c328 --- /dev/null +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -0,0 +1,1213 @@ +/* + * SPDX-License-Identifier: GPL-2.0 OR MIT + * + * Apple DockChannel HID transport driver + * + * Copyright The Asahi Linux Contributors + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../hid-ids.h" + +#define COMMAND_TIMEOUT_MS 1000 +#define START_TIMEOUT_MS 2000 + +#define MAX_INTERFACES 16 + +/* Data + checksum */ +#define MAX_PKT_SIZE (0xffff + 4) + +#define DCHID_CHANNEL_CMD 0x11 +#define DCHID_CHANNEL_REPORT 0x12 + +struct dchid_hdr { + u8 hdr_len; + u8 channel; + u16 length; + u8 seq; + u8 iface; + u16 pad; +} __packed; + +#define IFACE_COMM 0 + +#define FLAGS_GROUP GENMASK(7, 6) +#define FLAGS_REQ GENMASK(5, 0) + +#define REQ_SET_REPORT 0 +#define REQ_GET_REPORT 1 + +struct dchid_subhdr { + u8 flags; + u8 unk; + u16 length; + u32 retcode; +} __packed; + +#define EVENT_GPIO_CMD 0xa0 +#define EVENT_INIT 0xf0 +#define EVENT_READY 0xf1 + +struct dchid_init_hdr { + u8 type; + u8 unk1; + u8 unk2; + u8 iface; + char name[16]; + u8 more_packets; + u8 unkpad; +} __packed; + +#define INIT_HID_DESCRIPTOR 0 +#define INIT_GPIO_REQUEST 1 +#define INIT_TERMINATOR 2 +#define INIT_PRODUCT_NAME 7 + +#define CMD_RESET_INTERFACE 0x40 +#define CMD_SEND_FIRMWARE 0x95 +#define CMD_ENABLE_INTERFACE 0xb4 +#define CMD_ACK_GPIO_CMD 0xa1 + +struct dchid_init_block_hdr { + u16 type; + u16 length; +} __packed; + +#define MAX_GPIO_NAME 32 + +struct dchid_gpio_request { + u16 unk; + u16 id; + char name[MAX_GPIO_NAME]; +} __packed; + +struct dchid_gpio_cmd { + u8 type; + u8 iface; + u8 gpio; + u8 unk; + u8 cmd; +} __packed; + +struct dchid_gpio_ack { + u8 type; + u32 retcode; + u8 cmd[]; +} __packed; + +#define STM_REPORT_ID 0x10 +#define STM_REPORT_SERIAL 0x11 +#define STM_REPORT_KEYBTYPE 0x14 + +struct dchid_stm_id { + u8 unk; + u16 vendor_id; + u16 product_id; + u16 version_number; + u8 unk2; + u8 unk3; + u8 keyboard_type; + u8 serial_length; + /* Serial follows, but we grab it with a different report. */ +} __packed; + +#define FW_MAGIC 0x46444948 +#define FW_VER 1 + +struct fw_header { + u32 magic; + u32 version; + u32 hdr_length; + u32 data_length; + u32 iface_offset; +} __packed; + +struct dchid_work { + struct work_struct work; + struct dchid_iface *iface; + + struct dchid_hdr hdr; + u8 data[]; +}; + +struct dchid_iface { + struct dockchannel_hid *dchid; + struct hid_device *hid; + struct workqueue_struct *wq; + + bool creating; + struct work_struct create_work; + + int index; + const char *name; + const struct device_node *of_node; + + uint8_t tx_seq; + bool deferred; + bool starting; + bool open; + struct completion ready; + + void *hid_desc; + size_t hid_desc_len; + + struct gpio_desc *gpio; + char gpio_name[MAX_GPIO_NAME]; + int gpio_id; + + struct mutex out_mutex; + u32 out_flags; + int out_report; + u32 retcode; + void *resp_buf; + size_t resp_size; + struct completion out_complete; + + u32 keyboard_layout_id; +}; + +struct dockchannel_hid { + struct device *dev; + struct dockchannel *dc; + struct device_link *helper_link; + + bool id_ready; + struct dchid_stm_id device_id; + char serial[64]; + + struct dchid_iface *comm; + struct dchid_iface *ifaces[MAX_INTERFACES]; + + u8 pkt_buf[MAX_PKT_SIZE]; + + /* Workqueue to asynchronously create HID devices */ + struct workqueue_struct *new_iface_wq; +}; + +static ssize_t apple_layout_id_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct hid_device *hdev = to_hid_device(dev); + struct dchid_iface *iface = hdev->driver_data; + + return scnprintf(buf, PAGE_SIZE, "%d\n", iface->keyboard_layout_id); +} + +static DEVICE_ATTR_RO(apple_layout_id); + +static struct dchid_iface * +dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) +{ + struct dchid_iface *iface; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Interface index %d out of range\n", index); + return NULL; + } + + if (dchid->ifaces[index]) + return dchid->ifaces[index]; + + iface = devm_kzalloc(dchid->dev, sizeof(struct dchid_iface), GFP_KERNEL); + if (!iface) + return NULL; + + iface->index = index; + iface->name = devm_kstrdup(dchid->dev, name, GFP_KERNEL); + iface->dchid = dchid; + iface->out_report= -1; + init_completion(&iface->out_complete); + init_completion(&iface->ready); + mutex_init(&iface->out_mutex); + iface->wq = alloc_ordered_workqueue("dchid-%s", WQ_MEM_RECLAIM, iface->name); + if (!iface->wq) + return NULL; + + /* Comm is not a HID subdevice */ + if (!strcmp(name, "comm")) { + dchid->ifaces[index] = iface; + return iface; + } + + iface->of_node = of_get_child_by_name(dchid->dev->of_node, name); + if (!iface->of_node) { + dev_warn(dchid->dev, "No OF node for subdevice %s, ignoring.", name); + return NULL; + } + + dchid->ifaces[index] = iface; + return iface; +} + +static u32 dchid_checksum(void *p, size_t length) +{ + u32 sum = 0; + + while (length >= 4) { + sum += get_unaligned_le32(p); + p += 4; + length -= 4; + } + + WARN_ON_ONCE(length); + return sum; +} + +static int dchid_send(struct dchid_iface *iface, u32 flags, void *msg, size_t size) +{ + u32 checksum = 0xffffffff; + size_t wsize = round_down(size, 4); + size_t tsize = size - wsize; + int ret; + struct { + struct dchid_hdr hdr; + struct dchid_subhdr sub; + } __packed h; + + memset(&h, 0, sizeof(h)); + h.hdr.hdr_len = sizeof(h.hdr); + h.hdr.channel = DCHID_CHANNEL_CMD; + h.hdr.length = round_up(size, 4) + sizeof(h.sub); + h.hdr.seq = iface->tx_seq; + h.hdr.iface = iface->index; + h.sub.flags = flags; + h.sub.length = size; + + ret = dockchannel_send(iface->dchid->dc, &h, sizeof(h)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(&h, sizeof(h)); + + ret = dockchannel_send(iface->dchid->dc, msg, wsize); + if (ret < 0) + return ret; + checksum -= dchid_checksum(msg, wsize); + + if (tsize) { + u8 tail[4] = {0, 0, 0, 0}; + + memcpy(tail, msg + wsize, tsize); + ret = dockchannel_send(iface->dchid->dc, tail, sizeof(tail)); + if (ret < 0) + return ret; + checksum -= dchid_checksum(tail, sizeof(tail)); + } + + ret = dockchannel_send(iface->dchid->dc, &checksum, sizeof(checksum)); + if (ret < 0) + return ret; + + return 0; +} + +static int dchid_cmd(struct dchid_iface *iface, u32 type, u32 req, + void *data, size_t size, void *resp_buf, size_t resp_size) +{ + int ret; + int report_id = *(u8*)data; + + mutex_lock(&iface->out_mutex); + + WARN_ON(iface->out_report != -1); + iface->out_report = report_id; + iface->out_flags = FIELD_PREP(FLAGS_GROUP, type) | FIELD_PREP(FLAGS_REQ, req); + iface->resp_buf = resp_buf; + iface->resp_size = resp_size; + reinit_completion(&iface->out_complete); + + ret = dchid_send(iface, iface->out_flags, data, size); + if (ret < 0) + goto done; + + if (!wait_for_completion_timeout(&iface->out_complete, msecs_to_jiffies(COMMAND_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "output report 0x%x to iface %d (%s) timed out\n", + report_id, iface->index, iface->name); + ret = -ETIMEDOUT; + goto done; + } + + ret = iface->resp_size; + if (iface->retcode) { + dev_err(iface->dchid->dev, + "output report 0x%x to iface %d (%s) failed with err 0x%x\n", + report_id, iface->index, iface->name, iface->retcode); + ret = -EIO; + } + +done: + iface->tx_seq++; + iface->out_report = -1; + iface->out_flags = 0; + iface->resp_buf = NULL; + iface->resp_size = 0; + mutex_unlock(&iface->out_mutex); + return ret; +} + +static int dchid_comm_cmd(struct dockchannel_hid *dchid, void *cmd, size_t size) +{ + return dchid_cmd(dchid->comm, HID_FEATURE_REPORT, REQ_SET_REPORT, cmd, size, NULL, 0); +} + +static int dchid_enable_interface(struct dchid_iface *iface) +{ + u8 msg[] = { CMD_ENABLE_INTERFACE, iface->index }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_reset_interface(struct dchid_iface *iface, int state) +{ + u8 msg[] = { CMD_RESET_INTERFACE, 1, iface->index, state }; + + return dchid_comm_cmd(iface->dchid, msg, sizeof(msg)); +} + +static int dchid_send_firmware(struct dchid_iface *iface, void *firmware, size_t size) +{ + struct { + u8 cmd; + u8 unk1; + u8 unk2; + u8 iface; + u64 addr; + u32 size; + } __packed msg = { + .cmd = CMD_SEND_FIRMWARE, + .unk1 = 2, + .unk2 = 0, + .iface = iface->index, + .size = size, + }; + dma_addr_t addr; + void *buf = dmam_alloc_coherent(iface->dchid->dev, size, &addr, GFP_KERNEL); + + if (IS_ERR_OR_NULL(buf)) + return buf ? PTR_ERR(buf) : -ENOMEM; + + msg.addr = addr; + memcpy(buf, firmware, size); + wmb(); + + return dchid_comm_cmd(iface->dchid, &msg, sizeof(msg)); +} + +static int dchid_get_firmware(struct dchid_iface *iface, void **firmware, size_t *size) +{ + int ret; + const char *fw_name; + const struct firmware *fw; + struct fw_header *hdr; + u8 *fw_data; + + ret = of_property_read_string(iface->of_node, "firmware-name", &fw_name); + if (ret) { + /* Firmware is only for some devices */ + *firmware = NULL; + *size = 0; + return 0; + } + + ret = request_firmware(&fw, fw_name, iface->dchid->dev); + if (ret) + return ret; + + hdr = (struct fw_header *)fw->data; + + if (hdr->magic != FW_MAGIC || hdr->version != FW_VER || + hdr->hdr_length < sizeof(*hdr) || hdr->hdr_length > fw->size || + (hdr->hdr_length + (size_t)hdr->data_length) > fw->size || + hdr->iface_offset >= hdr->data_length) { + dev_warn(iface->dchid->dev, "%s: invalid firmware header\n", + fw_name); + ret = -EINVAL; + goto done; + } + + fw_data = devm_kmemdup(iface->dchid->dev, fw->data + hdr->hdr_length, + hdr->data_length, GFP_KERNEL); + if (!fw_data) { + ret = -ENOMEM; + goto done; + } + + if (hdr->iface_offset) + fw_data[hdr->iface_offset] = iface->index; + + *firmware = fw_data; + *size = hdr->data_length; + +done: + release_firmware(fw); + return ret; +} + +static int dchid_request_gpio(struct dchid_iface *iface) +{ + char prop_name[MAX_GPIO_NAME + 16]; + + if (iface->gpio) + return 0; + + dev_info(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_id, iface->gpio_name); + + snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + + iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + + if (IS_ERR_OR_NULL(iface->gpio)) { + dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios\n", prop_name); + iface->gpio = NULL; + return -1; + } + + return 0; +} + +static int dchid_start_interface(struct dchid_iface *iface) +{ + void *fw; + size_t size; + int ret; + + if (iface->starting) { + dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + return -EINPROGRESS; + } + + dev_info(iface->dchid->dev, "Starting interface %s\n", iface->name); + + iface->starting = true; + + /* Look to see if we need firmware */ + ret = dchid_get_firmware(iface, &fw, &size); + if (ret < 0) + goto err; + + /* If we need a GPIO, make sure we have it. */ + if (iface->gpio_id) { + ret = dchid_request_gpio(iface); + if (ret < 0) + goto err; + } + + /* Only multi-touch has firmware */ + if (fw && size) { + + /* Send firmware to the device */ + dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); + ret = dchid_send_firmware(iface, fw, size); + if (ret < 0) { + dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + goto err; + } + + /* After loading firmware, multi-touch needs a reset */ + dev_info(iface->dchid->dev, "Resetting %s\n", iface->name); + dchid_reset_interface(iface, 0); + dchid_reset_interface(iface, 2); + } + + return 0; + +err: + iface->starting = false; + return ret; +} + +static int dchid_start(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) { + int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); + if (ret) { + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + iface->keyboard_layout_id = 0; + } + } + + return 0; +}; + +static void dchid_stop(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + if (iface->keyboard_layout_id) + device_remove_file(&hdev->dev, &dev_attr_apple_layout_id); +} + +static int dchid_open(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + int ret; + + if (!completion_done(&iface->ready)) { + ret = dchid_start_interface(iface); + if (ret < 0) + return ret; + + if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); + return -ETIMEDOUT; + } + } + + iface->open = true; + return 0; +} + +static void dchid_close(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + iface->open = false; +} + +static int dchid_parse(struct hid_device *hdev) +{ + struct dchid_iface *iface = hdev->driver_data; + + return hid_parse_report(hdev, iface->hid_desc, iface->hid_desc_len); +} + +/* Note: buf excludes report number! For ease of fetching strings/etc. */ +static int dchid_get_report_cmd(struct dchid_iface *iface, u8 reportnum, void *buf, size_t len) +{ + int ret = dchid_cmd(iface, HID_FEATURE_REPORT, REQ_GET_REPORT, &reportnum, 1, buf, len); + + return ret <= 0 ? ret : ret - 1; +} + +/* Note: buf includes report number! */ +static int dchid_set_report(struct dchid_iface *iface, void *buf, size_t len) +{ + return dchid_cmd(iface, HID_OUTPUT_REPORT, REQ_SET_REPORT, buf, len, NULL, 0); +} + +static int dchid_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct dchid_iface *iface = hdev->driver_data; + + switch (reqtype) { + case HID_REQ_GET_REPORT: + buf[0] = reportnum; + return dchid_cmd(iface, rtype, REQ_GET_REPORT, &reportnum, 1, buf + 1, len - 1); + case HID_REQ_SET_REPORT: + return dchid_set_report(iface, buf, len); + default: + return -EIO; + } + + return 0; +} + +static struct hid_ll_driver dchid_ll = { + .start = &dchid_start, + .stop = &dchid_stop, + .open = &dchid_open, + .close = &dchid_close, + .parse = &dchid_parse, + .raw_request = &dchid_raw_request, +}; + +static void dchid_create_interface_work(struct work_struct *ws) +{ + struct dchid_iface *iface = container_of(ws, struct dchid_iface, create_work); + struct dockchannel_hid *dchid = iface->dchid; + struct hid_device *hid; + int ret; + + if (iface->hid) { + dev_warn(dchid->dev, "Interface %s already created!\n", + iface->name); + return; + } + + dev_info(dchid->dev, "New interface %s\n", iface->name); + + /* Start the interface. This is not the entire init process, as firmware is loaded later on device open. */ + ret = dchid_enable_interface(iface); + if (ret < 0) { + dev_warn(dchid->dev, "Failed to enable %s: %d\n", iface->name, ret); + return; + } + + iface->deferred = false; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return; + + snprintf(hid->name, sizeof(hid->name), "Apple MTP %s", iface->name); + snprintf(hid->phys, sizeof(hid->phys), "%s.%d (%s)", + dev_name(dchid->dev), iface->index, iface->name); + strscpy(hid->uniq, dchid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &dchid_ll; + hid->bus = BUS_HOST; + hid->vendor = dchid->device_id.vendor_id; + hid->product = dchid->device_id.product_id; + hid->version = dchid->device_id.version_number; + hid->type = HID_TYPE_OTHER; + if (!strcmp(iface->name, "multi-touch")) { + hid->type = HID_TYPE_SPI_MOUSE; + } else if (!strcmp(iface->name, "keyboard")) { + u32 country_code = 0; + + hid->type = HID_TYPE_SPI_KEYBOARD; + + /* + * We have to get the country code from the device tree, since the + * device provides no reliable way to get this info. + */ + if (!of_property_read_u32(iface->of_node, "hid-country-code", &country_code)) + hid->country = country_code; + + of_property_read_u32(iface->of_node, "apple,keyboard-layout-id", + &iface->keyboard_layout_id); + } + + hid->dev.parent = iface->dchid->dev; + hid->driver_data = iface; + + iface->hid = hid; + + ret = hid_add_device(hid); + if (ret < 0) { + iface->hid = NULL; + hid_destroy_device(hid); + dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + } +} + +static int dchid_create_interface(struct dchid_iface *iface) +{ + if (iface->creating) + return -EBUSY; + + iface->creating = true; + INIT_WORK(&iface->create_work, dchid_create_interface_work); + return queue_work(iface->dchid->new_iface_wq, &iface->create_work); +} + +static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) +{ + if (iface->hid) { + dev_warn(iface->dchid->dev, "Tried to initialize already started interface %s!\n", + iface->name); + return; + } + + iface->hid_desc = devm_kmemdup(iface->dchid->dev, hid_desc, desc_len, GFP_KERNEL); + if (!iface->hid_desc) + return; + + iface->hid_desc_len = desc_len; +} + +static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_iface *iface; + u8 *pkt = data; + u8 index; + int i, ret; + + if (length < 2) { + dev_err(dchid->dev, "Bad length for ready message: %zu\n", length); + return; + } + + index = pkt[1]; + + if (index >= MAX_INTERFACES) { + dev_err(dchid->dev, "Got ready notification for bad iface %d\n", index); + return; + } + + iface = dchid->ifaces[index]; + if (!iface) { + dev_err(dchid->dev, "Got ready notification for unknown iface %d\n", index); + return; + } + + dev_info(dchid->dev, "Interface %s is now ready\n", iface->name); + complete_all(&iface->ready); + + /* When STM is ready, grab global device info */ + if (!strcmp(iface->name, "stm")) { + ret = dchid_get_report_cmd(iface, STM_REPORT_ID, &dchid->device_id, + sizeof(dchid->device_id)); + if (ret < sizeof(dchid->device_id)) { + dev_warn(iface->dchid->dev, "Failed to get device ID from STM!\n"); + /* Fake it and keep going. Things might still work... */ + memset(&dchid->device_id, 0, sizeof(dchid->device_id)); + dchid->device_id.vendor_id = HOST_VENDOR_ID_APPLE; + } + ret = dchid_get_report_cmd(iface, STM_REPORT_SERIAL, dchid->serial, + sizeof(dchid->serial) - 1); + if (ret < 0) { + dev_warn(iface->dchid->dev, "Failed to get serial from STM!\n"); + dchid->serial[0] = 0; + } + + dchid->id_ready = true; + for (i = 0; i < MAX_INTERFACES; i++) { + if (!dchid->ifaces[i] || !dchid->ifaces[i]->deferred) + continue; + dchid_create_interface(dchid->ifaces[i]); + } + } +} + +static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_init_hdr *hdr = data; + struct dchid_iface *iface; + struct dchid_init_block_hdr *blk; + + if (length < sizeof(*hdr)) + return; + + iface = dchid_get_interface(dchid, hdr->iface, hdr->name); + if (!iface) + return; + + data += sizeof(*hdr); + length -= sizeof(*hdr); + + while (length >= sizeof(*blk)) { + blk = data; + data += sizeof(*blk); + length -= sizeof(*blk); + + if (blk->length > length) + break; + + switch (blk->type) { + case INIT_HID_DESCRIPTOR: + dchid_handle_descriptor(iface, data, blk->length); + break; + + case INIT_GPIO_REQUEST: { + struct dchid_gpio_request *req = data; + + if (sizeof(*req) > length) + break; + + if (iface->gpio_id) { + dev_err(dchid->dev, + "Cannot request more than one GPIO per interface!\n"); + break; + } + + strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); + iface->gpio_id = req->id; + break; + } + + case INIT_TERMINATOR: + break; + + case INIT_PRODUCT_NAME: { + char *product = data; + + if (product[blk->length - 1] != 0) { + dev_warn(dchid->dev, "Unterminated product name for %s\n", + iface->name); + } else { + dev_info(dchid->dev, "Product name for %s: %s\n", + iface->name, product); + } + break; + } + + default: + dev_warn(dchid->dev, "Unknown init packet %d for %s\n", + blk->type, iface->name); + break; + } + + data += blk->length; + length -= blk->length; + + if (blk->type == INIT_TERMINATOR) + break; + } + + if (hdr->more_packets) + return; + + /* We need to enable STM first, since it'll give us the device IDs */ + if (iface->dchid->id_ready || !strcmp(iface->name, "stm")) { + dchid_create_interface(iface); + } else { + iface->deferred = true; + } +} + +static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) +{ + struct dchid_gpio_cmd *cmd = data; + struct dchid_iface *iface; + u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ + struct dchid_gpio_ack *ack; + + if (length < sizeof(*cmd)) + return; + + if (cmd->iface >= MAX_INTERFACES || !(iface = dchid->ifaces[cmd->iface])) { + dev_err(dchid->dev, "Got GPIO command for bad inteface %d\n", cmd->iface); + goto err; + } + + if (dchid_request_gpio(iface) < 0) + goto err; + + if (!iface->gpio || cmd->gpio != iface->gpio_id) { + dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", + iface->name, cmd->gpio); + goto err; + } + + dev_info(dchid->dev, "GPIO command: %s#%d: %d\n", iface->name, cmd->gpio, cmd->cmd); + + switch (cmd->cmd) { + case 3: + /* Pulse. */ + gpiod_set_value_cansleep(iface->gpio, 1); + msleep(10); /* Random guess... */ + gpiod_set_value_cansleep(iface->gpio, 0); + retcode = 0; + break; + default: + dev_err(dchid->dev, "Unknown GPIO command %d\n", cmd->cmd ); + break; + } + +err: + /* Ack it */ + ack = kzalloc(sizeof(*ack) + length, GFP_KERNEL); + if (!ack) + return; + + ack->type = CMD_ACK_GPIO_CMD; + ack->retcode = retcode; + memcpy(ack->cmd, data, length); + + if (dchid_comm_cmd(dchid, ack, sizeof(*ack) + length) < 0) + dev_err(dchid->dev, "Failed to ACK GPIO command\n"); + + kfree(ack); +} + +static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t length) +{ + u8 *p = data; + switch (*p) { + case EVENT_INIT: + dchid_handle_init(dchid, data, length); + break; + case EVENT_READY: + dchid_handle_ready(dchid, data, length); + break; + case EVENT_GPIO_CMD: + dchid_handle_gpio(dchid, data, length); + break; + } +} + +static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) +{ + struct dockchannel_hid *dchid = iface->dchid; + + if (!iface->hid) { + dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); + return; + } + + if (!iface->open) + return; + + hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); +} + +static void dchid_packet_work(struct work_struct *ws) +{ + struct dchid_work *work = container_of(ws, struct dchid_work, work); + struct dchid_subhdr *shdr = (void *)work->data; + struct dockchannel_hid *dchid = work->iface->dchid; + int type = FIELD_GET(FLAGS_GROUP, shdr->flags); + u8 *payload = work->data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > work->hdr.length) { + dev_err(dchid->dev, "Bad sub header length (%hu > %zu)\n", + shdr->length, work->hdr.length - sizeof(*shdr)); + return; + } + + switch (type) { + case HID_INPUT_REPORT: + if (work->hdr.iface == IFACE_COMM) + dchid_handle_event(dchid, payload, shdr->length); + else + dchid_handle_report(work->iface, payload, shdr->length); + break; + default: + dev_err(dchid->dev, "Received unknown packet type %d\n", type); + break; + } + + kfree(work); +} + +static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, void *data) +{ + struct dchid_subhdr *shdr = (void *)data; + u8 *payload = data + sizeof(*shdr); + + if (shdr->length + sizeof(*shdr) > hdr->length) { + dev_err(iface->dchid->dev, "Bad sub header length (%hu > %zu)\n", + shdr->length, hdr->length - sizeof(*shdr)); + return; + } + if (shdr->flags != iface->out_flags) { + dev_err(iface->dchid->dev, + "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + shdr->flags, iface->out_flags); + return; + } + + if (shdr->length < 1) { + dev_err(iface->dchid->dev, "Received length 0 output report ack\n"); + return; + } + if (iface->tx_seq != hdr->seq) { + dev_err(iface->dchid->dev, "Received ACK with bad seq (expected %d, got %d)\n", + iface->tx_seq, hdr->seq); + return; + } + if (iface->out_report != payload[0]) { + dev_err(iface->dchid->dev, "Received ACK with bad report (expected %d, got %d\n", + iface->out_report, payload[0]); + return; + } + + if (iface->resp_buf && iface->resp_size) + memcpy(iface->resp_buf, payload + 1, min((size_t)shdr->length - 1, iface->resp_size)); + + iface->resp_size = shdr->length; + iface->out_report = -1; + iface->retcode = shdr->retcode; + complete(&iface->out_complete); +} + +static void dchid_handle_packet(void *cookie, size_t avail) +{ + struct dockchannel_hid *dchid = cookie; + struct dchid_hdr hdr; + struct dchid_work *work; + struct dchid_iface *iface; + u32 checksum; + + if (dockchannel_recv(dchid->dc, &hdr, sizeof(hdr)) != sizeof(hdr)) { + dev_err(dchid->dev, "Read failed (header)\n"); + return; + } + + if (hdr.hdr_len != sizeof(hdr)) { + dev_err(dchid->dev, "Bad header length %d\n", hdr.hdr_len); + goto done; + } + + if (dockchannel_recv(dchid->dc, dchid->pkt_buf, hdr.length + 4) != (hdr.length + 4)) { + dev_err(dchid->dev, "Read failed (body)\n"); + goto done; + } + + checksum = dchid_checksum(&hdr, sizeof(hdr)); + checksum += dchid_checksum(dchid->pkt_buf, hdr.length + 4); + + if (checksum != 0xffffffff) { + dev_err(dchid->dev, "Checksum mismatch (iface %d): 0x%08x != 0xffffffff\n", + hdr.iface, checksum); + goto done; + } + + + if (hdr.iface >= MAX_INTERFACES) { + dev_err(dchid->dev, "Bad iface %d\n", hdr.iface); + } + + iface = dchid->ifaces[hdr.iface]; + + if (!iface) { + dev_err(dchid->dev, "Received packet for uninitialized iface %d\n", hdr.iface); + goto done; + } + + switch (hdr.channel) { + case DCHID_CHANNEL_CMD: + dchid_handle_ack(iface, &hdr, dchid->pkt_buf); + goto done; + case DCHID_CHANNEL_REPORT: + break; + default: + dev_warn(dchid->dev, "Unknown channel 0x%x, treating as report...\n", + hdr.channel); + break; + } + + work = kzalloc(sizeof(*work) + hdr.length, GFP_KERNEL); + if (!work) + return; + + work->hdr = hdr; + work->iface = iface; + memcpy(work->data, dchid->pkt_buf, hdr.length); + INIT_WORK(&work->work, dchid_packet_work); + + queue_work(iface->wq, &work->work); + +done: + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); +} + +static int dockchannel_hid_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_hid *dchid; + struct device_node *child, *helper; + struct platform_device *helper_pdev; + struct property *prop; + int ret; + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + dchid = devm_kzalloc(dev, sizeof(*dchid), GFP_KERNEL); + if (!dchid) { + return -ENOMEM; + } + + dchid->dev = dev; + + /* + * First make sure all the GPIOs are available, in cased we need to defer. + * This is necessary because MTP will request them by name later, and by then + * it's too late to defer the probe. + */ + + for_each_child_of_node(dev->of_node, child) { + for_each_property_of_node(child, prop) { + size_t len = strlen(prop->name); + struct gpio_desc *gpio; + + if (len < 12 || strncmp("apple,", prop->name, 6) || + strcmp("-gpios", prop->name + len - 6)) + continue; + + gpio = fwnode_gpiod_get_index(&child->fwnode, prop->name, 0, GPIOD_ASIS, + prop->name); + if (IS_ERR_OR_NULL(gpio)) { + if (PTR_ERR(gpio) == -EPROBE_DEFER) { + of_node_put(child); + return -EPROBE_DEFER; + } + } else { + gpiod_put(gpio); + } + } + } + + /* + * Make sure we also have the MTP coprocessor available, and + * defer probe if the helper hasn't probed yet. + */ + helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); + if (!helper) { + dev_err(dev, "Missing apple,helper-cpu property"); + return -EINVAL; + } + + helper_pdev = of_find_device_by_node(helper); + of_node_put(helper); + if (!helper_pdev) { + dev_err(dev, "Failed to find helper device"); + return -EINVAL; + } + + dchid->helper_link = device_link_add(dev, &helper_pdev->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + put_device(&helper_pdev->dev); + if (!dchid->helper_link) { + dev_err(dev, "Failed to link to helper device"); + return -EINVAL; + } + + if (dchid->helper_link->supplier->links.status != DL_DEV_DRIVER_BOUND) + return -EPROBE_DEFER; + + /* Now it is safe to begin initializing */ + dchid->dc = dockchannel_init(pdev); + if (IS_ERR_OR_NULL(dchid->dc)) { + return PTR_ERR(dchid->dc); + } + dchid->new_iface_wq = alloc_workqueue("dchid-new", WQ_MEM_RECLAIM, 0); + if (!dchid->new_iface_wq) + return -ENOMEM; + + dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); + if (!dchid->comm) { + dev_err(dchid->dev, "Failed to initialize comm interface"); + return -EIO; + } + + dev_info(dchid->dev, "Initialized, awaiting packets\n"); + dockchannel_await(dchid->dc, dchid_handle_packet, dchid, sizeof(struct dchid_hdr)); + + return 0; +} + +static void dockchannel_hid_remove(struct platform_device *pdev) +{ + BUG_ON(1); +} + +static const struct of_device_id dockchannel_hid_of_match[] = { + { .compatible = "apple,dockchannel-hid" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_hid_of_match); +MODULE_FIRMWARE("apple/tpmtfw-*.bin"); + +static struct platform_driver dockchannel_hid_driver = { + .driver = { + .name = "dockchannel-hid", + .of_match_table = dockchannel_hid_of_match, + }, + .probe = dockchannel_hid_probe, + .remove = dockchannel_hid_remove, +}; +module_platform_driver(dockchannel_hid_driver); + +MODULE_DESCRIPTION("Apple DockChannel HID transport driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/hid/hid-alps.c b/drivers/hid/hid-alps.c index 21e55f3d0d1ba4..67179e3fe39b7d 100644 --- a/drivers/hid/hid-alps.c +++ b/drivers/hid/hid-alps.c @@ -437,6 +437,9 @@ static int alps_raw_event(struct hid_device *hdev, int ret = 0; struct alps_dev *hdata = hid_get_drvdata(hdev); + if (!(hdev->claimed & HID_CLAIMED_INPUT) || !hdata->input) + return 0; + switch (hdev->product) { case HID_PRODUCT_ID_T4_BTNLESS: ret = t4_raw_event(hdata, data, size); diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index fc5897a6bb5328..54c7488b6da4f2 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -54,10 +54,16 @@ #define APPLE_MAGIC_REPORT_ID_POWER 3 #define APPLE_MAGIC_REPORT_ID_BRIGHTNESS 1 +// DO NOT UPSTREAM: +// temporary Fn key mode until xkeyboard-config has keyboard layouts with media +// key mappings. At that point auto mode can drop function key mappings and this +// mode can be dropped. +#define FKEYS_IGNORE 5 + static unsigned int fnmode = 3; module_param(fnmode, uint, 0644); MODULE_PARM_DESC(fnmode, "Mode of fn key on Apple keyboards (0 = disabled, " - "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled)"); + "1 = fkeyslast, 2 = fkeysfirst, [3] = auto, 4 = fkeysdisabled, 5 = fkeysignore))"); static int iso_layout = -1; module_param(iso_layout, int, 0644); @@ -277,6 +283,16 @@ static const struct apple_key_translation apple_fn_keys[] = { { } }; +static const struct apple_key_translation apple_fn_keys_minimal[] = { + { KEY_BACKSPACE, KEY_DELETE }, + { KEY_ENTER, KEY_INSERT }, + { KEY_UP, KEY_PAGEUP }, + { KEY_DOWN, KEY_PAGEDOWN }, + { KEY_LEFT, KEY_HOME }, + { KEY_RIGHT, KEY_END }, + { } +}; + static const struct apple_key_translation powerbook_fn_keys[] = { { KEY_BACKSPACE, KEY_DELETE }, { KEY_F1, KEY_BRIGHTNESSDOWN, APPLE_FLAG_FKEY }, @@ -437,6 +453,8 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, real_fnmode = 2; else real_fnmode = 1; + } else if (fnmode == FKEYS_IGNORE) { + real_fnmode = 2; } else { real_fnmode = fnmode; } @@ -477,6 +495,23 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, asc->fn_on = !!value; if (real_fnmode) { + switch (hid->bus) { + case BUS_HOST: + case BUS_SPI: + switch (hid->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022: + table = macbookpro_dedicated_esc_fn_keys; + break; + default: + if (fnmode == FKEYS_IGNORE) + table = apple_fn_keys_minimal; + else + table = magic_keyboard_2021_and_2024_fn_keys; + break; + } + break; + default: switch (hid->product) { case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ANSI: case USB_DEVICE_ID_APPLE_ALU_WIRELESS_ISO: @@ -525,6 +560,7 @@ static int hidinput_apple_event(struct hid_device *hid, struct input_dev *input, else table = apple_fn_keys; } + } trans = apple_find_translation(table, code); @@ -706,6 +742,7 @@ static void apple_setup_input(struct input_dev *input) /* Enable all needed keys */ apple_setup_key_translation(input, apple_fn_keys); + apple_setup_key_translation(input, apple_fn_keys_minimal); apple_setup_key_translation(input, powerbook_fn_keys); apple_setup_key_translation(input, powerbook_numlock_keys); apple_setup_key_translation(input, apple_iso_keyboard); @@ -858,6 +895,7 @@ static int apple_backlight_init(struct hid_device *hdev) asc->backlight->cdev.name = "apple::kbd_backlight"; asc->backlight->cdev.max_brightness = rep->backlight_on_max; asc->backlight->cdev.brightness_set_blocking = apple_backlight_led_set; + asc->backlight->cdev.flags = LED_CORE_SUSPENDRESUME; ret = apple_backlight_set(hdev, 0, 0); if (ret < 0) { @@ -926,6 +964,7 @@ static int apple_magic_backlight_init(struct hid_device *hdev) backlight->cdev.name = ":white:" LED_FUNCTION_KBD_BACKLIGHT; backlight->cdev.max_brightness = backlight->brightness->field[0]->logical_maximum; backlight->cdev.brightness_set_blocking = apple_magic_backlight_led_set; + backlight->cdev.flags = LED_CORE_SUSPENDRESUME; apple_magic_backlight_set(backlight, 0, 0); @@ -940,6 +979,15 @@ static int apple_probe(struct hid_device *hdev, struct apple_sc *asc; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_KEYBOARD) + return -ENODEV; + + // key remapping will happen in xkeyboard-config so ignore + // APPLE_ISO_TILDE_QUIRK + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && fnmode == FKEYS_IGNORE) + quirks &= ~APPLE_ISO_TILDE_QUIRK; + asc = devm_kzalloc(&hdev->dev, sizeof(*asc), GFP_KERNEL); if (asc == NULL) { hid_err(hdev, "can't alloc apple descriptor\n"); @@ -1194,6 +1242,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_GEYSER1_TP_ONLY), .driver_data = APPLE_NUMLOCK_EMULATION | APPLE_HAS_FN }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2021), @@ -1206,6 +1256,8 @@ static const struct hid_device_id apple_devices[] = { .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_NUMPAD_2021), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK }, // TODO: remove APPLE_ISO_TILDE_QUIRK { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), .driver_data = APPLE_HAS_FN | APPLE_ISO_TILDE_QUIRK | APPLE_RDESC_BATTERY }, { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGIC_KEYBOARD_2024), diff --git a/drivers/hid/hid-appletb-kbd.c b/drivers/hid/hid-appletb-kbd.c index 0fdc0968b9ef26..462010a758993e 100644 --- a/drivers/hid/hid-appletb-kbd.c +++ b/drivers/hid/hid-appletb-kbd.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include "hid-ids.h" @@ -62,7 +62,8 @@ struct appletb_kbd { struct input_handle kbd_handle; struct input_handle tpd_handle; struct backlight_device *backlight_dev; - struct timer_list inactivity_timer; + struct delayed_work inactivity_work; + struct work_struct restore_brightness_work; bool has_dimmed; bool has_turned_off; u8 saved_mode; @@ -164,16 +165,18 @@ static int appletb_tb_key_to_slot(unsigned int code) } } -static void appletb_inactivity_timer(struct timer_list *t) +static void appletb_inactivity_work(struct work_struct *work) { - struct appletb_kbd *kbd = timer_container_of(kbd, t, inactivity_timer); + struct appletb_kbd *kbd = container_of(to_delayed_work(work), + struct appletb_kbd, + inactivity_work); if (kbd->backlight_dev && appletb_tb_autodim) { if (!kbd->has_dimmed) { backlight_device_set_brightness(kbd->backlight_dev, 1); kbd->has_dimmed = true; - mod_timer(&kbd->inactivity_timer, - jiffies + secs_to_jiffies(appletb_tb_idle_timeout)); + mod_delayed_work(system_wq, &kbd->inactivity_work, + secs_to_jiffies(appletb_tb_idle_timeout)); } else if (!kbd->has_turned_off) { backlight_device_set_brightness(kbd->backlight_dev, 0); kbd->has_turned_off = true; @@ -181,16 +184,25 @@ static void appletb_inactivity_timer(struct timer_list *t) } } +static void appletb_restore_brightness_work(struct work_struct *work) +{ + struct appletb_kbd *kbd = container_of(work, struct appletb_kbd, + restore_brightness_work); + + if (kbd->backlight_dev) + backlight_device_set_brightness(kbd->backlight_dev, 2); +} + static void reset_inactivity_timer(struct appletb_kbd *kbd) { if (kbd->backlight_dev && appletb_tb_autodim) { if (kbd->has_dimmed || kbd->has_turned_off) { - backlight_device_set_brightness(kbd->backlight_dev, 2); kbd->has_dimmed = false; kbd->has_turned_off = false; + schedule_work(&kbd->restore_brightness_work); } - mod_timer(&kbd->inactivity_timer, - jiffies + secs_to_jiffies(appletb_tb_dim_timeout)); + mod_delayed_work(system_wq, &kbd->inactivity_work, + secs_to_jiffies(appletb_tb_dim_timeout)); } } @@ -408,9 +420,11 @@ static int appletb_kbd_probe(struct hid_device *hdev, const struct hid_device_id dev_err_probe(dev, -ENODEV, "Failed to get backlight device\n"); } else { backlight_device_set_brightness(kbd->backlight_dev, 2); - timer_setup(&kbd->inactivity_timer, appletb_inactivity_timer, 0); - mod_timer(&kbd->inactivity_timer, - jiffies + secs_to_jiffies(appletb_tb_dim_timeout)); + INIT_DELAYED_WORK(&kbd->inactivity_work, appletb_inactivity_work); + INIT_WORK(&kbd->restore_brightness_work, + appletb_restore_brightness_work); + mod_delayed_work(system_wq, &kbd->inactivity_work, + secs_to_jiffies(appletb_tb_dim_timeout)); } kbd->inp_handler.event = appletb_kbd_inp_event; @@ -440,13 +454,14 @@ static int appletb_kbd_probe(struct hid_device *hdev, const struct hid_device_id unregister_handler: input_unregister_handler(&kbd->inp_handler); close_hw: - if (kbd->backlight_dev) { - put_device(&kbd->backlight_dev->dev); - timer_delete_sync(&kbd->inactivity_timer); - } hid_hw_close(hdev); stop_hw: hid_hw_stop(hdev); + if (kbd->backlight_dev) { + cancel_delayed_work_sync(&kbd->inactivity_work); + cancel_work_sync(&kbd->restore_brightness_work); + put_device(&kbd->backlight_dev->dev); + } return ret; } @@ -457,13 +472,14 @@ static void appletb_kbd_remove(struct hid_device *hdev) appletb_kbd_set_mode(kbd, APPLETB_KBD_MODE_OFF); input_unregister_handler(&kbd->inp_handler); + hid_hw_close(hdev); + hid_hw_stop(hdev); + if (kbd->backlight_dev) { + cancel_delayed_work_sync(&kbd->inactivity_work); + cancel_work_sync(&kbd->restore_brightness_work); put_device(&kbd->backlight_dev->dev); - timer_delete_sync(&kbd->inactivity_timer); } - - hid_hw_close(hdev); - hid_hw_stop(hdev); } static int appletb_kbd_suspend(struct hid_device *hdev, pm_message_t msg) diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index bc93b27f9b1369..b2332efabeb574 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -1163,7 +1163,8 @@ static int asus_start_multitouch(struct hid_device *hdev) return 0; } -static int __maybe_unused asus_resume(struct hid_device *hdev) { +static int __maybe_unused asus_resume(struct hid_device *hdev) +{ struct asus_drvdata *drvdata = hid_get_drvdata(hdev); int ret = 0; @@ -1311,22 +1312,17 @@ static int asus_probe(struct hid_device *hdev, const struct hid_device_id *id) * were freed during registration due to no usages being mapped, * leaving drvdata->input pointing to freed memory. */ - if (!drvdata->input || !(hdev->claimed & HID_CLAIMED_INPUT)) { - hid_err(hdev, "Asus input not registered\n"); - ret = -ENOMEM; - goto err_stop_hw; - } - - if (drvdata->tp) { - drvdata->input->name = "Asus TouchPad"; - } else { - drvdata->input->name = "Asus Keyboard"; - } + if (drvdata->input && (hdev->claimed & HID_CLAIMED_INPUT)) { + if (drvdata->tp) + drvdata->input->name = "Asus TouchPad"; + else + drvdata->input->name = "Asus Keyboard"; - if (drvdata->tp) { - ret = asus_start_multitouch(hdev); - if (ret) - goto err_stop_hw; + if (drvdata->tp) { + ret = asus_start_multitouch(hdev); + if (ret) + goto err_stop_hw; + } } return 0; diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 833df14ef68f15..a2e33505addd93 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -71,6 +71,9 @@ static u32 s32ton(__s32 value, unsigned int n) if (!value || !n) return 0; + if (n > 32) + n = 32; + a = value >> (n - 1); if (a && a != -1) return value < 0 ? 1 << (n - 1) : (1 << (n - 1)) - 1; @@ -468,7 +471,10 @@ static int hid_parser_global(struct hid_parser *parser, struct hid_item *item) case HID_GLOBAL_ITEM_TAG_REPORT_SIZE: parser->global.report_size = item_udata(item); - if (parser->global.report_size > 256) { + /* Arbitrary maximum. Some Apple devices have 16384 here. + * This * HID_MAX_USAGES must fit in a signed integer. + */ + if (parser->global.report_size > 16384) { hid_err(parser->device, "invalid report_size %d\n", parser->global.report_size); return -1; @@ -2026,24 +2032,32 @@ int __hid_request(struct hid_device *hid, struct hid_report *report, } EXPORT_SYMBOL_GPL(__hid_request); -int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, - int interrupt) +int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, + size_t bufsize, u32 size, int interrupt) { struct hid_report_enum *report_enum = hid->report_enum + type; struct hid_report *report; struct hid_driver *hdrv; int max_buffer_size = HID_MAX_BUFFER_SIZE; u32 rsize, csize = size; + size_t bsize = bufsize; u8 *cdata = data; int ret = 0; report = hid_get_report(report_enum, data); if (!report) - goto out; + return 0; + + if (unlikely(bsize < csize)) { + hid_warn_ratelimited(hid, "Event data for report %d is incorrect (%d vs %zu)\n", + report->id, csize, bsize); + return -EINVAL; + } if (report_enum->numbered) { cdata++; csize--; + bsize--; } rsize = hid_compute_report_size(report); @@ -2056,11 +2070,16 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 * else if (rsize > max_buffer_size) rsize = max_buffer_size; + if (bsize < rsize) { + hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %zu)\n", + report->id, rsize, bsize); + return -EINVAL; + } + if (csize < rsize) { - hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %d)\n", - report->id, rsize, csize); - ret = -EINVAL; - goto out; + dbg_hid("report %d is too short, (%d < %d)\n", report->id, + csize, rsize); + memset(cdata + csize, 0, rsize - csize); } if ((hid->claimed & HID_CLAIMED_HIDDEV) && hid->hiddev_report_event) @@ -2068,7 +2087,7 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 * if (hid->claimed & HID_CLAIMED_HIDRAW) { ret = hidraw_report_event(hid, data, size); if (ret) - goto out; + return ret; } if (hid->claimed != HID_CLAIMED_HIDRAW && report->maxfield) { @@ -2080,15 +2099,15 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 * if (hid->claimed & HID_CLAIMED_INPUT) hidinput_report_event(hid, report); -out: + return ret; } EXPORT_SYMBOL_GPL(hid_report_raw_event); static int __hid_input_report(struct hid_device *hid, enum hid_report_type type, - u8 *data, u32 size, int interrupt, u64 source, bool from_bpf, - bool lock_already_taken) + u8 *data, size_t bufsize, u32 size, int interrupt, u64 source, + bool from_bpf, bool lock_already_taken) { struct hid_report_enum *report_enum; struct hid_driver *hdrv; @@ -2113,7 +2132,8 @@ static int __hid_input_report(struct hid_device *hid, enum hid_report_type type, report_enum = hid->report_enum + type; hdrv = hid->driver; - data = dispatch_hid_bpf_device_event(hid, type, data, &size, interrupt, source, from_bpf); + data = dispatch_hid_bpf_device_event(hid, type, data, &bufsize, &size, interrupt, + source, from_bpf); if (IS_ERR(data)) { ret = PTR_ERR(data); goto unlock; @@ -2142,7 +2162,7 @@ static int __hid_input_report(struct hid_device *hid, enum hid_report_type type, goto unlock; } - ret = hid_report_raw_event(hid, type, data, size, interrupt); + ret = hid_report_raw_event(hid, type, data, bufsize, size, interrupt); unlock: if (!lock_already_taken) @@ -2160,16 +2180,41 @@ static int __hid_input_report(struct hid_device *hid, enum hid_report_type type, * @interrupt: distinguish between interrupt and control transfers * * This is data entry for lower layers. + * Legacy, please use hid_safe_input_report() instead. */ int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, int interrupt) { - return __hid_input_report(hid, type, data, size, interrupt, 0, + return __hid_input_report(hid, type, data, size, size, interrupt, 0, false, /* from_bpf */ false /* lock_already_taken */); } EXPORT_SYMBOL_GPL(hid_input_report); +/** + * hid_safe_input_report - report data from lower layer (usb, bt...) + * + * @hid: hid device + * @type: HID report type (HID_*_REPORT) + * @data: report contents + * @bufsize: allocated size of the data buffer + * @size: useful size of data parameter + * @interrupt: distinguish between interrupt and control transfers + * + * This is data entry for lower layers. + * Please use this function instead of the non safe version because we provide + * here the size of the buffer, allowing hid-core to make smarter decisions + * regarding the incoming buffer. + */ +int hid_safe_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, + size_t bufsize, u32 size, int interrupt) +{ + return __hid_input_report(hid, type, data, bufsize, size, interrupt, 0, + false, /* from_bpf */ + false /* lock_already_taken */); +} +EXPORT_SYMBOL_GPL(hid_safe_input_report); + bool hid_match_one_id(const struct hid_device *hdev, const struct hid_device_id *id) { @@ -2320,6 +2365,12 @@ int hid_connect(struct hid_device *hdev, unsigned int connect_mask) case BUS_SDW: bus = "SOUNDWIRE"; break; + case BUS_SPI: + bus = "SPI"; + break; + case BUS_HOST: + bus = "HOST"; + break; case BUS_VIRTUAL: bus = "VIRTUAL"; break; diff --git a/drivers/hid/hid-gfrm.c b/drivers/hid/hid-gfrm.c index 699186ff2349e9..d2a56bf92b416e 100644 --- a/drivers/hid/hid-gfrm.c +++ b/drivers/hid/hid-gfrm.c @@ -66,7 +66,7 @@ static int gfrm_raw_event(struct hid_device *hdev, struct hid_report *report, switch (data[1]) { case GFRM100_SEARCH_KEY_DOWN: ret = hid_report_raw_event(hdev, HID_INPUT_REPORT, search_key_dn, - sizeof(search_key_dn), 1); + sizeof(search_key_dn), sizeof(search_key_dn), 1); break; case GFRM100_SEARCH_KEY_AUDIO_DATA: @@ -74,7 +74,7 @@ static int gfrm_raw_event(struct hid_device *hdev, struct hid_report *report, case GFRM100_SEARCH_KEY_UP: ret = hid_report_raw_event(hdev, HID_INPUT_REPORT, search_key_up, - sizeof(search_key_up), 1); + sizeof(search_key_up), sizeof(search_key_up), 1); break; default: diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h index c1e4a6ce96317a..405d32fc827a28 100644 --- a/drivers/hid/hid-ids.h +++ b/drivers/hid/hid-ids.h @@ -96,6 +96,8 @@ #define USB_VENDOR_ID_APPLE 0x05ac #define BT_VENDOR_ID_APPLE 0x004c +#define SPI_VENDOR_ID_APPLE 0x05ac +#define HOST_VENDOR_ID_APPLE 0x05ac #define USB_DEVICE_ID_APPLE_MIGHTYMOUSE 0x0304 #define USB_DEVICE_ID_APPLE_MAGICMOUSE 0x030d #define USB_DEVICE_ID_APPLE_MAGICMOUSE2 0x0269 @@ -200,6 +202,14 @@ #define USB_DEVICE_ID_APPLE_IRCONTROL5 0x8243 #define USB_DEVICE_ID_APPLE_TOUCHBAR_BACKLIGHT 0x8102 #define USB_DEVICE_ID_APPLE_TOUCHBAR_DISPLAY 0x8302 +#define SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020 0x0281 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020 0x0341 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021 0x0342 +#define SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021 0x0343 +#define HOST_DEVICE_ID_APPLE_MACBOOK_AIR13_2022 0x0351 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO14_2023 0x0352 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO16_2023 0x0353 +#define HOST_DEVICE_ID_APPLE_MACBOOK_PRO13_2022 0x0354 #define USB_VENDOR_ID_ASETEK 0x2433 #define USB_DEVICE_ID_ASETEK_INVICTA 0xf300 diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index d1dea7297712d5..e9aa99ade5aacc 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -3665,7 +3665,7 @@ static int hidpp10_consumer_keys_raw_event(struct hidpp_device *hidpp, memcpy(&consumer_report[1], &data[3], 4); /* We are called from atomic context */ hid_report_raw_event(hidpp->hid_dev, HID_INPUT_REPORT, - consumer_report, 5, 1); + consumer_report, sizeof(consumer_report), 5, 1); return 1; } diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 9eadf3252d0dc8..2ea04d133f5a41 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -60,8 +60,14 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define MOUSE_REPORT_ID 0x29 #define MOUSE2_REPORT_ID 0x12 #define DOUBLE_REPORT_ID 0xf7 +#define SPI_REPORT_ID 0x02 +#define SPI_RESET_REPORT_ID 0x60 +#define MTP_REPORT_ID 0x75 +#define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_SEC 60 +#define MAX_CONTACTS 16 + /* These definitions are not precise, but they're close enough. (Bits * 0x03 seem to indicate the aspect ratio of the touch, bits 0x70 seem * to be some kind of bit mask -- 0x20 may be a near-field reading, @@ -112,30 +118,53 @@ MODULE_PARM_DESC(report_undeciphered, "Report undeciphered multi-touch state fie #define TRACKPAD2_RES_Y \ ((TRACKPAD2_MAX_Y - TRACKPAD2_MIN_Y) / (TRACKPAD2_DIMENSION_Y / 100)) +/* These are fallback values, since the real values will be queried from the device. */ +#define J314_TP_DIMENSION_X (float)13000 +#define J314_TP_MIN_X -5900 +#define J314_TP_MAX_X 6500 +#define J314_TP_RES_X \ + ((J314_TP_MAX_X - J314_TP_MIN_X) / (J314_TP_DIMENSION_X / 100)) +#define J314_TP_DIMENSION_Y (float)8100 +#define J314_TP_MIN_Y -200 +#define J314_TP_MAX_Y 7400 +#define J314_TP_RES_Y \ + ((J314_TP_MAX_Y - J314_TP_MIN_Y) / (J314_TP_DIMENSION_Y / 100)) + +#define J314_TP_MAX_FINGER_ORIENTATION 16384 + +struct magicmouse_input_ops { + int (*raw_event)(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size); + int (*setup_input)(struct input_dev *input, struct hid_device *hdev); +}; + /** * struct magicmouse_sc - Tracks Magic Mouse-specific data. * @input: Input device through which we report events. * @quirks: Currently unused. + * @query_dimensions: Whether to query and update dimensions on first open * @ntouches: Number of touches in most recent touch report. * @scroll_accel: Number of consecutive scroll motions. * @scroll_jiffies: Time of last scroll motion. + * @pos: multi touch position data of the last report. * @touches: Most recent data for a touch, indexed by tracking ID. * @tracking_ids: Mapping of current touch input data to @touches. * @hdev: Pointer to the underlying HID device. * @work: Workqueue to handle initialization retry for quirky devices. * @battery_timer: Timer for obtaining battery level information. + * @input_ops: Input ops based on device type. */ struct magicmouse_sc { struct input_dev *input; unsigned long quirks; + bool query_dimensions; int ntouches; int scroll_accel; unsigned long scroll_jiffies; + struct input_mt_pos pos[MAX_CONTACTS]; struct { - short x; - short y; short scroll_x; short scroll_y; short scroll_x_hr; @@ -143,14 +172,112 @@ struct magicmouse_sc { u8 size; bool scroll_x_active; bool scroll_y_active; - } touches[16]; - int tracking_ids[16]; + } touches[MAX_CONTACTS]; + int tracking_ids[MAX_CONTACTS]; struct hid_device *hdev; struct delayed_work work; struct timer_list battery_timer; + struct magicmouse_input_ops input_ops; }; +static int magicmouse_enable_multitouch(struct hid_device *hdev); + +static inline int le16_to_int(__le16 x) +{ + return (signed short)le16_to_cpu(x); +} + +static int magicmouse_open(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + int ret; + + ret = hid_hw_open(hdev); + if (ret) + return ret; + + /* + * Some devices repond with 'invalid report id' when feature + * report switching it into multitouch mode is sent to it. + * + * This results in -EIO from the _raw low-level transport callback, + * but there seems to be no other way of switching the mode. + * Thus the super-ugly hacky success check below. + * + * MTP devices do not need this. + */ + if (hdev->bus != BUS_HOST) { + ret = magicmouse_enable_multitouch(hdev); + if (ret != -EIO && ret < 0) { + hid_err(hdev, "unable to request touch data (%d)\n", ret); + return ret; + } + if (ret == -EIO && (hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || + hdev->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + } + } + + /* + * For Apple Silicon trackpads, we want to query the dimensions on + * device open. This is because doing so requires the firmware, but + * we don't want to force a firmware load until the device is opened + * for the first time. So do that here and update the input properties + * just in time before userspace queries them. + */ + if (msc->query_dimensions) { + struct input_dev *input = msc->input; + u8 buf[32]; + struct { + __le32 width; + __le32 height; + __le16 min_x; + __le16 min_y; + __le16 max_x; + __le16 max_y; + } dim; + uint32_t x_span, y_span; + + ret = hid_hw_raw_request(hdev, SENSOR_DIMENSIONS_REPORT_ID, buf, sizeof(buf), HID_FEATURE_REPORT, HID_REQ_GET_REPORT); + if (ret < (int)(1 + sizeof(dim))) { + hid_err(hdev, "unable to request dimensions (%d)\n", ret); + return ret; + } + + memcpy(&dim, buf + 1, sizeof(dim)); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, + le16_to_int(dim.min_x), le16_to_int(dim.max_x), 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, + -le16_to_int(dim.max_y), -le16_to_int(dim.min_y), 0, 0); + x_span = le16_to_int(dim.max_x) - le16_to_int(dim.min_x); + y_span = le16_to_int(dim.max_y) - le16_to_int(dim.min_y); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, 100 * x_span / le32_to_cpu(dim.width) ); + input_abs_set_res(input, ABS_MT_POSITION_Y, 100 * y_span / le32_to_cpu(dim.height) ); + + /* copy info, as input_mt_init_slots() does */ + dev->absinfo[ABS_X] = dev->absinfo[ABS_MT_POSITION_X]; + dev->absinfo[ABS_Y] = dev->absinfo[ABS_MT_POSITION_Y]; + + msc->query_dimensions = false; + } + + return 0; +} + +static void magicmouse_close(struct input_dev *dev) +{ + struct hid_device *hdev = input_get_drvdata(dev); + + hid_hw_close(hdev); +} + static int magicmouse_firm_touch(struct magicmouse_sc *msc) { int touch = -1; @@ -192,7 +319,7 @@ static void magicmouse_emit_buttons(struct magicmouse_sc *msc, int state) } else if (last_state != 0) { state = last_state; } else if ((id = magicmouse_firm_touch(msc)) >= 0) { - int x = msc->touches[id].x; + int x = msc->pos[id].x; if (x < middle_button_start) state = 1; else if (x > middle_button_stop) @@ -256,8 +383,8 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda /* Store tracking ID and other fields. */ msc->tracking_ids[raw_id] = id; - msc->touches[id].x = x; - msc->touches[id].y = y; + msc->pos[id].x = x; + msc->pos[id].y = y; msc->touches[id].size = size; /* If requested, emulate a scroll wheel by detecting small @@ -387,6 +514,14 @@ static int magicmouse_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.raw_event(hdev, report, data, size); +} + +static int magicmouse_raw_event_usb(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; @@ -518,6 +653,174 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +/** + * struct tp_finger - single trackpad finger structure, le16-aligned + * + * @unknown1: unknown + * @unknown2: unknown + * @abs_x: absolute x coordinate + * @abs_y: absolute y coordinate + * @rel_x: relative x coordinate + * @rel_y: relative y coordinate + * @tool_major: tool area, major axis + * @tool_minor: tool area, minor axis + * @orientation: 16384 when point, else 15 bit angle + * @touch_major: touch area, major axis + * @touch_minor: touch area, minor axis + * @unused: zeros + * @pressure: pressure on forcetouch touchpad + * @multi: one finger: varies, more fingers: constant + */ +struct tp_finger { + __le16 unknown1; + __le16 unknown2; + __le16 abs_x; + __le16 abs_y; + __le16 rel_x; + __le16 rel_y; + __le16 tool_major; + __le16 tool_minor; + __le16 orientation; + __le16 touch_major; + __le16 touch_minor; + __le16 unused[2]; + __le16 pressure; + __le16 multi; +} __attribute__((packed, aligned(2))); + +/** + * struct vendor trackpad report + * + * @num_fingers: the number of fingers being reported in @fingers + * @buttons: same as HID buttons + */ +struct tp_header { + // HID vendor part, up to 1751 bytes + u8 unknown[22]; + u8 num_fingers; + u8 buttons; + u8 unknown3[14]; +}; + +/** + * struct standard HID mouse report + * + * @report_id: reportid + * @buttons: HID Usage Buttons 3 1-bit reports + */ +struct tp_mouse_report { + // HID mouse report + u8 report_id; + u8 buttons; + u8 rel_x; + u8 rel_y; + u8 padding[4]; +}; + +static void report_finger_data(struct input_dev *input, int slot, + const struct input_mt_pos *pos, + const struct tp_finger *f) +{ + input_mt_slot(input, slot); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + + input_report_abs(input, ABS_MT_TOUCH_MAJOR, + le16_to_int(f->touch_major) << 1); + input_report_abs(input, ABS_MT_TOUCH_MINOR, + le16_to_int(f->touch_minor) << 1); + input_report_abs(input, ABS_MT_WIDTH_MAJOR, + le16_to_int(f->tool_major) << 1); + input_report_abs(input, ABS_MT_WIDTH_MINOR, + le16_to_int(f->tool_minor) << 1); + input_report_abs(input, ABS_MT_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION - le16_to_int(f->orientation)); + input_report_abs(input, ABS_MT_PRESSURE, le16_to_int(f->pressure)); + input_report_abs(input, ABS_MT_POSITION_X, pos->x); + input_report_abs(input, ABS_MT_POSITION_Y, pos->y); +} + +static int magicmouse_raw_event_mtp(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + struct tp_header *tp_hdr; + struct tp_finger *f; + int i, n; + u32 npoints; + const size_t hdr_sz = sizeof(struct tp_header); + const size_t touch_sz = sizeof(struct tp_finger); + u8 map_contacs[MAX_CONTACTS]; + + // hid_warn(hdev, "%s\n", __func__); + // print_hex_dump_debug("appleft ev: ", DUMP_PREFIX_OFFSET, 16, 1, data, + // size, false); + + /* Expect 46 bytes of prefix, and N * 30 bytes of touch data. */ + if (size < hdr_sz || ((size - hdr_sz) % touch_sz) != 0) + return 0; + + tp_hdr = (struct tp_header *)data; + + npoints = (size - hdr_sz) / touch_sz; + if (npoints < tp_hdr->num_fingers || npoints > MAX_CONTACTS) { + hid_warn(hdev, + "unexpected number of touches (%u) for " + "report\n", + npoints); + return 0; + } + + n = 0; + for (i = 0; i < tp_hdr->num_fingers; i++) { + f = (struct tp_finger *)(data + hdr_sz + i * touch_sz); + if (le16_to_int(f->touch_major) == 0) + continue; + + hid_dbg(hdev, "ev x:%04x y:%04x\n", le16_to_int(f->abs_x), + le16_to_int(f->abs_y)); + msc->pos[n].x = le16_to_int(f->abs_x); + msc->pos[n].y = -le16_to_int(f->abs_y); + map_contacs[n] = i; + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + int idx = map_contacs[i]; + f = (struct tp_finger *)(data + hdr_sz + idx * touch_sz); + report_finger_data(input, msc->tracking_ids[i], &msc->pos[i], f); + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, tp_hdr->buttons & 1); + + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_spi(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + const size_t hdr_sz = sizeof(struct tp_mouse_report); + + if (!size) + return 0; + + if (data[0] == SPI_RESET_REPORT_ID) { + hid_info(hdev, "Touch controller was reset, re-enabling touch mode\n"); + schedule_delayed_work(&msc->work, msecs_to_jiffies(10)); + return 1; + } + + if (data[0] != SPI_REPORT_ID || size < hdr_sz) + return 0; + + return magicmouse_raw_event_mtp(hdev, report, data + hdr_sz, size - hdr_sz); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -536,7 +839,17 @@ static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, return 0; } -static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hdev) + +static int magicmouse_setup_input(struct input_dev *input, + struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + return msc->input_ops.setup_input(input, hdev); +} + +static int magicmouse_setup_input_usb(struct input_dev *input, + struct hid_device *hdev) { int error; int mt_flags = 0; @@ -615,7 +928,7 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd __set_bit(EV_ABS, input->evbit); - error = input_mt_init_slots(input, 16, mt_flags); + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); if (error) return error; input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 255 << 2, @@ -695,6 +1008,109 @@ static int magicmouse_setup_input(struct input_dev *input, struct hid_device *hd */ __clear_bit(EV_REP, input->evbit); + /* + * This isn't strictly speaking needed for USB, but enabling MT on + * device open is probably more robust than only doing it once on probe + * even if USB devices are not known to suffer from the SPI reset issue. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + return 0; +} + +static int magicmouse_setup_input_mtp(struct input_dev *input, + struct hid_device *hdev) +{ + int error; + int mt_flags = 0; + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); + __clear_bit(BTN_0, input->keybit); + __clear_bit(BTN_RIGHT, input->keybit); + __clear_bit(BTN_MIDDLE, input->keybit); + __clear_bit(EV_REL, input->evbit); + __clear_bit(REL_X, input->relbit); + __clear_bit(REL_Y, input->relbit); + + mt_flags = INPUT_MT_POINTER | INPUT_MT_DROP_UNUSED | INPUT_MT_TRACK; + + /* finger touch area */ + input_set_abs_params(input, ABS_MT_TOUCH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_TOUCH_MINOR, 0, 5000, 0, 0); + + /* finger approach area */ + input_set_abs_params(input, ABS_MT_WIDTH_MAJOR, 0, 5000, 0, 0); + input_set_abs_params(input, ABS_MT_WIDTH_MINOR, 0, 5000, 0, 0); + + /* Note: Touch Y position from the device is inverted relative + * to how pointer motion is reported (and relative to how USB + * HID recommends the coordinates work). This driver keeps + * the origin at the same position, and just uses the additive + * inverse of the reported Y. + */ + + input_set_abs_params(input, ABS_MT_PRESSURE, 0, 6000, 0, 0); + + /* + * This makes libinput recognize this as a PressurePad and + * stop trying to use pressure for touch size. Pressure unit + * seems to be ~grams on these touchpads. + */ + input_abs_set_res(input, ABS_MT_PRESSURE, 1); + + /* finger orientation */ + input_set_abs_params(input, ABS_MT_ORIENTATION, -J314_TP_MAX_FINGER_ORIENTATION, + J314_TP_MAX_FINGER_ORIENTATION, 0, 0); + + /* finger position */ + input_set_abs_params(input, ABS_MT_POSITION_X, J314_TP_MIN_X, J314_TP_MAX_X, + 0, 0); + /* Y axis is inverted */ + input_set_abs_params(input, ABS_MT_POSITION_Y, -J314_TP_MAX_Y, -J314_TP_MIN_Y, + 0, 0); + + /* X/Y resolution */ + input_abs_set_res(input, ABS_MT_POSITION_X, J314_TP_RES_X); + input_abs_set_res(input, ABS_MT_POSITION_Y, J314_TP_RES_Y); + + input_set_events_per_packet(input, 60); + + /* touchpad button */ + input_set_capability(input, EV_KEY, BTN_MOUSE); + + /* + * hid-input may mark device as using autorepeat, but the trackpad does + * not actually want it. + */ + __clear_bit(EV_REP, input->evbit); + + error = input_mt_init_slots(input, MAX_CONTACTS, mt_flags); + if (error) + return error; + + /* + * Override the default input->open function to send the MT + * enable every time the device is opened. This ensures it works + * even if we missed a reset event due to the device being closed. + * input->close is overridden for symmetry. + * + * This also takes care of the dimensions query. + */ + input->open = magicmouse_open; + input->close = magicmouse_close; + msc->query_dimensions = true; + + return 0; +} + +static int magicmouse_setup_input_spi(struct input_dev *input, + struct hid_device *hdev) +{ + int ret = magicmouse_setup_input_mtp(input, hdev); + if (ret) + return ret; + return 0; } @@ -753,6 +1169,10 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) int feature_size; switch (hdev->product) { + case SPI_DEVICE_ID_APPLE_MACBOOK_AIR_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO13_2020: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO14_2021: + case SPI_DEVICE_ID_APPLE_MACBOOK_PRO16_2021: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2: case USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC: switch (hdev->vendor) { @@ -760,7 +1180,7 @@ static int magicmouse_enable_multitouch(struct hid_device *hdev) feature_size = sizeof(feature_mt_trackpad2_bt); feature = feature_mt_trackpad2_bt; break; - default: /* USB_VENDOR_ID_APPLE */ + default: /* USB_VENDOR_ID_APPLE || SPI_VENDOR_ID_APPLE */ feature_size = sizeof(feature_mt_trackpad2_usb); feature = feature_mt_trackpad2_usb; } @@ -857,12 +1277,29 @@ static int magicmouse_probe(struct hid_device *hdev, struct hid_report *report; int ret; + if ((id->bus == BUS_SPI || id->bus == BUS_HOST) && id->vendor == SPI_VENDOR_ID_APPLE && + hdev->type != HID_TYPE_SPI_MOUSE) + return -ENODEV; + msc = devm_kzalloc(&hdev->dev, sizeof(*msc), GFP_KERNEL); if (msc == NULL) { hid_err(hdev, "can't alloc magicmouse descriptor\n"); return -ENOMEM; } + // internal trackpad use a data format use input ops to avoid + // conflicts with the report ID. + if (id->bus == BUS_HOST) { + msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.setup_input = magicmouse_setup_input_mtp; + } else if (id->bus == BUS_SPI) { + msc->input_ops.raw_event = magicmouse_raw_event_spi; + msc->input_ops.setup_input = magicmouse_setup_input_spi; + } else { + msc->input_ops.raw_event = magicmouse_raw_event_usb; + msc->input_ops.setup_input = magicmouse_setup_input_usb; + } + msc->scroll_accel = SCROLL_ACCEL_DEFAULT; msc->hdev = hdev; INIT_DEFERRABLE_WORK(&msc->work, magicmouse_enable_mt_work); @@ -921,11 +1358,20 @@ static int magicmouse_probe(struct hid_device *hdev, TRACKPAD2_USB_REPORT_ID, 0); } break; - default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ - report = hid_register_report(hdev, HID_INPUT_REPORT, - TRACKPAD_REPORT_ID, 0); - report = hid_register_report(hdev, HID_INPUT_REPORT, - DOUBLE_REPORT_ID, 0); + default: + switch (id->bus) { + case BUS_HOST: + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + break; + case BUS_SPI: + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); + break; + default: /* USB_DEVICE_ID_APPLE_MAGICTRACKPAD */ + report = hid_register_report(hdev, HID_INPUT_REPORT, + TRACKPAD_REPORT_ID, 0); + report = hid_register_report(hdev, HID_INPUT_REPORT, + DOUBLE_REPORT_ID, 0); + } } if (!report) { @@ -935,22 +1381,14 @@ static int magicmouse_probe(struct hid_device *hdev, } report->size = 6; - /* - * Some devices repond with 'invalid report id' when feature - * report switching it into multitouch mode is sent to it. - * - * This results in -EIO from the _raw low-level transport callback, - * but there seems to be no other way of switching the mode. - * Thus the super-ugly hacky success check below. - */ - ret = magicmouse_enable_multitouch(hdev); - if (ret != -EIO && ret < 0) { - hid_err(hdev, "unable to request touch data (%d)\n", ret); - goto err_stop_hw; - } - if (ret == -EIO && (id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2 || - id->product == USB_DEVICE_ID_APPLE_MAGICMOUSE2_USBC)) { - schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + /* MTP devices do not need the MT enable, this is handled by the MTP driver */ + if (id->bus == BUS_HOST) + return 0; + + /* SPI devices need to watch for reset events to re-send the MT enable */ + if (id->bus == BUS_SPI) { + report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_RESET_REPORT_ID, 0); + report->size = 2; } return 0; @@ -1026,10 +1464,24 @@ static const struct hid_device_id magic_mice[] = { USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, { HID_USB_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), .driver_data = 0 }, + { HID_SPI_DEVICE(SPI_VENDOR_ID_APPLE, HID_ANY_ID), + .driver_data = 0 }, + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, + HID_ANY_ID), .driver_data = 0 }, { } }; MODULE_DEVICE_TABLE(hid, magic_mice); +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + if (hdev->bus == BUS_SPI) + return magicmouse_enable_multitouch(hdev); + + return 0; +} +#endif + static struct hid_driver magicmouse_driver = { .name = "magicmouse", .id_table = magic_mice, @@ -1040,6 +1492,10 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif + }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index e82a3c4e5b44ef..eeab0b6e32ccce 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -533,7 +533,7 @@ static void mt_get_feature(struct hid_device *hdev, struct hid_report *report) } ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, buf, - size, 0); + size, size, 0); if (ret) dev_warn(&hdev->dev, "failed to report feature\n"); } diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c index 3c0db8f93c829a..8d06ddff356a97 100644 --- a/drivers/hid/hid-playstation.c +++ b/drivers/hid/hid-playstation.c @@ -2378,7 +2378,8 @@ static int dualshock4_parse_report(struct ps_device *ps_dev, struct hid_report * (struct dualshock4_input_report_usb *)data; ds4_report = &usb->common; - num_touch_reports = usb->num_touch_reports; + num_touch_reports = min_t(u8, usb->num_touch_reports, + ARRAY_SIZE(usb->touch_reports)); touch_reports = usb->touch_reports; } else if (hdev->bus == BUS_BLUETOOTH && report->id == DS4_INPUT_REPORT_BT && size == DS4_INPUT_REPORT_BT_SIZE) { @@ -2392,7 +2393,8 @@ static int dualshock4_parse_report(struct ps_device *ps_dev, struct hid_report * } ds4_report = &bt->common; - num_touch_reports = bt->num_touch_reports; + num_touch_reports = min_t(u8, bt->num_touch_reports, + ARRAY_SIZE(bt->touch_reports)); touch_reports = bt->touch_reports; } else if (hdev->bus == BUS_BLUETOOTH && report->id == DS4_INPUT_REPORT_BT_MINIMAL && diff --git a/drivers/hid/hid-primax.c b/drivers/hid/hid-primax.c index e44d79dff8de63..8db054280afbcd 100644 --- a/drivers/hid/hid-primax.c +++ b/drivers/hid/hid-primax.c @@ -44,7 +44,7 @@ static int px_raw_event(struct hid_device *hid, struct hid_report *report, data[0] |= (1 << (data[idx] - 0xE0)); data[idx] = 0; } - hid_report_raw_event(hid, HID_INPUT_REPORT, data, size, 0); + hid_report_raw_event(hid, HID_INPUT_REPORT, data, size, size, 0); return 1; default: /* unknown report */ diff --git a/drivers/hid/hid-vivaldi-common.c b/drivers/hid/hid-vivaldi-common.c index bf734055d4b69d..b12bb5cc091aa3 100644 --- a/drivers/hid/hid-vivaldi-common.c +++ b/drivers/hid/hid-vivaldi-common.c @@ -85,7 +85,7 @@ void vivaldi_feature_mapping(struct hid_device *hdev, } ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, report_data, - report_len, 0); + report_len, report_len, 0); if (ret) { dev_warn(&hdev->dev, "failed to report feature %d\n", field->report->id); diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index 5a183af3d5c6a6..e0a302544cef4b 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -574,9 +574,10 @@ static void i2c_hid_get_input(struct i2c_hid *ihid) if (ihid->hid->group != HID_GROUP_RMI) pm_wakeup_event(&ihid->client->dev, 0); - hid_input_report(ihid->hid, HID_INPUT_REPORT, - ihid->inbuf + sizeof(__le16), - ret_size - sizeof(__le16), 1); + hid_safe_input_report(ihid->hid, HID_INPUT_REPORT, + ihid->inbuf + sizeof(__le16), + ihid->bufsize - sizeof(__le16), + ret_size - sizeof(__le16), 1); } return; diff --git a/drivers/hid/spi-hid/Kconfig b/drivers/hid/spi-hid/Kconfig new file mode 100644 index 00000000000000..59076c6ebeed9b --- /dev/null +++ b/drivers/hid/spi-hid/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "SPI HID support" + depends on SPI + +config SPI_HID_APPLE_OF + tristate "HID over SPI transport layer for Apple Silicon SoCs" + depends on INPUT && OF + select SPI_HID_APPLE_CORE + help + Say Y here if you use Apple Silicon based laptop. The keyboard and + touchpad are HID based devices connected via SPI. + + If unsure, say N. + + This support is also available as a module. If so, the module + will be called spi-hid-apple-of. It will also build/depend on the + module spi-hid-apple. + +endmenu + +config SPI_HID_APPLE_CORE + tristate + select HID + select CRC16 diff --git a/drivers/hid/spi-hid/Makefile b/drivers/hid/spi-hid/Makefile new file mode 100644 index 00000000000000..f276ee12cb94fc --- /dev/null +++ b/drivers/hid/spi-hid/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for SPI HID tarnsport drivers +# + +obj-$(CONFIG_SPI_HID_APPLE_CORE) += spi-hid-apple.o + +spi-hid-apple-objs = spi-hid-apple-core.o + +obj-$(CONFIG_SPI_HID_APPLE_OF) += spi-hid-apple-of.o diff --git a/drivers/hid/spi-hid/spi-hid-apple-core.c b/drivers/hid/spi-hid/spi-hid-apple-core.c new file mode 100644 index 00000000000000..2ed909895391c8 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-core.c @@ -0,0 +1,1194 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on: drivers/input/applespi.c + * + * MacBook (Pro) SPI keyboard and touchpad driver + * + * Copyright (c) 2015-2018 Federico Lorenzi + * Copyright (c) 2017-2018 Ronald Tschalär + * + */ + +//#define DEBUG 2 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "spi-hid-apple.h" + +#define SPIHID_DEF_WAIT msecs_to_jiffies(1000) + +#define SPIHID_MAX_INPUT_REPORT_SIZE 0x800 + +/* support only keyboard, trackpad and management dev for now */ +#define SPIHID_MAX_DEVICES 3 + +#define SPIHID_DEVICE_ID_MNGT 0x0 +#define SPIHID_DEVICE_ID_KBD 0x1 +#define SPIHID_DEVICE_ID_TP 0x2 +#define SPIHID_DEVICE_ID_INFO 0xd0 + +#define SPIHID_READ_PACKET 0x20 +#define SPIHID_WRITE_PACKET 0x40 + +#define SPIHID_DESC_MAX 512 + +#define SPIHID_SET_LEDS 0x0151 /* caps lock */ + +#define SPI_RW_CHG_DELAY_US 200 /* 'Inter Stage Us'? */ + +static const u8 spi_hid_apple_booted[4] = { 0xa0, 0x80, 0x00, 0x00 }; +static const u8 spi_hid_apple_status_ok[4] = { 0xac, 0x27, 0x68, 0xd5 }; + +struct spihid_interface { + struct hid_device *hid; + u8 *hid_desc; + u32 hid_desc_len; + u32 id; + unsigned country; + u32 max_control_report_len; + u32 max_input_report_len; + u32 max_output_report_len; + u8 name[32]; + u8 reply_buf[SPIHID_DESC_MAX]; + u32 reply_len; + bool ready; +}; + +struct spihid_input_report { + u8 *buf; + u32 length; + u32 offset; + u8 device; + u8 flags; +}; + +struct spihid_apple { + struct spi_device *spidev; + + struct spihid_apple_ops *ops; + + struct spihid_interface mngt; + struct spihid_interface kbd; + struct spihid_interface tp; + + wait_queue_head_t wait; + struct mutex tx_lock; //< protects against concurrent SPI writes + + struct spi_message rx_msg; + struct spi_message tx_msg; + struct spi_transfer rx_transfer; + struct spi_transfer tx_transfer; + struct spi_transfer status_transfer; + + u8 *rx_buf; + u8 *tx_buf; + u8 *status_buf; + + u8 vendor[32]; + u8 product[64]; + u8 serial[32]; + + u32 num_devices; + + u32 vendor_id; + u32 product_id; + u32 version_number; + + u8 msg_id; + + /* fragmented HID report */ + struct spihid_input_report report; + + /* state tracking flags */ + bool status_booted; + +#ifdef IRQ_WAKE_SUPPORT + bool irq_wake_enabled; +#endif +}; + +/** + * struct spihid_msg_hdr - common header of protocol messages. + * + * Each message begins with fixed header, followed by a message-type specific + * payload, and ends with a 16-bit crc. Because of the varying lengths of the + * payload, the crc is defined at the end of each payload struct, rather than + * in this struct. + * + * @unknown0: request type? output, input (0x10), feature, protocol + * @unknown1: maybe report id? + * @unknown2: mostly zero, in info request maybe device num + * @id: incremented on each message, rolls over after 255; there is a + * separate counter for each message type. + * @rsplen: response length (the exact nature of this field is quite + * speculative). On a request/write this is often the same as + * @length, though in some cases it has been seen to be much larger + * (e.g. 0x400); on a response/read this the same as on the + * request; for reads that are not responses it is 0. + * @length: length of the remainder of the data in the whole message + * structure (after re-assembly in case of being split over + * multiple spi-packets), minus the trailing crc. The total size + * of a message is therefore @length + 10. + */ + +struct spihid_msg_hdr { + u8 unknown0; + u8 unknown1; + u8 unknown2; + u8 id; + __le16 rsplen; + __le16 length; +}; + +/** + * struct spihid_transfer_packet - a complete spi packet; always 256 bytes. This carries + * the (parts of the) message in the data. But note that this does not + * necessarily contain a complete message, as in some cases (e.g. many + * fingers pressed) the message is split over multiple packets (see the + * @offset, @remain, and @length fields). In general the data parts in + * spihid_transfer_packet's are concatenated until @remaining is 0, and the + * result is an message. + * + * @flags: 0x40 = write (to device), 0x20 = read (from device); note that + * the response to a write still has 0x40. + * @device: 1 = keyboard, 2 = touchpad + * @offset: specifies the offset of this packet's data in the complete + * message; i.e. > 0 indicates this is a continuation packet (in + * the second packet for a message split over multiple packets + * this would then be the same as the @length in the first packet) + * @remain: number of message bytes remaining in subsequents packets (in + * the first packet of a message split over two packets this would + * then be the same as the @length in the second packet) + * @length: length of the valid data in the @data in this packet + * @data: all or part of a message + * @crc16: crc over this whole structure minus this @crc16 field. This + * covers just this packet, even on multi-packet messages (in + * contrast to the crc in the message). + */ +struct spihid_transfer_packet { + u8 flags; + u8 device; + __le16 offset; + __le16 remain; + __le16 length; + u8 data[246]; + __le16 crc16; +}; + +/* + * how HID is mapped onto the protocol is not fully clear. This are the known + * reports/request: + * + * pkt.flags pkt.dev? msg.u0 msg.u1 msg.u2 + * info 0x40 0xd0 0x20 0x01 0xd0 + * + * info mngt: 0x40 0xd0 0x20 0x10 0x00 + * info kbd: 0x40 0xd0 0x20 0x10 0x01 + * info tp: 0x40 0xd0 0x20 0x10 0x02 + * + * desc kbd: 0x40 0xd0 0x20 0x10 0x01 + * desc trackpad: 0x40 0xd0 0x20 0x10 0x02 + * + * mt mode: 0x40 0x02 0x52 0x02 0x00 set protocol? + * capslock led 0x40 0x01 0x51 0x01 0x00 output report + * + * report kbd: 0x20 0x01 0x10 0x01 0x00 input report + * report tp: 0x20 0x02 0x10 0x02 0x00 input report + * + */ + + +static int spihid_apple_request(struct spihid_apple *spihid, u8 target, u8 unk0, + u8 unk1, u8 unk2, u16 resp_len, u8 *buf, + size_t len) +{ + struct spihid_transfer_packet *pkt; + struct spihid_msg_hdr *hdr; + u16 crc; + int err; + + /* know reports are small enoug to fit in a single packet */ + if (len > sizeof(pkt->data) - sizeof(*hdr) - sizeof(__le16)) + return -EINVAL; + + err = mutex_lock_interruptible(&spihid->tx_lock); + if (err < 0) + return err; + + pkt = (struct spihid_transfer_packet *)spihid->tx_buf; + + memset(pkt, 0, sizeof(*pkt)); + pkt->flags = SPIHID_WRITE_PACKET; + pkt->device = target; + pkt->length = cpu_to_le16(sizeof(*hdr) + len + sizeof(__le16)); + + hdr = (struct spihid_msg_hdr *)&pkt->data[0]; + hdr->unknown0 = unk0; + hdr->unknown1 = unk1; + hdr->unknown2 = unk2; + hdr->id = spihid->msg_id++; + hdr->rsplen = cpu_to_le16(resp_len); + hdr->length = cpu_to_le16(len); + + if (len) + memcpy(pkt->data + sizeof(*hdr), buf, len); + crc = crc16(0, &pkt->data[0], sizeof(*hdr) + len); + put_unaligned_le16(crc, pkt->data + sizeof(*hdr) + len); + + pkt->crc16 = cpu_to_le16(crc16(0, spihid->tx_buf, + offsetof(struct spihid_transfer_packet, crc16))); + + memset(spihid->status_buf, 0, sizeof(spi_hid_apple_status_ok)); + + err = spi_sync(spihid->spidev, &spihid->tx_msg); + + if (memcmp(spihid->status_buf, spi_hid_apple_status_ok, + sizeof(spi_hid_apple_status_ok))) { + u8 *b = spihid->status_buf; + dev_warn_ratelimited(&spihid->spidev->dev, "status message " + "mismatch: %02x %02x %02x %02x\n", + b[0], b[1], b[2], b[3]); + } + mutex_unlock(&spihid->tx_lock); + if (err < 0) + return err; + + return (int)len; +} + +static struct spihid_apple *spihid_get_data(struct spihid_interface *idev) +{ + switch (idev->id) { + case SPIHID_DEVICE_ID_KBD: + return container_of(idev, struct spihid_apple, kbd); + case SPIHID_DEVICE_ID_TP: + return container_of(idev, struct spihid_apple, tp); + default: + return NULL; + } +} + +static int apple_ll_start(struct hid_device *hdev) +{ + /* no-op SPI transport is already setup */ + return 0; +}; + +static void apple_ll_stop(struct hid_device *hdev) +{ + /* no-op, devices will be desstroyed on driver destruction */ +} + +static int apple_ll_open(struct hid_device *hdev) +{ + struct spihid_apple *spihid; + struct spihid_interface *idev = hdev->driver_data; + + if (idev->hid_desc_len == 0) { + spihid = spihid_get_data(idev); + dev_warn(&spihid->spidev->dev, + "HID descriptor missing for dev %u", idev->id); + } else + idev->ready = true; + + return 0; +} + +static void apple_ll_close(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + idev->ready = false; +} + +static int apple_ll_parse(struct hid_device *hdev) +{ + struct spihid_interface *idev = hdev->driver_data; + + return hid_parse_report(hdev, idev->hid_desc, idev->hid_desc_len); +} + +static int apple_ll_raw_request(struct hid_device *hdev, + unsigned char reportnum, __u8 *buf, size_t len, + unsigned char rtype, int reqtype) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + int ret; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_raw_request: device:%u reportnum:%hhu rtype:%hhu", + idev->id, reportnum, rtype); + + switch (reqtype) { + case HID_REQ_GET_REPORT: + if (rtype != HID_FEATURE_REPORT) + return -EINVAL; + + idev->reply_len = 0; + ret = spihid_apple_request(spihid, idev->id, 0x32, reportnum, 0x00, len, NULL, 0); + if (ret < 0) + return ret; + + ret = wait_event_interruptible_timeout(spihid->wait, idev->reply_len, + SPIHID_DEF_WAIT); + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) { + dev_err(&spihid->spidev->dev, "waiting for get report failed: %d", ret); + return ret; + } + memcpy(buf, idev->reply_buf, max_t(size_t, len, idev->reply_len)); + return idev->reply_len; + + case HID_REQ_SET_REPORT: + if (buf[0] != reportnum) + return -EINVAL; + if (reportnum != idev->id) { + dev_warn(&spihid->spidev->dev, + "device:%u reportnum:" + "%hhu mismatch", + idev->id, reportnum); + return -EINVAL; + } + return spihid_apple_request(spihid, idev->id, 0x52, reportnum, 0x00, 2, buf, len); + default: + return -EIO; + } +} + +static int apple_ll_output_report(struct hid_device *hdev, __u8 *buf, + size_t len) +{ + struct spihid_interface *idev = hdev->driver_data; + struct spihid_apple *spihid = spihid_get_data(idev); + if (!spihid) + return -1; + + dev_dbg(&spihid->spidev->dev, + "apple_ll_output_report: device:%u len:%zu:", + idev->id, len); + // second idev->id should maybe be buf[0]? + return spihid_apple_request(spihid, idev->id, 0x51, idev->id, 0x00, 0, buf, len); +} + +static struct hid_ll_driver apple_hid_ll = { + .start = &apple_ll_start, + .stop = &apple_ll_stop, + .open = &apple_ll_open, + .close = &apple_ll_close, + .parse = &apple_ll_parse, + .raw_request = &apple_ll_raw_request, + .output_report = &apple_ll_output_report, + .max_buffer_size = SPIHID_MAX_INPUT_REPORT_SIZE, +}; + +static struct spihid_interface *spihid_get_iface(struct spihid_apple *spihid, + u32 iface) +{ + switch (iface) { + case SPIHID_DEVICE_ID_MNGT: + return &spihid->mngt; + case SPIHID_DEVICE_ID_KBD: + return &spihid->kbd; + case SPIHID_DEVICE_ID_TP: + return &spihid->tp; + default: + return NULL; + } +} + +static int spihid_verify_msg(struct spihid_apple *spihid, u8 *buf, size_t len) +{ + u16 msg_crc, crc; + struct device *dev = &spihid->spidev->dev; + + crc = crc16(0, buf, len - sizeof(__le16)); + msg_crc = get_unaligned_le16(buf + len - sizeof(__le16)); + if (crc != msg_crc) { + dev_warn_ratelimited(dev, "Read message crc mismatch\n"); + return 0; + } + return 1; +} + +static bool spihid_status_report(struct spihid_apple *spihid, u8 *pl, + size_t len) +{ + struct device *dev = &spihid->spidev->dev; + dev_dbg(dev, "%s: len: %zu", __func__, len); + if (len == 5 && pl[0] == 0xe0) + return true; + + return false; +} + +static bool spihid_process_input_report(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + //dev_dbg(&spihid>spidev->dev, "input report: req:%hx iface:%u ", hdr->unknown0, device); + if (hdr->unknown0 != 0x10) + return false; + + /* HID device as well but Vendor usage only, handle it internally for now */ + if (device == 0) { + if (hdr->unknown1 == 0xe0) { + return spihid_status_report(spihid, payload, len); + } + } else if (device < SPIHID_MAX_DEVICES) { + struct spihid_interface *iface = + spihid_get_iface(spihid, device); + if (iface && iface->hid && iface->ready) { + hid_input_report(iface->hid, HID_INPUT_REPORT, payload, + len, 1); + return true; + } + } else + dev_dbg(&spihid->spidev->dev, + "unexpected iface:%u for input report", device); + + return false; +} + +struct spihid_device_info { + __le16 u0[2]; + __le16 num_devices; + __le16 vendor_id; + __le16 product_id; + __le16 version_number; + __le16 vendor_str[2]; //< offset and string length + __le16 product_str[2]; //< offset and string length + __le16 serial_str[2]; //< offset and string length +}; + +static bool spihid_process_device_info(struct spihid_apple *spihid, u32 iface, + u8 *payload, size_t len) +{ + struct device *dev = &spihid->spidev->dev; + + if (iface != SPIHID_DEVICE_ID_INFO) + return false; + + if (spihid->vendor_id == 0 && + len >= sizeof(struct spihid_device_info)) { + struct spihid_device_info *info = + (struct spihid_device_info *)payload; + u16 voff, vlen, poff, plen, soff, slen; + u32 num_devices; + + num_devices = __le16_to_cpu(info->num_devices); + + if (num_devices < SPIHID_MAX_DEVICES) { + dev_err(dev, + "Device info reports %u devices, expecting at least 3", + num_devices); + return false; + } + spihid->num_devices = num_devices; + + if (spihid->num_devices > SPIHID_MAX_DEVICES) { + dev_info( + dev, + "limiting the number of devices to mngt, kbd and mouse"); + spihid->num_devices = SPIHID_MAX_DEVICES; + } + + spihid->vendor_id = __le16_to_cpu(info->vendor_id); + spihid->product_id = __le16_to_cpu(info->product_id); + spihid->version_number = __le16_to_cpu(info->version_number); + + voff = __le16_to_cpu(info->vendor_str[0]); + vlen = __le16_to_cpu(info->vendor_str[1]); + + if (voff < len && vlen <= len - voff && + vlen < sizeof(spihid->vendor)) { + memcpy(spihid->vendor, payload + voff, vlen); + spihid->vendor[vlen] = '\0'; + } + + poff = __le16_to_cpu(info->product_str[0]); + plen = __le16_to_cpu(info->product_str[1]); + + if (poff < len && plen <= len - poff && + plen < sizeof(spihid->product)) { + memcpy(spihid->product, payload + poff, plen); + spihid->product[plen] = '\0'; + } + + soff = __le16_to_cpu(info->serial_str[0]); + slen = __le16_to_cpu(info->serial_str[1]); + + if (soff < len && slen <= len - soff && + slen < sizeof(spihid->serial)) { + memcpy(spihid->vendor, payload + soff, slen); + spihid->serial[slen] = '\0'; + } + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +struct spihid_iface_info { + u8 u_0; + u8 interface_num; + u8 u_2; + u8 u_3; + u8 u_4; + u8 country_code; + __le16 max_input_report_len; + __le16 max_output_report_len; + __le16 max_control_report_len; + __le16 name_offset; + __le16 name_length; +}; + +static bool spihid_process_iface_info(struct spihid_apple *spihid, u32 num, + u8 *payload, size_t len) +{ + struct spihid_iface_info *info; + struct spihid_interface *iface = spihid_get_iface(spihid, num); + u32 name_off, name_len; + + if (!iface) + return false; + + if (!iface->max_input_report_len) { + if (len < sizeof(*info)) + return false; + + info = (struct spihid_iface_info *)payload; + + iface->max_input_report_len = + le16_to_cpu(info->max_input_report_len); + iface->max_output_report_len = + le16_to_cpu(info->max_output_report_len); + iface->max_control_report_len = + le16_to_cpu(info->max_control_report_len); + iface->country = info->country_code; + + name_off = le16_to_cpu(info->name_offset); + name_len = le16_to_cpu(info->name_length); + + if (name_off < len && name_len <= len - name_off && + name_len < sizeof(iface->name)) { + memcpy(iface->name, payload + name_off, name_len); + iface->name[name_len] = '\0'; + } + + dev_dbg(&spihid->spidev->dev, "Info for %s, country code: 0x%x", + iface->name, iface->country); + + wake_up_interruptible(&spihid->wait); + } + + return true; +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *idev, u8 device); + +static bool spihid_process_iface_hid_report_desc(struct spihid_apple *spihid, + u32 num, u8 *payload, + size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, num); + + if (!iface) + return false; + + if (iface->hid_desc_len == 0) { + if (len > SPIHID_DESC_MAX) + return false; + memcpy(iface->hid_desc, payload, len); + iface->hid_desc_len = len; + + /* do not register the mngt iface as HID device */ + if (num > 0) + spihid_register_hid_device(spihid, iface, num); + + wake_up_interruptible(&spihid->wait); + } + return true; +} + +static bool spihid_process_iface_get_report(struct spihid_apple *spihid, + u32 device, u8 report, + u8 *payload, size_t len) +{ + struct spihid_interface *iface = spihid_get_iface(spihid, device); + + if (!iface) + return false; + + if (len > sizeof(iface->reply_buf) || len < 1) + return false; + + memcpy(iface->reply_buf, payload, len); + iface->reply_len = len; + + wake_up_interruptible(&spihid->wait); + + return true; +} + +static bool spihid_process_response(struct spihid_apple *spihid, u32 device, + struct spihid_msg_hdr *hdr, u8 *payload, + size_t len) +{ + if (hdr->unknown0 == 0x20) { + switch (hdr->unknown1) { + case 0x01: + return spihid_process_device_info(spihid, hdr->unknown2, + payload, len); + case 0x02: + return spihid_process_iface_info(spihid, hdr->unknown2, + payload, len); + case 0x10: + return spihid_process_iface_hid_report_desc( + spihid, hdr->unknown2, payload, len); + default: + break; + } + } + + if (hdr->unknown0 == 0x32) { + return spihid_process_iface_get_report(spihid, device, hdr->unknown1, payload, len); + } + + return false; +} + +static void spihid_process_message(struct spihid_apple *spihid, u8 *data, + size_t length, u8 device, u8 flags) +{ + struct device *dev = &spihid->spidev->dev; + struct spihid_msg_hdr *hdr; + bool handled = false; + size_t payload_len; + u8 *payload; + + if (!spihid_verify_msg(spihid, data, length)) + return; + + hdr = (struct spihid_msg_hdr *)data; + payload_len = le16_to_cpu(hdr->length); + + if (payload_len == 0 || + (payload_len + sizeof(struct spihid_msg_hdr) + 2) > length) + return; + + payload = data + sizeof(struct spihid_msg_hdr); + + switch (flags) { + case SPIHID_READ_PACKET: + handled = spihid_process_input_report(spihid, device, hdr, + payload, payload_len); + break; + case SPIHID_WRITE_PACKET: + handled = spihid_process_response(spihid, device, hdr, payload, + payload_len); + break; + default: + break; + } + +#if defined(DEBUG) && DEBUG > 1 + { + dev_dbg(dev, + "R msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#else + if (!handled) { + dev_dbg(dev, + "R unhandled msg: req:%02hhx rep:%02hhx dev:%02hhx id:%hu len:%hu\n", + hdr->unknown0, hdr->unknown1, hdr->unknown2, hdr->id, + hdr->length); + print_hex_dump_debug("spihid msg: ", DUMP_PREFIX_OFFSET, 16, 1, + payload, le16_to_cpu(hdr->length), true); + } +#endif +} + +static void spihid_assemble_message(struct spihid_apple *spihid, + struct spihid_transfer_packet *pkt) +{ + size_t length, offset, remain; + struct device *dev = &spihid->spidev->dev; + struct spihid_input_report *rep = &spihid->report; + + length = le16_to_cpu(pkt->length); + remain = le16_to_cpu(pkt->remain); + offset = le16_to_cpu(pkt->offset); + + if (offset + length + remain > U16_MAX) { + return; + } + + if (pkt->device != rep->device || pkt->flags != rep->flags || + offset != rep->offset) { + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + + if (offset == 0) { + if (rep->offset != 0) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + } + memcpy(rep->buf, pkt->data, length); + rep->offset = length; + rep->length = length + remain; + rep->device = pkt->device; + rep->flags = pkt->flags; + } else if (offset == rep->offset) { + if (offset + length + remain != rep->length) { + dev_warn(dev, "incomplete report off:%u len:%u", + rep->offset, rep->length); + return; + } + memcpy(rep->buf + offset, pkt->data, length); + rep->offset += length; + + if (rep->offset == rep->length) { + spihid_process_message(spihid, rep->buf, rep->length, + rep->device, rep->flags); + rep->device = 0; + rep->flags = 0; + rep->offset = 0; + rep->length = 0; + } + } +} + +static void spihid_process_read(struct spihid_apple *spihid) +{ + u16 crc; + size_t length; + struct device *dev = &spihid->spidev->dev; + struct spihid_transfer_packet *pkt; + + pkt = (struct spihid_transfer_packet *)spihid->rx_buf; + + /* check transfer packet crc */ + crc = crc16(0, spihid->rx_buf, + offsetof(struct spihid_transfer_packet, crc16)); + if (crc != le16_to_cpu(pkt->crc16)) { + dev_warn_ratelimited(dev, "Read package crc mismatch\n"); + return; + } + + length = le16_to_cpu(pkt->length); + + if (length < sizeof(struct spihid_msg_hdr) + 2) { + if (length == sizeof(spi_hid_apple_booted) && + !memcmp(pkt->data, spi_hid_apple_booted, length)) { + if (!spihid->status_booted) { + spihid->status_booted = true; + wake_up_interruptible(&spihid->wait); + } + } else { + dev_info(dev, "R short packet: len:%zu\n", length); + print_hex_dump(KERN_INFO, "spihid pkt:", + DUMP_PREFIX_OFFSET, 16, 1, pkt->data, + length, false); + } + return; + } + +#if defined(DEBUG) && DEBUG > 1 + dev_dbg(dev, + "R pkt: flags:%02hhx dev:%02hhx off:%hu remain:%hu, len:%zu\n", + pkt->flags, pkt->device, pkt->offset, pkt->remain, length); +#if defined(DEBUG) && DEBUG > 2 + print_hex_dump_debug("spihid pkt: ", DUMP_PREFIX_OFFSET, 16, 1, + spihid->rx_buf, + sizeof(struct spihid_transfer_packet), true); +#endif +#endif + + if (length > sizeof(pkt->data)) { + dev_warn_ratelimited(dev, "Invalid pkt len:%zu", length); + return; + } + + /* short message */ + if (pkt->offset == 0 && pkt->remain == 0) { + spihid_process_message(spihid, pkt->data, length, pkt->device, + pkt->flags); + } else { + spihid_assemble_message(spihid, pkt); + } +} + +static void spihid_read_packet_sync(struct spihid_apple *spihid) +{ + int err; + + err = spi_sync(spihid->spidev, &spihid->rx_msg); + if (!err) { + spihid_process_read(spihid); + } else { + dev_warn(&spihid->spidev->dev, "RX failed: %d\n", err); + } +} + +irqreturn_t spihid_apple_core_irq(int irq, void *data) +{ + struct spi_device *spi = data; + struct spihid_apple *spihid = spi_get_drvdata(spi); + + spihid_read_packet_sync(spihid); + + return IRQ_HANDLED; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_irq); + +static void spihid_apple_setup_spi_msgs(struct spihid_apple *spihid) +{ + memset(&spihid->rx_transfer, 0, sizeof(spihid->rx_transfer)); + + spihid->rx_transfer.rx_buf = spihid->rx_buf; + spihid->rx_transfer.len = sizeof(struct spihid_transfer_packet); + + spi_message_init(&spihid->rx_msg); + spi_message_add_tail(&spihid->rx_transfer, &spihid->rx_msg); + + memset(&spihid->tx_transfer, 0, sizeof(spihid->rx_transfer)); + memset(&spihid->status_transfer, 0, sizeof(spihid->status_transfer)); + + spihid->tx_transfer.tx_buf = spihid->tx_buf; + spihid->tx_transfer.len = sizeof(struct spihid_transfer_packet); + spihid->tx_transfer.delay.unit = SPI_DELAY_UNIT_USECS; + spihid->tx_transfer.delay.value = SPI_RW_CHG_DELAY_US; + + spihid->status_transfer.rx_buf = spihid->status_buf; + spihid->status_transfer.len = sizeof(spi_hid_apple_status_ok); + + spi_message_init(&spihid->tx_msg); + spi_message_add_tail(&spihid->tx_transfer, &spihid->tx_msg); + spi_message_add_tail(&spihid->status_transfer, &spihid->tx_msg); +} + +static int spihid_apple_setup_spi(struct spihid_apple *spihid) +{ + spihid_apple_setup_spi_msgs(spihid); + + return spihid->ops->power_on(spihid->ops); +} + +static int spihid_register_hid_device(struct spihid_apple *spihid, + struct spihid_interface *iface, u8 device) +{ + int ret; + char *suffix; + struct hid_device *hid; + + iface->id = device; + + hid = hid_allocate_device(); + if (IS_ERR(hid)) + return PTR_ERR(hid); + + /* + * Use 'Apple SPI Keyboard' and 'Apple SPI Trackpad' as input device + * names. The device names need to be distinct since at least Kwin uses + * the tripple Vendor ID, Product ID, Name to identify devices. + */ + snprintf(hid->name, sizeof(hid->name), "Apple SPI %s", iface->name); + // strip ' / Boot' suffix from the name + suffix = strstr(hid->name, " / Boot"); + if (suffix) + suffix[0] = '\0'; + snprintf(hid->phys, sizeof(hid->phys), "%s (%hhx)", + dev_name(&spihid->spidev->dev), device); + strscpy(hid->uniq, spihid->serial, sizeof(hid->uniq)); + + hid->ll_driver = &apple_hid_ll; + hid->bus = BUS_SPI; + hid->vendor = spihid->vendor_id; + hid->product = spihid->product_id; + hid->version = spihid->version_number; + + if (device == SPIHID_DEVICE_ID_KBD) + hid->type = HID_TYPE_SPI_KEYBOARD; + else if (device == SPIHID_DEVICE_ID_TP) + hid->type = HID_TYPE_SPI_MOUSE; + + hid->country = iface->country; + hid->dev.parent = &spihid->spidev->dev; + hid->driver_data = iface; + + ret = hid_add_device(hid); + if (ret < 0) { + hid_destroy_device(hid); + dev_warn(&spihid->spidev->dev, + "Failed to register hid device %hhu", device); + return ret; + } + + iface->hid = hid; + + return 0; +} + +static void spihid_destroy_hid_device(struct spihid_interface *iface) +{ + if (iface->hid) { + hid_destroy_device(iface->hid); + iface->hid = NULL; + } + iface->ready = false; +} + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops) +{ + struct device *dev = &spi->dev; + struct spihid_apple *spihid; + int err, i; + + if (!ops || !ops->power_on || !ops->power_off || !ops->enable_irq || !ops->disable_irq) + return -EINVAL; + + spihid = devm_kzalloc(dev, sizeof(*spihid), GFP_KERNEL); + if (!spihid) + return -ENOMEM; + + spihid->ops = ops; + spihid->spidev = spi; + + // init spi + spi_set_drvdata(spi, spihid); + + /* + * allocate SPI buffers + * Overallocate the receice buffer since it passed directly into + * hid_input_report / hid_report_raw_event. The later expects the buffer + * to be HID_MAX_BUFFER_SIZE (16k) or hid_ll_driver.max_buffer_size if + * set. + */ + spihid->rx_buf = devm_kmalloc( + &spi->dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + spihid->tx_buf = devm_kmalloc( + &spi->dev, sizeof(struct spihid_transfer_packet), GFP_KERNEL); + spihid->status_buf = devm_kmalloc( + &spi->dev, sizeof(spi_hid_apple_status_ok), GFP_KERNEL); + + if (!spihid->rx_buf || !spihid->tx_buf || !spihid->status_buf) + return -ENOMEM; + + spihid->report.buf = + devm_kmalloc(dev, SPIHID_MAX_INPUT_REPORT_SIZE, GFP_KERNEL); + + spihid->kbd.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + spihid->tp.hid_desc = devm_kmalloc(dev, SPIHID_DESC_MAX, GFP_KERNEL); + + if (!spihid->report.buf || !spihid->kbd.hid_desc || + !spihid->tp.hid_desc) + return -ENOMEM; + + init_waitqueue_head(&spihid->wait); + + mutex_init(&spihid->tx_lock); + + /* Init spi transfer buffers and power device on */ + err = spihid_apple_setup_spi(spihid); + if (err < 0) + goto error; + + /* enable HID irq */ + spihid->ops->enable_irq(spihid->ops); + + // wait for boot message + err = wait_event_interruptible_timeout(spihid->wait, + spihid->status_booted, + msecs_to_jiffies(1000)); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device boot failed: %d", err); + goto error; + } + + /* request device information */ + dev_dbg(dev, "request device info"); + spihid_apple_request(spihid, 0xd0, 0x20, 0x01, 0xd0, 0, NULL, 0); + err = wait_event_interruptible_timeout(spihid->wait, spihid->vendor_id, + SPIHID_DEF_WAIT); + if (err == 0) + err = -ENODEV; + if (err < 0) { + dev_err(dev, "waiting for device info failed: %d", err); + goto error; + } + + /* request interface information */ + for (i = 0; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request interface info 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x02, i, + SPIHID_DESC_MAX, NULL, 0); + err = wait_event_interruptible_timeout( + spihid->wait, iface->max_input_report_len, + SPIHID_DEF_WAIT); + } + + /* request HID report descriptors */ + for (i = 1; i < spihid->num_devices; i++) { + struct spihid_interface *iface = spihid_get_iface(spihid, i); + if (!iface) + continue; + dev_dbg(dev, "request hid report desc 0x%02x", i); + spihid_apple_request(spihid, 0xd0, 0x20, 0x10, i, + SPIHID_DESC_MAX, NULL, 0); + wait_event_interruptible_timeout( + spihid->wait, iface->hid_desc_len, SPIHID_DEF_WAIT); + } + + return 0; +error: + return err; +} +EXPORT_SYMBOL_GPL(spihid_apple_core_probe); + +void spihid_apple_core_remove(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* destroy input devices */ + + spihid_destroy_hid_device(&spihid->tp); + spihid_destroy_hid_device(&spihid->kbd); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_remove); + +void spihid_apple_core_shutdown(struct spi_device *spi) +{ + struct spihid_apple *spihid = spi_get_drvdata(spi); + + /* disable irq */ + spihid->ops->disable_irq(spihid->ops); + + /* power SPI device down */ + spihid->ops->power_off(spihid->ops); +} +EXPORT_SYMBOL_GPL(spihid_apple_core_shutdown); + +#ifdef CONFIG_PM_SLEEP +static int spihid_apple_core_suspend(struct device *dev) +{ + int ret; +#ifdef IRQ_WAKE_SUPPORT + int wake_status; +#endif + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); + + if (spihid->tp.hid) { + ret = hid_driver_suspend(spihid->tp.hid, PMSG_SUSPEND); + if (ret < 0) + return ret; + } + + if (spihid->kbd.hid) { + ret = hid_driver_suspend(spihid->kbd.hid, PMSG_SUSPEND); + if (ret < 0) { + if (spihid->tp.hid) + hid_driver_resume(spihid->tp.hid); + return ret; + } + } + + /* Save some power */ + spihid->ops->disable_irq(spihid->ops); + +#ifdef IRQ_WAKE_SUPPORT + if (device_may_wakeup(dev)) { + wake_status = spihid->ops->enable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = true; + else + dev_warn(dev, "Failed to enable irq wake: %d\n", + wake_status); + } else { + spihid->ops->power_off(spihid->ops); + } +#else + spihid->ops->power_off(spihid->ops); +#endif + + return 0; +} + +static int spihid_apple_core_resume(struct device *dev) +{ + int ret_tp = 0, ret_kbd = 0; + struct spihid_apple *spihid = spi_get_drvdata(to_spi_device(dev)); +#ifdef IRQ_WAKE_SUPPORT + int wake_status; + + if (!device_may_wakeup(dev)) { + spihid->ops->power_on(spihid->ops); + } else if (spihid->irq_wake_enabled) { + wake_status = spihid->ops->disable_irq_wake(spihid->ops); + if (!wake_status) + spihid->irq_wake_enabled = false; + else + dev_warn(dev, "Failed to disable irq wake: %d\n", + wake_status); + } +#endif + + spihid->ops->enable_irq(spihid->ops); + spihid->ops->power_on(spihid->ops); + + if (spihid->tp.hid) + ret_tp = hid_driver_reset_resume(spihid->tp.hid); + if (spihid->kbd.hid) + ret_kbd = hid_driver_reset_resume(spihid->kbd.hid); + + if (ret_tp < 0) + return ret_tp; + + return ret_kbd; +} +#endif + +const struct dev_pm_ops spihid_apple_core_pm = { + SET_SYSTEM_SLEEP_PM_OPS(spihid_apple_core_suspend, + spihid_apple_core_resume) +}; +EXPORT_SYMBOL_GPL(spihid_apple_core_pm); + +MODULE_DESCRIPTION("Apple SPI HID transport driver"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple-of.c b/drivers/hid/spi-hid/spi-hid-apple-of.c new file mode 100644 index 00000000000000..b631212b836d30 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple-of.c @@ -0,0 +1,153 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Apple SPI HID transport driver - Open Firmware + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include + +#include "spi-hid-apple.h" + + +struct spihid_apple_of { + struct spihid_apple_ops ops; + + struct gpio_desc *enable_gpio; + int irq; +}; + +static int spihid_apple_of_power_on(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* reset the controller on boot */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(5); + gpiod_direction_output(sh_of->enable_gpio, 0); + msleep(5); + /* turn SPI device on */ + gpiod_direction_output(sh_of->enable_gpio, 1); + msleep(50); + + return 0; +} + +static int spihid_apple_of_power_off(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + /* turn SPI device off */ + gpiod_direction_output(sh_of->enable_gpio, 0); + + return 0; +} + +static int spihid_apple_of_enable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + enable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_disable_irq(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + disable_irq(sh_of->irq); + + return 0; +} + +static int spihid_apple_of_enable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return enable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_disable_irq_wake(struct spihid_apple_ops *ops) +{ + struct spihid_apple_of *sh_of = container_of(ops, struct spihid_apple_of, ops); + + return disable_irq_wake(sh_of->irq); +} + +static int spihid_apple_of_probe(struct spi_device *spi) +{ + struct device *dev = &spi->dev; + struct spihid_apple_of *spihid_of; + int err; + + spihid_of = devm_kzalloc(dev, sizeof(*spihid_of), GFP_KERNEL); + if (!spihid_of) + return -ENOMEM; + + spihid_of->ops.power_on = spihid_apple_of_power_on; + spihid_of->ops.power_off = spihid_apple_of_power_off; + spihid_of->ops.enable_irq = spihid_apple_of_enable_irq; + spihid_of->ops.disable_irq = spihid_apple_of_disable_irq; + spihid_of->ops.enable_irq_wake = spihid_apple_of_enable_irq_wake; + spihid_of->ops.disable_irq_wake = spihid_apple_of_disable_irq_wake; + + spihid_of->enable_gpio = devm_gpiod_get_index(dev, "spien", 0, 0); + if (IS_ERR(spihid_of->enable_gpio)) { + err = PTR_ERR(spihid_of->enable_gpio); + dev_err(dev, "failed to get 'spien' gpio pin: %d", err); + return err; + } + + spihid_of->irq = of_irq_get(dev->of_node, 0); + if (spihid_of->irq < 0) { + err = spihid_of->irq; + dev_err(dev, "failed to get 'extended-irq': %d", err); + return err; + } + err = devm_request_threaded_irq(dev, spihid_of->irq, NULL, + spihid_apple_core_irq, IRQF_ONESHOT | IRQF_NO_AUTOEN, + "spi-hid-apple-irq", spi); + if (err < 0) { + dev_err(dev, "failed to request extended-irq %d: %d", + spihid_of->irq, err); + return err; + } + + return spihid_apple_core_probe(spi, &spihid_of->ops); +} + +static const struct of_device_id spihid_apple_of_match[] = { + { .compatible = "apple,spi-hid-transport" }, + {}, +}; +MODULE_DEVICE_TABLE(of, spihid_apple_of_match); + +static struct spi_device_id spihid_apple_of_id[] = { + { "spi-hid-transport", 0 }, + {} +}; +MODULE_DEVICE_TABLE(spi, spihid_apple_of_id); + +static struct spi_driver spihid_apple_of_driver = { + .driver = { + .name = "spi-hid-apple-of", + .pm = &spihid_apple_core_pm, + .of_match_table = of_match_ptr(spihid_apple_of_match), + }, + + .id_table = spihid_apple_of_id, + .probe = spihid_apple_of_probe, + .remove = spihid_apple_core_remove, + .shutdown = spihid_apple_core_shutdown, +}; + +module_spi_driver(spihid_apple_of_driver); + +MODULE_DESCRIPTION("Apple SPI HID transport driver for OpenFirmware systems"); +MODULE_AUTHOR("Janne Grunau "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/spi-hid/spi-hid-apple.h b/drivers/hid/spi-hid/spi-hid-apple.h new file mode 100644 index 00000000000000..9abecd1ba78028 --- /dev/null +++ b/drivers/hid/spi-hid/spi-hid-apple.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +#ifndef SPI_HID_APPLE_H +#define SPI_HID_APPLE_H + +#include +#include + +/** + * struct spihid_apple_ops - Ops to control the device from the core driver. + * + * @power_on: reset and power the device on. + * @power_off: power the device off. + * @enable_irq: enable irq or ACPI gpe. + * @disable_irq: disable irq or ACPI gpe. + */ + +struct spihid_apple_ops { + int (*power_on)(struct spihid_apple_ops *ops); + int (*power_off)(struct spihid_apple_ops *ops); + int (*enable_irq)(struct spihid_apple_ops *ops); + int (*disable_irq)(struct spihid_apple_ops *ops); + int (*enable_irq_wake)(struct spihid_apple_ops *ops); + int (*disable_irq_wake)(struct spihid_apple_ops *ops); +}; + +irqreturn_t spihid_apple_core_irq(int irq, void *data); + +int spihid_apple_core_probe(struct spi_device *spi, struct spihid_apple_ops *ops); +void spihid_apple_core_remove(struct spi_device *spi); +void spihid_apple_core_shutdown(struct spi_device *spi); + +extern const struct dev_pm_ops spihid_apple_core_pm; + +#endif /* SPI_HID_APPLE_H */ diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c index ddd5d77fb5a5e8..047d390d5aae13 100644 --- a/drivers/hid/usbhid/hid-core.c +++ b/drivers/hid/usbhid/hid-core.c @@ -283,9 +283,9 @@ static void hid_irq_in(struct urb *urb) break; usbhid_mark_busy(usbhid); if (!test_bit(HID_RESUME_RUNNING, &usbhid->iofl)) { - hid_input_report(urb->context, HID_INPUT_REPORT, - urb->transfer_buffer, - urb->actual_length, 1); + hid_safe_input_report(urb->context, HID_INPUT_REPORT, + urb->transfer_buffer, urb->transfer_buffer_length, + urb->actual_length, 1); /* * autosuspend refused while keys are pressed * because most keyboards don't wake up when @@ -482,9 +482,10 @@ static void hid_ctrl(struct urb *urb) switch (status) { case 0: /* success */ if (usbhid->ctrl[usbhid->ctrltail].dir == USB_DIR_IN) - hid_input_report(urb->context, + hid_safe_input_report(urb->context, usbhid->ctrl[usbhid->ctrltail].report->type, - urb->transfer_buffer, urb->actual_length, 0); + urb->transfer_buffer, urb->transfer_buffer_length, + urb->actual_length, 0); break; case -ESHUTDOWN: /* unplug */ unplug = 1; @@ -1552,7 +1553,7 @@ static int hid_post_reset(struct usb_interface *intf) * configuration descriptors passed, we already know that * the size of the HID report descriptor has not changed. */ - rdesc = kmalloc(hid->dev_rsize, GFP_KERNEL); + rdesc = kmalloc(hid->dev_rsize, GFP_NOIO); if (!rdesc) return -ENOMEM; diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index fbf3dbc92e6693..fad166ab21d680 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -11,6 +11,7 @@ #include "hid-pidff.h" #include #include +#include #include #include #include @@ -325,8 +326,10 @@ static s32 pidff_clamp(s32 i, struct hid_field *field) */ static int pidff_rescale(int i, int max, struct hid_field *field) { - return i * (field->logical_maximum - field->logical_minimum) / max + - field->logical_minimum; + /* 64 bits needed for big values during rescale */ + s64 result = field->logical_maximum - field->logical_minimum; + + return div_s64(result * i, max) + field->logical_minimum; } /* diff --git a/drivers/hid/wacom_sys.c b/drivers/hid/wacom_sys.c index 0d1c6d90fe21c5..a32320b351e3ee 100644 --- a/drivers/hid/wacom_sys.c +++ b/drivers/hid/wacom_sys.c @@ -90,7 +90,7 @@ static void wacom_wac_queue_flush(struct hid_device *hdev, kfree(buf); continue; } - err = hid_report_raw_event(hdev, HID_INPUT_REPORT, buf, size, false); + err = hid_report_raw_event(hdev, HID_INPUT_REPORT, buf, size, size, false); if (err) { hid_warn(hdev, "%s: unable to flush event due to error %d\n", __func__, err); @@ -334,7 +334,7 @@ static void wacom_feature_mapping(struct hid_device *hdev, data, n, WAC_CMD_RETRIES); if (ret == n && features->type == HID_GENERIC) { ret = hid_report_raw_event(hdev, - HID_FEATURE_REPORT, data, n, 0); + HID_FEATURE_REPORT, data, n, n, 0); } else if (ret == 2 && features->type != HID_GENERIC) { features->touch_max = data[1]; } else { @@ -395,7 +395,7 @@ static void wacom_feature_mapping(struct hid_device *hdev, data, n, WAC_CMD_RETRIES); if (ret == n) { ret = hid_report_raw_event(hdev, HID_FEATURE_REPORT, - data, n, 0); + data, n, n, 0); } else { hid_warn(hdev, "%s: could not retrieve sensor offsets\n", __func__); diff --git a/drivers/hte/Kconfig b/drivers/hte/Kconfig index 641af722b555dc..f57bad67deef02 100644 --- a/drivers/hte/Kconfig +++ b/drivers/hte/Kconfig @@ -16,13 +16,13 @@ if HTE config HTE_TEGRA194 tristate "NVIDIA Tegra194 HTE Support" - depends on (ARCH_TEGRA_194_SOC || COMPILE_TEST) + depends on (ARCH_TEGRA || COMPILE_TEST) depends on GPIOLIB help Enable this option for integrated hardware timestamping engine also known as generic timestamping engine (GTE) support on NVIDIA Tegra194 - systems-on-chip. The driver supports 352 LIC IRQs and 39 AON GPIOs - lines for timestamping in realtime. + and later systems-on-chip. The driver supports 352 LIC IRQs and 39 + AON GPIOs lines for timestamping in realtime. config HTE_TEGRA194_TEST tristate "NVIDIA Tegra194 HTE Test" diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig index 7937ac0cbd0ff8..2d0b3fcb0ff874 100644 --- a/drivers/hv/Kconfig +++ b/drivers/hv/Kconfig @@ -9,7 +9,6 @@ config HYPERV select PARAVIRT select X86_HV_CALLBACK_VECTOR if X86 select OF_EARLY_FLATTREE if OF - select SYSFB if EFI && !HYPERV_VTL_MODE select IRQ_MSI_LIB if X86 help Select this option to run Linux as a Hyper-V client operating @@ -62,6 +61,7 @@ config HYPERV_VMBUS tristate "Microsoft Hyper-V VMBus driver" depends on HYPERV default HYPERV + select SYSFB if EFI && !HYPERV_VTL_MODE help Select this option to enable Hyper-V Vmbus driver. diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c index 5856975f32e128..c1940070146783 100644 --- a/drivers/hv/mshv_vtl_main.c +++ b/drivers/hv/mshv_vtl_main.c @@ -386,7 +386,6 @@ static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem))) return -EFAULT; - /* vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design */ if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) { dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n", vtl0_mem.start_pfn, vtl0_mem.last_pfn); @@ -397,6 +396,10 @@ static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) if (!pgmap) return -ENOMEM; + /* + * vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design. + * last_pfn is not reserved or wasted, and reflects 'start_pfn + size' of pagemap range. + */ pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn); pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1; pgmap->nr_range = 1; @@ -405,8 +408,11 @@ static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) /* * Determine the highest page order that can be used for the given memory range. * This works best when the range is aligned; i.e. both the start and the length. + * Clamp to MAX_FOLIO_ORDER to avoid a WARN in memremap_pages() when the range + * alignment exceeds the maximum supported folio order for this kernel config. */ - pgmap->vmemmap_shift = count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn); + pgmap->vmemmap_shift = min(count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn), + MAX_FOLIO_ORDER); dev_dbg(vtl->module_dev, "Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n", vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift); @@ -415,7 +421,7 @@ static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) if (IS_ERR(addr)) { dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr)); kfree(pgmap); - return -EFAULT; + return PTR_ERR(addr); } /* Don't free pgmap, since it has to stick around until the memory diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index bc4fc1951ae1cf..3d2827477f0a54 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1430,7 +1430,6 @@ static int vmbus_alloc_synic_and_connect(void) { int ret, cpu; struct work_struct __percpu *works; - int hyperv_cpuhp_online; ret = hv_synic_alloc(); if (ret < 0) diff --git a/drivers/hwmon/aspeed-g6-pwm-tach.c b/drivers/hwmon/aspeed-g6-pwm-tach.c index 44e1ecba205d7a..4f6e6d440dd405 100644 --- a/drivers/hwmon/aspeed-g6-pwm-tach.c +++ b/drivers/hwmon/aspeed-g6-pwm-tach.c @@ -517,13 +517,6 @@ static int aspeed_pwm_tach_probe(struct platform_device *pdev) return 0; } -static void aspeed_pwm_tach_remove(struct platform_device *pdev) -{ - struct aspeed_pwm_tach_data *priv = platform_get_drvdata(pdev); - - reset_control_assert(priv->reset); -} - static const struct of_device_id aspeed_pwm_tach_match[] = { { .compatible = "aspeed,ast2600-pwm-tach", @@ -537,7 +530,6 @@ MODULE_DEVICE_TABLE(of, aspeed_pwm_tach_match); static struct platform_driver aspeed_pwm_tach_driver = { .probe = aspeed_pwm_tach_probe, - .remove = aspeed_pwm_tach_remove, .driver = { .name = "aspeed-g6-pwm-tach", .of_match_table = aspeed_pwm_tach_match, diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c index dddbd2463f8da7..76f3e1da68d09e 100644 --- a/drivers/hwmon/corsair-psu.c +++ b/drivers/hwmon/corsair-psu.c @@ -796,13 +796,13 @@ static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id ret = corsairpsu_init(priv); if (ret < 0) { dev_err(&hdev->dev, "unable to initialize device (%d)\n", ret); - goto fail_and_stop; + goto fail_and_close; } ret = corsairpsu_fwinfo(priv); if (ret < 0) { dev_err(&hdev->dev, "unable to query firmware (%d)\n", ret); - goto fail_and_stop; + goto fail_and_close; } corsairpsu_get_criticals(priv); diff --git a/drivers/hwmon/isl28022.c b/drivers/hwmon/isl28022.c index c2e559dde63f61..c5a34ceedcdb28 100644 --- a/drivers/hwmon/isl28022.c +++ b/drivers/hwmon/isl28022.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -185,8 +186,8 @@ static int isl28022_read_power(struct device *dev, u32 attr, long *val) ISL28022_REG_POWER, ®val); if (err < 0) return err; - *val = ((51200000L * ((long)data->gain)) / - (long)data->shunt) * (long)regval; + *val = min(div_u64(51200000ULL * data->gain * regval, + data->shunt), LONG_MAX); break; default: return -EOPNOTSUPP; diff --git a/drivers/hwmon/ltc2992.c b/drivers/hwmon/ltc2992.c index 1fcd320d616197..2617c4538af91d 100644 --- a/drivers/hwmon/ltc2992.c +++ b/drivers/hwmon/ltc2992.c @@ -431,10 +431,16 @@ static int ltc2992_get_voltage(struct ltc2992_state *st, u32 reg, u32 scale, lon static int ltc2992_set_voltage(struct ltc2992_state *st, u32 reg, u32 scale, long val) { - val = DIV_ROUND_CLOSEST(val * 1000, scale); - val = val << 4; + u32 reg_val; + long vmax; + + vmax = DIV_ROUND_CLOSEST_ULL(0xFFFULL * scale, 1000); + val = max(val, 0L); + val = min(val, vmax); + reg_val = min(DIV_ROUND_CLOSEST_ULL((u64)val * 1000, scale), + 0xFFFULL) << 4; - return ltc2992_write_reg(st, reg, 2, val); + return ltc2992_write_reg(st, reg, 2, reg_val); } static int ltc2992_read_gpio_alarm(struct ltc2992_state *st, int nr_gpio, u32 attr, long *val) @@ -559,9 +565,15 @@ static int ltc2992_get_current(struct ltc2992_state *st, u32 reg, u32 channel, l static int ltc2992_set_current(struct ltc2992_state *st, u32 reg, u32 channel, long val) { u32 reg_val; + long cmax; - reg_val = DIV_ROUND_CLOSEST(val * st->r_sense_uohm[channel], LTC2992_IADC_NANOV_LSB); - reg_val = reg_val << 4; + cmax = DIV_ROUND_CLOSEST_ULL(0xFFFULL * LTC2992_IADC_NANOV_LSB, + st->r_sense_uohm[channel]); + val = max(val, 0L); + val = min(val, cmax); + reg_val = min(DIV_ROUND_CLOSEST_ULL((u64)val * st->r_sense_uohm[channel], + LTC2992_IADC_NANOV_LSB), + 0xFFFULL) << 4; return ltc2992_write_reg(st, reg, 2, reg_val); } @@ -625,8 +637,10 @@ static int ltc2992_get_power(struct ltc2992_state *st, u32 reg, u32 channel, lon if (reg_val < 0) return reg_val; - *val = mul_u64_u32_div(reg_val, LTC2992_VADC_UV_LSB * LTC2992_IADC_NANOV_LSB, - st->r_sense_uohm[channel] * 1000); + *val = mul_u64_u32_div(reg_val, + LTC2992_VADC_UV_LSB / 1000 * + LTC2992_IADC_NANOV_LSB, + st->r_sense_uohm[channel]); return 0; } @@ -634,9 +648,18 @@ static int ltc2992_get_power(struct ltc2992_state *st, u32 reg, u32 channel, lon static int ltc2992_set_power(struct ltc2992_state *st, u32 reg, u32 channel, long val) { u32 reg_val; - - reg_val = mul_u64_u32_div(val, st->r_sense_uohm[channel] * 1000, - LTC2992_VADC_UV_LSB * LTC2992_IADC_NANOV_LSB); + u64 pmax, uval; + + uval = max(val, 0L); + pmax = mul_u64_u32_div(0xFFFFFFULL, + LTC2992_VADC_UV_LSB / 1000 * + LTC2992_IADC_NANOV_LSB, + st->r_sense_uohm[channel]); + uval = min(uval, pmax); + reg_val = min(mul_u64_u32_div(uval, st->r_sense_uohm[channel], + LTC2992_VADC_UV_LSB / 1000 * + LTC2992_IADC_NANOV_LSB), + 0xFFFFFFULL); return ltc2992_write_reg(st, reg, 3, reg_val); } diff --git a/drivers/hwmon/powerz.c b/drivers/hwmon/powerz.c index 4e663d5b4e330b..6e1359144cabef 100644 --- a/drivers/hwmon/powerz.c +++ b/drivers/hwmon/powerz.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -33,7 +34,9 @@ struct powerz_sensor_data { } __packed; struct powerz_priv { - char transfer_buffer[64]; /* first member to satisfy DMA alignment */ + __dma_from_device_group_begin(); + char transfer_buffer[64]; + __dma_from_device_group_end(); struct mutex mutex; struct completion completion; struct urb *urb; @@ -106,8 +109,12 @@ static void powerz_usb_cmd_complete(struct urb *urb) static int powerz_read_data(struct usb_device *udev, struct powerz_priv *priv) { + long rc; int ret; + if (!priv->urb) + return -ENODEV; + priv->status = -ETIMEDOUT; reinit_completion(&priv->completion); @@ -124,8 +131,14 @@ static int powerz_read_data(struct usb_device *udev, struct powerz_priv *priv) if (ret) return ret; - if (!wait_for_completion_interruptible_timeout - (&priv->completion, msecs_to_jiffies(5))) { + rc = wait_for_completion_interruptible_timeout(&priv->completion, + msecs_to_jiffies(5)); + if (rc < 0) { + usb_kill_urb(priv->urb); + return rc; + } + + if (rc == 0) { usb_kill_urb(priv->urb); return -EIO; } @@ -224,6 +237,8 @@ static int powerz_probe(struct usb_interface *intf, mutex_init(&priv->mutex); init_completion(&priv->completion); + usb_set_intfdata(intf, priv); + hwmon_dev = devm_hwmon_device_register_with_info(parent, DRIVER_NAME, priv, &powerz_chip_info, NULL); @@ -232,8 +247,6 @@ static int powerz_probe(struct usb_interface *intf, return PTR_ERR(hwmon_dev); } - usb_set_intfdata(intf, priv); - return 0; } @@ -244,6 +257,7 @@ static void powerz_disconnect(struct usb_interface *intf) mutex_lock(&priv->mutex); usb_kill_urb(priv->urb); usb_free_urb(priv->urb); + priv->urb = NULL; mutex_unlock(&priv->mutex); } diff --git a/drivers/hwmon/pt5161l.c b/drivers/hwmon/pt5161l.c index 20e3cfa625f170..89d4da8aa4c093 100644 --- a/drivers/hwmon/pt5161l.c +++ b/drivers/hwmon/pt5161l.c @@ -121,7 +121,7 @@ static int pt5161l_read_block_data(struct pt5161l_data *data, u32 address, int ret, tries; u8 remain_len = len; u8 curr_len; - u8 wbuf[16], rbuf[24]; + u8 wbuf[16], rbuf[I2C_SMBUS_BLOCK_MAX]; u8 cmd = 0x08; /* [7]:pec_en, [4:2]:func, [1]:start, [0]:end */ u8 config = 0x00; /* [6]:cfg_type, [4:1]:burst_len, [0]:address bit16 */ @@ -151,7 +151,7 @@ static int pt5161l_read_block_data(struct pt5161l_data *data, u32 address, break; } if (tries >= 3) - return ret; + return ret < 0 ? ret : -EIO; memcpy(val, rbuf, curr_len); val += curr_len; diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 8138f5ef40f06b..15e14a6fe6dce9 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -503,8 +503,13 @@ static void i2c_s3c_irq_nextbyte(struct s3c24xx_i2c *i2c, unsigned long iicstat) i2c->msg->buf[i2c->msg_ptr++] = byte; /* Add actual length to read for smbus block read */ - if (i2c->msg->flags & I2C_M_RECV_LEN && i2c->msg->len == 1) + if (i2c->msg->flags & I2C_M_RECV_LEN && i2c->msg->len == 1) { + if (byte == 0 || byte > I2C_SMBUS_BLOCK_MAX) { + s3c24xx_i2c_stop(i2c, -EPROTO); + break; + } i2c->msg->len += byte; + } prepare_read: if (is_msglast(i2c)) { /* last byte of buffer */ diff --git a/drivers/i2c/i2c-core-of.c b/drivers/i2c/i2c-core-of.c index eb7fb202355f87..354a88d0599e3e 100644 --- a/drivers/i2c/i2c-core-of.c +++ b/drivers/i2c/i2c-core-of.c @@ -180,7 +180,7 @@ static int of_i2c_notify(struct notifier_block *nb, unsigned long action, * Clear the flag before adding the device so that fw_devlink * doesn't skip adding consumers to this device. */ - rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE; + fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); client = of_i2c_register_device(adap, rd->dn); if (IS_ERR(client)) { dev_err(&adap->dev, "failed to create client for '%pOF'\n", diff --git a/drivers/i3c/master/adi-i3c-master.c b/drivers/i3c/master/adi-i3c-master.c index 6616f751075ae2..545ddd79a45dbd 100644 --- a/drivers/i3c/master/adi-i3c-master.c +++ b/drivers/i3c/master/adi-i3c-master.c @@ -361,7 +361,7 @@ static int adi_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, cmd->err = adi_i3c_cmd_get_err(&xfer->cmds[0]); - return 0; + return xfer->ret; } static int adi_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index d6bdb32397fb92..259e4f5276655a 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -924,7 +925,6 @@ static int dw_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, struct i3c_master_controller *m = i3c_dev_get_master(dev); struct dw_i3c_master *master = to_dw_i3c_master(m); unsigned int nrxwords = 0, ntxwords = 0; - struct dw_i3c_xfer *xfer; int i, ret = 0; if (!i3c_nxfers) @@ -944,7 +944,7 @@ static int dw_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, nrxwords > master->caps.datafifodepth) return -EOPNOTSUPP; - xfer = dw_i3c_master_alloc_xfer(master, i3c_nxfers); + struct dw_i3c_xfer *xfer __free(kfree) = dw_i3c_master_alloc_xfer(master, i3c_nxfers); if (!xfer) return -ENOMEM; @@ -995,7 +995,6 @@ static int dw_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, } ret = xfer->ret; - dw_i3c_master_free_xfer(xfer); pm_runtime_put_autosuspend(master->dev); return ret; @@ -1606,13 +1605,11 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, if (IS_ERR(master->pclk)) return PTR_ERR(master->pclk); - master->core_rst = devm_reset_control_get_optional_exclusive(&pdev->dev, - "core_rst"); + master->core_rst = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev, + "core_rst"); if (IS_ERR(master->core_rst)) return PTR_ERR(master->core_rst); - reset_control_deassert(master->core_rst); - spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); @@ -1624,7 +1621,7 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, dw_i3c_master_irq_handler, 0, dev_name(&pdev->dev), master); if (ret) - goto err_assert_rst; + return ret; platform_set_drvdata(pdev, master); @@ -1669,13 +1666,12 @@ int dw_i3c_common_probe(struct dw_i3c_master *master, return 0; err_disable_pm: + if (master->quirks & DW_I3C_DISABLE_RUNTIME_PM_QUIRK) + pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); -err_assert_rst: - reset_control_assert(master->core_rst); - return ret; } EXPORT_SYMBOL_GPL(dw_i3c_common_probe); diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c index e487ef52f6b4e2..e4daaa6120550d 100644 --- a/drivers/i3c/master/mipi-i3c-hci/dma.c +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -754,7 +754,10 @@ static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) if (!(ibi_status & IBI_LAST_STATUS)) { ibi_size += chunks * rh->ibi_chunk_sz; } else { - ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status); + if (chunks) { + ibi_size += (chunks - 1) * rh->ibi_chunk_sz; + ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status); + } last_ptr = ptr; break; } diff --git a/drivers/i3c/master/renesas-i3c.c b/drivers/i3c/master/renesas-i3c.c index d9f5b30a4b2f3a..a8a9e89a9710ab 100644 --- a/drivers/i3c/master/renesas-i3c.c +++ b/drivers/i3c/master/renesas-i3c.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -817,13 +818,12 @@ static int renesas_i3c_i3c_xfers(struct i3c_dev_desc *dev, struct i3c_xfer *i3c_ struct i3c_master_controller *m = i3c_dev_get_master(dev); struct renesas_i3c *i3c = to_renesas_i3c(m); struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); - struct renesas_i3c_xfer *xfer; int i; /* Enable I3C bus. */ renesas_i3c_bus_enable(m, true); - xfer = renesas_i3c_alloc_xfer(i3c, 1); + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); if (!xfer) return -ENOMEM; diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c index e16dede687d3f6..dd02077f76126c 100644 --- a/drivers/iio/adc/ad7768-1.c +++ b/drivers/iio/adc/ad7768-1.c @@ -465,10 +465,6 @@ static int ad7768_scan_direct(struct iio_dev *indio_dev) reinit_completion(&st->completion); - ret = ad7768_set_mode(st, AD7768_ONE_SHOT); - if (ret < 0) - return ret; - ret = wait_for_completion_timeout(&st->completion, msecs_to_jiffies(1000)); if (!ret) @@ -487,14 +483,6 @@ static int ad7768_scan_direct(struct iio_dev *indio_dev) if (st->oversampling_ratio == 8) readval >>= 8; - /* - * Any SPI configuration of the AD7768-1 can only be - * performed in continuous conversion mode. - */ - ret = ad7768_set_mode(st, AD7768_CONTINUOUS); - if (ret < 0) - return ret; - return readval; } @@ -1252,6 +1240,10 @@ static int ad7768_setup(struct iio_dev *indio_dev) return ret; } + ret = ad7768_set_mode(st, AD7768_CONTINUOUS); + if (ret) + return ret; + /* For backwards compatibility, try the adi,sync-in-gpios property */ st->gpio_sync_in = devm_gpiod_get_optional(&st->spi->dev, "adi,sync-in", GPIOD_OUT_LOW); diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c index cdc62488955933..418452aaca8103 100644 --- a/drivers/iio/adc/ti-ads7950.c +++ b/drivers/iio/adc/ti-ads7950.c @@ -47,8 +47,6 @@ #define TI_ADS7950_MAX_CHAN 16 #define TI_ADS7950_NUM_GPIOS 4 -#define TI_ADS7950_TIMESTAMP_SIZE (sizeof(int64_t) / sizeof(__be16)) - /* val = value, dec = left shift, bits = number of bits of the mask */ #define TI_ADS7950_EXTRACT(val, dec, bits) \ (((val) >> (dec)) & ((1 << (bits)) - 1)) @@ -105,8 +103,7 @@ struct ti_ads7950_state { * DMA (thus cache coherency maintenance) may require the * transfer buffers to live in their own cache lines. */ - u16 rx_buf[TI_ADS7950_MAX_CHAN + 2 + TI_ADS7950_TIMESTAMP_SIZE] - __aligned(IIO_DMA_MINALIGN); + u16 rx_buf[TI_ADS7950_MAX_CHAN + 2] __aligned(IIO_DMA_MINALIGN); u16 tx_buf[TI_ADS7950_MAX_CHAN + 2]; u16 single_tx; u16 single_rx; @@ -313,8 +310,10 @@ static irqreturn_t ti_ads7950_trigger_handler(int irq, void *p) if (ret < 0) goto out; - iio_push_to_buffers_with_timestamp(indio_dev, &st->rx_buf[2], - iio_get_time_ns(indio_dev)); + iio_push_to_buffers_with_ts_unaligned(indio_dev, &st->rx_buf[2], + sizeof(*st->rx_buf) * + TI_ADS7950_MAX_CHAN, + iio_get_time_ns(indio_dev)); out: mutex_unlock(&st->slock); diff --git a/drivers/iio/common/Kconfig b/drivers/iio/common/Kconfig index 1ccb5ccf370660..e3818ef567822b 100644 --- a/drivers/iio/common/Kconfig +++ b/drivers/iio/common/Kconfig @@ -3,6 +3,7 @@ # IIO common modules # +source "drivers/iio/common/aop_sensors/Kconfig" source "drivers/iio/common/cros_ec_sensors/Kconfig" source "drivers/iio/common/hid-sensors/Kconfig" source "drivers/iio/common/inv_sensors/Kconfig" diff --git a/drivers/iio/common/Makefile b/drivers/iio/common/Makefile index d3e952239a6219..5f99a429725d66 100644 --- a/drivers/iio/common/Makefile +++ b/drivers/iio/common/Makefile @@ -8,6 +8,7 @@ # # When adding new entries keep the list in alphabetical order +obj-y += aop_sensors/ obj-y += cros_ec_sensors/ obj-y += hid-sensors/ obj-y += inv_sensors/ diff --git a/drivers/iio/common/aop_sensors/Kconfig b/drivers/iio/common/aop_sensors/Kconfig new file mode 100644 index 00000000000000..b2c4397787dbfd --- /dev/null +++ b/drivers/iio/common/aop_sensors/Kconfig @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +config IIO_AOP_SENSOR_LAS + tristate "AOP Lid angle sensor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + help + Module to handle the lid angle sensor attached to the AOP + coprocessor on Apple laptops. + +config IIO_AOP_SENSOR_ALS + tristate "AOP Ambient light sensor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + depends on SYSFS + select APPLE_AOP + select RUST_FW_LOADER_ABSTRACTIONS + help + Module to handle the ambient light sensor attached to the AOP + coprocessor on Apple laptops. diff --git a/drivers/iio/common/aop_sensors/Makefile b/drivers/iio/common/aop_sensors/Makefile new file mode 100644 index 00000000000000..8da5a19efe0f0c --- /dev/null +++ b/drivers/iio/common/aop_sensors/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT + +obj-$(CONFIG_IIO_AOP_SENSOR_LAS) += aop_las.o +obj-$(CONFIG_IIO_AOP_SENSOR_ALS) += aop_als.o diff --git a/drivers/iio/common/aop_sensors/aop_als.rs b/drivers/iio/common/aop_sensors/aop_als.rs new file mode 100644 index 00000000000000..87a4686f3ca03a --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_als.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP ambient light sensor driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use kernel::{ + bindings, c_str, + device::Core, + firmware::Firmware, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{EPICService, AOP}, + sync::Arc, + types::ForeignOwnable, +}; + +const EPIC_SUBTYPE_GET_AOP_PROPERTY: u16 = 0xa; +const EPIC_SUBTYPE_SET_ALS_PROPERTY: u16 = 0x4; +const LUX_OFFSET_CT720: usize = 0x1d; +const LUX_OFFSET_VD6286: usize = 0x28; + +fn get_lux_offset(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result { + let name = get_aop_property(aop, svc, 0xf, 16)?.1; + match name.as_slice() { + b"Redbird\0" => Ok(LUX_OFFSET_VD6286), + b"FireFish2\0" => Ok(LUX_OFFSET_CT720), + _ => { + dev_warn!( + dev.as_ref(), + "Unknown sensor type {:?}", + core::str::from_utf8(&name) + ); + Err(EIO) + } + } +} + +fn enable_als(aop: &dyn AOP, dev: &platform::Device, svc: &EPICService) -> Result<()> { + let fw = Firmware::request(c_str!("apple/aop-als-cal.bin"), dev.as_ref())?; + set_als_property(aop, svc, 0xb, fw.data())?; + set_als_property(aop, svc, 0, &200000u32.to_le_bytes())?; + + Ok(()) +} + +fn get_aop_property( + aop: &dyn AOP, + svc: &EPICService, + tag: u32, + data_len: usize, +) -> Result<(u32, KVec)> { + let mut buf = KVec::new(); + buf.resize(8, 0, GFP_KERNEL)?; + buf[4..8].copy_from_slice(&tag.to_le_bytes()); + aop.epic_call_ret(svc, EPIC_SUBTYPE_GET_AOP_PROPERTY, &buf, data_len) +} + +fn set_als_property(aop: &dyn AOP, svc: &EPICService, tag: u32, data: &[u8]) -> Result { + let mut buf = KVec::new(); + buf.resize(data.len() + 8, 0, GFP_KERNEL)?; + buf[8..].copy_from_slice(data); + buf[4..8].copy_from_slice(&tag.to_le_bytes()); + aop.epic_call(svc, EPIC_SUBTYPE_SET_ALS_PROPERTY, &buf) +} + +fn f32_to_u32(f: u32) -> u32 { + if f & 0x80000000 != 0 { + return 0; + } + let exp = ((f & 0x7f800000) >> 23) as i32 - 127; + if exp < 0 { + return 0; + } + if exp == 128 && f & 0x7fffff != 0 { + return 0; + } + let mant = f & 0x7fffff | 0x800000; + if exp <= 23 { + return mant >> (23 - exp); + } + if exp >= 32 { + return u32::MAX; + } + mant << (exp - 23) +} + +struct MsgProc(usize); + +impl MessageProcessor for MsgProc { + fn process(&self, message: &[u8]) -> u32 { + let offset = self.0; + let raw = u32::from_le_bytes(message[offset..offset + 4].try_into().unwrap()); + f32_to_u32(raw) + } +} + +#[repr(transparent)] +struct IIOAopAlsDriver(IIORegistration); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-als")), ())] +); + +impl platform::Driver for IIOAopAlsDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = pdev.as_ref(); + let parent = dev.parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + let ty = bindings::BINDINGS_IIO_LIGHT; + let offset = get_lux_offset(adata.as_ref(), pdev, &service)?; + let data = AopSensorData::new(dev.into(), ty, MsgProc(offset))?; + adata.add_fakehid_listener(service, data.clone())?; + enable_als(adata.as_ref(), pdev, &service)?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED; + Ok(IIOAopAlsDriver(IIORegistration::::new( + data, + c"aop-sensors-als", + ty, + info_mask, + &THIS_MODULE, + )?)) + } +} + +module_platform_driver! { + type: IIOAopAlsDriver, + name: "iio_aop_als", + description: "AOP ambient light sensor driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_als"], + firmware: ["apple/aop-als-cal.bin"], +} diff --git a/drivers/iio/common/aop_sensors/aop_las.rs b/drivers/iio/common/aop_sensors/aop_las.rs new file mode 100644 index 00000000000000..9256d61aed9d83 --- /dev/null +++ b/drivers/iio/common/aop_sensors/aop_las.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP lid angle sensor driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use kernel::{ + bindings, c_str, + device::Core, + iio::common::aop_sensors::{AopSensorData, IIORegistration, MessageProcessor}, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{EPICService, AOP}, + sync::Arc, + types::ForeignOwnable, +}; + +struct MsgProc; + +impl MessageProcessor for MsgProc { + fn process(&self, message: &[u8]) -> u32 { + message[1] as u32 + } +} + +#[repr(transparent)] +struct IIOAopLasDriver(IIORegistration); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-las")), ())] +); + +impl platform::Driver for IIOAopLasDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = pdev.as_ref(); + let parent = dev.parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let service = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + + let ty = bindings::BINDINGS_IIO_ANGL; + let data = AopSensorData::new(dev.into(), ty, MsgProc)?; + adata.add_fakehid_listener(service, data.clone())?; + let info_mask = 1 << bindings::BINDINGS_IIO_CHAN_INFO_RAW; + Ok(IIOAopLasDriver(IIORegistration::::new( + data, + c"aop-sensors-las", + ty, + info_mask, + &THIS_MODULE, + )?)) + } +} + +module_platform_driver! { + type: IIOAopLasDriver, + name: "iio_aop_las", + description: "AOP lid angle sensor driver", + license: "Dual MIT/GPL", + alias: ["platform:iio_aop_las"], +} diff --git a/drivers/iio/frequency/admv1013.c b/drivers/iio/frequency/admv1013.c index d8e8d541990f87..5cea2c98879053 100644 --- a/drivers/iio/frequency/admv1013.c +++ b/drivers/iio/frequency/admv1013.c @@ -85,9 +85,9 @@ enum { }; enum { - ADMV1013_SE_MODE_POS = 6, - ADMV1013_SE_MODE_NEG = 9, - ADMV1013_SE_MODE_DIFF = 12 + ADMV1013_SE_MODE_POS, + ADMV1013_SE_MODE_NEG, + ADMV1013_SE_MODE_DIFF, }; struct admv1013_state { @@ -470,10 +470,23 @@ static int admv1013_init(struct admv1013_state *st, int vcm_uv) if (ret) return ret; - data = FIELD_PREP(ADMV1013_QUAD_SE_MODE_MSK, st->quad_se_mode); + switch (st->quad_se_mode) { + case ADMV1013_SE_MODE_POS: + data = 6; + break; + case ADMV1013_SE_MODE_NEG: + data = 9; + break; + case ADMV1013_SE_MODE_DIFF: + data = 12; + break; + default: + return -EINVAL; + } ret = __admv1013_spi_update_bits(st, ADMV1013_REG_QUAD, - ADMV1013_QUAD_SE_MODE_MSK, data); + ADMV1013_QUAD_SE_MODE_MSK, + FIELD_PREP(ADMV1013_QUAD_SE_MODE_MSK, data)); if (ret) return ret; @@ -514,43 +527,39 @@ static void admv1013_powerdown(void *data) admv1013_spi_update_bits(data, ADMV1013_REG_ENABLE, enable_reg_msk, enable_reg); } +static const char * const admv1013_input_modes[] = { + [ADMV1013_IQ_MODE] = "iq", + [ADMV1013_IF_MODE] = "if", +}; + +static const char * const admv1013_quad_se_modes[] = { + [ADMV1013_SE_MODE_POS] = "se-pos", + [ADMV1013_SE_MODE_NEG] = "se-neg", + [ADMV1013_SE_MODE_DIFF] = "diff", +}; + static int admv1013_properties_parse(struct admv1013_state *st) { int ret; - const char *str; - struct spi_device *spi = st->spi; + struct device *dev = &st->spi->dev; - st->det_en = device_property_read_bool(&spi->dev, "adi,detector-enable"); + st->det_en = device_property_read_bool(dev, "adi,detector-enable"); - ret = device_property_read_string(&spi->dev, "adi,input-mode", &str); - if (ret) - st->input_mode = ADMV1013_IQ_MODE; + ret = device_property_match_property_string(dev, "adi,input-mode", + admv1013_input_modes, + ARRAY_SIZE(admv1013_input_modes)); + st->input_mode = ret >= 0 ? ret : ADMV1013_IQ_MODE; - if (!strcmp(str, "iq")) - st->input_mode = ADMV1013_IQ_MODE; - else if (!strcmp(str, "if")) - st->input_mode = ADMV1013_IF_MODE; - else - return -EINVAL; - - ret = device_property_read_string(&spi->dev, "adi,quad-se-mode", &str); - if (ret) - st->quad_se_mode = ADMV1013_SE_MODE_DIFF; - - if (!strcmp(str, "diff")) - st->quad_se_mode = ADMV1013_SE_MODE_DIFF; - else if (!strcmp(str, "se-pos")) - st->quad_se_mode = ADMV1013_SE_MODE_POS; - else if (!strcmp(str, "se-neg")) - st->quad_se_mode = ADMV1013_SE_MODE_NEG; - else - return -EINVAL; + ret = device_property_match_property_string(dev, "adi,quad-se-mode", + admv1013_quad_se_modes, + ARRAY_SIZE(admv1013_quad_se_modes)); + st->quad_se_mode = ret >= 0 ? ret : ADMV1013_SE_MODE_DIFF; - ret = devm_regulator_bulk_get_enable(&st->spi->dev, + ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(admv1013_vcc_regs), admv1013_vcc_regs); if (ret) { - dev_err_probe(&spi->dev, ret, + dev_err_probe(dev, ret, "Failed to request VCC regulators\n"); return ret; } @@ -562,9 +571,10 @@ static int admv1013_probe(struct spi_device *spi) { struct iio_dev *indio_dev; struct admv1013_state *st; + struct device *dev = &spi->dev; int ret, vcm_uv; - indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st)); + indio_dev = devm_iio_device_alloc(dev, sizeof(*st)); if (!indio_dev) return -ENOMEM; @@ -581,20 +591,20 @@ static int admv1013_probe(struct spi_device *spi) if (ret) return ret; - ret = devm_regulator_get_enable_read_voltage(&spi->dev, "vcm"); + ret = devm_regulator_get_enable_read_voltage(dev, "vcm"); if (ret < 0) - return dev_err_probe(&spi->dev, ret, + return dev_err_probe(dev, ret, "failed to get the common-mode voltage\n"); vcm_uv = ret; - st->clkin = devm_clk_get_enabled(&spi->dev, "lo_in"); + st->clkin = devm_clk_get_enabled(dev, "lo_in"); if (IS_ERR(st->clkin)) - return dev_err_probe(&spi->dev, PTR_ERR(st->clkin), + return dev_err_probe(dev, PTR_ERR(st->clkin), "failed to get the LO input clock\n"); st->nb.notifier_call = admv1013_freq_change; - ret = devm_clk_notifier_register(&spi->dev, st->clkin, &st->nb); + ret = devm_clk_notifier_register(dev, st->clkin, &st->nb); if (ret) return ret; @@ -606,11 +616,11 @@ static int admv1013_probe(struct spi_device *spi) return ret; } - ret = devm_add_action_or_reset(&spi->dev, admv1013_powerdown, st); + ret = devm_add_action_or_reset(dev, admv1013_powerdown, st); if (ret) return ret; - return devm_iio_device_register(&spi->dev, indio_dev); + return devm_iio_device_register(dev, indio_dev); } static const struct spi_device_id admv1013_id[] = { diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c index 866746695712ae..6526fda8f9c0bb 100644 --- a/drivers/infiniband/core/addr.c +++ b/drivers/infiniband/core/addr.c @@ -321,11 +321,14 @@ static int dst_fetch_ha(const struct dst_entry *dst, if (!n) return -ENODATA; + read_lock_bh(&n->lock); if (!(n->nud_state & NUD_VALID)) { + read_unlock_bh(&n->lock); neigh_event_send(n, NULL); ret = -ENODATA; } else { neigh_ha_snapshot(dev_addr->dst_dev_addr, n, dst->dev); + read_unlock_bh(&n->lock); } neigh_release(n); diff --git a/drivers/infiniband/core/iwpm_msg.c b/drivers/infiniband/core/iwpm_msg.c index 69c85249b46506..4625abd29ac063 100644 --- a/drivers/infiniband/core/iwpm_msg.c +++ b/drivers/infiniband/core/iwpm_msg.c @@ -365,9 +365,9 @@ int iwpm_remove_mapping(struct sockaddr_storage *local_addr, u8 nl_client) /* netlink attribute policy for the received response to register pid request */ static const struct nla_policy resp_reg_policy[IWPM_NLA_RREG_PID_MAX] = { [IWPM_NLA_RREG_PID_SEQ] = { .type = NLA_U32 }, - [IWPM_NLA_RREG_IBDEV_NAME] = { .type = NLA_STRING, + [IWPM_NLA_RREG_IBDEV_NAME] = { .type = NLA_NUL_STRING, .len = IWPM_DEVNAME_SIZE - 1 }, - [IWPM_NLA_RREG_ULIB_NAME] = { .type = NLA_STRING, + [IWPM_NLA_RREG_ULIB_NAME] = { .type = NLA_NUL_STRING, .len = IWPM_ULIBNAME_SIZE - 1 }, [IWPM_NLA_RREG_ULIB_VER] = { .type = NLA_U16 }, [IWPM_NLA_RREG_PID_ERR] = { .type = NLA_U16 } @@ -677,7 +677,7 @@ int iwpm_remote_info_cb(struct sk_buff *skb, struct netlink_callback *cb) /* netlink attribute policy for the received request for mapping info */ static const struct nla_policy resp_mapinfo_policy[IWPM_NLA_MAPINFO_REQ_MAX] = { - [IWPM_NLA_MAPINFO_ULIB_NAME] = { .type = NLA_STRING, + [IWPM_NLA_MAPINFO_ULIB_NAME] = { .type = NLA_NUL_STRING, .len = IWPM_ULIBNAME_SIZE - 1 }, [IWPM_NLA_MAPINFO_ULIB_VER] = { .type = NLA_U16 } }; diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index edc34c69f0f235..acf4ce2891b76d 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -55,8 +55,7 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d if (dirty) ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt, - DMA_BIDIRECTIONAL, - DMA_ATTR_REQUIRE_COHERENT); + DMA_BIDIRECTIONAL, umem->dma_attrs); for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) { unpin_user_page_range_dirty_lock(sg_page(sg), @@ -170,7 +169,6 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, unsigned long lock_limit; unsigned long new_pinned; unsigned long cur_base; - unsigned long dma_attr = DMA_ATTR_REQUIRE_COHERENT; struct mm_struct *mm; unsigned long npages; int pinned, ret; @@ -203,6 +201,10 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, umem->iova = addr; umem->writable = ib_access_writable(access); umem->owning_mm = mm = current->mm; + umem->dma_attrs = DMA_ATTR_REQUIRE_COHERENT; + if (access & IB_ACCESS_RELAXED_ORDERING) + umem->dma_attrs |= DMA_ATTR_WEAK_ORDERING; + mmgrab(mm); page_list = (struct page **) __get_free_page(GFP_KERNEL); @@ -255,11 +257,8 @@ struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, } } - if (access & IB_ACCESS_RELAXED_ORDERING) - dma_attr |= DMA_ATTR_WEAK_ORDERING; - ret = ib_dma_map_sgtable_attrs(device, &umem->sgt_append.sgt, - DMA_BIDIRECTIONAL, dma_attr); + DMA_BIDIRECTIONAL, umem->dma_attrs); if (ret) goto umem_release; goto out; diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 5f7ea6c1664460..44300f7db5b16e 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -1178,6 +1178,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct hns_roce_ib_create_qp_resp resp = {}; struct ib_device *ibdev = &hr_dev->ib_dev; struct hns_roce_ib_create_qp ucmd = {}; + unsigned long flags; int ret; mutex_init(&hr_qp->mutex); @@ -1264,7 +1265,13 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, return 0; err_flow_ctrl: + spin_lock_irqsave(&hr_dev->qp_list_lock, flags); + hns_roce_lock_cqs(init_attr->send_cq ? to_hr_cq(init_attr->send_cq) : NULL, + init_attr->recv_cq ? to_hr_cq(init_attr->recv_cq) : NULL); hns_roce_qp_remove(hr_dev, hr_qp); + hns_roce_unlock_cqs(init_attr->send_cq ? to_hr_cq(init_attr->send_cq) : NULL, + init_attr->recv_cq ? to_hr_cq(init_attr->recv_cq) : NULL); + spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags); err_store: free_qpc(hr_dev, hr_qp); err_qpc: diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.c b/drivers/infiniband/hw/ionic/ionic_ibdev.c index bd4c73e530d081..73a616ae350236 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.c +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.c @@ -185,7 +185,7 @@ static ssize_t hca_type_show(struct device *device, struct ionic_ibdev *dev = rdma_device_to_drv_device(device, struct ionic_ibdev, ibdev); - return sysfs_emit(buf, "%s\n", dev->ibdev.node_desc); + return sysfs_emit(buf, "%.64s\n", dev->ibdev.node_desc); } static DEVICE_ATTR_RO(hca_type); diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index b2749f971cd0af..25e5b904e7c81d 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -142,8 +142,9 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq) if (cq->queue.id >= gc->max_num_cqs) return -EINVAL; - /* Create CQ table entry */ - WARN_ON(gc->cq_table[cq->queue.id]); + /* Create CQ table entry, sharing a CQ between WQs is not supported */ + if (gc->cq_table[cq->queue.id]) + return -EINVAL; if (cq->queue.kmem) gdma_cq = cq->queue.kmem; else diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index 82f84f7ad37a90..123e298dcb8566 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -21,6 +21,9 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev *dev, gc = mdev_to_gc(dev); + if (rx_hash_key_len > sizeof(req->hashkey)) + return -EINVAL; + req_buf_size = struct_size(req, indir_tab, MANA_INDIRECT_TABLE_DEF_SIZE); req = kzalloc(req_buf_size, GFP_KERNEL); if (!req) @@ -193,11 +196,8 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ, &wq_spec, &cq_spec, &wq->rx_object); - if (ret) { - /* Do cleanup starting with index i-1 */ - i--; + if (ret) goto fail; - } /* The GDMA regions are now owned by the WQ object */ wq->queue.gdma_region = GDMA_INVALID_DMA_REGION; @@ -217,8 +217,10 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, /* Create CQ table entry */ ret = mana_ib_install_cq_cb(mdev, cq); - if (ret) + if (ret) { + mana_destroy_wq_obj(mpc, GDMA_RQ, wq->rx_object); goto fail; + } } resp.num_entries = i; @@ -235,13 +237,15 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd, ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata create rss-qp, %d\n", ret); - goto fail; + goto err_disable_vport_rx; } kfree(mana_ind_table); return 0; +err_disable_vport_rx: + mana_disable_vport_rx(mpc); fail: while (i-- > 0) { ibwq = ind_tbl->ind_tbl[i]; @@ -822,6 +826,21 @@ static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, ndev = mana_ib_get_netdev(qp->ibqp.device, qp->port); mpc = netdev_priv(ndev); + /* Disable vPort RX steering before destroying RX WQ objects. + * Otherwise firmware still routes traffic to the destroyed queues, + * which can cause bogus completions on reused CQ IDs when the + * ethernet driver later creates new queues on mana_open(). + * + * Unlike the ethernet teardown path, mana_fence_rqs() cannot be + * used here because the fence completion CQE is delivered on the + * CQ which is polled by userspace (e.g. DPDK), so there is no way + * for the kernel to wait for fence completion. + * + * This is best effort — if it fails there is not much we can do, + * and mana_cfg_vport_steering() already logs the error. + */ + mana_disable_vport_rx(mpc); + for (i = 0; i < (1 << ind_tbl->log_ind_tbl_size); i++) { ibwq = ind_tbl->ind_tbl[i]; wq = container_of(ibwq, struct mana_ib_wq, ibwq); diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index c4cf91235eee3a..68e8b04c538808 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -193,13 +193,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, if (udata) if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { err = -EFAULT; - goto err_wrid; + goto err_srq; } init_attr->attr.max_wr = srq->msrq.max - 1; return 0; +err_srq: + mlx4_srq_free(dev->dev, &srq->msrq); err_wrid: if (udata) mlx4_ib_db_unmap_user(ucontext, &srq->db); diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 635002e684a55f..356a7c7856e7b9 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -3380,6 +3380,7 @@ int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev) "Couldn't create SRQ 1 for res init, err=%pe\n", s1); ib_destroy_srq(s0); + goto unlock; } devr->s0 = s0; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index e89be2fbd5eb27..ea04b8cace99a3 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -620,9 +620,9 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd, ucopy_err: if (pd->dpp_enabled) - ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE); + ocrdma_del_mmap(uctx, dpp_page_addr, PAGE_SIZE); dpp_map_err: - ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size); + ocrdma_del_mmap(uctx, db_page_addr, db_page_size); return status; } diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c index bcd43dc30e21c6..c7c2b41060e526 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c @@ -322,7 +322,7 @@ int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) uresp.qp_tab_size = vdev->dsr->caps.max_qp; ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); if (ret) { - pvrdma_uar_free(vdev, &context->uar); + /* pvrdma_dealloc_ucontext() also frees the UAR */ pvrdma_dealloc_ucontext(&context->ibucontext); return -EFAULT; } diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c index c1199ea5d41fda..3c7ee7ddc5dd78 100644 --- a/drivers/infiniband/sw/rdmavt/qp.c +++ b/drivers/infiniband/sw/rdmavt/qp.c @@ -92,12 +92,10 @@ static int rvt_wss_llc_size(void) static void cacheless_memcpy(void *dst, void *src, size_t n) { /* - * Use the only available X64 cacheless copy. Add a __user cast - * to quiet sparse. The src agument is already in the kernel so - * there are no security issues. The extra fault recovery machinery - * is not invoked. + * Use the only available X64 cacheless copy. + * The extra fault recovery machinery is not invoked. */ - __copy_user_nocache(dst, (void __user *)src, n); + copy_to_nontemporal(dst, src, n); } void rvt_wss_exit(struct rvt_dev_info *rdi) diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c index 5861e42440490d..2d5e701ff961af 100644 --- a/drivers/infiniband/sw/rxe/rxe_recv.c +++ b/drivers/infiniband/sw/rxe/rxe_recv.c @@ -330,7 +330,19 @@ void rxe_rcv(struct sk_buff *skb) pkt->qp = NULL; pkt->mask |= rxe_opcode[pkt->opcode].mask; - if (unlikely(skb->len < header_size(pkt))) + /* + * Unknown opcodes have a zero-initialized rxe_opcode[] entry, so + * both mask and length are 0. Reject them before any length math: + * rxe_icrc_hdr() would otherwise compute length - RXE_BTH_BYTES + * and pass the underflowed value to rxe_crc32(), producing an + * out-of-bounds read. + */ + if (unlikely(!rxe_opcode[pkt->opcode].mask || + !rxe_opcode[pkt->opcode].length)) + goto drop; + + if (unlikely(pkt->paylen < header_size(pkt) + bth_pad(pkt) + + RXE_ICRC_SIZE)) goto drop; err = hdr_check(pkt); diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 711f73e0bbb1c3..09ba21d0f3c4fe 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -526,7 +526,19 @@ static enum resp_states check_rkey(struct rxe_qp *qp, } skip_check_range: - if (pkt->mask & (RXE_WRITE_MASK | RXE_ATOMIC_WRITE_MASK)) { + if (pkt->mask & RXE_ATOMIC_WRITE_MASK) { + /* IBA oA19-28: ATOMIC_WRITE payload is exactly 8 bytes. + * Reject any other length before the responder reads + * sizeof(u64) bytes from payload_addr(pkt); a shorter + * payload would read past the logical end of the packet + * into skb->head tailroom. + */ + if (resid != sizeof(u64) || pktlen != sizeof(u64) || + bth_pad(pkt)) { + state = RESPST_ERR_LENGTH; + goto err; + } + } else if (pkt->mask & RXE_WRITE_MASK) { if (resid > mtu) { if (pktlen != mtu || bth_pad(pkt)) { state = RESPST_ERR_LENGTH; diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig index 94a753fcb64fa5..c327a21b418ce5 100644 --- a/drivers/input/misc/Kconfig +++ b/drivers/input/misc/Kconfig @@ -1003,4 +1003,15 @@ config INPUT_STPMIC1_ONKEY To compile this driver as a module, choose M here: the module will be called stpmic1_onkey. +config INPUT_MACSMC_INPUT + tristate "Apple Mac SMC lid/buttons" + depends on MFD_MACSMC + help + Say Y here if you want to use the input events delivered via the + SMC controller on Apple Mac machines using the macsmc driver. + This includes lid open/close and the power button. + + To compile this driver as a module, choose M here: the + module will be called macsmc-input. + endif diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile index 415fc4e2918be8..aa148f6a0f3233 100644 --- a/drivers/input/misc/Makefile +++ b/drivers/input/misc/Makefile @@ -52,6 +52,7 @@ obj-$(CONFIG_INPUT_IQS7222) += iqs7222.o obj-$(CONFIG_INPUT_KEYSPAN_REMOTE) += keyspan_remote.o obj-$(CONFIG_INPUT_KXTJ9) += kxtj9.o obj-$(CONFIG_INPUT_M68K_BEEP) += m68kspkr.o +obj-$(CONFIG_INPUT_MACSMC_INPUT) += macsmc-input.o obj-$(CONFIG_INPUT_MAX7360_ROTARY) += max7360-rotary.o obj-$(CONFIG_INPUT_MAX77650_ONKEY) += max77650-onkey.o obj-$(CONFIG_INPUT_MAX77693_HAPTIC) += max77693-haptic.o diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c new file mode 100644 index 00000000000000..2cead3b7f45fed --- /dev/null +++ b/drivers/input/misc/macsmc-input.c @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC input event driver + * Copyright The Asahi Linux Contributors + * + * This driver exposes HID events from the SMC as an input device. + * This includes the lid open/close and power button notifications. + */ + +#include +#include +#include +#include +#include +#include + +/** + * struct macsmc_input + * @dev: Underlying struct device for the input sub-device + * @smc: Pointer to apple_smc struct of the mfd parent + * @input: Allocated input_dev; devres managed + * @nb: Notifier block used for incoming events from SMC (e.g. button pressed down) + * @wakeup_mode: Set to true when system is suspended and power button events should wake it + */ +struct macsmc_input { + struct device *dev; + struct apple_smc *smc; + struct input_dev *input; + struct notifier_block nb; + bool wakeup_mode; +}; + +#define SMC_EV_BTN 0x7201 +#define SMC_EV_LID 0x7203 + +#define BTN_POWER 0x01 /* power button on e.g. Mac Mini chasis pressed */ +#define BTN_TOUCHID 0x06 /* combined TouchID / power button on MacBooks pressed */ +#define BTN_POWER_HELD_SHORT 0xfe /* power button briefly held down */ +#define BTN_POWER_HELD_LONG 0x00 /* power button held down; sent just before forced poweroff */ + +static void macsmc_input_event_button(struct macsmc_input *smcin, unsigned long event) +{ + u8 button = (event >> 8) & 0xff; + u8 state = !!(event & 0xff); + + switch (button) { + case BTN_POWER: + case BTN_TOUCHID: + pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && state)); + /* + * Suppress KEY_POWER reports when suspended to avoid powering down + * immediately after waking from s2idle. + * */ + if (smcin->wakeup_mode) + return; + + input_report_key(smcin->input, KEY_POWER, state); + input_sync(smcin->input); + break; + case BTN_POWER_HELD_SHORT: /* power button held down; ignore */ + break; + case BTN_POWER_HELD_LONG: + /* + * If we get here the power button has been held down for a while and + * we have about 4 seconds before forced power-off is triggered by SMC. + * Try to do an emergency shutdown to make sure the NVMe cache is + * flushed. macOS actually does this by panicing (!)... + */ + if (state) { + dev_crit(smcin->dev, "Triggering forced shutdown!\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? */ + kernel_restart("SMC power button triggered restart"); + } + break; + default: + dev_warn(smcin->dev, "Unknown SMC button event: %04lx\n", event & 0xffff); + } +} + +static void macsmc_input_event_lid(struct macsmc_input *smcin, unsigned long event) +{ + u8 lid_state = !!((event >> 8) & 0xff); + + pm_wakeup_dev_event(smcin->dev, 0, (smcin->wakeup_mode && !lid_state)); + input_report_switch(smcin->input, SW_LID, lid_state); + input_sync(smcin->input); +} + +static int macsmc_input_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_input *smcin = container_of(nb, struct macsmc_input, nb); + u16 type = event >> 16; + + switch (type) { + case SMC_EV_BTN: + macsmc_input_event_button(smcin, event); + return NOTIFY_OK; + case SMC_EV_LID: + macsmc_input_event_lid(smcin, event); + return NOTIFY_OK; + default: + /* SMC event meant for another driver */ + return NOTIFY_DONE; + } +} + +static int macsmc_input_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct macsmc_input *smcin; + bool have_lid, have_power; + int error; + + /* Bail early if this SMC neither supports power button nor lid events */ + have_lid = apple_smc_key_exists(smc, SMC_KEY(MSLD)); + have_power = apple_smc_key_exists(smc, SMC_KEY(bHLD)); + if (!have_lid && !have_power) + return -ENODEV; + + smcin = devm_kzalloc(&pdev->dev, sizeof(*smcin), GFP_KERNEL); + if (!smcin) + return -ENOMEM; + + smcin->dev = &pdev->dev; + smcin->smc = smc; + platform_set_drvdata(pdev, smcin); + + smcin->input = devm_input_allocate_device(&pdev->dev); + if (!smcin->input) + return -ENOMEM; + + smcin->input->phys = "macsmc-input (0)"; + smcin->input->name = "Apple SMC power/lid events"; + + if (have_lid) + input_set_capability(smcin->input, EV_SW, SW_LID); + if (have_power) + input_set_capability(smcin->input, EV_KEY, KEY_POWER); + + if (have_lid) { + u8 val; + + error = apple_smc_read_u8(smc, SMC_KEY(MSLD), &val); + if (error < 0) + dev_warn(&pdev->dev, "Failed to read initial lid state\n"); + else + input_report_switch(smcin->input, SW_LID, val); + } + + if (have_power) { + u32 val; + + error = apple_smc_read_u32(smc, SMC_KEY(bHLD), &val); + if (error < 0) + dev_warn(&pdev->dev, "Failed to read initial power button state\n"); + else + input_report_key(smcin->input, KEY_POWER, val & 1); + } + + error = input_register_device(smcin->input); + if (error) { + dev_err(&pdev->dev, "Failed to register input device: %d\n", error); + return error; + } + + input_sync(smcin->input); + + smcin->nb.notifier_call = macsmc_input_event; + blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + + device_init_wakeup(&pdev->dev, true); + + return 0; +} + +static int macsmc_input_pm_prepare(struct device *dev) +{ + struct macsmc_input *smcin = dev_get_drvdata(dev); + + smcin->wakeup_mode = true; + return 0; +} + +static void macsmc_input_pm_complete(struct device *dev) +{ + struct macsmc_input *smcin = dev_get_drvdata(dev); + + smcin->wakeup_mode = false; +} + +static const struct dev_pm_ops macsmc_input_pm_ops = { + .prepare = macsmc_input_pm_prepare, + .complete = macsmc_input_pm_complete, +}; + +static struct platform_driver macsmc_input_driver = { + .driver = { + .name = "macsmc-input", + .pm = &macsmc_input_pm_ops, + }, + .probe = macsmc_input_probe, +}; +module_platform_driver(macsmc_input_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC input driver"); +MODULE_ALIAS("platform:macsmc-input"); diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c index d0ab644be0069b..0042361d17ad46 100644 --- a/drivers/input/touchscreen/edt-ft5x06.c +++ b/drivers/input/touchscreen/edt-ft5x06.c @@ -829,7 +829,10 @@ static void edt_ft5x06_ts_prepare_debugfs(struct edt_ft5x06_ts_data *tsdata) static void edt_ft5x06_ts_teardown_debugfs(struct edt_ft5x06_ts_data *tsdata) { + guard(mutex)(&tsdata->mutex); + kfree(tsdata->raw_buffer); + tsdata->raw_buffer = NULL; } #else diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 760d5f4623b553..72f1f6e28138df 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -351,8 +351,12 @@ static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) struct amd_iommu_pci_seg *pci_seg; for_each_pci_segment(pci_seg) { - if (pci_seg->id == seg) - return pci_seg->rlookup_table[devid]; + if (pci_seg->id != seg) + continue; + /* IVRS may not describe every device on the bus */ + if (devid > pci_seg->last_bdf) + return NULL; + return pci_seg->rlookup_table[devid]; } return NULL; } @@ -403,11 +407,12 @@ struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) return NULL; } -static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) +static int clone_alias(struct pci_dev *pdev_origin, u16 alias, void *data) { struct dev_table_entry new; struct amd_iommu *iommu; struct iommu_dev_data *dev_data, *alias_data; + struct pci_dev *pdev = data; u16 devid = pci_dev_id(pdev); int ret = 0; @@ -454,9 +459,9 @@ static void clone_aliases(struct amd_iommu *iommu, struct device *dev) * part of the PCI DMA aliases if it's bus differs * from the original device. */ - clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); + clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], pdev); - pci_for_each_dma_alias(pdev, clone_alias, NULL); + pci_for_each_dma_alias(pdev, clone_alias, pdev); } static void setup_aliases(struct amd_iommu *iommu, struct device *dev) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 17bdadb6b50468..2b05e570c421ea 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -201,6 +203,7 @@ struct apple_dart_hw { * @lock: lock for hardware operations involving this dart * @pgsize: pagesize supported by this DART * @supports_bypass: indicates if this DART supports bypass mode + * @locked: indicates if this DART is locked * @sid2group: maps stream ids to iommu_groups * @iommu: iommu core device */ @@ -222,12 +225,18 @@ struct apple_dart { u32 num_streams; u32 supports_bypass : 1; u32 four_level : 1; + u32 locked : 1; + + dma_addr_t dma_min; + dma_addr_t dma_max; struct iommu_group *sid2group[DART_MAX_STREAMS]; struct iommu_device iommu; u32 save_tcr[DART_MAX_STREAMS]; u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; + + u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; }; /* @@ -267,6 +276,7 @@ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; bool finalized; + u64 mask; struct mutex init_lock; struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; @@ -284,6 +294,7 @@ struct apple_dart_domain { struct apple_dart_master_cfg { /* Intersection of DART capabilitles */ u32 supports_bypass : 1; + u32 locked : 1; struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; }; @@ -378,6 +389,82 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) apple_dart_hw_clear_ttbr(stream_map, i); } +static int +apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + u32 ttbr; + phys_addr_t phys; + u64 *l1_tbl; + + ttbr = readl(dart->regs + DART_TTBR(dart, sid, idx)); + + if (!(ttbr & dart->hw->ttbr_valid)) { + dev_err(dart->dev, "Invalid ttbr[%u] for locked dart\n", + idx); + return -EIO; + } + + ttbr &= ~dart->hw->ttbr_valid; + + if (dart->hw->ttbr_addr_field_shift) + ttbr >>= dart->hw->ttbr_addr_field_shift; + phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift; + + l1_tbl = devm_memremap(dart->dev, phys, dart->pgsize, + MEMREMAP_WB); + if (!l1_tbl) + return -ENOMEM; + + dart->locked_ttbr[sid][idx] = l1_tbl; + } + + return 0; +} + +static int +apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, + u8 idx) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + /* TODO: locked L1 table might need to be restored to boot state */ + if (dart->locked_ttbr[sid][idx]) { + memset(dart->locked_ttbr[sid][idx], 0, dart->pgsize); + devm_memunmap(dart->dev, dart->locked_ttbr[sid][idx]); + } + dart->locked_ttbr[sid][idx] = NULL; + } + + return 0; +} + +static int +apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, + struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + int sid; + + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + for (int idx = 0; idx < dart->hw->ttbr_count; idx++) { + u64 *ttbrep = dart->locked_ttbr[sid][idx]; + u64 *ptep = cfg->apple_dart_cfg.ttbr[idx]; + if (!ttbrep || !ptep) + continue; + for (int entry = 0; entry < dart->pgsize / sizeof(*ptep); entry++) + ttbrep[entry] = ptep[entry]; + } + } + + return 0; +} + static int apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, u32 command) @@ -464,17 +551,9 @@ apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static int apple_dart_hw_reset(struct apple_dart *dart) { - u32 config; struct apple_dart_stream_map stream_map; int i; - config = readl(dart->regs + dart->hw->lock); - if (config & dart->hw->lock_bit) { - dev_err(dart->dev, "DART is locked down until reboot: %08x\n", - config); - return -EINVAL; - } - stream_map.dart = dart; bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); bitmap_set(stream_map.sidmap, 0, dart->num_streams); @@ -499,6 +578,8 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) int i, j; struct apple_dart_atomic_stream_map *domain_stream_map; struct apple_dart_stream_map stream_map; + struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; for_each_stream_map(i, domain, domain_stream_map) { stream_map.dart = domain_stream_map->dart; @@ -506,7 +587,13 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); + WARN_ON(pm_runtime_get_sync(stream_map.dart->dev) < 0); + + if (stream_map.dart->locked) + apple_dart_hw_sync_locked(pgtbl_cfg, &stream_map); + stream_map.dart->hw->invalidate_tlb(&stream_map); + pm_runtime_put(stream_map.dart->dev); } } @@ -537,7 +624,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (!ops) return 0; - return ops->iova_to_phys(ops, iova); + return ops->iova_to_phys(ops, iova & dart_domain->mask); } static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, @@ -551,8 +638,8 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, if (!ops) return -ENODEV; - return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, - mapped); + return ops->map_pages(ops, iova & dart_domain->mask, paddr, pgsize, + pgcount, prot, gfp, mapped); } static size_t apple_dart_unmap_pages(struct iommu_domain *domain, @@ -563,7 +650,8 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; - return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); + return ops->unmap_pages(ops, iova & dart_domain->mask, pgsize, pgcount, + gather); } static void @@ -574,9 +662,10 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) - apple_dart_hw_set_ttbr(stream_map, i, - pgtbl_cfg->apple_dart_cfg.ttbr[i]); + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) { + u64 ttbr = virt_to_phys(pgtbl_cfg->apple_dart_cfg.ttbr[i]); + apple_dart_hw_set_ttbr(stream_map, i, ttbr); + } for (; i < stream_map->dart->hw->ttbr_count; ++i) apple_dart_hw_clear_ttbr(stream_map, i); @@ -585,11 +674,31 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, stream_map->dart->hw->invalidate_tlb(stream_map); } +static void +apple_dart_setup_translation_locked(struct apple_dart_domain *domain, + struct apple_dart_stream_map *stream_map) +{ + int i; + struct io_pgtable_cfg *pgtbl_cfg = + &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + + /* Locked DARTs are set up by the bootloader. */ + for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) + apple_dart_hw_map_locked_ttbr(stream_map, i); + for (; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + + apple_dart_hw_sync_locked(pgtbl_cfg, stream_map); + stream_map->dart->hw->invalidate_tlb(stream_map); +} + static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, struct apple_dart_master_cfg *cfg) { struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; + dma_addr_t dma_max = dart->dma_max; + u32 ias = min_t(u32, dart->ias, fls64(dma_max)); int ret = 0; int i, j; @@ -610,12 +719,48 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, pgtbl_cfg = (struct io_pgtable_cfg){ .pgsize_bitmap = dart->pgsize, - .ias = dart->ias, + .ias = ias, .oas = dart->oas, .coherent_walk = 1, .iommu_dev = dart->dev, }; + if (dart->locked) { + unsigned long *sidmap; + int sid; + u32 ttbr; + + /* Locked DARTs can only have a single stream bound */ + sidmap = cfg->stream_maps[0].sidmap; + sid = find_first_bit(sidmap, dart->num_streams); + + WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1); + ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + + WARN_ON(!(ttbr & dart->hw->ttbr_valid)); + + /* If the DART is locked, we need to keep the translation level count. */ + if (dart->hw->tcr_4level && dart->ias > 36) { + if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) { + if (ias < 37) { + dev_info(dart->dev, "Expanded to ias=37 due to lock\n"); + pgtbl_cfg.ias = 37; + } + } else if (ias > 36) { + dev_info(dart->dev, "Limited to ias=36 due to lock\n"); + pgtbl_cfg.ias = 36; + if (dart->dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) { + dma_max = DMA_BIT_MASK(pgtbl_cfg.ias); + } else if ((dart->dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) { + dev_err(dart->dev, + "Invalid DMA range for locked 3-level PT\n"); + ret = -ENOMEM; + goto done; + } + } + } + } + dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, &dart_domain->domain); if (!dart_domain->pgtbl_ops) { @@ -623,10 +768,16 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, goto done; } + if (pgtbl_cfg.pgsize_bitmap == SZ_4K) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 32)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 3) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 36)); + else if (pgtbl_cfg.apple_dart_cfg.n_levels == 4) + dart_domain->mask = DMA_BIT_MASK(min_t(u32, dart->ias, 47)); + dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; - dart_domain->domain.geometry.aperture_start = 0; - dart_domain->domain.geometry.aperture_end = - (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias); + dart_domain->domain.geometry.aperture_start = dart->dma_min; + dart_domain->domain.geometry.aperture_end = dma_max; dart_domain->domain.geometry.force_aperture = true; dart_domain->finalized = true; @@ -680,17 +831,29 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_domain *dart_domain = to_dart_domain(domain); + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + ret = apple_dart_finalize_domain(dart_domain, cfg); if (ret) - return ret; + goto err; ret = apple_dart_domain_add_streams(dart_domain, cfg); if (ret) - return ret; + goto err; + + for_each_stream_map(i, cfg, stream_map) { + if (!stream_map->dart->locked) + apple_dart_setup_translation(dart_domain, stream_map); + else + apple_dart_setup_translation_locked(dart_domain, + stream_map); + } +err: for_each_stream_map(i, cfg, stream_map) - apple_dart_setup_translation(dart_domain, stream_map); - return 0; + pm_runtime_put(stream_map->dart->dev); + return ret; } static int apple_dart_attach_dev_identity(struct iommu_domain *domain, @@ -704,8 +867,17 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, if (!cfg->supports_bypass) return -EINVAL; + if (cfg->locked) + return -EINVAL; + + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + for_each_stream_map(i, cfg, stream_map) apple_dart_hw_enable_bypass(stream_map); + + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return 0; } @@ -726,8 +898,17 @@ static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; int i; + if (cfg->locked) + return -EINVAL; + + for_each_stream_map(i, cfg, stream_map) + WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); + for_each_stream_map(i, cfg, stream_map) apple_dart_hw_disable_dma(stream_map); + + for_each_stream_map(i, cfg, stream_map) + pm_runtime_put(stream_map->dart->dev); return 0; } @@ -746,21 +927,29 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) struct apple_dart_stream_map *stream_map; int i; - if (!cfg) + if (!dev_iommu_fwspec_get(dev) || !cfg) return ERR_PTR(-ENODEV); for_each_stream_map(i, cfg, stream_map) - device_link_add( - dev, stream_map->dart->dev, - DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); + device_link_add(dev, stream_map->dart->dev, + DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER | + DL_FLAG_RPM_ACTIVE); return &cfg->stream_maps[0].dart->iommu; } static void apple_dart_release_device(struct device *dev) { + int i, j; + struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + for_each_stream_map(j, cfg, stream_map) { + if (stream_map->dart->locked) + for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) + apple_dart_hw_unmap_locked_ttbr(stream_map, i); + } + kfree(cfg); } @@ -817,6 +1006,8 @@ static int apple_dart_of_xlate(struct device *dev, return -ENOMEM; /* Will be ANDed with DART capabilities */ cfg->supports_bypass = true; + /* Will be ORed with DART capabilities*/ + cfg->locked = false; } dev_iommu_priv_set(dev, cfg); @@ -829,6 +1020,7 @@ static int apple_dart_of_xlate(struct device *dev, } cfg->supports_bypass &= dart->supports_bypass; + cfg->locked |= dart->locked; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (cfg->stream_maps[i].dart == dart) { @@ -971,6 +1163,8 @@ static int apple_dart_def_domain_type(struct device *dev) return IOMMU_DOMAIN_IDENTITY; if (!cfg->supports_bypass) return IOMMU_DOMAIN_DMA; + if (cfg->locked) + return IOMMU_DOMAIN_DMA; return 0; } @@ -1003,12 +1197,12 @@ static void apple_dart_get_resv_regions(struct device *dev, static const struct iommu_ops apple_dart_iommu_ops = { .identity_domain = &apple_dart_identity_domain, .blocked_domain = &apple_dart_blocked_domain, + .def_domain_type = apple_dart_def_domain_type, .domain_alloc_paging = apple_dart_domain_alloc_paging, .probe_device = apple_dart_probe_device, .release_device = apple_dart_release_device, .device_group = apple_dart_device_group, .of_xlate = apple_dart_of_xlate, - .def_domain_type = apple_dart_def_domain_type, .get_resv_regions = apple_dart_get_resv_regions, .owner = THIS_MODULE, .default_domain_ops = &(const struct iommu_domain_ops) { @@ -1102,6 +1296,22 @@ static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) return IRQ_HANDLED; } +static irqreturn_t apple_dart_irq(int irq, void *dev) +{ + irqreturn_t ret; + struct apple_dart *dart = dev; + + WARN_ON(pm_runtime_get_sync(dart->dev) < 0); + ret = dart->hw->irq_handler(irq, dev); + pm_runtime_put(dart->dev); + return ret; +} + +static bool apple_dart_is_locked(struct apple_dart *dart) +{ + return !!(readl(dart->regs + dart->hw->lock) & dart->hw->lock_bit); +} + static int apple_dart_probe(struct platform_device *pdev) { int ret; @@ -1109,6 +1319,7 @@ static int apple_dart_probe(struct platform_device *pdev) struct resource *res; struct apple_dart *dart; struct device *dev = &pdev->dev; + u64 dma_range[2]; dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); if (!dart) @@ -1140,6 +1351,14 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) return ret; + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_irq_safe(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + goto err_clk_disable; + dart_params[0] = readl(dart->regs + DART_PARAMS1); dart_params[1] = readl(dart->regs + DART_PARAMS2); dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); @@ -1163,6 +1382,27 @@ static int apple_dart_probe(struct platform_device *pdev) break; } + dart->dma_min = 0; + dart->dma_max = DMA_BIT_MASK(dart->ias); + + ret = of_property_read_u64_array(dev->of_node, "apple,dma-range", dma_range, 2); + if (ret == -EINVAL) { + ret = 0; + } else if (ret) { + goto err_clk_disable; + } else { + dart->dma_min = dma_range[0]; + dart->dma_max = dma_range[0] + dma_range[1] - 1; + if ((dart->dma_min ^ dart->dma_max) & ~DMA_BIT_MASK(dart->ias)) { + dev_err(&pdev->dev, "Invalid DMA range for ias=%d\n", + dart->ias); + ret = -EINVAL; + goto err_clk_disable; + } + dev_info(&pdev->dev, "Limiting DMA range to %pad..%pad\n", + &dart->dma_min, &dart->dma_max); + } + if (dart->num_streams > DART_MAX_STREAMS) { dev_err(&pdev->dev, "Too many streams (%d > %d)\n", dart->num_streams, DART_MAX_STREAMS); @@ -1170,11 +1410,14 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } - ret = apple_dart_hw_reset(dart); - if (ret) - goto err_clk_disable; + dart->locked = apple_dart_is_locked(dart); + if (!dart->locked) { + ret = apple_dart_hw_reset(dart); + if (ret) + goto err_clk_disable; + } - ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, + ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) goto err_clk_disable; @@ -1190,11 +1433,13 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) goto err_sysfs_remove; + pm_runtime_put(dev); + dev_info( &pdev->dev, - "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n", + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d, AS %d -> %d] initialized\n", dart->pgsize, dart->num_streams, dart->supports_bypass, - dart->pgsize > PAGE_SIZE, dart->ias, dart->oas); + dart->pgsize > PAGE_SIZE, dart->locked, dart->ias, dart->oas); return 0; err_sysfs_remove: @@ -1202,6 +1447,7 @@ static int apple_dart_probe(struct platform_device *pdev) err_free_irq: free_irq(dart->irq, dart); err_clk_disable: + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -1211,7 +1457,9 @@ static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); - apple_dart_hw_reset(dart); + if (!dart->locked) + apple_dart_hw_reset(dart); + free_irq(dart->irq, dart); iommu_device_unregister(&dart->iommu); @@ -1329,6 +1577,10 @@ static __maybe_unused int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; + /* Locked DARTs can't be restored so skip saving their registers. */ + if (dart->locked) + return 0; + for (sid = 0; sid < dart->num_streams; sid++) { dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); for (idx = 0; idx < dart->hw->ttbr_count; idx++) @@ -1345,6 +1597,10 @@ static __maybe_unused int apple_dart_resume(struct device *dev) unsigned int sid, idx; int ret; + /* Locked DARTs can't be restored, and they should not need it */ + if (dart->locked) + return 0; + ret = apple_dart_hw_reset(dart); if (ret) { dev_err(dev, "Failed to reset DART on resume\n"); @@ -1361,7 +1617,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) return 0; } -static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); +static DEFINE_RUNTIME_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume, NULL); static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, @@ -1377,7 +1633,7 @@ static struct platform_driver apple_dart_driver = { .name = "apple-dart", .of_match_table = apple_dart_of_match, .suppress_bind_attrs = true, - .pm = pm_sleep_ptr(&apple_dart_pm_ops), + .pm = pm_ptr(&apple_dart_pm_ops), }, .probe = apple_dart_probe, .remove = apple_dart_remove, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 4d00d796f07830..606abe051e68b4 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1236,6 +1236,13 @@ void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry, __le64 unused_update[NUM_ENTRY_QWORDS]; u8 used_qword_diff; + /* + * Many of the entry structures have pointers to other structures that + * need to have their updates be visible before any writes of the entry + * happen. + */ + dma_wmb(); + used_qword_diff = arm_smmu_entry_qword_diff(writer, entry, target, unused_update); if (hweight8(used_qword_diff) == 1) { diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 6fe5563eaf9eb2..83f6e9f6c51d6b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -479,6 +479,10 @@ static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq) /* Reset VCMDQ */ tegra241_vcmdq_hw_deinit(vcmdq); + /* vintf->hyp_own is a HW state finalized in tegra241_vintf_hw_init() */ + if (!vcmdq->vintf->hyp_own) + vcmdq->cmdq.supports_cmd = tegra241_guest_vcmdq_supports_cmd; + /* Configure and enable VCMDQ */ writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE)); @@ -639,9 +643,6 @@ static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq) q->q_base = q->base_dma & VCMDQ_ADDR; q->q_base |= FIELD_PREP(VCMDQ_LOG2SIZE, q->llq.max_n_shift); - if (!vcmdq->vintf->hyp_own) - cmdq->supports_cmd = tegra241_guest_vcmdq_supports_cmd; - return arm_smmu_cmdq_init(smmu, cmdq); } diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 94d5141696424d..6fc9be107cc86c 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -572,8 +572,13 @@ static int iova_reserve_iommu_regions(struct device *dev, if (region->type == IOMMU_RESV_SW_MSI) continue; - lo = iova_pfn(iovad, region->start); - hi = iova_pfn(iovad, region->start + region->length - 1); + if (region->type == IOMMU_RESV_TRANSLATED) { + lo = iova_pfn(iovad, region->dva); + hi = iova_pfn(iovad, region->dva + region->length - 1); + } else { + lo = iova_pfn(iovad, region->start); + hi = iova_pfn(iovad, region->start + region->length - 1); + } reserve_iova(iovad, lo, hi); if (region->type == IOMMU_RESV_MSI) diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index ef7613b177b9a9..88fabca898bb79 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -3529,8 +3529,8 @@ void domain_remove_dev_pasid(struct iommu_domain *domain, if (!domain) return; - /* Identity domain has no meta data for pasid. */ - if (domain->type == IOMMU_DOMAIN_IDENTITY) + /* Identity domain and blocked domain have no meta data for pasid. */ + if (domain->type == IOMMU_DOMAIN_IDENTITY || domain->type == IOMMU_DOMAIN_BLOCKED) return; dmar_domain = to_dmar_domain(domain); @@ -3544,12 +3544,13 @@ void domain_remove_dev_pasid(struct iommu_domain *domain, } spin_unlock_irqrestore(&dmar_domain->lock, flags); + if (WARN_ON_ONCE(!dev_pasid)) + return; + cache_tag_unassign_domain(dmar_domain, dev, pasid); domain_detach_iommu(dmar_domain, iommu); - if (!WARN_ON_ONCE(!dev_pasid)) { - intel_iommu_debugfs_remove_dev_pasid(dev_pasid); - kfree(dev_pasid); - } + intel_iommu_debugfs_remove_dev_pasid(dev_pasid); + kfree(dev_pasid); } static int blocking_domain_set_dev_pasid(struct iommu_domain *domain, @@ -3933,6 +3934,9 @@ static void quirk_iommu_igfx(struct pci_dev *dev) disable_igfx_iommu = 1; } +/* Q35 integrated gfx dmar support is totally busted. */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x29b2, quirk_iommu_igfx); + /* G4x/GM45 integrated gfx dmar support is totally busted. */ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx); diff --git a/drivers/iommu/intel/nested.c b/drivers/iommu/intel/nested.c index 2b979bec56cefe..16c82ba47d302b 100644 --- a/drivers/iommu/intel/nested.c +++ b/drivers/iommu/intel/nested.c @@ -148,6 +148,7 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, { struct device_domain_info *info = dev_iommu_priv_get(dev); struct dmar_domain *dmar_domain = to_dmar_domain(domain); + struct iommu_domain *s2_domain = &dmar_domain->s2_domain->domain; struct intel_iommu *iommu = info->iommu; struct dev_pasid_info *dev_pasid; int ret; @@ -155,10 +156,13 @@ static int intel_nested_set_dev_pasid(struct iommu_domain *domain, if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev)) return -EOPNOTSUPP; + if (s2_domain->dirty_ops) + return -EINVAL; + if (context_copied(iommu, info->bus, info->devfn)) return -EBUSY; - ret = paging_domain_compatible(&dmar_domain->s2_domain->domain, dev); + ret = paging_domain_compatible(s2_domain, dev); if (ret) return ret; diff --git a/drivers/iommu/io-pgtable-dart.c b/drivers/iommu/io-pgtable-dart.c index cbc5d6aa2daa23..fdaf0584f535e7 100644 --- a/drivers/iommu/io-pgtable-dart.c +++ b/drivers/iommu/io-pgtable-dart.c @@ -435,7 +435,7 @@ apple_dart_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) iommu_alloc_pages_sz(GFP_KERNEL, DART_GRANULE(data)); if (!data->pgd[i]) goto out_free_data; - cfg->apple_dart_cfg.ttbr[i] = virt_to_phys(data->pgd[i]); + cfg->apple_dart_cfg.ttbr[i] = data->pgd[i]; } return &data->iop; diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index ee83850c70605e..7b7772adad3962 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -61,14 +61,14 @@ struct iommu_group { int id; struct iommu_domain *default_domain; struct iommu_domain *blocking_domain; - /* - * During a group device reset, @resetting_domain points to the physical - * domain, while @domain points to the attached domain before the reset. - */ - struct iommu_domain *resetting_domain; struct iommu_domain *domain; struct list_head entry; unsigned int owner_cnt; + /* + * Number of devices in the group undergoing or awaiting recovery. + * If non-zero, concurrent domain attachments are rejected. + */ + unsigned int recovery_cnt; void *owner; }; @@ -76,12 +76,33 @@ struct group_device { struct list_head list; struct device *dev; char *name; + /* + * Device is blocked for a pending recovery while its group->domain is + * retained. This can happen when: + * - Device is undergoing a reset + */ + bool blocked; + unsigned int reset_depth; }; /* Iterate over each struct group_device in a struct iommu_group */ #define for_each_group_device(group, pos) \ list_for_each_entry(pos, &(group)->devices, list) +static struct group_device *__dev_to_gdev(struct device *dev) +{ + struct iommu_group *group = dev->iommu_group; + struct group_device *gdev; + + lockdep_assert_held(&group->mutex); + + for_each_group_device(group, gdev) { + if (gdev->dev == dev) + return gdev; + } + return NULL; +} + struct iommu_group_attribute { struct attribute attr; ssize_t (*show)(struct iommu_group *group, char *buf); @@ -95,6 +116,7 @@ static const char * const iommu_group_resv_type_string[] = { [IOMMU_RESV_RESERVED] = "reserved", [IOMMU_RESV_MSI] = "msi", [IOMMU_RESV_SW_MSI] = "msi", + [IOMMU_RESV_TRANSLATED] = "translated", }; #define IOMMU_CMD_LINE_DMA_API BIT(0) @@ -138,8 +160,8 @@ static void __iommu_group_set_domain_nofail(struct iommu_group *group, static int iommu_setup_default_domain(struct iommu_group *group, int target_type); -static int iommu_create_device_direct_mappings(struct iommu_domain *domain, - struct device *dev); +static int iommu_create_device_fw_mappings(struct iommu_domain *domain, + struct device *dev); static ssize_t iommu_group_store_type(struct iommu_group *group, const char *buf, size_t count); static struct group_device *iommu_group_alloc_device(struct iommu_group *group, @@ -647,7 +669,7 @@ static int __iommu_probe_device(struct device *dev, struct list_head *group_list list_add_tail(&gdev->list, &group->devices); WARN_ON(group->default_domain && !group->domain); if (group->default_domain) - iommu_create_device_direct_mappings(group->default_domain, dev); + iommu_create_device_fw_mappings(group->default_domain, dev); if (group->domain) { ret = __iommu_device_set_domain(group, dev, group->domain, NULL, 0); @@ -1176,8 +1198,8 @@ int iommu_group_set_name(struct iommu_group *group, const char *name) } EXPORT_SYMBOL_GPL(iommu_group_set_name); -static int iommu_create_device_direct_mappings(struct iommu_domain *domain, - struct device *dev) +static int iommu_create_device_fw_mappings(struct iommu_domain *domain, + struct device *dev) { struct iommu_resv_region *entry; LIST_HEAD(mappings); @@ -1193,21 +1215,29 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, /* We need to consider overlapping regions for different devices */ list_for_each_entry(entry, &mappings, list) { - dma_addr_t start, end, addr; + dma_addr_t start, end, addr, iova; size_t map_size = 0; if (entry->type == IOMMU_RESV_DIRECT) dev->iommu->require_direct = 1; + if (entry->type == IOMMU_RESV_TRANSLATED) + dev->iommu->require_translated = 1; if ((entry->type != IOMMU_RESV_DIRECT && - entry->type != IOMMU_RESV_DIRECT_RELAXABLE) || + entry->type != IOMMU_RESV_DIRECT_RELAXABLE && + entry->type != IOMMU_RESV_TRANSLATED) || !iommu_is_dma_domain(domain)) continue; start = ALIGN(entry->start, pg_size); end = ALIGN(entry->start + entry->length, pg_size); - for (addr = start; addr <= end; addr += pg_size) { + if (entry->type == IOMMU_RESV_TRANSLATED) + iova = ALIGN(entry->dva, pg_size); + else + iova = start; + + for (addr = start; addr <= end; addr += pg_size, iova += pg_size) { phys_addr_t phys_addr; if (addr == end) @@ -1217,7 +1247,7 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, * Return address by iommu_iova_to_phys for 0 is * ambiguous. Offset to address 1 if addr is 0. */ - phys_addr = iommu_iova_to_phys(domain, addr ? addr : 1); + phys_addr = iommu_iova_to_phys(domain, iova ? iova : 1); if (!phys_addr) { map_size += pg_size; continue; @@ -1225,7 +1255,7 @@ static int iommu_create_device_direct_mappings(struct iommu_domain *domain, map_end: if (map_size) { - ret = iommu_map(domain, addr - map_size, + ret = iommu_map(domain, iova - map_size, addr - map_size, map_size, entry->prot, GFP_KERNEL); if (ret) @@ -2195,6 +2225,8 @@ EXPORT_SYMBOL_GPL(iommu_attach_device); int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) { + struct group_device *gdev; + /* * This is called on the dma mapping fast path so avoid locking. This is * racy, but we have an expectation that the driver will setup its DMAs @@ -2205,14 +2237,18 @@ int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain) guard(mutex)(&dev->iommu_group->mutex); + gdev = __dev_to_gdev(dev); + if (WARN_ON(!gdev)) + return -ENODEV; + /* - * This is a concurrent attach during a device reset. Reject it until + * This is a concurrent attach during device recovery. Reject it until * pci_dev_reset_iommu_done() attaches the device to group->domain. * * Note that this might fail the iommu_dma_map(). But there's nothing * more we can do here. */ - if (dev->iommu_group->resetting_domain) + if (gdev->blocked) return -EBUSY; return __iommu_attach_device(domain, dev, NULL); } @@ -2269,19 +2305,24 @@ EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev) { struct iommu_group *group = dev->iommu_group; + struct group_device *gdev; lockdep_assert_held(&group->mutex); + gdev = __dev_to_gdev(dev); + if (WARN_ON(!gdev)) + return NULL; + /* * Driver handles the low-level __iommu_attach_device(), including the * one invoked by pci_dev_reset_iommu_done() re-attaching the device to * the cached group->domain. In this case, the driver must get the old - * domain from group->resetting_domain rather than group->domain. This + * domain from group->blocking_domain rather than group->domain. This * prevents it from re-attaching the device from group->domain (old) to * group->domain (new). */ - if (group->resetting_domain) - return group->resetting_domain; + if (gdev->blocked) + return group->blocking_domain; return group->domain; } @@ -2383,6 +2424,19 @@ static int __iommu_device_set_domain(struct iommu_group *group, "Firmware has requested this device have a 1:1 IOMMU mapping, rejecting configuring the device without a 1:1 mapping. Contact your platform vendor.\n"); return -EINVAL; } + /* + * If the device requires IOMMU_RESV_TRANSLATED then we cannot allow + * the identy or blocking domain to be attached as it does not contain + * the required translated mapping. + */ + if (dev->iommu->require_translated && + (new_domain->type == IOMMU_DOMAIN_IDENTITY || + new_domain->type == IOMMU_DOMAIN_BLOCKED || + new_domain == group->blocking_domain)) { + dev_warn(dev, + "Firmware has requested this device have a translated IOMMU mapping, rejecting configuring the device without a translated mapping. Contact your platform vendor.\n"); + return -EINVAL; + } if (dev->iommu->attach_deferred) { if (new_domain == group->default_domain) @@ -2440,10 +2494,11 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group, return -EINVAL; /* - * This is a concurrent attach during a device reset. Reject it until - * pci_dev_reset_iommu_done() attaches the device to group->domain. + * This is a concurrent attach during device recovery. Reject it until + * pci_dev_reset_iommu_done() attaches the device to group->domain, if + * IOMMU_SET_DOMAIN_MUST_SUCCEED is not set. */ - if (group->resetting_domain) + if (group->recovery_cnt && !(flags & IOMMU_SET_DOMAIN_MUST_SUCCEED)) return -EBUSY; /* @@ -2454,6 +2509,13 @@ static int __iommu_group_set_domain_internal(struct iommu_group *group, */ result = 0; for_each_group_device(group, gdev) { + /* + * Device under recovery is attached to group->blocking_domain. + * Don't change that. pci_dev_reset_iommu_done() will re-attach + * its domain to the updated group->domain, after the recovery. + */ + if (gdev->blocked) + continue; ret = __iommu_device_set_domain(group, gdev->dev, new_domain, group->domain, flags); if (ret) { @@ -2942,10 +3004,11 @@ void iommu_put_resv_regions(struct device *dev, struct list_head *list) } EXPORT_SYMBOL(iommu_put_resv_regions); -struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, - size_t length, int prot, - enum iommu_resv_type type, - gfp_t gfp) +struct iommu_resv_region *iommu_alloc_resv_region_tr(phys_addr_t start, + dma_addr_t dva_start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) { struct iommu_resv_region *region; @@ -2955,11 +3018,25 @@ struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, INIT_LIST_HEAD(®ion->list); region->start = start; + if (type == IOMMU_RESV_TRANSLATED) + region->dva = dva_start; region->length = length; region->prot = prot; region->type = type; return region; } +EXPORT_SYMBOL_GPL(iommu_alloc_resv_region_tr); + +struct iommu_resv_region *iommu_alloc_resv_region(phys_addr_t start, + size_t length, int prot, + enum iommu_resv_type type, + gfp_t gfp) +{ + if (type == IOMMU_RESV_TRANSLATED) + return NULL; + + return iommu_alloc_resv_region_tr(start, 0, length, prot, type, gfp); +} EXPORT_SYMBOL_GPL(iommu_alloc_resv_region); void iommu_set_default_passthrough(bool cmd_line) @@ -3084,7 +3161,7 @@ static int iommu_setup_default_domain(struct iommu_group *group, struct iommu_domain *old_dom = group->default_domain; struct group_device *gdev; struct iommu_domain *dom; - bool direct_failed; + bool fw_failed; int req_type; int ret; @@ -3114,10 +3191,10 @@ static int iommu_setup_default_domain(struct iommu_group *group, * mapped before their device is attached, in order to guarantee * continuity with any FW activity */ - direct_failed = false; + fw_failed = false; for_each_group_device(group, gdev) { - if (iommu_create_device_direct_mappings(dom, gdev->dev)) { - direct_failed = true; + if (iommu_create_device_fw_mappings(dom, gdev->dev)) { + fw_failed = true; dev_warn_once( gdev->dev->iommu->iommu_dev->dev, "IOMMU driver was not able to establish FW requested direct mapping."); @@ -3149,9 +3226,9 @@ static int iommu_setup_default_domain(struct iommu_group *group, * trying again after attaching. If this happens it means the device * will not continuously have the IOMMU_RESV_DIRECT map. */ - if (direct_failed) { + if (fw_failed) { for_each_group_device(group, gdev) { - ret = iommu_create_device_direct_mappings(dom, gdev->dev); + ret = iommu_create_device_fw_mappings(dom, gdev->dev); if (ret) goto err_restore_domain; } @@ -3532,7 +3609,12 @@ static void __iommu_remove_group_pasid(struct iommu_group *group, struct group_device *device; for_each_group_device(group, device) { - if (device->dev->iommu->max_pasids > 0) + /* + * A group-level detach cannot fail, even if there is a blocked + * device. In fact, blocked devices must be already detached for + * a pending device recovery. + */ + if (!device->blocked && device->dev->iommu->max_pasids > 0) iommu_remove_dev_pasid(device->dev, pasid, domain); } } @@ -3577,10 +3659,10 @@ int iommu_attach_device_pasid(struct iommu_domain *domain, mutex_lock(&group->mutex); /* - * This is a concurrent attach during a device reset. Reject it until + * This is a concurrent attach during device recovery. Reject it until * pci_dev_reset_iommu_done() attaches the device to group->domain. */ - if (group->resetting_domain) { + if (group->recovery_cnt) { ret = -EBUSY; goto out_unlock; } @@ -3670,10 +3752,10 @@ int iommu_replace_device_pasid(struct iommu_domain *domain, mutex_lock(&group->mutex); /* - * This is a concurrent attach during a device reset. Reject it until + * This is a concurrent attach during device recovery. Reject it until * pci_dev_reset_iommu_done() attaches the device to group->domain. */ - if (group->resetting_domain) { + if (group->recovery_cnt) { ret = -EBUSY; goto out_unlock; } @@ -3944,12 +4026,12 @@ EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL"); * routine wants to block any IOMMU activity: translation and ATS invalidation. * * This function attaches the device's RID/PASID(s) the group->blocking_domain, - * setting the group->resetting_domain. This allows the IOMMU driver pausing any + * incrementing the group->recovery_cnt, to allow the IOMMU driver pausing any * IOMMU activity while leaving the group->domain pointer intact. Later when the * reset is finished, pci_dev_reset_iommu_done() can restore everything. * * Caller must use pci_dev_reset_iommu_prepare() with pci_dev_reset_iommu_done() - * before/after the core-level reset routine, to unset the resetting_domain. + * before/after the core-level reset routine, to decrement the recovery_cnt. * * Return: 0 on success or negative error code if the preparation failed. * @@ -3962,6 +4044,7 @@ EXPORT_SYMBOL_NS_GPL(iommu_replace_group_handle, "IOMMUFD_INTERNAL"); int pci_dev_reset_iommu_prepare(struct pci_dev *pdev) { struct iommu_group *group = pdev->dev.iommu_group; + struct group_device *gdev; unsigned long pasid; void *entry; int ret; @@ -3971,33 +4054,52 @@ int pci_dev_reset_iommu_prepare(struct pci_dev *pdev) guard(mutex)(&group->mutex); - /* Re-entry is not allowed */ - if (WARN_ON(group->resetting_domain)) - return -EBUSY; + gdev = __dev_to_gdev(&pdev->dev); + if (WARN_ON(!gdev)) + return -ENODEV; + + if (gdev->reset_depth++) + return 0; ret = __iommu_group_alloc_blocking_domain(group); - if (ret) + if (ret) { + gdev->reset_depth--; return ret; + } /* Stage RID domain at blocking_domain while retaining group->domain */ if (group->domain != group->blocking_domain) { ret = __iommu_attach_device(group->blocking_domain, &pdev->dev, group->domain); - if (ret) + if (ret) { + gdev->reset_depth--; return ret; + } } + /* + * Update gdev->blocked upon the domain change, as it is used to return + * the correct domain in iommu_driver_get_domain_for_dev() that might be + * called in a set_dev_pasid callback function. + */ + gdev->blocked = true; + /* * Stage PASID domains at blocking_domain while retaining pasid_array. * * The pasid_array is mostly fenced by group->mutex, except one reader * in iommu_attach_handle_get(), so it's safe to read without xa_lock. */ - xa_for_each_start(&group->pasid_array, pasid, entry, 1) - iommu_remove_dev_pasid(&pdev->dev, pasid, - pasid_array_entry_to_domain(entry)); + if (pdev->dev.iommu->max_pasids > 0) { + xa_for_each_start(&group->pasid_array, pasid, entry, 1) { + struct iommu_domain *pasid_dom = + pasid_array_entry_to_domain(entry); + + iommu_remove_dev_pasid(&pdev->dev, pasid, pasid_dom); + } + } - group->resetting_domain = group->blocking_domain; + group->recovery_cnt++; return ret; } EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_prepare); @@ -4019,6 +4121,7 @@ EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_prepare); void pci_dev_reset_iommu_done(struct pci_dev *pdev) { struct iommu_group *group = pdev->dev.iommu_group; + struct group_device *gdev; unsigned long pasid; void *entry; @@ -4027,32 +4130,56 @@ void pci_dev_reset_iommu_done(struct pci_dev *pdev) guard(mutex)(&group->mutex); - /* pci_dev_reset_iommu_prepare() was bypassed for the device */ - if (!group->resetting_domain) + gdev = __dev_to_gdev(&pdev->dev); + if (WARN_ON(!gdev)) + return; + + /* Unbalanced done() calls would underflow the counter */ + if (WARN_ON(gdev->reset_depth == 0)) + return; + if (--gdev->reset_depth) return; - /* pci_dev_reset_iommu_prepare() was not successfully called */ if (WARN_ON(!group->blocking_domain)) return; - /* Re-attach RID domain back to group->domain */ - if (group->domain != group->blocking_domain) { + /* + * Re-attach RID domain back to group->domain + * + * Leave the device parked in the blocking_domain if group->domain isn't + * initialized yet + */ + if (group->domain && group->domain != group->blocking_domain) { WARN_ON(__iommu_attach_device(group->domain, &pdev->dev, group->blocking_domain)); } + /* + * Update gdev->blocked upon the domain change, as it is used to return + * the correct domain in iommu_driver_get_domain_for_dev() that might be + * called in a set_dev_pasid callback function. + */ + gdev->blocked = false; + /* * Re-attach PASID domains back to the domains retained in pasid_array. * * The pasid_array is mostly fenced by group->mutex, except one reader * in iommu_attach_handle_get(), so it's safe to read without xa_lock. */ - xa_for_each_start(&group->pasid_array, pasid, entry, 1) - WARN_ON(__iommu_set_group_pasid( - pasid_array_entry_to_domain(entry), group, pasid, - group->blocking_domain)); + if (pdev->dev.iommu->max_pasids > 0) { + xa_for_each_start(&group->pasid_array, pasid, entry, 1) { + struct iommu_domain *pasid_dom = + pasid_array_entry_to_domain(entry); + + WARN_ON(pasid_dom->ops->set_dev_pasid( + pasid_dom, &pdev->dev, pasid, + group->blocking_domain)); + } + } - group->resetting_domain = NULL; + if (!WARN_ON(group->recovery_cnt == 0)) + group->recovery_cnt--; } EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_done); diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index f1e686b3a2657b..710eef0b600454 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -187,9 +187,10 @@ static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *b mutex_lock(&fault->mutex); while (count > done) { - rc = copy_from_user(&response, buf + done, response_size); - if (rc) + if (copy_from_user(&response, buf + done, response_size)) { + rc = -EFAULT; break; + } static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS == (int)IOMMU_PAGE_RESP_SUCCESS); diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index ee003bb2f647f0..24d4917105d942 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -814,6 +814,16 @@ static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start, unmapped_bytes += area_last - area_first + 1; down_write(&iopt->iova_rwsem); + + /* + * After releasing the iova_rwsem concurrent allocation could + * place new areas at IOVAs we have already unmapped. Keep + * moving the start of the search forward to ignore the area + * already unmapped. + */ + if (area_last >= last) + break; + start = area_last + 1; } out_unlock_iova: diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index 7823142097d472..83e2215e7800d0 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -636,7 +636,7 @@ static void mock_viommu_destroy(struct iommufd_viommu *viommu) if (mock_viommu->mmap_offset) iommufd_viommu_destroy_mmap(&mock_viommu->core, mock_viommu->mmap_offset); - free_page((unsigned long)mock_viommu->page); + free_pages((unsigned long)mock_viommu->page, 1); mutex_destroy(&mock_viommu->queue_mutex); /* iommufd core frees mock_viommu and viommu */ @@ -870,7 +870,7 @@ static int mock_viommu_init(struct iommufd_viommu *viommu, iommufd_viommu_destroy_mmap(&mock_viommu->core, mock_viommu->mmap_offset); err_free_page: - free_page((unsigned long)mock_viommu->page); + free_pages((unsigned long)mock_viommu->page, 1); return rc; } diff --git a/drivers/iommu/iommufd/vfio_compat.c b/drivers/iommu/iommufd/vfio_compat.c index a258ee2f4579f5..acb48cdd3b005a 100644 --- a/drivers/iommu/iommufd/vfio_compat.c +++ b/drivers/iommu/iommufd/vfio_compat.c @@ -283,7 +283,7 @@ static int iommufd_vfio_check_extension(struct iommufd_ctx *ictx, case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: case VFIO_UNMAP_ALL: - return 1; + return !ictx->no_iommu_mode; case VFIO_NOIOMMU_IOMMU: return IS_ENABLED(CONFIG_VFIO_NOIOMMU); diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 6b989a62def20e..69377addd6cebb 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -147,6 +147,8 @@ int of_iommu_configure(struct device *dev, struct device_node *master_np, of_pci_check_device_ats(dev, master_np); } else { err = of_iommu_configure_device(master_np, dev, id); + if (err == -EPROBE_DEFER) + iommu_fwspec_free(dev); } if (err && dev_iommu_present) @@ -187,9 +189,7 @@ iommu_resv_region_get_type(struct device *dev, if (start == phys->start && end == phys->end) return IOMMU_RESV_DIRECT; - dev_warn(dev, "treating non-direct mapping [%pr] -> [%pap-%pap] as reservation\n", phys, - &start, &end); - return IOMMU_RESV_RESERVED; + return IOMMU_RESV_TRANSLATED; } /** @@ -260,8 +260,13 @@ void of_iommu_get_resv_regions(struct device *dev, struct list_head *list) } type = iommu_resv_region_get_type(dev, &phys, iova, length); - region = iommu_alloc_resv_region(iova, length, prot, type, + if (type == IOMMU_RESV_TRANSLATED) + region = iommu_alloc_resv_region_tr(phys.start, iova, length, prot, type, + GFP_KERNEL); + else + region = iommu_alloc_resv_region(iova, length, prot, type, GFP_KERNEL); + if (region) list_add_tail(®ion->list, list); } diff --git a/drivers/iommu/riscv/Kconfig b/drivers/iommu/riscv/Kconfig index c071816f59a67b..fb8e217edc3d33 100644 --- a/drivers/iommu/riscv/Kconfig +++ b/drivers/iommu/riscv/Kconfig @@ -4,6 +4,7 @@ config RISCV_IOMMU bool "RISC-V IOMMU Support" depends on RISCV && 64BIT + depends on GENERIC_MSI_IRQ default y select IOMMU_API help diff --git a/drivers/iommu/riscv/iommu-platform.c b/drivers/iommu/riscv/iommu-platform.c index 83a28c83f99145..399ba8fe1b3e55 100644 --- a/drivers/iommu/riscv/iommu-platform.c +++ b/drivers/iommu/riscv/iommu-platform.c @@ -68,12 +68,7 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev) iommu->caps = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_CAPABILITIES); iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL); - iommu->irqs_count = platform_irq_count(pdev); - if (iommu->irqs_count <= 0) - return dev_err_probe(dev, -ENODEV, - "no IRQ resources provided\n"); - if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT) - iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; + iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; igs = FIELD_GET(RISCV_IOMMU_CAPABILITIES_IGS, iommu->caps); switch (igs) { @@ -120,6 +115,16 @@ static int riscv_iommu_platform_probe(struct platform_device *pdev) fallthrough; case RISCV_IOMMU_CAPABILITIES_IGS_WSI: + ret = platform_irq_count(pdev); + if (ret <= 0) + return dev_err_probe(dev, -ENODEV, + "no IRQ resources provided\n"); + + iommu->irqs_count = ret; + + if (iommu->irqs_count > RISCV_IOMMU_INTR_COUNT) + iommu->irqs_count = RISCV_IOMMU_INTR_COUNT; + for (vec = 0; vec < iommu->irqs_count; vec++) iommu->irqs[vec] = platform_get_irq(pdev, vec); diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index fa2ebfd2f912e1..3ec99c979d4736 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -368,6 +368,8 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue, unsigned int timeout_us) { unsigned int cons = atomic_read(&queue->head); + unsigned int flags = RISCV_IOMMU_CQCSR_CQMF | RISCV_IOMMU_CQCSR_CMD_TO | + RISCV_IOMMU_CQCSR_CMD_ILL; /* Already processed by the consumer */ if ((int)(cons - index) > 0) @@ -375,6 +377,7 @@ static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue, /* Monitor consumer index */ return readx_poll_timeout(riscv_iommu_queue_cons, queue, cons, + (riscv_iommu_readl(queue->iommu, queue->qcr) & flags) || (int)(cons - index) > 0, 0, timeout_us); } @@ -928,8 +931,6 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, struct riscv_iommu_bond *bond; struct riscv_iommu_device *iommu, *prev; struct riscv_iommu_command cmd; - unsigned long len = end - start + 1; - unsigned long iova; /* * For each IOMMU linked with this protection domain (via bonds->dev), @@ -972,11 +973,14 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, riscv_iommu_cmd_inval_vma(&cmd); riscv_iommu_cmd_inval_set_pscid(&cmd, domain->pscid); - if (len && len < RISCV_IOMMU_IOTLB_INVAL_LIMIT) { - for (iova = start; iova < end; iova += PAGE_SIZE) { + if (end - start < RISCV_IOMMU_IOTLB_INVAL_LIMIT - 1) { + unsigned long iova = start; + + do { riscv_iommu_cmd_inval_set_addr(&cmd, iova); riscv_iommu_cmd_send(iommu, &cmd); - } + } while (!check_add_overflow(iova, PAGE_SIZE, &iova) && + iova < end); } else { riscv_iommu_cmd_send(iommu, &cmd); } @@ -996,7 +1000,67 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain, } #define RISCV_IOMMU_FSC_BARE 0 +/* + * This function sends IOTINVAL commands as required by the RISC-V + * IOMMU specification (Section 6.3.1 and 6.3.2 in 1.0 spec version) + * after modifying DDT or PDT entries + */ +static void riscv_iommu_iodir_iotinval(struct riscv_iommu_device *iommu, + bool inval_pdt, unsigned long iohgatp, + struct riscv_iommu_dc *dc, + struct riscv_iommu_pc *pc) +{ + struct riscv_iommu_command cmd; + riscv_iommu_cmd_inval_vma(&cmd); + + if (FIELD_GET(RISCV_IOMMU_DC_IOHGATP_MODE, iohgatp) == + RISCV_IOMMU_DC_IOHGATP_MODE_BARE) { + if (inval_pdt) { + /* + * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and + * PSCID=PC.PSCID + */ + riscv_iommu_cmd_inval_set_pscid(&cmd, + FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta)); + } else { + if (!FIELD_GET(RISCV_IOMMU_DC_TC_PDTV, dc->tc) && + FIELD_GET(RISCV_IOMMU_DC_FSC_MODE, dc->fsc) != + RISCV_IOMMU_DC_FSC_MODE_BARE) { + /* + * DC.tc.PDTV == 0 && DC.fsc.MODE != Bare + * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and + * PSCID=DC.ta.PSCID + */ + riscv_iommu_cmd_inval_set_pscid(&cmd, + FIELD_GET(RISCV_IOMMU_DC_TA_PSCID, dc->ta)); + } + /* else: IOTINVAL.VMA with GV=AV=PSCV=0 */ + } + } else { + riscv_iommu_cmd_inval_set_gscid(&cmd, + FIELD_GET(RISCV_IOMMU_DC_IOHGATP_GSCID, iohgatp)); + + if (inval_pdt) { + /* + * IOTINVAL.VMA with GV=1, AV=0, and PSCV=1, and + * GSCID=DC.iohgatp.GSCID, PSCID=PC.PSCID + */ + riscv_iommu_cmd_inval_set_pscid(&cmd, + FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta)); + } + /* + * else: IOTINVAL.VMA with GV=1,AV=PSCV=0,and + * GSCID=DC.iohgatp.GSCID + * + * IOTINVAL.GVMA with GV=1,AV=0,and + * GSCID=DC.iohgatp.GSCID + * TODO: For now, the Second-Stage feature have not yet been merged, + * also issue IOTINVAL.GVMA once second-stage support is merged. + */ + } + riscv_iommu_cmd_send(iommu, &cmd); +} /* * Update IODIR for the device. * @@ -1031,6 +1095,11 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu, riscv_iommu_cmd_iodir_inval_ddt(&cmd); riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]); riscv_iommu_cmd_send(iommu, &cmd); + /* + * For now, the SVA and PASID features have not yet been merged, the + * default configuration is inval_pdt=false and pc=NULL. + */ + riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL); sync_required = true; } @@ -1056,6 +1125,11 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu, riscv_iommu_cmd_iodir_inval_ddt(&cmd); riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]); riscv_iommu_cmd_send(iommu, &cmd); + /* + * For now, the SVA and PASID features have not yet been merged, the + * default configuration is inval_pdt=false and pc=NULL. + */ + riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL); } riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT); diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 2b24c82bb0df5b..4a3141d9f91416 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -134,8 +134,12 @@ #define AIC2_IRQ_CFG 0x2000 +/* AIC v3 registers (MMIO) */ +#define AIC3_IRQ_CFG 0x10000 + /* * AIC2 registers are laid out like this, starting at AIC2_IRQ_CFG: + * AIC3 registers use the same layout but start at AIC3_IRQ_CFG: * * Repeat for each die: * IRQ_CFG: u32 * MAX_IRQS @@ -293,6 +297,15 @@ static const struct aic_info aic2_info __initconst = { .local_fast_ipi = true, }; +static const struct aic_info aic3_info __initconst = { + .version = 3, + + .irq_cfg = AIC3_IRQ_CFG, + + .fast_ipi = true, + .local_fast_ipi = true, +}; + static const struct of_device_id aic_info_match[] = { { .compatible = "apple,t8103-aic", @@ -310,6 +323,10 @@ static const struct of_device_id aic_info_match[] = { .compatible = "apple,aic2", .data = &aic2_info, }, + { + .compatible = "apple,t8122-aic3", + .data = &aic3_info, + }, {} }; @@ -620,7 +637,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, u32 type = FIELD_GET(AIC_EVENT_TYPE, hw); struct irq_chip *chip = &aic_chip; - if (ic->info.version == 2) + if (ic->info.version == 2 || ic->info.version == 3) chip = &aic2_chip; if (type == AIC_EVENT_TYPE_IRQ) { @@ -991,7 +1008,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p break; } - case 2: { + case 2 ... 3: { u32 info1, info3; info1 = aic_ic_read(irqc, AIC2_INFO1); @@ -1065,7 +1082,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p off += irqc->info.die_stride; } - if (irqc->info.version == 2) { + if (irqc->info.version == 2 || irqc->info.version == 3) { u32 config = aic_ic_read(irqc, AIC2_CONFIG); config |= AIC2_CONFIG_ENABLE; @@ -1116,3 +1133,4 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p IRQCHIP_DECLARE(apple_aic, "apple,aic", aic_of_ic_init); IRQCHIP_DECLARE(apple_aic2, "apple,aic2", aic_of_ic_init); +IRQCHIP_DECLARE(apple_aic3, "apple,t8122-aic3", aic_of_ic_init); diff --git a/drivers/irqchip/irq-gic-v5-its.c b/drivers/irqchip/irq-gic-v5-its.c index 36a8d1368f0e44..28e39b065de0ee 100644 --- a/drivers/irqchip/irq-gic-v5-its.c +++ b/drivers/irqchip/irq-gic-v5-its.c @@ -929,14 +929,15 @@ static void gicv5_its_free_eventid(struct gicv5_its_dev *its_dev, u32 event_id_b static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { - u32 device_id, event_id_base, lpi; struct gicv5_its_dev *its_dev; + u32 device_id, event_id_base; msi_alloc_info_t *info = arg; irq_hw_number_t hwirq; struct irq_data *irqd; int ret, i; its_dev = info->scratchpad[0].ptr; + device_id = its_dev->device_id; ret = gicv5_its_alloc_eventid(its_dev, info, nr_irqs, &event_id_base); if (ret) @@ -946,22 +947,11 @@ static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int vi if (ret) goto out_eventid; - device_id = its_dev->device_id; + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, NULL); + if (ret) + goto out_eventid; for (i = 0; i < nr_irqs; i++) { - ret = gicv5_alloc_lpi(); - if (ret < 0) { - pr_debug("Failed to find free LPI!\n"); - goto out_free_irqs; - } - lpi = ret; - - ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi); - if (ret) { - gicv5_free_lpi(lpi); - goto out_free_irqs; - } - /* * Store eventid and deviceid into the hwirq for later use. * @@ -980,13 +970,6 @@ static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int vi return 0; -out_free_irqs: - while (--i >= 0) { - irqd = irq_domain_get_irq_data(domain, virq + i); - gicv5_free_lpi(irqd->parent_data->hwirq); - irq_domain_reset_irq_data(irqd); - irq_domain_free_irqs_parent(domain, virq + i, 1); - } out_eventid: gicv5_its_free_eventid(its_dev, event_id_base, nr_irqs); return ret; @@ -1009,15 +992,14 @@ static void gicv5_its_irq_domain_free(struct irq_domain *domain, unsigned int vi bitmap_release_region(its_dev->event_map, event_id_base, get_count_order(nr_irqs)); - /* Hierarchically free irq data */ for (i = 0; i < nr_irqs; i++) { d = irq_domain_get_irq_data(domain, virq + i); - - gicv5_free_lpi(d->parent_data->hwirq); irq_domain_reset_irq_data(d); - irq_domain_free_irqs_parent(domain, virq + i, 1); } + /* Hierarchically free irq data */ + irq_domain_free_irqs_parent(domain, virq, nr_irqs); + gicv5_its_syncr(its, its_dev); gicv5_irs_syncr(); } diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index 405a5eee847b6a..386daf5be316e7 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -59,16 +59,6 @@ static void release_lpi(u32 lpi) ida_free(&lpi_ida, lpi); } -int gicv5_alloc_lpi(void) -{ - return alloc_lpi(); -} - -void gicv5_free_lpi(u32 lpi) -{ - release_lpi(lpi); -} - static void gicv5_ppi_priority_init(void) { write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR0_EL1); @@ -788,38 +778,64 @@ static void gicv5_lpi_config_reset(struct irq_data *d) gicv5_lpi_irq_write_pending_state(d, false); } +static void gicv5_irq_lpi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d; + + for (unsigned int i = 0; i < nr_irqs; i++, virq++) { + d = irq_domain_get_irq_data(domain, virq); + + release_lpi(d->hwirq); + + irq_set_handler(virq, NULL); + irq_domain_reset_irq_data(d); + } +} + static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *arg) { irq_hw_number_t hwirq; struct irq_data *irqd; - u32 *lpi = arg; + unsigned int i; int ret; - if (WARN_ON_ONCE(nr_irqs != 1)) - return -EINVAL; + for (i = 0; i < nr_irqs; i++) { + ret = alloc_lpi(); + if (ret < 0) + goto out_free_lpis; + hwirq = ret; + + ret = gicv5_irs_iste_alloc(hwirq); + if (ret < 0) { + /* Undo partial state first, then clean up the rest */ + release_lpi(hwirq); + goto out_free_lpis; + } - hwirq = *lpi; + irqd = irq_domain_get_irq_data(domain, virq + i); - irqd = irq_domain_get_irq_data(domain, virq); + irq_domain_set_info(domain, virq + i, hwirq, &gicv5_lpi_irq_chip, + NULL, handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irqd); - irq_domain_set_info(domain, virq, hwirq, &gicv5_lpi_irq_chip, NULL, - handle_fasteoi_irq, NULL, NULL); - irqd_set_single_target(irqd); + gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI); + gicv5_lpi_config_reset(irqd); + } - ret = gicv5_irs_iste_alloc(hwirq); - if (ret < 0) - return ret; + return 0; - gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI); - gicv5_lpi_config_reset(irqd); +out_free_lpis: + if (i) + gicv5_irq_lpi_domain_free(domain, virq, i); - return 0; + return ret; } static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = { .alloc = gicv5_irq_lpi_domain_alloc, - .free = gicv5_irq_domain_free, + .free = gicv5_irq_lpi_domain_free, }; void __init gicv5_init_lpi_domain(void) @@ -840,30 +856,21 @@ static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int vi unsigned int nr_irqs, void *arg) { struct irq_data *irqd; - int ret, i; - u32 lpi; - - for (i = 0; i < nr_irqs; i++) { - ret = gicv5_alloc_lpi(); - if (ret < 0) - return ret; - - lpi = ret; + int ret; - ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi); - if (ret) { - gicv5_free_lpi(lpi); - return ret; - } + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); + if (ret) + return ret; - irqd = irq_domain_get_irq_data(domain, virq + i); + for (unsigned int i = 0; i < nr_irqs; i++, virq++) { + irqd = irq_domain_get_irq_data(domain, virq); - irq_domain_set_hwirq_and_chip(domain, virq + i, i, - &gicv5_ipi_irq_chip, NULL); + irq_domain_set_hwirq_and_chip(domain, virq, i, + &gicv5_ipi_irq_chip, NULL); irqd_set_single_target(irqd); - irq_set_handler(virq + i, handle_percpu_irq); + irq_set_handler(virq, handle_percpu_irq); } return 0; @@ -881,12 +888,11 @@ static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int vi if (!d) return; - gicv5_free_lpi(d->parent_data->hwirq); - irq_set_handler(virq + i, NULL); irq_domain_reset_irq_data(d); - irq_domain_free_irqs_parent(domain, virq + i, 1); } + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); } static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = { diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c index f722e9c57e2e40..74a376ef452e21 100644 --- a/drivers/irqchip/irq-meson-gpio.c +++ b/drivers/irqchip/irq-meson-gpio.c @@ -415,8 +415,7 @@ static int meson_s4_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl, if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING)) val |= BIT(ctl->params->edge_single_offset + idx); - meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, - BIT(idx) | BIT(12 + idx), val); + meson_gpio_irq_update_bits(ctl, REG_EDGE_POL, BIT(idx) | BIT(12 + idx), val); return 0; }; diff --git a/drivers/irqchip/irq-pic32-evic.c b/drivers/irqchip/irq-pic32-evic.c index e85c3e3007018f..325b97a0287f63 100644 --- a/drivers/irqchip/irq-pic32-evic.c +++ b/drivers/irqchip/irq-pic32-evic.c @@ -196,7 +196,7 @@ static void __init pic32_ext_irq_of_init(struct irq_domain *domain) of_property_for_each_u32(node, pname, hwirq) { if (i >= ARRAY_SIZE(priv->ext_irqs)) { - pr_warn("More than %d external irq, skip rest\n", + pr_warn("More than %zu external irq, skip rest\n", ARRAY_SIZE(priv->ext_irqs)); break; } diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index e73d426cea6d37..eb01d4c5aca75c 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -577,7 +577,7 @@ static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_n irq_domain = irq_domain_create_hierarchy(parent_domain, 0, IRQC_NUM_IRQ, dev_fwnode(dev), &rzg2l_irqc_domain_ops, rzg2l_irqc_data); if (!irq_domain) { - pm_runtime_put(dev); + pm_runtime_put_sync(dev); return -ENOMEM; } diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index ba903fa689bd52..a7a1852b548c48 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -158,6 +158,8 @@ static int imsic_dying_cpu(unsigned int cpu) /* Cleanup IPIs */ imsic_ipi_dying_cpu(); + imsic_local_sync_all(false); + /* Mark per-CPU IMSIC state as offline */ imsic_state_offline(); diff --git a/drivers/leds/blink/leds-lgm-sso.c b/drivers/leds/blink/leds-lgm-sso.c index 8923d2df47049b..3d9ef9a54805cb 100644 --- a/drivers/leds/blink/leds-lgm-sso.c +++ b/drivers/leds/blink/leds-lgm-sso.c @@ -808,8 +808,6 @@ static int intel_sso_led_probe(struct platform_device *pdev) priv->fpid_clkrate = clk_get_rate(priv->clocks[1].clk); - priv->mmap = syscon_node_to_regmap(dev->of_node); - priv->mmap = syscon_node_to_regmap(dev->of_node); if (IS_ERR(priv->mmap)) { dev_err(dev, "Failed to map iomem!\n"); diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c index 016bf468e09413..f6061c47f863d6 100644 --- a/drivers/leds/rgb/leds-qcom-lpg.c +++ b/drivers/leds/rgb/leds-qcom-lpg.c @@ -1273,7 +1273,12 @@ static int lpg_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, return ret; if (chan->subtype == LPG_SUBTYPE_HI_RES_PWM) { - refclk = lpg_clk_rates_hi_res[FIELD_GET(PWM_CLK_SELECT_HI_RES_MASK, val)]; + unsigned int clk_idx = FIELD_GET(PWM_CLK_SELECT_HI_RES_MASK, val); + + if (clk_idx >= ARRAY_SIZE(lpg_clk_rates_hi_res)) + return -EINVAL; + + refclk = lpg_clk_rates_hi_res[clk_idx]; resolution = lpg_pwm_resolution_hi_res[FIELD_GET(PWM_SIZE_HI_RES_MASK, val)]; } else { refclk = lpg_clk_rates[FIELD_GET(PWM_CLK_SELECT_MASK, val)]; diff --git a/drivers/mailbox/mailbox-test.c b/drivers/mailbox/mailbox-test.c index 3a28ab5c42e575..41c8c7f3da9d89 100644 --- a/drivers/mailbox/mailbox-test.c +++ b/drivers/mailbox/mailbox-test.c @@ -28,8 +28,6 @@ #define MBOX_HEXDUMP_MAX_LEN (MBOX_HEXDUMP_LINE_LEN * \ (MBOX_MAX_MSG_LEN / MBOX_BYTES_PER_LINE)) -static bool mbox_data_ready; - struct mbox_test_device { struct device *dev; void __iomem *tx_mmio; @@ -42,6 +40,7 @@ struct mbox_test_device { spinlock_t lock; struct mutex mutex; wait_queue_head_t waitq; + bool data_ready; struct fasync_struct *async_queue; struct dentry *root_debugfs_dir; }; @@ -162,7 +161,7 @@ static bool mbox_test_message_data_ready(struct mbox_test_device *tdev) unsigned long flags; spin_lock_irqsave(&tdev->lock, flags); - data_ready = mbox_data_ready; + data_ready = tdev->data_ready; spin_unlock_irqrestore(&tdev->lock, flags); return data_ready; @@ -227,7 +226,7 @@ static ssize_t mbox_test_message_read(struct file *filp, char __user *userbuf, *(touser + l) = '\0'; memset(tdev->rx_buffer, 0, MBOX_MAX_MSG_LEN); - mbox_data_ready = false; + tdev->data_ready = false; spin_unlock_irqrestore(&tdev->lock, flags); @@ -297,7 +296,7 @@ static void mbox_test_receive_message(struct mbox_client *client, void *message) message, MBOX_MAX_MSG_LEN); memcpy(tdev->rx_buffer, message, MBOX_MAX_MSG_LEN); } - mbox_data_ready = true; + tdev->data_ready = true; spin_unlock_irqrestore(&tdev->lock, flags); wake_up_interruptible(&tdev->waitq); @@ -336,7 +335,7 @@ mbox_test_request_channel(struct platform_device *pdev, const char *name) client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL); if (!client) - return ERR_PTR(-ENOMEM); + return NULL; client->dev = &pdev->dev; client->rx_callback = mbox_test_receive_message; @@ -366,6 +365,12 @@ static int mbox_test_probe(struct platform_device *pdev) if (!tdev) return -ENOMEM; + tdev->dev = &pdev->dev; + spin_lock_init(&tdev->lock); + mutex_init(&tdev->mutex); + init_waitqueue_head(&tdev->waitq); + platform_set_drvdata(pdev, tdev); + /* It's okay for MMIO to be NULL */ tdev->tx_mmio = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (PTR_ERR(tdev->tx_mmio) == -EBUSY) { @@ -388,34 +393,36 @@ static int mbox_test_probe(struct platform_device *pdev) tdev->tx_channel = mbox_test_request_channel(pdev, "tx"); tdev->rx_channel = mbox_test_request_channel(pdev, "rx"); - if (IS_ERR_OR_NULL(tdev->tx_channel) && IS_ERR_OR_NULL(tdev->rx_channel)) + if (!tdev->tx_channel && !tdev->rx_channel) return -EPROBE_DEFER; /* If Rx is not specified but has Rx MMIO, then Rx = Tx */ if (!tdev->rx_channel && (tdev->rx_mmio != tdev->tx_mmio)) tdev->rx_channel = tdev->tx_channel; - tdev->dev = &pdev->dev; - platform_set_drvdata(pdev, tdev); - - spin_lock_init(&tdev->lock); - mutex_init(&tdev->mutex); - if (tdev->rx_channel) { tdev->rx_buffer = devm_kzalloc(&pdev->dev, MBOX_MAX_MSG_LEN, GFP_KERNEL); - if (!tdev->rx_buffer) - return -ENOMEM; + if (!tdev->rx_buffer) { + ret = -ENOMEM; + goto err_free_chans; + } } ret = mbox_test_add_debugfs(pdev, tdev); if (ret) - return ret; + goto err_free_chans; - init_waitqueue_head(&tdev->waitq); dev_info(&pdev->dev, "Successfully registered\n"); return 0; + +err_free_chans: + if (tdev->tx_channel) + mbox_free_channel(tdev->tx_channel); + if (tdev->rx_channel && tdev->rx_channel != tdev->tx_channel) + mbox_free_channel(tdev->rx_channel); + return ret; } static void mbox_test_remove(struct platform_device *pdev) @@ -426,7 +433,7 @@ static void mbox_test_remove(struct platform_device *pdev) if (tdev->tx_channel) mbox_free_channel(tdev->tx_channel); - if (tdev->rx_channel) + if (tdev->rx_channel && tdev->rx_channel != tdev->tx_channel) mbox_free_channel(tdev->rx_channel); } diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 617ba505691d37..b77162db509f2d 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -505,8 +505,7 @@ int mbox_controller_register(struct mbox_controller *mbox) { int i, txdone; - /* Sanity check */ - if (!mbox || !mbox->dev || !mbox->ops || !mbox->num_chans) + if (!mbox || !mbox->dev || !mbox->ops || !mbox->chans || !mbox->num_chans) return -EINVAL; if (mbox->txdone_irq) diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c index d7c6b38888a379..547a10a8fad3ac 100644 --- a/drivers/mailbox/mtk-cmdq-mailbox.c +++ b/drivers/mailbox/mtk-cmdq-mailbox.c @@ -493,14 +493,14 @@ static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data) if (curr_pa == end_pa - CMDQ_INST_SIZE || curr_pa == end_pa) { /* set to this task directly */ - writel(task->pa_base >> cmdq->pdata->shift, - thread->base + CMDQ_THR_CURR_ADDR); + gce_addr = cmdq_convert_gce_addr(task->pa_base, cmdq->pdata); + writel(gce_addr, thread->base + CMDQ_THR_CURR_ADDR); } else { cmdq_task_insert_into_thread(task); smp_mb(); /* modify jump before enable thread */ } - writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->pdata->shift, - thread->base + CMDQ_THR_END_ADDR); + gce_addr = cmdq_convert_gce_addr(task->pa_base + pkt->cmd_buf_size, cmdq->pdata); + writel(gce_addr, thread->base + CMDQ_THR_END_ADDR); cmdq_thread_resume(thread); } list_move_tail(&task->list_entry, &thread->task_busy_list); diff --git a/drivers/mailbox/mtk-vcp-mailbox.c b/drivers/mailbox/mtk-vcp-mailbox.c index cedad575528fb4..1b291b8ea15ac0 100644 --- a/drivers/mailbox/mtk-vcp-mailbox.c +++ b/drivers/mailbox/mtk-vcp-mailbox.c @@ -50,7 +50,7 @@ static struct mbox_chan *mtk_vcp_mbox_xlate(struct mbox_controller *mbox, const struct of_phandle_args *sp) { if (sp->args_count) - return NULL; + return ERR_PTR(-EINVAL); return &mbox->chans[0]; } diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 64bb38c9589565..6627a381f65ae7 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1373,6 +1373,13 @@ static CLOSURE_CALLBACK(cached_dev_free) mutex_unlock(&bch_register_lock); + /* + * Wait for any pending sb_write to complete before free. + * The sb_bio is embedded in struct cached_dev, so we must + * ensure no I/O is in progress. + */ + closure_sync(&dc->sb_write); + if (dc->sb_disk) folio_put(virt_to_folio(dc->sb_disk)); diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c index 57158c02d096ed..ddfc1a3cf2f536 100644 --- a/drivers/md/dm-cache-metadata.c +++ b/drivers/md/dm-cache-metadata.c @@ -1023,6 +1023,12 @@ static bool cmd_write_lock(struct dm_cache_metadata *cmd) return; \ } while (0) +#define WRITE_LOCK_OR_GOTO(cmd, label) \ + do { \ + if (!cmd_write_lock((cmd))) \ + goto label; \ + } while (0) + #define WRITE_UNLOCK(cmd) \ up_write(&(cmd)->root_lock) @@ -1714,17 +1720,6 @@ int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy * return r; } -int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result) -{ - int r; - - READ_LOCK(cmd); - r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result); - READ_UNLOCK(cmd); - - return r; -} - void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd) { WRITE_LOCK_VOID(cmd); @@ -1791,11 +1786,8 @@ int dm_cache_metadata_abort(struct dm_cache_metadata *cmd) new_bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT, CACHE_MAX_CONCURRENT_LOCKS); - WRITE_LOCK(cmd); - if (cmd->fail_io) { - WRITE_UNLOCK(cmd); - goto out; - } + /* cmd_write_lock() already checks fail_io with cmd->root_lock held */ + WRITE_LOCK_OR_GOTO(cmd, out); __destroy_persistent_data_objects(cmd, false); old_bm = cmd->bm; diff --git a/drivers/md/dm-cache-metadata.h b/drivers/md/dm-cache-metadata.h index 5f77890207fede..2f107e7c67d0a8 100644 --- a/drivers/md/dm-cache-metadata.h +++ b/drivers/md/dm-cache-metadata.h @@ -135,11 +135,6 @@ int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd, */ int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *p); -/* - * Query method. Are all the blocks in the cache clean? - */ -int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result); - int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result); int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd); void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd); diff --git a/drivers/md/dm-cache-policy-smq.c b/drivers/md/dm-cache-policy-smq.c index b328d9601046b2..dd77a93fd68d2d 100644 --- a/drivers/md/dm-cache-policy-smq.c +++ b/drivers/md/dm-cache-policy-smq.c @@ -1589,14 +1589,18 @@ static int smq_invalidate_mapping(struct dm_cache_policy *p, dm_cblock_t cblock) { struct smq_policy *mq = to_smq_policy(p); struct entry *e = get_entry(&mq->cache_alloc, from_cblock(cblock)); + unsigned long flags; if (!e->allocated) return -ENODATA; + spin_lock_irqsave(&mq->lock, flags); // FIXME: what if this block has pending background work? del_queue(mq, e); h_remove(&mq->table, e); free_entry(&mq->cache_alloc, e); + spin_unlock_irqrestore(&mq->lock, flags); + return 0; } diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 935ab79b1d0cd4..af7a2571988b43 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -1462,11 +1462,19 @@ static void invalidate_complete(struct dm_cache_migration *mg, bool success) struct cache *cache = mg->cache; bio_list_init(&bios); - if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) - free_prison_cell(cache, mg->cell); + if (mg->cell) { + if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios)) + free_prison_cell(cache, mg->cell); + } - if (!success && mg->overwrite_bio) - bio_io_error(mg->overwrite_bio); + if (mg->overwrite_bio) { + // Set generic error if the bio hasn't been issued yet, + // e.g., invalidation or metadata commit failed before bio + // submission. Otherwise preserve the bio's own error status. + if (!success && !mg->overwrite_bio->bi_status) + mg->overwrite_bio->bi_status = BLK_STS_IOERR; + bio_endio(mg->overwrite_bio); + } free_migration(mg); defer_bios(cache, &bios); @@ -1506,6 +1514,24 @@ static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock) return r; } +static void invalidate_committed(struct work_struct *ws) +{ + struct dm_cache_migration *mg = ws_to_mg(ws); + struct cache *cache = mg->cache; + struct bio *bio = mg->overwrite_bio; + struct per_bio_data *pb = get_per_bio_data(bio); + + if (mg->k.input) { + invalidate_complete(mg, false); + return; + } + + init_continuation(&mg->k, invalidate_completed); + remap_to_origin_clear_discard(cache, bio, mg->invalidate_oblock); + dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg); + dm_submit_bio_remap(bio, NULL); +} + static void invalidate_remove(struct work_struct *ws) { int r; @@ -1518,10 +1544,8 @@ static void invalidate_remove(struct work_struct *ws) return; } - init_continuation(&mg->k, invalidate_completed); + init_continuation(&mg->k, invalidate_committed); continue_after_commit(&cache->committer, &mg->k); - remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock); - mg->overwrite_bio = NULL; schedule_commit(&cache->committer); } @@ -1539,6 +1563,15 @@ static int invalidate_lock(struct dm_cache_migration *mg) READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell); if (r < 0) { free_prison_cell(cache, prealloc); + + /* Defer the bio for retrying the cell lock */ + if (mg->overwrite_bio) { + struct bio *bio = mg->overwrite_bio; + + mg->overwrite_bio = NULL; + defer_bio(cache, bio); + } + invalidate_complete(mg, false); return r; } @@ -1701,6 +1734,7 @@ static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block, bio_drop_shared_lock(cache, bio); atomic_inc(&cache->stats.demotion); invalidate_start(cache, cblock, block, bio); + return DM_MAPIO_SUBMITTED; } else remap_to_origin_clear_discard(cache, bio, block); } else { @@ -2467,23 +2501,8 @@ static int cache_create(struct cache_args *ca, struct cache **result) goto bad; } - if (passthrough_mode(cache)) { - bool all_clean; - - r = dm_cache_metadata_all_clean(cache->cmd, &all_clean); - if (r) { - *error = "dm_cache_metadata_all_clean() failed"; - goto bad; - } - - if (!all_clean) { - *error = "Cannot enter passthrough mode unless all blocks are clean"; - r = -EINVAL; - goto bad; - } - + if (passthrough_mode(cache)) policy_allow_migrations(cache->policy, false); - } spin_lock_init(&cache->lock); bio_list_init(&cache->deferred_bios); @@ -2810,6 +2829,12 @@ static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock, struct cache *cache = context; if (dirty) { + if (passthrough_mode(cache)) { + DMERR("%s: cannot enter passthrough mode unless all blocks are clean", + cache_device_name(cache)); + return -EBUSY; + } + set_bit(from_cblock(cblock), cache->dirty_bitset); atomic_inc(&cache->nr_dirty); } else @@ -3043,7 +3068,7 @@ static int cache_preresume(struct dm_target *ti) load_filtered_mapping, cache); if (r) { DMERR("%s: could not load cache mappings", cache_device_name(cache)); - if (r != -EFBIG) + if (r != -EFBIG && r != -EBUSY) metadata_operation_failed(cache, "dm_cache_load_mappings", r); return r; } diff --git a/drivers/md/dm-init.c b/drivers/md/dm-init.c index 7403823384c575..c1bacba92c65cc 100644 --- a/drivers/md/dm-init.c +++ b/drivers/md/dm-init.c @@ -303,8 +303,10 @@ static int __init dm_init_init(void) } } - if (waitfor[0]) + if (waitfor[0]) { + wait_for_device_probe(); DMINFO("all devices available"); + } list_for_each_entry(dev, &devices, list) { if (dm_early_create(&dev->dmi, dev->table, diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 3ab8b4beff8641..e0c574862d0610 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -384,7 +384,7 @@ static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool up_write(&_hash_lock); - if (dev_skipped) + if (dev_skipped && !only_deferred) DMWARN("remove_all left %d open device(s)", dev_skipped); } @@ -1341,6 +1341,10 @@ static void retrieve_status(struct dm_table *table, used = param->data_start + (outptr - outbuf); outptr = align_ptr(outptr); + if (!outptr || outptr > outbuf + len) { + param->flags |= DM_BUFFER_FULL_FLAG; + break; + } spec->next = outptr - outbuf; } diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index 1aa6a4a7d2323a..d316757a328bec 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -373,7 +373,7 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti, struct log_c *lc; uint32_t region_size; - unsigned int region_count; + sector_t region_count; size_t bitset_size, buf_size; int r; char dummy; @@ -401,6 +401,10 @@ static int create_log_context(struct dm_dirty_log *log, struct dm_target *ti, } region_count = dm_sector_div_up(ti->len, region_size); + if (region_count > UINT_MAX) { + DMWARN("region count exceeds limit of %u", UINT_MAX); + return -EINVAL; + } lc = kmalloc_obj(*lc); if (!lc) { diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c index 8f4ae2f5154533..7cb7bb6233b645 100644 --- a/drivers/md/dm-mpath.c +++ b/drivers/md/dm-mpath.c @@ -102,7 +102,6 @@ struct multipath { struct bio_list queued_bios; struct timer_list nopath_timer; /* Timeout for queue_if_no_path */ - bool is_suspending; }; /* @@ -1749,9 +1748,6 @@ static void multipath_presuspend(struct dm_target *ti) { struct multipath *m = ti->private; - spin_lock_irq(&m->lock); - m->is_suspending = true; - spin_unlock_irq(&m->lock); /* FIXME: bio-based shouldn't need to always disable queue_if_no_path */ if (m->queue_mode == DM_TYPE_BIO_BASED || !dm_noflush_suspending(m->ti)) queue_if_no_path(m, false, true, __func__); @@ -1774,7 +1770,6 @@ static void multipath_resume(struct dm_target *ti) struct multipath *m = ti->private; spin_lock_irq(&m->lock); - m->is_suspending = false; if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) { set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); @@ -2098,7 +2093,7 @@ static int probe_active_paths(struct multipath *m) if (m->current_pg == m->last_probed_pg) goto skip_probe; } - if (!m->current_pg || m->is_suspending || + if (!m->current_pg || dm_suspended(m->ti) || test_bit(MPATHF_QUEUE_IO, &m->flags)) goto skip_probe; set_bit(MPATHF_DELAY_PG_SWITCH, &m->flags); @@ -2107,7 +2102,7 @@ static int probe_active_paths(struct multipath *m) list_for_each_entry(pgpath, &pg->pgpaths, list) { if (pg != READ_ONCE(m->current_pg) || - READ_ONCE(m->is_suspending)) + dm_suspended(m->ti)) goto out; if (!pgpath->is_active) continue; diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index 80a5c412770799..de5c00704e69c6 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -993,13 +993,13 @@ static struct dm_dirty_log *create_dirty_log(struct dm_target *ti, return NULL; } - *args_used = 2 + param_count; - - if (argc < *args_used) { + if (param_count > argc - 2) { ti->error = "Insufficient mirror log arguments"; return NULL; } + *args_used = 2 + param_count; + dl = dm_dirty_log_create(argv[0], ti, mirror_flush, param_count, argv + 2); if (!dl) { diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 14be4d888af388..e5d38bb3f16f13 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -33,36 +33,6 @@ static inline u64 fec_interleave(struct dm_verity *v, u64 offset) return offset + mod * (v->fec->rounds << v->data_dev_block_bits); } -/* - * Read error-correcting codes for the requested RS block. Returns a pointer - * to the data block. Caller is responsible for releasing buf. - */ -static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, - unsigned int *offset, unsigned int par_buf_offset, - struct dm_buffer **buf, unsigned short ioprio) -{ - u64 position, block, rem; - u8 *res; - - /* We have already part of parity bytes read, skip to the next block */ - if (par_buf_offset) - index++; - - position = (index + rsb) * v->fec->roots; - block = div64_u64_rem(position, v->fec->io_size, &rem); - *offset = par_buf_offset ? 0 : (unsigned int)rem; - - res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); - if (IS_ERR(res)) { - DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", - v->data_dev->name, (unsigned long long)rsb, - (unsigned long long)block, PTR_ERR(res)); - *buf = NULL; - } - - return res; -} - /* Loop over each allocated buffer. */ #define fec_for_each_buffer(io, __i) \ for (__i = 0; __i < (io)->nbufs; __i++) @@ -102,15 +72,29 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, { int r, corrected = 0, res; struct dm_buffer *buf; - unsigned int n, i, j, offset, par_buf_offset = 0; + unsigned int n, i, j, parity_pos, to_copy; uint16_t par_buf[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; u8 *par, *block; + u64 parity_block; struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); - par = fec_read_parity(v, rsb, block_offset, &offset, - par_buf_offset, &buf, bio->bi_ioprio); - if (IS_ERR(par)) + /* + * Compute the index of the first parity block that will be needed and + * the starting position in that block. Then read that block. + * + * io_size is always a power of 2, but roots might not be. Note that + * when it's not, a codeword's parity bytes can span a block boundary. + */ + parity_block = (rsb + block_offset) * v->fec->roots; + parity_pos = parity_block & (v->fec->io_size - 1); + parity_block >>= v->data_dev_block_bits; + par = dm_bufio_read_with_ioprio(v->fec->bufio, parity_block, &buf, + bio->bi_ioprio); + if (IS_ERR(par)) { + DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", + v->data_dev->name, rsb, parity_block, PTR_ERR(par)); return PTR_ERR(par); + } /* * Decode the RS blocks we have in bufs. Each RS block results in @@ -118,8 +102,32 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, */ fec_for_each_buffer_rs_block(fio, n, i) { block = fec_buffer_rs_block(v, fio, n, i); - for (j = 0; j < v->fec->roots - par_buf_offset; j++) - par_buf[par_buf_offset + j] = par[offset + j]; + + /* + * Copy the next 'roots' parity bytes to 'par_buf', reading + * another parity block if needed. + */ + to_copy = min(v->fec->io_size - parity_pos, v->fec->roots); + for (j = 0; j < to_copy; j++) + par_buf[j] = par[parity_pos++]; + if (to_copy < v->fec->roots) { + parity_block++; + parity_pos = 0; + + dm_bufio_release(buf); + par = dm_bufio_read_with_ioprio(v->fec->bufio, + parity_block, &buf, + bio->bi_ioprio); + if (IS_ERR(par)) { + DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", + v->data_dev->name, rsb, parity_block, + PTR_ERR(par)); + return PTR_ERR(par); + } + for (; j < v->fec->roots; j++) + par_buf[j] = par[parity_pos++]; + } + /* Decode an RS block using Reed-Solomon */ res = decode_rs8(fio->rs, block, par_buf, v->fec->rsn, NULL, neras, fio->erasures, 0, NULL); @@ -134,26 +142,6 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, block_offset++; if (block_offset >= 1 << v->data_dev_block_bits) goto done; - - /* Read the next block when we run out of parity bytes */ - offset += (v->fec->roots - par_buf_offset); - /* Check if parity bytes are split between blocks */ - if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) { - par_buf_offset = v->fec->io_size - offset; - for (j = 0; j < par_buf_offset; j++) - par_buf[j] = par[offset + j]; - offset += par_buf_offset; - } else - par_buf_offset = 0; - - if (offset >= v->fec->io_size) { - dm_bufio_release(buf); - - par = fec_read_parity(v, rsb, block_offset, &offset, - par_buf_offset, &buf, bio->bi_ioprio); - if (IS_ERR(par)) - return PTR_ERR(par); - } } done: r = corrected; @@ -163,11 +151,9 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, if (r < 0 && neras) DMERR_LIMIT("%s: FEC %llu: failed to correct: %d", v->data_dev->name, (unsigned long long)rsb, r); - else if (r > 0) { + else if (r > 0) DMWARN_LIMIT("%s: FEC %llu: corrected %d errors", v->data_dev->name, (unsigned long long)rsb, r); - atomic64_inc(&v->fec->corrected); - } return r; } @@ -439,6 +425,7 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, } memcpy(dest, fio->output, 1 << v->data_dev_block_bits); + atomic64_inc(&v->fec->corrected); done: fio->level--; @@ -625,7 +612,7 @@ int verity_fec_ctr(struct dm_verity *v) { struct dm_verity_fec *f = v->fec; struct dm_target *ti = v->ti; - u64 hash_blocks, fec_blocks; + u64 hash_blocks; int ret; if (!verity_fec_is_enabled(v)) { @@ -688,7 +675,8 @@ int verity_fec_ctr(struct dm_verity *v) * it to be large enough. */ f->hash_blocks = f->blocks - v->data_blocks; - if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { + if (dm_bufio_get_device_size(v->bufio) < + v->hash_start + f->hash_blocks) { ti->error = "Hash device is too small for " DM_VERITY_OPT_FEC_BLOCKS; return -E2BIG; @@ -706,8 +694,7 @@ int verity_fec_ctr(struct dm_verity *v) dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); - fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); - if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { + if (dm_bufio_get_device_size(f->bufio) < f->rounds * f->roots) { ti->error = "FEC device is too small"; return -E2BIG; } diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h index 35d28d9f8a9b09..32ca2bfee1db79 100644 --- a/drivers/md/dm-verity-fec.h +++ b/drivers/md/dm-verity-fec.h @@ -47,7 +47,8 @@ struct dm_verity_fec { /* per-bio data */ struct dm_verity_fec_io { struct rs_control *rs; /* Reed-Solomon state */ - int erasures[DM_VERITY_FEC_MAX_RSN]; /* erasures for decode_rs8 */ + /* erasures for decode_rs8 */ + int erasures[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN + 1]; u8 *output; /* buffer for corrected output */ unsigned int level; /* recursion level */ unsigned int nbufs; /* number of buffers allocated */ diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 83378c033c7285..028b9ca8ce52db 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -216,6 +216,7 @@ struct bitmap { }; static struct workqueue_struct *md_bitmap_wq; +static struct attribute_group md_bitmap_internal_group; static int __bitmap_resize(struct bitmap *bitmap, sector_t blocks, int chunksize, bool init); @@ -2580,6 +2581,30 @@ static int bitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) return __bitmap_resize(bitmap, blocks, chunksize, false); } +static bool bitmap_none_enabled(void *data, bool flush) +{ + return false; +} + +static int bitmap_none_create(struct mddev *mddev) +{ + return 0; +} + +static int bitmap_none_load(struct mddev *mddev) +{ + return 0; +} + +static void bitmap_none_destroy(struct mddev *mddev) +{ +} + +static int bitmap_none_get_stats(void *data, struct md_bitmap_stats *stats) +{ + return -ENOENT; +} + static ssize_t location_show(struct mddev *mddev, char *page) { @@ -2618,7 +2643,11 @@ location_store(struct mddev *mddev, const char *buf, size_t len) goto out; } - bitmap_destroy(mddev); + sysfs_unmerge_group(&mddev->kobj, &md_bitmap_internal_group); + md_bitmap_destroy_nosysfs(mddev); + mddev->bitmap_id = ID_BITMAP_NONE; + if (!mddev_set_bitmap_ops_nosysfs(mddev)) + goto none_err; mddev->bitmap_info.offset = 0; if (mddev->bitmap_info.file) { struct file *f = mddev->bitmap_info.file; @@ -2654,16 +2683,25 @@ location_store(struct mddev *mddev, const char *buf, size_t len) } mddev->bitmap_info.offset = offset; - rv = bitmap_create(mddev); + md_bitmap_destroy_nosysfs(mddev); + mddev->bitmap_id = ID_BITMAP; + if (!mddev_set_bitmap_ops_nosysfs(mddev)) + goto bitmap_err; + + rv = md_bitmap_create_nosysfs(mddev); if (rv) - goto out; + goto create_err; - rv = bitmap_load(mddev); + rv = mddev->bitmap_ops->load(mddev); if (rv) { mddev->bitmap_info.offset = 0; - bitmap_destroy(mddev); - goto out; + goto load_err; } + + rv = sysfs_merge_group(&mddev->kobj, + &md_bitmap_internal_group); + if (rv) + goto merge_err; } } if (!mddev->external) { @@ -2679,6 +2717,22 @@ location_store(struct mddev *mddev, const char *buf, size_t len) if (rv) return rv; return len; + +merge_err: + mddev->bitmap_info.offset = 0; +load_err: + md_bitmap_destroy_nosysfs(mddev); +create_err: + mddev->bitmap_info.offset = 0; + mddev->bitmap_id = ID_BITMAP_NONE; + if (!mddev_set_bitmap_ops_nosysfs(mddev)) + rv = -ENOENT; + goto out; +bitmap_err: + rv = -ENOENT; +none_err: + mddev->bitmap_info.offset = 0; + goto out; } static struct md_sysfs_entry bitmap_location = @@ -2955,8 +3009,12 @@ static struct md_sysfs_entry max_backlog_used = __ATTR(max_backlog_used, S_IRUGO | S_IWUSR, behind_writes_used_show, behind_writes_used_reset); -static struct attribute *md_bitmap_attrs[] = { +static struct attribute *md_bitmap_common_attrs[] = { &bitmap_location.attr, + NULL +}; + +static struct attribute *md_bitmap_internal_attrs[] = { &bitmap_space.attr, &bitmap_timeout.attr, &bitmap_backlog.attr, @@ -2967,9 +3025,41 @@ static struct attribute *md_bitmap_attrs[] = { NULL }; -static struct attribute_group md_bitmap_group = { +static struct attribute_group md_bitmap_common_group = { .name = "bitmap", - .attrs = md_bitmap_attrs, + .attrs = md_bitmap_common_attrs, +}; + +static struct attribute_group md_bitmap_internal_group = { + .name = "bitmap", + .attrs = md_bitmap_internal_attrs, +}; + +static const struct attribute_group *bitmap_groups[] = { + &md_bitmap_common_group, + &md_bitmap_internal_group, + NULL, +}; + +static const struct attribute_group *bitmap_none_groups[] = { + &md_bitmap_common_group, + NULL, +}; + +static struct bitmap_operations bitmap_none_ops = { + .head = { + .type = MD_BITMAP, + .id = ID_BITMAP_NONE, + .name = "none", + }, + + .enabled = bitmap_none_enabled, + .create = bitmap_none_create, + .load = bitmap_none_load, + .destroy = bitmap_none_destroy, + .get_stats = bitmap_none_get_stats, + + .groups = bitmap_none_groups, }; static struct bitmap_operations bitmap_ops = { @@ -3013,21 +3103,38 @@ static struct bitmap_operations bitmap_ops = { .set_pages = bitmap_set_pages, .free = md_bitmap_free, - .group = &md_bitmap_group, + .groups = bitmap_groups, }; int md_bitmap_init(void) { + int err; + md_bitmap_wq = alloc_workqueue("md_bitmap", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); if (!md_bitmap_wq) return -ENOMEM; - return register_md_submodule(&bitmap_ops.head); + err = register_md_submodule(&bitmap_none_ops.head); + if (err) + goto err_wq; + + err = register_md_submodule(&bitmap_ops.head); + if (err) + goto err_none; + + return 0; + +err_none: + unregister_md_submodule(&bitmap_none_ops.head); +err_wq: + destroy_workqueue(md_bitmap_wq); + return err; } void md_bitmap_exit(void) { - destroy_workqueue(md_bitmap_wq); unregister_md_submodule(&bitmap_ops.head); + unregister_md_submodule(&bitmap_none_ops.head); + destroy_workqueue(md_bitmap_wq); } diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index b42a28fa83a0f5..214f623c7e790b 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -125,7 +125,7 @@ struct bitmap_operations { void (*set_pages)(void *data, unsigned long pages); void (*free)(void *data); - struct attribute_group *group; + const struct attribute_group **groups; }; /* the bitmap API */ diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index bf398d7476b35a..aeb061166e732c 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -459,7 +459,8 @@ static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx) rdev_for_each(rdev, mddev) { sector_t sector; - if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags)) + if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags) || + !test_bit(In_sync, &rdev->flags)) continue; sector = mddev->bitmap_info.offset + @@ -1069,12 +1070,12 @@ static void llbitmap_start_write(struct mddev *mddev, sector_t offset, int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; - llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite); - while (page_start <= page_end) { llbitmap_raise_barrier(llbitmap, page_start); page_start++; } + + llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite); } static void llbitmap_end_write(struct mddev *mddev, sector_t offset, @@ -1101,12 +1102,12 @@ static void llbitmap_start_discard(struct mddev *mddev, sector_t offset, int page_start = (start + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; int page_end = (end + BITMAP_DATA_OFFSET) >> PAGE_SHIFT; - llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard); - while (page_start <= page_end) { llbitmap_raise_barrier(llbitmap, page_start); page_start++; } + + llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard); } static void llbitmap_end_discard(struct mddev *mddev, sector_t offset, @@ -1561,6 +1562,11 @@ static struct attribute_group md_llbitmap_group = { .attrs = md_llbitmap_attrs, }; +static const struct attribute_group *md_llbitmap_groups[] = { + &md_llbitmap_group, + NULL, +}; + static struct bitmap_operations llbitmap_ops = { .head = { .type = MD_BITMAP, @@ -1597,7 +1603,7 @@ static struct bitmap_operations llbitmap_ops = { .dirty_bits = llbitmap_dirty_bits, .write_all = llbitmap_write_all, - .group = &md_llbitmap_group, + .groups = md_llbitmap_groups, }; int md_llbitmap_init(void) diff --git a/drivers/md/md.c b/drivers/md/md.c index 3ce6f9e9d38e6e..32927c24ebf7cd 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -84,7 +84,6 @@ static DEFINE_XARRAY(md_submodule); static const struct kobj_type md_ktype; static DECLARE_WAIT_QUEUE_HEAD(resync_wait); -static struct workqueue_struct *md_wq; /* * This workqueue is used for sync_work to register new sync_thread, and for @@ -489,6 +488,17 @@ int mddev_suspend(struct mddev *mddev, bool interruptible) } percpu_ref_kill(&mddev->active_io); + + /* + * RAID456 IO can sleep in wait_for_reshape while still holding an + * active_io reference. If reshape is already interrupted or frozen, + * wake those waiters so they can abort and drop the reference instead + * of deadlocking suspend. + */ + if (mddev->pers && mddev->pers->prepare_suspend && + reshape_interrupted(mddev)) + mddev->pers->prepare_suspend(mddev); + if (interruptible) err = wait_event_interruptible(mddev->sb_wait, percpu_ref_is_zero(&mddev->active_io)); @@ -678,13 +688,38 @@ static void active_io_release(struct percpu_ref *ref) static void no_op(struct percpu_ref *r) {} -static bool mddev_set_bitmap_ops(struct mddev *mddev) +static void md_bitmap_sysfs_add(struct mddev *mddev) +{ + if (sysfs_update_groups(&mddev->kobj, mddev->bitmap_ops->groups)) + pr_warn("md: cannot register extra bitmap attributes for %s\n", + mdname(mddev)); + else + /* + * Inform user with KOBJ_CHANGE about new bitmap + * attributes. + */ + kobject_uevent(&mddev->kobj, KOBJ_CHANGE); +} + +static void md_bitmap_sysfs_del(struct mddev *mddev) +{ + int nr_groups = 0; + + for (nr_groups = 0; mddev->bitmap_ops->groups[nr_groups]; nr_groups++) + ; + + while (--nr_groups >= 1) + sysfs_unmerge_group(&mddev->kobj, + mddev->bitmap_ops->groups[nr_groups]); + sysfs_remove_group(&mddev->kobj, mddev->bitmap_ops->groups[0]); +} + +bool mddev_set_bitmap_ops_nosysfs(struct mddev *mddev) { - struct bitmap_operations *old = mddev->bitmap_ops; struct md_submodule_head *head; - if (mddev->bitmap_id == ID_BITMAP_NONE || - (old && old->head.id == mddev->bitmap_id)) + if (mddev->bitmap_ops && + mddev->bitmap_ops->head.id == mddev->bitmap_id) return true; xa_lock(&md_submodule); @@ -702,18 +737,6 @@ static bool mddev_set_bitmap_ops(struct mddev *mddev) mddev->bitmap_ops = (void *)head; xa_unlock(&md_submodule); - - if (!mddev_is_dm(mddev) && mddev->bitmap_ops->group) { - if (sysfs_create_group(&mddev->kobj, mddev->bitmap_ops->group)) - pr_warn("md: cannot register extra bitmap attributes for %s\n", - mdname(mddev)); - else - /* - * Inform user with KOBJ_CHANGE about new bitmap - * attributes. - */ - kobject_uevent(&mddev->kobj, KOBJ_CHANGE); - } return true; err: @@ -721,15 +744,6 @@ static bool mddev_set_bitmap_ops(struct mddev *mddev) return false; } -static void mddev_clear_bitmap_ops(struct mddev *mddev) -{ - if (!mddev_is_dm(mddev) && mddev->bitmap_ops && - mddev->bitmap_ops->group) - sysfs_remove_group(&mddev->kobj, mddev->bitmap_ops->group); - - mddev->bitmap_ops = NULL; -} - int mddev_init(struct mddev *mddev) { int err = 0; @@ -2788,7 +2802,9 @@ void md_update_sb(struct mddev *mddev, int force_change) if (!md_is_rdwr(mddev)) { if (force_change) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); - pr_err("%s: can't update sb for read-only array %s\n", __func__, mdname(mddev)); + if (!mddev_is_dm(mddev)) + pr_err_ratelimited("%s: can't update sb for read-only array %s\n", + __func__, mdname(mddev)); return; } @@ -4268,7 +4284,7 @@ bitmap_type_show(struct mddev *mddev, char *page) xa_lock(&md_submodule); xa_for_each(&md_submodule, i, head) { - if (head->type != MD_BITMAP) + if (head->type != MD_BITMAP || head->id == ID_BITMAP_NONE) continue; if (mddev->bitmap_id == head->id) @@ -6128,10 +6144,16 @@ md_attr_store(struct kobject *kobj, struct attribute *attr, } spin_unlock(&all_mddevs_lock); rv = entry->store(mddev, page, length); - mddev_put(mddev); + /* + * For "array_state=clear", dropping the extra kobject reference from + * sysfs_break_active_protection() can trigger md kobject deletion. + * Restore active protection before mddev_put() so deletion happens + * after the sysfs write path fully unwinds. + */ if (kn) sysfs_unbreak_active_protection(kn); + mddev_put(mddev); return rv; } @@ -6447,24 +6469,170 @@ static void md_safemode_timeout(struct timer_list *t) static int start_dirty_degraded; -static int md_bitmap_create(struct mddev *mddev) +/* + * Read bitmap superblock and return the bitmap_id based on disk version. + * This is used as fallback when default bitmap version and on-disk version + * doesn't match, and mdadm is not the latest version to set bitmap_type. + */ +static enum md_submodule_id md_bitmap_get_id_from_sb(struct mddev *mddev) { + struct md_rdev *rdev; + struct page *sb_page; + bitmap_super_t *sb; + enum md_submodule_id id = ID_BITMAP_NONE; + sector_t sector; + u32 version; + + if (!mddev->bitmap_info.offset) + return ID_BITMAP_NONE; + + sb_page = alloc_page(GFP_KERNEL); + if (!sb_page) { + pr_warn("md: %s: failed to allocate memory for bitmap\n", + mdname(mddev)); + return ID_BITMAP_NONE; + } + + sector = mddev->bitmap_info.offset; + + rdev_for_each(rdev, mddev) { + u32 iosize; + + if (!test_bit(In_sync, &rdev->flags) || + test_bit(Faulty, &rdev->flags) || + test_bit(Bitmap_sync, &rdev->flags)) + continue; + + iosize = roundup(sizeof(bitmap_super_t), + bdev_logical_block_size(rdev->bdev)); + if (sync_page_io(rdev, sector, iosize, sb_page, REQ_OP_READ, + true)) + goto read_ok; + } + pr_warn("md: %s: failed to read bitmap from any device\n", + mdname(mddev)); + goto out; + +read_ok: + sb = kmap_local_page(sb_page); + if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) { + pr_warn("md: %s: invalid bitmap magic 0x%x\n", + mdname(mddev), le32_to_cpu(sb->magic)); + goto out_unmap; + } + + version = le32_to_cpu(sb->version); + switch (version) { + case BITMAP_MAJOR_LO: + case BITMAP_MAJOR_HI: + case BITMAP_MAJOR_CLUSTERED: + id = ID_BITMAP; + break; + case BITMAP_MAJOR_LOCKLESS: + id = ID_LLBITMAP; + break; + default: + pr_warn("md: %s: unknown bitmap version %u\n", + mdname(mddev), version); + break; + } + +out_unmap: + kunmap_local(sb); +out: + __free_page(sb_page); + return id; +} + +int md_bitmap_create_nosysfs(struct mddev *mddev) +{ + enum md_submodule_id orig_id = mddev->bitmap_id; + enum md_submodule_id sb_id; + int err; + if (mddev->bitmap_id == ID_BITMAP_NONE) return -EINVAL; - if (!mddev_set_bitmap_ops(mddev)) + if (!mddev_set_bitmap_ops_nosysfs(mddev)) { + mddev->bitmap_id = orig_id; return -ENOENT; + } + + err = mddev->bitmap_ops->create(mddev); + if (!err) + return 0; - return mddev->bitmap_ops->create(mddev); + /* + * Create failed, if default bitmap version and on-disk version + * doesn't match, and mdadm is not the latest version to set + * bitmap_type, set bitmap_ops based on the disk version. + */ + mddev->bitmap_ops = NULL; + + sb_id = md_bitmap_get_id_from_sb(mddev); + if (sb_id == ID_BITMAP_NONE || sb_id == orig_id) { + mddev->bitmap_id = orig_id; + return err; + } + + pr_info("md: %s: bitmap version mismatch, switching from %d to %d\n", + mdname(mddev), orig_id, sb_id); + + mddev->bitmap_id = sb_id; + if (!mddev_set_bitmap_ops_nosysfs(mddev)) { + mddev->bitmap_id = orig_id; + return -ENOENT; + } + + err = mddev->bitmap_ops->create(mddev); + if (err) { + mddev->bitmap_ops = NULL; + mddev->bitmap_id = orig_id; + } + + return err; } -static void md_bitmap_destroy(struct mddev *mddev) +static int md_bitmap_create(struct mddev *mddev) +{ + int err; + + err = md_bitmap_create_nosysfs(mddev); + if (err) + return err; + + if (!mddev_is_dm(mddev) && mddev->bitmap_ops->groups) + md_bitmap_sysfs_add(mddev); + + return 0; +} + +void md_bitmap_destroy_nosysfs(struct mddev *mddev) { if (!md_bitmap_registered(mddev)) return; mddev->bitmap_ops->destroy(mddev); - mddev_clear_bitmap_ops(mddev); + mddev->bitmap_ops = NULL; +} + +static void md_bitmap_destroy(struct mddev *mddev) +{ + if (!mddev_is_dm(mddev) && mddev->bitmap_ops && + mddev->bitmap_ops->groups) + md_bitmap_sysfs_del(mddev); + + md_bitmap_destroy_nosysfs(mddev); +} + +static void md_bitmap_set_none(struct mddev *mddev) +{ + mddev->bitmap_id = ID_BITMAP_NONE; + if (!mddev_set_bitmap_ops_nosysfs(mddev)) + return; + + if (!mddev_is_dm(mddev) && mddev->bitmap_ops->groups) + md_bitmap_sysfs_add(mddev); } int md_run(struct mddev *mddev) @@ -6676,6 +6844,10 @@ int md_run(struct mddev *mddev) if (mddev->sb_flags) md_update_sb(mddev, 0); + if (IS_ENABLED(CONFIG_MD_BITMAP) && !mddev->bitmap_info.file && + !mddev->bitmap_info.offset) + md_bitmap_set_none(mddev); + md_new_event(); return 0; @@ -7621,7 +7793,8 @@ static int set_bitmap_file(struct mddev *mddev, int fd) { int err = 0; - if (!md_bitmap_registered(mddev)) + if (!md_bitmap_registered(mddev) || + mddev->bitmap_id == ID_BITMAP_NONE) return -EINVAL; if (mddev->pers) { @@ -7686,10 +7859,12 @@ static int set_bitmap_file(struct mddev *mddev, int fd) if (err) { md_bitmap_destroy(mddev); + md_bitmap_set_none(mddev); fd = -1; } } else if (fd < 0) { md_bitmap_destroy(mddev); + md_bitmap_set_none(mddev); } } @@ -7996,12 +8171,16 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info) mddev->bitmap_info.default_offset; mddev->bitmap_info.space = mddev->bitmap_info.default_space; + mddev->bitmap_id = ID_BITMAP; rv = md_bitmap_create(mddev); if (!rv) rv = mddev->bitmap_ops->load(mddev); - if (rv) + if (rv) { md_bitmap_destroy(mddev); + mddev->bitmap_info.offset = 0; + md_bitmap_set_none(mddev); + } } else { struct md_bitmap_stats stats; @@ -8029,6 +8208,7 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info) } md_bitmap_destroy(mddev); mddev->bitmap_info.offset = 0; + md_bitmap_set_none(mddev); } } md_update_sb(mddev, 1); @@ -10503,10 +10683,6 @@ static int __init md_init(void) goto err_bitmap; ret = -ENOMEM; - md_wq = alloc_workqueue("md", WQ_MEM_RECLAIM | WQ_PERCPU, 0); - if (!md_wq) - goto err_wq; - md_misc_wq = alloc_workqueue("md_misc", WQ_PERCPU, 0); if (!md_misc_wq) goto err_misc_wq; @@ -10531,8 +10707,6 @@ static int __init md_init(void) err_md: destroy_workqueue(md_misc_wq); err_misc_wq: - destroy_workqueue(md_wq); -err_wq: md_llbitmap_exit(); err_bitmap: md_bitmap_exit(); @@ -10841,7 +11015,6 @@ static __exit void md_exit(void) spin_unlock(&all_mddevs_lock); destroy_workqueue(md_misc_wq); - destroy_workqueue(md_wq); md_bitmap_exit(); } diff --git a/drivers/md/md.h b/drivers/md/md.h index ac84289664cd7e..409c8f61695d32 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -932,6 +932,9 @@ extern void md_allow_write(struct mddev *mddev); extern void md_wait_for_blocked_rdev(struct md_rdev *rdev, struct mddev *mddev); extern void md_set_array_sectors(struct mddev *mddev, sector_t array_sectors); extern int md_check_no_bitmap(struct mddev *mddev); +bool mddev_set_bitmap_ops_nosysfs(struct mddev *mddev); +int md_bitmap_create_nosysfs(struct mddev *mddev); +void md_bitmap_destroy_nosysfs(struct mddev *mddev); extern int md_integrity_register(struct mddev *mddev); extern int strict_strtoul_scaled(const char *cp, unsigned long *res, int scale); diff --git a/drivers/md/persistent-data/dm-btree-remove.c b/drivers/md/persistent-data/dm-btree-remove.c index 942cd47eb52dac..aeec5b9a1dd5c5 100644 --- a/drivers/md/persistent-data/dm-btree-remove.c +++ b/drivers/md/persistent-data/dm-btree-remove.c @@ -490,12 +490,20 @@ static int rebalance_children(struct shadow_spine *s, if (le32_to_cpu(n->header.nr_entries) == 1) { struct dm_block *child; + int is_shared; dm_block_t b = value64(n, 0); + r = dm_tm_block_is_shared(info->tm, b, &is_shared); + if (r) + return r; + r = dm_tm_read_lock(info->tm, b, &btree_node_validator, &child); if (r) return r; + if (is_shared) + inc_children(info->tm, dm_block_data(child), vt); + memcpy(n, dm_block_data(child), dm_bm_block_size(dm_tm_get_bm(info->tm))); diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c index c33099925f230e..56a56a4da4f83f 100644 --- a/drivers/md/raid1-10.c +++ b/drivers/md/raid1-10.c @@ -293,8 +293,13 @@ static inline bool raid1_should_read_first(struct mddev *mddev, * bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is * submitted to the underlying disks, hence don't record badblocks or retry * in this case. + * + * BLK_STS_INVAL means the bio was not valid for the underlying device. This + * is a user error, not a device failure, so retrying or recording bad blocks + * would be wrong. */ static inline bool raid1_should_handle_error(struct bio *bio) { - return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)); + return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) && + bio->bi_status != BLK_STS_INVAL; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 181400e147c0df..be2565dee4202d 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -62,7 +62,7 @@ static int check_and_add_serial(struct md_rdev *rdev, struct r1bio *r1_bio, unsigned long flags; int ret = 0; sector_t lo = r1_bio->sector; - sector_t hi = lo + r1_bio->sectors; + sector_t hi = lo + r1_bio->sectors - 1; struct serial_in_rdev *serial = &rdev->serial[idx]; spin_lock_irqsave(&serial->serial_lock, flags); @@ -452,7 +452,7 @@ static void raid1_end_write_request(struct bio *bio) int mirror = find_bio_disk(r1_bio, bio); struct md_rdev *rdev = conf->mirrors[mirror].rdev; sector_t lo = r1_bio->sector; - sector_t hi = r1_bio->sector + r1_bio->sectors; + sector_t hi = r1_bio->sector + r1_bio->sectors - 1; bool ignore_error = !raid1_should_handle_error(bio) || (bio->bi_status && bio_op(bio) == REQ_OP_DISCARD); diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 0653b5d8545a6c..698d169628a02d 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1184,7 +1184,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { - raid_end_bio_io(r10_bio); + free_r10bio(r10_bio); return; } @@ -1372,7 +1372,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, sectors = r10_bio->sectors; if (!regular_request_wait(mddev, conf, bio, sectors)) { - raid_end_bio_io(r10_bio); + free_r10bio(r10_bio); return; } @@ -3791,6 +3791,8 @@ static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) nc = layout & 255; fc = (layout >> 8) & 255; fo = layout & (1<<16); + if (!nc || !fc) + return -1; geo->raid_disks = disks; geo->near_copies = nc; geo->far_copies = fc; diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c index 66b10cbda96d66..7b7546bfa21fb0 100644 --- a/drivers/md/raid5-cache.c +++ b/drivers/md/raid5-cache.c @@ -2002,15 +2002,27 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log, return -ENOMEM; while (mb_offset < le32_to_cpu(mb->meta_size)) { + sector_t payload_len; + payload = (void *)mb + mb_offset; payload_flush = (void *)mb + mb_offset; if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) { + payload_len = sizeof(struct r5l_payload_data_parity) + + (sector_t)sizeof(__le32) * + (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9)); + if (mb_offset + payload_len > le32_to_cpu(mb->meta_size)) + goto mismatch; if (r5l_recovery_verify_data_checksum( log, ctx, page, log_offset, payload->checksum[0]) < 0) goto mismatch; } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) { + payload_len = sizeof(struct r5l_payload_data_parity) + + (sector_t)sizeof(__le32) * + (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9)); + if (mb_offset + payload_len > le32_to_cpu(mb->meta_size)) + goto mismatch; if (r5l_recovery_verify_data_checksum( log, ctx, page, log_offset, payload->checksum[0]) < 0) @@ -2023,22 +2035,18 @@ r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log, payload->checksum[1]) < 0) goto mismatch; } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) { - /* nothing to do for R5LOG_PAYLOAD_FLUSH here */ + payload_len = sizeof(struct r5l_payload_flush) + + (sector_t)le32_to_cpu(payload_flush->size); + if (mb_offset + payload_len > le32_to_cpu(mb->meta_size)) + goto mismatch; } else /* not R5LOG_PAYLOAD_DATA/PARITY/FLUSH */ goto mismatch; - if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) { - mb_offset += sizeof(struct r5l_payload_flush) + - le32_to_cpu(payload_flush->size); - } else { - /* DATA or PARITY payload */ + if (le16_to_cpu(payload->header.type) != R5LOG_PAYLOAD_FLUSH) { log_offset = r5l_ring_add(log, log_offset, le32_to_cpu(payload->size)); - mb_offset += sizeof(struct r5l_payload_data_parity) + - sizeof(__le32) * - (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9)); } - + mb_offset += payload_len; } put_page(page); @@ -2089,6 +2097,7 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS); while (mb_offset < le32_to_cpu(mb->meta_size)) { + sector_t payload_len; int dd; payload = (void *)mb + mb_offset; @@ -2097,6 +2106,12 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) { int i, count; + payload_len = sizeof(struct r5l_payload_flush) + + (sector_t)le32_to_cpu(payload_flush->size); + if (mb_offset + payload_len > + le32_to_cpu(mb->meta_size)) + return -EINVAL; + count = le32_to_cpu(payload_flush->size) / sizeof(__le64); for (i = 0; i < count; ++i) { stripe_sect = le64_to_cpu(payload_flush->flush_stripes[i]); @@ -2110,12 +2125,17 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, } } - mb_offset += sizeof(struct r5l_payload_flush) + - le32_to_cpu(payload_flush->size); + mb_offset += payload_len; continue; } /* DATA or PARITY payload */ + payload_len = sizeof(struct r5l_payload_data_parity) + + (sector_t)sizeof(__le32) * + (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9)); + if (mb_offset + payload_len > le32_to_cpu(mb->meta_size)) + return -EINVAL; + stripe_sect = (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) ? raid5_compute_sector( conf, le64_to_cpu(payload->location), 0, &dd, @@ -2180,9 +2200,7 @@ r5c_recovery_analyze_meta_block(struct r5l_log *log, log_offset = r5l_ring_add(log, log_offset, le32_to_cpu(payload->size)); - mb_offset += sizeof(struct r5l_payload_data_parity) + - sizeof(__le32) * - (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9)); + mb_offset += payload_len; } return 0; diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index a8e8d431071bac..335d2b6b107969 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6641,7 +6641,13 @@ static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio, } if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { - raid5_release_stripe(sh); + int hash; + + spin_lock_irq(&conf->device_lock); + hash = sh->hash_lock_index; + __release_stripe(conf, sh, + &conf->temp_inactive_list[hash]); + spin_unlock_irq(&conf->device_lock); conf->retry_read_aligned = raid_bio; conf->retry_read_offset = scnt; return handled; diff --git a/drivers/media/common/videobuf2/videobuf2-dma-sg.c b/drivers/media/common/videobuf2/videobuf2-dma-sg.c index 982021d547e53b..b1d0695cda2604 100644 --- a/drivers/media/common/videobuf2/videobuf2-dma-sg.c +++ b/drivers/media/common/videobuf2/videobuf2-dma-sg.c @@ -345,6 +345,7 @@ static int vb2_dma_sg_mmap(void *buf_priv, struct vm_area_struct *vma) return err; } + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); /* * Use common vm_area operations to track buffer refcount. */ diff --git a/drivers/media/dvb-frontends/dib8000.c b/drivers/media/dvb-frontends/dib8000.c index ebef27bcc98990..d291113291e049 100644 --- a/drivers/media/dvb-frontends/dib8000.c +++ b/drivers/media/dvb-frontends/dib8000.c @@ -2695,7 +2695,7 @@ static void dib8000_viterbi_state(struct dib8000_state *state, u8 onoff) static void dib8000_set_dds(struct dib8000_state *state, s32 offset_khz) { - s16 unit_khz_dds_val; + s32 unit_khz_dds_val; u32 abs_offset_khz = abs(offset_khz); u32 dds = state->cfg.pll->ifreq & 0x1ffffff; u8 invert = !!(state->cfg.pll->ifreq & (1 << 25)); @@ -2716,7 +2716,7 @@ static void dib8000_set_dds(struct dib8000_state *state, s32 offset_khz) dds = (1<<26) - dds; } else { ratio = 2; - unit_khz_dds_val = (u16) (67108864 / state->cfg.pll->internal); + unit_khz_dds_val = 67108864 / state->cfg.pll->internal; if (offset_khz < 0) unit_khz_dds_val *= -1; diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index fee63bc106d9e6..7da02ce5da1544 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -1218,6 +1218,9 @@ static int imx219_probe(struct i2c_client *client) /* Request optional enable pin */ imx219->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(imx219->reset_gpio)) + return dev_err_probe(dev, PTR_ERR(imx219->reset_gpio), + "failed to get reset gpio\n"); /* * The sensor must be powered for imx219_identify_module() diff --git a/drivers/media/i2c/imx283.c b/drivers/media/i2c/imx283.c index 8ab63ad8f385f6..1be6164c2d1552 100644 --- a/drivers/media/i2c/imx283.c +++ b/drivers/media/i2c/imx283.c @@ -129,7 +129,8 @@ /* Master Mode Operation Control */ #define IMX283_REG_XMSTA CCI_REG8(0x3105) -#define IMX283_XMSTA BIT(0) +#define IMX283_XMSTA_START 0 +#define IMX283_XMSTA_STOP BIT(0) #define IMX283_REG_SYNCDRV CCI_REG8(0x3107) #define IMX283_SYNCDRV_XHS_XVS (0xa0 | 0x02) @@ -1023,8 +1024,6 @@ static int imx283_standby_cancel(struct imx283 *imx283) usleep_range(19000, 20000); cci_write(imx283->cci, IMX283_REG_CLAMP, IMX283_CLPSQRST, &ret); - cci_write(imx283->cci, IMX283_REG_XMSTA, 0, &ret); - cci_write(imx283->cci, IMX283_REG_SYNCDRV, IMX283_SYNCDRV_XHS_XVS, &ret); return ret; } @@ -1117,6 +1116,10 @@ static int imx283_start_streaming(struct imx283 *imx283, /* Apply customized values from controls (HMAX/VMAX/SHR) */ ret = __v4l2_ctrl_handler_setup(imx283->sd.ctrl_handler); + /* Start master mode */ + cci_write(imx283->cci, IMX283_REG_XMSTA, IMX283_XMSTA_START, &ret); + cci_write(imx283->cci, IMX283_REG_SYNCDRV, IMX283_SYNCDRV_XHS_XVS, &ret); + return ret; } @@ -1153,12 +1156,14 @@ static int imx283_disable_streams(struct v4l2_subdev *sd, u64 streams_mask) { struct imx283 *imx283 = to_imx283(sd); - int ret; + int ret = 0; if (pad != IMAGE_PAD) return -EINVAL; - ret = cci_write(imx283->cci, IMX283_REG_STANDBY, IMX283_STBLOGIC, NULL); + cci_write(imx283->cci, IMX283_REG_XMSTA, IMX283_XMSTA_STOP, &ret); + cci_write(imx283->cci, IMX283_REG_STANDBY, IMX283_STANDBY, &ret); + if (ret) dev_err(imx283->dev, "Failed to stop stream\n"); diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c index b3826f80354703..aa63dfc349181b 100644 --- a/drivers/media/i2c/imx412.c +++ b/drivers/media/i2c/imx412.c @@ -925,7 +925,7 @@ static int imx412_parse_hw_config(struct imx412 *imx412) /* Request optional reset pin */ imx412->reset_gpio = devm_gpiod_get_optional(imx412->dev, "reset", - GPIOD_OUT_LOW); + GPIOD_OUT_HIGH); if (IS_ERR(imx412->reset_gpio)) { dev_err(imx412->dev, "failed to get reset gpio %pe\n", imx412->reset_gpio); diff --git a/drivers/media/i2c/og01a1b.c b/drivers/media/i2c/og01a1b.c index c7184de6251ae6..7b892b26203c04 100644 --- a/drivers/media/i2c/og01a1b.c +++ b/drivers/media/i2c/og01a1b.c @@ -1042,6 +1042,7 @@ static void og01a1b_remove(struct i2c_client *client) struct og01a1b *og01a1b = to_og01a1b(sd); v4l2_async_unregister_subdev(sd); + v4l2_subdev_cleanup(&og01a1b->sd); media_entity_cleanup(&sd->entity); v4l2_ctrl_handler_free(sd->ctrl_handler); pm_runtime_disable(og01a1b->dev); @@ -1153,11 +1154,18 @@ static int og01a1b_probe(struct i2c_client *client) goto probe_error_v4l2_ctrl_handler_free; } + ret = v4l2_subdev_init_finalize(&og01a1b->sd); + if (ret < 0) { + dev_err_probe(og01a1b->dev, ret, + "failed to finalize subdevice init\n"); + goto probe_error_media_entity_cleanup; + } + ret = v4l2_async_register_subdev_sensor(&og01a1b->sd); if (ret < 0) { dev_err(og01a1b->dev, "failed to register V4L2 subdev: %d", ret); - goto probe_error_media_entity_cleanup; + goto probe_error_v4l2_subdev_cleanup; } /* Enable runtime PM and turn off the device */ @@ -1167,6 +1175,9 @@ static int og01a1b_probe(struct i2c_client *client) return 0; +probe_error_v4l2_subdev_cleanup: + v4l2_subdev_cleanup(&og01a1b->sd); + probe_error_media_entity_cleanup: media_entity_cleanup(&og01a1b->sd.entity); diff --git a/drivers/media/i2c/ov08d10.c b/drivers/media/i2c/ov08d10.c index 43ec2a1f2fcffb..5e1b8b58b3d64b 100644 --- a/drivers/media/i2c/ov08d10.c +++ b/drivers/media/i2c/ov08d10.c @@ -217,7 +217,7 @@ static const struct ov08d10_reg lane_2_mode_3280x2460[] = { {0x9a, 0x30}, {0xa8, 0x02}, {0xfd, 0x02}, - {0xa1, 0x01}, + {0xa1, 0x00}, {0xa2, 0x09}, {0xa3, 0x9c}, {0xa5, 0x00}, @@ -335,7 +335,7 @@ static const struct ov08d10_reg lane_2_mode_3264x2448[] = { {0x9a, 0x30}, {0xa8, 0x02}, {0xfd, 0x02}, - {0xa1, 0x09}, + {0xa1, 0x08}, {0xa2, 0x09}, {0xa3, 0x90}, {0xa5, 0x08}, @@ -467,7 +467,7 @@ static const struct ov08d10_reg lane_2_mode_1632x1224[] = { {0xaa, 0xd0}, {0xab, 0x06}, {0xac, 0x68}, - {0xa1, 0x09}, + {0xa1, 0x04}, {0xa2, 0x04}, {0xa3, 0xc8}, {0xa5, 0x04}, @@ -613,8 +613,8 @@ static const struct ov08d10_lane_cfg lane_cfg_2 = { static u32 ov08d10_get_format_code(struct ov08d10 *ov08d10) { static const u32 codes[2][2] = { - { MEDIA_BUS_FMT_SGRBG10_1X10, MEDIA_BUS_FMT_SRGGB10_1X10}, - { MEDIA_BUS_FMT_SBGGR10_1X10, MEDIA_BUS_FMT_SGBRG10_1X10}, + { MEDIA_BUS_FMT_SBGGR10_1X10, MEDIA_BUS_FMT_SGBRG10_1X10 }, + { MEDIA_BUS_FMT_SGRBG10_1X10, MEDIA_BUS_FMT_SRGGB10_1X10 }, }; return codes[ov08d10->vflip->val][ov08d10->hflip->val]; @@ -1430,6 +1430,9 @@ static int ov08d10_probe(struct i2c_client *client) goto probe_error_v4l2_ctrl_handler_free; } + pm_runtime_set_active(ov08d10->dev); + pm_runtime_enable(ov08d10->dev); + ret = v4l2_async_register_subdev_sensor(&ov08d10->sd); if (ret < 0) { dev_err(ov08d10->dev, "failed to register V4L2 subdev: %d", @@ -1437,17 +1440,13 @@ static int ov08d10_probe(struct i2c_client *client) goto probe_error_media_entity_cleanup; } - /* - * Device is already turned on by i2c-core with ACPI domain PM. - * Enable runtime PM and turn off the device. - */ - pm_runtime_set_active(ov08d10->dev); - pm_runtime_enable(ov08d10->dev); pm_runtime_idle(ov08d10->dev); return 0; probe_error_media_entity_cleanup: + pm_runtime_disable(ov08d10->dev); + pm_runtime_set_suspended(ov08d10->dev); media_entity_cleanup(&ov08d10->sd.entity); probe_error_v4l2_ctrl_handler_free: diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c index 6a46ef7233ac3f..db9bd2892140de 100644 --- a/drivers/media/i2c/ov5647.c +++ b/drivers/media/i2c/ov5647.c @@ -967,21 +967,21 @@ static int ov5647_s_ctrl(struct v4l2_ctrl *ctrl) break; case V4L2_CID_AUTOGAIN: /* Non-zero turns on AGC by clearing bit 1.*/ - return cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(1), - ctrl->val ? 0 : BIT(1), NULL); + ret = cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(1), + ctrl->val ? 0 : BIT(1), NULL); break; case V4L2_CID_EXPOSURE_AUTO: /* * Everything except V4L2_EXPOSURE_MANUAL turns on AEC by * clearing bit 0. */ - return cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(0), - ctrl->val == V4L2_EXPOSURE_MANUAL ? BIT(0) : 0, NULL); + ret = cci_update_bits(sensor->regmap, OV5647_REG_AEC_AGC, BIT(0), + ctrl->val == V4L2_EXPOSURE_MANUAL ? BIT(0) : 0, NULL); break; case V4L2_CID_ANALOGUE_GAIN: /* 10 bits of gain, 2 in the high register. */ - return cci_write(sensor->regmap, OV5647_REG_GAIN, - ctrl->val & 0x3ff, NULL); + ret = cci_write(sensor->regmap, OV5647_REG_GAIN, + ctrl->val & 0x3ff, NULL); break; case V4L2_CID_EXPOSURE: /* diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c index e2998cfa0d18a3..dd01e1d515ff6d 100644 --- a/drivers/media/i2c/ov8856.c +++ b/drivers/media/i2c/ov8856.c @@ -1951,12 +1951,18 @@ static int ov8856_init_controls(struct ov8856 *ov8856) V4L2_CID_HFLIP, 0, 1, 1, 0); v4l2_ctrl_new_std(ctrl_hdlr, &ov8856_ctrl_ops, V4L2_CID_VFLIP, 0, 1, 1, 0); - if (ctrl_hdlr->error) - return ctrl_hdlr->error; + if (ctrl_hdlr->error) { + ret = ctrl_hdlr->error; + goto err_ctrl_handler_free; + } ov8856->sd.ctrl_handler = ctrl_hdlr; return 0; + +err_ctrl_handler_free: + v4l2_ctrl_handler_free(ctrl_hdlr); + return ret; } static void ov8856_update_pad_format(struct ov8856 *ov8856, diff --git a/drivers/media/pci/intel/ipu-bridge.c b/drivers/media/pci/intel/ipu-bridge.c index 32cc95a766b713..3028293eeb7526 100644 --- a/drivers/media/pci/intel/ipu-bridge.c +++ b/drivers/media/pci/intel/ipu-bridge.c @@ -104,6 +104,13 @@ static const struct ipu_sensor_config ipu_supported_sensors[] = { * without reporting a rotation of 180° in neither the SSDB nor the _PLD. */ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = { + { + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 13 9340"), + }, + .driver_data = "OVTI02C1", + }, { .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), @@ -111,6 +118,13 @@ static const struct dmi_system_id upside_down_sensor_dmi_ids[] = { }, .driver_data = "OVTI02C1", }, + { + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "XPS 14 9440"), + }, + .driver_data = "OVTI02C1", + }, { .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Dell Inc."), diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index 34f67f4f1bb567..d033d461816927 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -686,7 +686,7 @@ static int ipu6_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) out_ipu6_rpm_put: pm_runtime_put_sync(&isp->psys->auxdev.dev); out_ipu6_bus_del_devices: - if (isp->psys) { + if (!IS_ERR_OR_NULL(isp->psys)) { ipu6_cpd_free_pkg_dir(isp->psys); ipu6_buttress_unmap_fw_image(isp->psys, &isp->psys->fw_sgt); } diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c index 74406d5ea0a507..6bcde506adf506 100644 --- a/drivers/media/pci/saa7164/saa7164-core.c +++ b/drivers/media/pci/saa7164/saa7164-core.c @@ -888,6 +888,15 @@ static int get_resources(struct saa7164_dev *dev) return -EBUSY; } +static void release_resources(struct saa7164_dev *dev) +{ + release_mem_region(pci_resource_start(dev->pci, 0), + pci_resource_len(dev->pci, 0)); + + release_mem_region(pci_resource_start(dev->pci, 2), + pci_resource_len(dev->pci, 2)); +} + static int saa7164_port_init(struct saa7164_dev *dev, int portnr) { struct saa7164_port *port = NULL; @@ -947,9 +956,9 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) snprintf(dev->name, sizeof(dev->name), "saa7164[%d]", dev->nr); - mutex_lock(&devlist); - list_add_tail(&dev->devlist, &saa7164_devlist); - mutex_unlock(&devlist); + scoped_guard(mutex, &devlist) { + list_add_tail(&dev->devlist, &saa7164_devlist); + } /* board config */ dev->board = UNSET; @@ -996,11 +1005,17 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) } /* PCI/e allocations */ - dev->lmmio = ioremap(pci_resource_start(dev->pci, 0), - pci_resource_len(dev->pci, 0)); + dev->lmmio = pci_ioremap_bar(dev->pci, 0); + if (!dev->lmmio) { + dev_err(&dev->pci->dev, "Failed to remap MMIO BAR 0\n"); + goto err_ioremap_bar0; + } - dev->lmmio2 = ioremap(pci_resource_start(dev->pci, 2), - pci_resource_len(dev->pci, 2)); + dev->lmmio2 = pci_ioremap_bar(dev->pci, 2); + if (!dev->lmmio2) { + dev_err(&dev->pci->dev, "Failed to remap MMIO BAR 2\n"); + goto err_ioremap_bar2; + } dev->bmmio = (u8 __iomem *)dev->lmmio; dev->bmmio2 = (u8 __iomem *)dev->lmmio2; @@ -1019,17 +1034,25 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) saa7164_pci_quirks(dev); return 0; + +err_ioremap_bar2: + iounmap(dev->lmmio); +err_ioremap_bar0: + release_resources(dev); + + scoped_guard(mutex, &devlist) { + list_del(&dev->devlist); + } + saa7164_devcount--; + + return -ENODEV; } static void saa7164_dev_unregister(struct saa7164_dev *dev) { dprintk(1, "%s()\n", __func__); - release_mem_region(pci_resource_start(dev->pci, 0), - pci_resource_len(dev->pci, 0)); - - release_mem_region(pci_resource_start(dev->pci, 2), - pci_resource_len(dev->pci, 2)); + release_resources(dev); if (!atomic_dec_and_test(&dev->refcount)) return; diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c index d81facf735d920..f707bdc1fb0f1d 100644 --- a/drivers/media/pci/zoran/zoran_card.c +++ b/drivers/media/pci/zoran/zoran_card.c @@ -1373,7 +1373,7 @@ static int zoran_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } if (zr->codec->type != zr->card.video_codec) { pci_err(pdev, "%s - wrong codec\n", __func__); - goto zr_unreg_videocodec; + goto zr_detach_codec; } } if (zr->card.video_vfe != 0) { diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig index 3f0b7bb68cc940..8b31f087e7da20 100644 --- a/drivers/media/platform/Kconfig +++ b/drivers/media/platform/Kconfig @@ -65,6 +65,7 @@ config VIDEO_MUX source "drivers/media/platform/allegro-dvt/Kconfig" source "drivers/media/platform/amlogic/Kconfig" source "drivers/media/platform/amphion/Kconfig" +source "drivers/media/platform/apple/Kconfig" source "drivers/media/platform/arm/Kconfig" source "drivers/media/platform/aspeed/Kconfig" source "drivers/media/platform/atmel/Kconfig" diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index 6d5f79ddfcc37a..0e93f7d860caf4 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -8,6 +8,7 @@ obj-y += allegro-dvt/ obj-y += amlogic/ obj-y += amphion/ +obj-y += apple/ obj-y += arm/ obj-y += aspeed/ obj-y += atmel/ diff --git a/drivers/media/platform/amphion/vpu_v4l2.c b/drivers/media/platform/amphion/vpu_v4l2.c index 64fc88d89cccdc..7cccc994fc5029 100644 --- a/drivers/media/platform/amphion/vpu_v4l2.c +++ b/drivers/media/platform/amphion/vpu_v4l2.c @@ -447,17 +447,14 @@ static void vpu_m2m_device_run(void *priv) { } -static void vpu_m2m_job_abort(void *priv) +static int vpu_m2m_job_ready(void *priv) { - struct vpu_inst *inst = priv; - struct v4l2_m2m_ctx *m2m_ctx = inst->fh.m2m_ctx; - - v4l2_m2m_job_finish(m2m_ctx->m2m_dev, m2m_ctx); + return 0; } static const struct v4l2_m2m_ops vpu_m2m_ops = { .device_run = vpu_m2m_device_run, - .job_abort = vpu_m2m_job_abort + .job_ready = vpu_m2m_job_ready, }; static int vpu_vb2_queue_setup(struct vb2_queue *vq, diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig new file mode 100644 index 00000000000000..f16508bff5242a --- /dev/null +++ b/drivers/media/platform/apple/Kconfig @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only + +comment "Apple media platform drivers" + +source "drivers/media/platform/apple/isp/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile new file mode 100644 index 00000000000000..d8fe985b0e6c37 --- /dev/null +++ b/drivers/media/platform/apple/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += isp/ diff --git a/drivers/media/platform/apple/isp/.gitignore b/drivers/media/platform/apple/isp/.gitignore new file mode 100644 index 00000000000000..bd7fab40e0d98a --- /dev/null +++ b/drivers/media/platform/apple/isp/.gitignore @@ -0,0 +1 @@ +.clang-format diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig new file mode 100644 index 00000000000000..8e339db43bc418 --- /dev/null +++ b/drivers/media/platform/apple/isp/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_ISP + tristate "Apple Silicon Image Signal Processor driver" + select VIDEOBUF2_CORE + select VIDEOBUF2_V4L2 + select VIDEOBUF2_DMA_SG + select APPLE_PMP_REPORT + depends on ARCH_APPLE || COMPILE_TEST + depends on OF_ADDRESS + depends on V4L_PLATFORM_DRIVERS + depends on VIDEO_DEV diff --git a/drivers/media/platform/apple/isp/Makefile b/drivers/media/platform/apple/isp/Makefile new file mode 100644 index 00000000000000..4649f32987f025 --- /dev/null +++ b/drivers/media/platform/apple/isp/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +apple-isp-y := isp-cam.o isp-cmd.o isp-drv.o isp-fw.o isp-iommu.o isp-ipc.o isp-v4l2.o +obj-$(CONFIG_VIDEO_APPLE_ISP) += apple-isp.o diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c new file mode 100644 index 00000000000000..f84f0603260d47 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include + +#include "isp-cam.h" +#include "isp-cmd.h" +#include "isp-fw.h" +#include "isp-iommu.h" + +#define ISP_MAX_PRESETS 32 + +struct isp_setfile { + u32 version; + u32 magic; + const char *path; + size_t size; +}; + +// clang-format off +static const struct isp_setfile isp_setfiles[] = { + [ISP_IMX248_1820_01] = {0x248, 0x18200103, "apple/isp_1820_01XX.dat", 0x442c}, + [ISP_IMX248_1822_02] = {0x248, 0x18220201, "apple/isp_1822_02XX.dat", 0x442c}, + [ISP_IMX343_5221_02] = {0x343, 0x52210211, "apple/isp_5221_02XX.dat", 0x4870}, + [ISP_IMX354_9251_02] = {0x354, 0x92510208, "apple/isp_9251_02XX.dat", 0xa5ec}, + [ISP_IMX356_4820_01] = {0x356, 0x48200107, "apple/isp_4820_01XX.dat", 0x9324}, + [ISP_IMX356_4820_02] = {0x356, 0x48200206, "apple/isp_4820_02XX.dat", 0x9324}, + [ISP_IMX364_8720_01] = {0x364, 0x87200103, "apple/isp_8720_01XX.dat", 0x36ac}, + [ISP_IMX364_8723_01] = {0x364, 0x87230101, "apple/isp_8723_01XX.dat", 0x361c}, + [ISP_IMX372_3820_01] = {0x372, 0x38200108, "apple/isp_3820_01XX.dat", 0xfdb0}, + [ISP_IMX372_3820_02] = {0x372, 0x38200205, "apple/isp_3820_02XX.dat", 0xfdb0}, + [ISP_IMX372_3820_11] = {0x372, 0x38201104, "apple/isp_3820_11XX.dat", 0xfdb0}, + [ISP_IMX372_3820_12] = {0x372, 0x38201204, "apple/isp_3820_12XX.dat", 0xfdb0}, + [ISP_IMX405_9720_01] = {0x405, 0x97200102, "apple/isp_9720_01XX.dat", 0x92c8}, + [ISP_IMX405_9721_01] = {0x405, 0x97210102, "apple/isp_9721_01XX.dat", 0x9818}, + [ISP_IMX405_9723_01] = {0x405, 0x97230101, "apple/isp_9723_01XX.dat", 0x92c8}, + [ISP_IMX414_2520_01] = {0x414, 0x25200102, "apple/isp_2520_01XX.dat", 0xa444}, + [ISP_IMX503_7820_01] = {0x503, 0x78200109, "apple/isp_7820_01XX.dat", 0xb268}, + [ISP_IMX503_7820_02] = {0x503, 0x78200206, "apple/isp_7820_02XX.dat", 0xb268}, + [ISP_IMX505_3921_01] = {0x505, 0x39210102, "apple/isp_3921_01XX.dat", 0x89b0}, + [ISP_IMX514_2820_01] = {0x514, 0x28200108, "apple/isp_2820_01XX.dat", 0xa198}, + [ISP_IMX514_2820_02] = {0x514, 0x28200205, "apple/isp_2820_02XX.dat", 0xa198}, + [ISP_IMX514_2820_03] = {0x514, 0x28200305, "apple/isp_2820_03XX.dat", 0xa198}, + [ISP_IMX514_2820_04] = {0x514, 0x28200405, "apple/isp_2820_04XX.dat", 0xa198}, + [ISP_IMX558_1921_01] = {0x558, 0x19210106, "apple/isp_1921_01XX.dat", 0xad40}, + [ISP_IMX558_1922_02] = {0x558, 0x19220201, "apple/isp_1922_02XX.dat", 0xad40}, + [ISP_IMX603_7920_01] = {0x603, 0x79200109, "apple/isp_7920_01XX.dat", 0xad2c}, + [ISP_IMX603_7920_02] = {0x603, 0x79200205, "apple/isp_7920_02XX.dat", 0xad2c}, + [ISP_IMX603_7921_01] = {0x603, 0x79210104, "apple/isp_7921_01XX.dat", 0xad90}, + [ISP_IMX613_4920_01] = {0x613, 0x49200108, "apple/isp_4920_01XX.dat", 0x9324}, + [ISP_IMX613_4920_02] = {0x613, 0x49200204, "apple/isp_4920_02XX.dat", 0x9324}, + [ISP_IMX614_2921_01] = {0x614, 0x29210107, "apple/isp_2921_01XX.dat", 0xed6c}, + [ISP_IMX614_2921_02] = {0x614, 0x29210202, "apple/isp_2921_02XX.dat", 0xed6c}, + [ISP_IMX614_2922_02] = {0x614, 0x29220201, "apple/isp_2922_02XX.dat", 0xed6c}, + [ISP_IMX633_3622_01] = {0x633, 0x36220111, "apple/isp_3622_01XX.dat", 0x100d4}, + [ISP_IMX703_7721_01] = {0x703, 0x77210106, "apple/isp_7721_01XX.dat", 0x936c}, + [ISP_IMX703_7722_01] = {0x703, 0x77220106, "apple/isp_7722_01XX.dat", 0xac20}, + [ISP_IMX713_4721_01] = {0x713, 0x47210107, "apple/isp_4721_01XX.dat", 0x936c}, + [ISP_IMX713_4722_01] = {0x713, 0x47220109, "apple/isp_4722_01XX.dat", 0x9218}, + [ISP_IMX714_2022_01] = {0x714, 0x20220107, "apple/isp_2022_01XX.dat", 0xa198}, + [ISP_IMX772_3721_01] = {0x772, 0x37210106, "apple/isp_3721_01XX.dat", 0xfdf8}, + [ISP_IMX772_3721_11] = {0x772, 0x37211106, "apple/isp_3721_11XX.dat", 0xfe14}, + [ISP_IMX772_3722_01] = {0x772, 0x37220104, "apple/isp_3722_01XX.dat", 0xfca4}, + [ISP_IMX772_3723_01] = {0x772, 0x37230106, "apple/isp_3723_01XX.dat", 0xfca4}, + [ISP_IMX814_2123_01] = {0x814, 0x21230101, "apple/isp_2123_01XX.dat", 0xed54}, + [ISP_IMX853_7622_01] = {0x853, 0x76220112, "apple/isp_7622_01XX.dat", 0x247f8}, + [ISP_IMX913_7523_01] = {0x913, 0x75230107, "apple/isp_7523_01XX.dat", 0x247f8}, + [ISP_VD56G0_6221_01] = {0xd56, 0x62210102, "apple/isp_6221_01XX.dat", 0x1b80}, + [ISP_VD56G0_6222_01] = {0xd56, 0x62220102, "apple/isp_6222_01XX.dat", 0x1b80}, +}; +// clang-format on + +static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + enum isp_sensor_id id; + int err = 0; + + /* TODO need more datapoints to figure out the sub-versions + * Defaulting to 1st release for now, the calib files aren't too different. + */ + switch (fmt->version) { + case 0x248: + id = ISP_IMX248_1820_01; + break; + case 0x343: + id = ISP_IMX343_5221_02; + break; + case 0x354: + id = ISP_IMX354_9251_02; + break; + case 0x356: + id = ISP_IMX356_4820_01; + break; + case 0x364: + id = ISP_IMX364_8720_01; + break; + case 0x372: + id = ISP_IMX372_3820_01; + break; + case 0x405: + id = ISP_IMX405_9720_01; + break; + case 0x414: + id = ISP_IMX414_2520_01; + break; + case 0x503: + id = ISP_IMX503_7820_01; + break; + case 0x505: + id = ISP_IMX505_3921_01; + break; + case 0x514: + id = ISP_IMX514_2820_01; + break; + case 0x558: + id = ISP_IMX558_1921_01; + break; + case 0x603: + id = ISP_IMX603_7920_01; + break; + case 0x613: + id = ISP_IMX613_4920_01; + break; + case 0x614: + id = ISP_IMX614_2921_01; + break; + case 0x633: + id = ISP_IMX633_3622_01; + break; + case 0x703: + id = ISP_IMX703_7721_01; + break; + case 0x713: + id = ISP_IMX713_4721_01; + break; + case 0x714: + id = ISP_IMX714_2022_01; + break; + case 0x772: + id = ISP_IMX772_3721_01; + break; + case 0x814: + id = ISP_IMX814_2123_01; + break; + case 0x853: + id = ISP_IMX853_7622_01; + break; + case 0x913: + id = ISP_IMX913_7523_01; + break; + case 0xd56: + id = ISP_VD56G0_6221_01; + break; + default: + err = -EINVAL; + break; + } + + if (err) + dev_err(isp->dev, "invalid sensor version: 0x%x\n", + fmt->version); + else + fmt->id = id; + + return err; +} + +static int isp_ch_get_camera_preset(struct apple_isp *isp, u32 ch, u32 ps) +{ + int err = 0; + + struct cmd_ch_camera_config *args; /* Too big to allocate on stack */ + args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!args) + return -ENOMEM; + + err = isp_cmd_ch_camera_config_get(isp, ch, ps, args); + if (err) + goto exit; + + pr_info("apple-isp: ps: CISP_CMD_CH_CAMERA_CONFIG_GET: %d\n", ps); + print_hex_dump(KERN_INFO, "apple-isp: ps: ", DUMP_PREFIX_NONE, 32, 4, + args, sizeof(*args), false); + +exit: + kfree(args); + + return err; +} + +static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err = 0; + + struct cmd_ch_info *args; /* Too big to allocate on stack */ + args = kzalloc(sizeof(*args), GFP_KERNEL); + if (!args) + return -ENOMEM; + + err = isp_cmd_ch_info_get(isp, ch, args); + if (err) + goto exit; + + dev_info(isp->dev, "found sensor %x %s on ch %d\n", args->version, + args->module_sn, ch); + + fmt->version = args->version; + + pr_info("apple-isp: ch: CISP_CMD_CH_INFO_GET: %d\n", ch); + print_hex_dump(KERN_INFO, "apple-isp: ch: ", DUMP_PREFIX_NONE, 32, 4, + args, sizeof(*args), false); + + for (u32 ps = 0; ps < args->num_presets; ps++) { + isp_ch_get_camera_preset(isp, ch, ps); + } + + err = isp_ch_get_sensor_id(isp, ch); + if (err || + (fmt->id != ISP_IMX248_1820_01 && fmt->id != ISP_IMX558_1921_01 && + fmt->id != ISP_IMX364_8720_01)) { + dev_err(isp->dev, + "ch %d: unsupported sensor. Please file a bug report with hardware info & dmesg trace.\n", + ch); + return -ENODEV; + } + +exit: + kfree(args); + + return err; +} + +static int isp_detect_camera(struct apple_isp *isp) +{ + int err; + + struct cmd_config_get args; + memset(&args, 0, sizeof(args)); + + err = isp_cmd_config_get(isp, &args); + if (err) + return err; + + pr_info("apple-isp: CISP_CMD_CONFIG_GET: \n"); + print_hex_dump(KERN_INFO, "apple-isp: ", DUMP_PREFIX_NONE, 32, 4, &args, + sizeof(args), false); + + if (!args.num_channels) { + dev_err(isp->dev, "did not detect any channels\n"); + return -ENODEV; + } + + if (args.num_channels > ISP_MAX_CHANNELS) { + dev_warn(isp->dev, "found %d channels when maximum is %d\n", + args.num_channels, ISP_MAX_CHANNELS); + args.num_channels = ISP_MAX_CHANNELS; + } + + if (args.num_channels > 1) { + dev_warn( + isp->dev, + "warning: driver doesn't support multiple channels. Please file a bug report with hardware info & dmesg trace.\n"); + } + + isp->num_channels = args.num_channels; + isp->current_ch = 0; + + err = isp_ch_cache_sensor_info(isp, isp->current_ch); + if (err) { + dev_err(isp->dev, "failed to cache sensor info\n"); + return err; + } + + return 0; +} + +int apple_isp_detect_camera(struct apple_isp *isp) +{ + int err; + + /* RPM must be enabled prior to calling this */ + err = apple_isp_firmware_boot(isp); + if (err) { + dev_err(isp->dev, + "failed to boot firmware for initial sensor detection: %d\n", + err); + return -EPROBE_DEFER; + } + + err = isp_detect_camera(isp); + + isp_cmd_flicker_sensor_set(isp, 0); + + isp_cmd_ch_stop(isp, 0); + isp_cmd_ch_buffer_return(isp, isp->current_ch); + + apple_isp_firmware_shutdown(isp); + + return err; +} + +static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + const struct isp_setfile *setfile = &isp_setfiles[fmt->id]; + const struct firmware *fw; + u32 magic; + int err; + + err = request_firmware(&fw, setfile->path, isp->dev); + if (err) { + dev_err(isp->dev, "failed to request setfile '%s': %d\n", + setfile->path, err); + return err; + } + + if (fw->size < setfile->size) { + dev_err(isp->dev, "setfile too small (0x%zx/0x%zx)\n", fw->size, + setfile->size); + release_firmware(fw); + return -EINVAL; + } + + magic = be32_to_cpup((__be32 *)fw->data); + if (magic != setfile->magic) { + dev_err(isp->dev, "setfile '%s' corrupted?\n", setfile->path); + release_firmware(fw); + return -EINVAL; + } + + memcpy(isp->data_surf->virt, (void *)fw->data, setfile->size); + release_firmware(fw); + + return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, + setfile->size); +} + +static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err; + + isp_cmd_flicker_sensor_set(isp, 0); + + /* The setfile isn't requisite but then we don't get calibration */ + err = isp_ch_load_setfile(isp, ch); + if (err) { + dev_err(isp->dev, "warning: calibration data not loaded: %d\n", + err); + + /* If this failed due to a signal, propagate */ + if (err == -EINTR) + return err; + } + + if (isp->hw->lpdp) { + err = isp_cmd_ch_lpdp_hs_receiver_tuning_set(isp, ch, 1, 15); + if (err) + return err; + } + + err = isp_cmd_ch_sbs_enable(isp, ch, 1); + if (err) + return err; + + err = isp_cmd_ch_camera_config_select(isp, ch, fmt->preset->index); + if (err) + return err; + + err = isp_cmd_ch_buffer_recycle_mode_set( + isp, ch, CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY); + if (err) + return err; + + err = isp_cmd_ch_buffer_recycle_start(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_crop_set(isp, ch, fmt->preset->crop_offset.x, + fmt->preset->crop_offset.y, + fmt->preset->crop_size.x, + fmt->preset->crop_size.y); + if (err) + return err; + + err = isp_cmd_ch_output_config_set(isp, ch, fmt->preset->output_dim.x, + fmt->preset->output_dim.y, + fmt->strides, CISP_COLORSPACE_REC709, + CISP_OUTPUT_FORMAT_YUV_2PLANE); + if (err) + return err; + + err = isp_cmd_ch_preview_stream_set(isp, ch, 1); + if (err) + return err; + + err = isp_cmd_ch_cnr_start(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_mbnr_enable(isp, ch, 0, ISP_MBNR_MODE_ENABLE, 1); + if (err) + return err; + + err = isp_cmd_apple_ch_ae_fd_scene_metering_config_set(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_ae_metering_mode_set(isp, ch, 3); + if (err) + return err; + + err = isp_cmd_ch_ae_stability_set(isp, ch, 32); + if (err) + return err; + + err = isp_cmd_ch_ae_stability_to_stable_set(isp, ch, 20); + if (err) + return err; + + err = isp_cmd_ch_sif_pixel_format_set(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_ae_frame_rate_max_set(isp, ch, ISP_FRAME_RATE_DEN); + if (err) + return err; + + err = isp_cmd_ch_ae_frame_rate_min_set(isp, ch, ISP_FRAME_RATE_DEN2); + if (err) + return err; + + err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, isp->temporal_filter); + if (err) + return err; + + err = isp_cmd_apple_ch_motion_history_start(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_temporal_filter_enable(isp, ch); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); + if (err) + return err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, + CISP_POOL_TYPE_META_CAPTURE); + if (err) + return err; + + return 0; +} + +static int isp_configure_capture(struct apple_isp *isp) +{ + return isp_ch_configure_capture(isp, isp->current_ch); +} + +int apple_isp_start_camera(struct apple_isp *isp) +{ + int err; + + err = apple_isp_firmware_boot(isp); + if (err < 0) { + dev_err(isp->dev, "failed to boot firmware: %d\n", err); + return err; + } + + err = isp_configure_capture(isp); + if (err) { + dev_err(isp->dev, "failed to configure capture: %d\n", err); + apple_isp_firmware_shutdown(isp); + return err; + } + + return 0; +} + +void apple_isp_stop_camera(struct apple_isp *isp) +{ + apple_isp_firmware_shutdown(isp); +} + +int apple_isp_start_capture(struct apple_isp *isp) +{ + return isp_cmd_ch_start(isp, 0); // TODO channel mask +} + +void apple_isp_stop_capture(struct apple_isp *isp) +{ + isp_cmd_ch_stop(isp, 0); // TODO channel mask + isp_cmd_ch_buffer_return(isp, isp->current_ch); +} diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h new file mode 100644 index 00000000000000..f4fa4224c7a934 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_CAM_H__ +#define __ISP_CAM_H__ + +#include "isp-drv.h" + +#define ISP_FRAME_RATE_NUM 256 +#define ISP_FRAME_RATE_DEN 7680 +#define ISP_FRAME_RATE_DEN2 3840 + +int apple_isp_detect_camera(struct apple_isp *isp); + +int apple_isp_start_camera(struct apple_isp *isp); +void apple_isp_stop_camera(struct apple_isp *isp); + +int apple_isp_start_capture(struct apple_isp *isp); +void apple_isp_stop_capture(struct apple_isp *isp); + +#endif /* __ISP_CAM_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c new file mode 100644 index 00000000000000..ee491d2cb42c5b --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -0,0 +1,634 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-cmd.h" +#include "isp-drv.h" +#include "isp-iommu.h" +#include "isp-ipc.h" + +#define CISP_OPCODE_SHIFT 32UL +#define CISP_OPCODE(x) (((u64)(x)) << CISP_OPCODE_SHIFT) +#define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) + +#define CISP_TIMEOUT msecs_to_jiffies(3000) +#define CISP_SEND_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) +#define CISP_SEND_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) +#define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) +#define CISP_POST_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, 0)) +#define CISP_POST_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), 0)) + +static int cisp_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +{ + struct isp_channel *chan = isp->chan_io; + struct isp_message *req = &chan->req; + int err; + + req->arg0 = isp->cmd_iova; + req->arg1 = insize; + req->arg2 = outsize; + + memcpy(isp->cmd_virt, args, insize); + err = ipc_chan_send(isp, chan, timeout); + if (err) { + u64 opcode; + memcpy(&opcode, args, sizeof(opcode)); + dev_err(isp->dev, + "%s: failed to send OPCODE 0x%04llx: [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, CISP_OPCODE_GET(opcode), req->arg0, + req->arg1, req->arg2); + } + + return err; +} + +static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, + u32 outsize) +{ + /* TODO do I need to lock the iova space? */ + int err = cisp_send(isp, args, insize, outsize, CISP_TIMEOUT); + if (err) + return err; + + memcpy(args, isp->cmd_virt, outsize); + return 0; +} + +int isp_cmd_start(struct apple_isp *isp, u32 mode) +{ + struct cmd_start args = { + .opcode = CISP_OPCODE(CISP_CMD_START), + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_stop(struct apple_isp *isp, u32 mode) +{ + struct cmd_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_STOP), + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_power_down(struct apple_isp *isp) +{ + struct cmd_power_down args = { + .opcode = CISP_OPCODE(CISP_CMD_POWER_DOWN), + }; + return CISP_POST_INOUT(isp, args); +} + +int isp_cmd_suspend(struct apple_isp *isp) +{ + struct cmd_suspend args = { + .opcode = CISP_OPCODE(CISP_CMD_SUSPEND), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_print_enable(struct apple_isp *isp, u32 enable) +{ + struct cmd_print_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_PRINT_ENABLE), + .enable = enable, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable) +{ + struct cmd_trace_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_TRACE_ENABLE), + .enable = enable, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CONFIG_GET); + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base) +{ + struct cmd_set_isp_pmu_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_ISP_PMU_BASE), + .pmu_base = pmu_base, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, + u64 dsid_clr_base1, u64 dsid_clr_base2, + u64 dsid_clr_base3, u32 dsid_clr_range0, + u32 dsid_clr_range1, u32 dsid_clr_range2, + u32 dsid_clr_range3) +{ + struct cmd_set_dsid_clr_req_base2 args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_REG_BASE2), + .dsid_clr_base0 = dsid_clr_base0, + .dsid_clr_base1 = dsid_clr_base1, + .dsid_clr_base2 = dsid_clr_base2, + .dsid_clr_base3 = dsid_clr_base3, + .dsid_clr_range0 = dsid_clr_range0, + .dsid_clr_range1 = dsid_clr_range1, + .dsid_clr_range2 = dsid_clr_range2, + .dsid_clr_range3 = dsid_clr_range3, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range) +{ + struct cmd_set_dsid_clr_req_base args = { + .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_REG_BASE), + .dsid_clr_base = dsid_clr_base, + .dsid_clr_range = dsid_clr_range, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, + u64 clock_base, u8 clock_bit, u8 clock_size, + u64 bandwidth_scratch, u64 bandwidth_base, + u8 bandwidth_bit, u8 bandwidth_size) +{ + struct cmd_pmp_ctrl_set args = { + .opcode = CISP_OPCODE(CISP_CMD_PMP_CTRL_SET), + .clock_scratch = clock_scratch, + .clock_base = clock_base, + .clock_bit = clock_bit, + .clock_size = clock_size, + .clock_pad = 0, + .bandwidth_scratch = bandwidth_scratch, + .bandwidth_base = bandwidth_base, + .bandwidth_bit = bandwidth_bit, + .bandwidth_size = bandwidth_size, + .bandwidth_pad = 0, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_fid_enter(struct apple_isp *isp) +{ + struct cmd_fid_enter args = { + .opcode = CISP_OPCODE(CISP_CMD_FID_ENTER), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_fid_exit(struct apple_isp *isp) +{ + struct cmd_fid_exit args = { + .opcode = CISP_OPCODE(CISP_CMD_FID_EXIT), + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode) +{ + struct cmd_flicker_sensor_set args = { + .opcode = CISP_OPCODE(CISP_CMD_FLICKER_SENSOR_SET), + .mode = mode, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_info *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_INFO_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, + struct cmd_ch_camera_config *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_GET); + args->preset = preset; + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_camera_config *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_CURRENT_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, u32 preset) +{ + struct cmd_ch_camera_config_select args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_SELECT), + .chan = chan, + .preset = preset, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_return args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RETURN), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, + u32 size) +{ + if (isp->fw_compat >= ISP_FIRMWARE_V_13_5) { + struct cmd_ch_set_file_load64 args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), + .chan = chan, + .addr = addr, + .size = size, + }; + return CISP_SEND_IN(isp, args); + } else { + struct cmd_ch_set_file_load args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SET_FILE_LOAD), + .chan = chan, + .addr = addr, + .size = size, + }; + return CISP_SEND_IN(isp, args); + } +} + +int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable) +{ + struct cmd_ch_sbs_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SBS_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, + u32 y2) +{ + struct cmd_ch_crop_set args = { + .opcode = CISP_OPCODE(isp->hw->scl1 ? CISP_CMD_CH_CROP_SCL1_SET + : CISP_CMD_CH_CROP_SET), + .chan = chan, + .x1 = x1, + .y1 = y1, + .x2 = x2, + .y2 = y2, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, + u32 height, u32 strides[3], u32 colorspace, u32 format) +{ + struct cmd_ch_output_config_set args = { + .opcode = CISP_OPCODE(isp->hw->scl1 ? CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET + : CISP_CMD_CH_OUTPUT_CONFIG_SET), + .chan = chan, + .width = width, + .height = height, + .colorspace = colorspace, + .format = format, + .padding_rows = 0, + .unk_h0 = height, + .compress = 0, + .unk_w2 = width, + }; + memcpy(args.strides, strides, sizeof(args.strides)); + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_preview_stream_set(struct apple_isp *isp, u32 chan, u32 stream) +{ + struct cmd_ch_preview_stream_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PREVIEW_STREAM_SET), + .chan = chan, + .stream = stream, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_als_disable(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_als_disable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_ALS_DISABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_cnr_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_cnr_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_CNR_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_mbnr_enable(struct apple_isp *isp, u32 chan, u32 use_case, + u32 mode, u32 enable_chroma) +{ + struct cmd_ch_mbnr_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_MBNR_ENABLE), + .chan = chan, + .use_case = use_case, + .mode = mode, + .enable_chroma = enable_chroma, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_sif_pixel_format_set(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_sif_pixel_format_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SIF_PIXEL_FORMAT_SET), + .chan = chan, + .format = 3, + .type = 1, + .compress = 0, + .unk_10 = 0, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_ch_buffer_recycle_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RECYCLE_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_recycle_start args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_RECYCLE_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type) +{ + struct cmd_ch_buffer_pool_config_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), + .chan = chan, + .type = type, + .count = ISP_MAX_BUFFERS, + .meta_size0 = isp->hw->meta_size, + .meta_size1 = isp->hw->meta_size, + .unk0 = 0, + .unk1 = 0, + .unk2 = 0, + .data_blocks = 1, + .compress = 0, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan) +{ + struct cmd_ch_buffer_pool_return args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_RETURN), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_start(struct apple_isp *isp, u32 chan, u32 arg) +{ + struct cmd_apple_ch_temporal_filter_start args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START), + .chan = chan, + .unk_c = 1, + .unk_10 = arg, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_motion_history_start args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_MOTION_HISTORY_START), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_motion_history_stop args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_disable args = { + .opcode = + CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE), + .chan = chan, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability) +{ + struct cmd_ch_ae_stability_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_STABILITY_SET), + .chan = chan, + .stability = stability, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_stability_to_stable_set(struct apple_isp *isp, u32 chan, + u32 stability) +{ + struct cmd_ch_ae_stability_to_stable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_STABILITY_TO_STABLE_SET), + .chan = chan, + .stability = stability, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_max_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_ae_frame_rate_max_get *args) +{ + args->opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MAX_GET); + args->chan = chan; + return CISP_SEND_OUT(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_max_set(struct apple_isp *isp, u32 chan, + u32 framerate) +{ + struct cmd_ch_ae_frame_rate_max_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MAX_SET), + .chan = chan, + .framerate = framerate, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_ae_frame_rate_min_set(struct apple_isp *isp, u32 chan, + u32 framerate) +{ + struct cmd_ch_ae_frame_rate_min_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_AE_FRAME_RATE_MIN_SET), + .chan = chan, + .framerate = framerate, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, + u32 chan) +{ + struct cmd_apple_ch_ae_fd_scene_metering_config_set args = { + .opcode = CISP_OPCODE( + CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET), + .chan = chan, + .unk_c = 0xb8, + .unk_10 = 0x2000200, + .unk_14 = 0x280800, + .unk_18 = 0xe10028, + .unk_1c = 0xa0399, + .unk_20 = 0x3cc02cc, + }; + return CISP_SEND_INOUT(isp, args); +} + +int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, + u32 mode) +{ + struct cmd_apple_ch_ae_metering_mode_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_AE_METERING_MODE_SET), + .chan = chan, + .mode = mode, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, + u32 chan, u32 freq) +{ + struct cmd_apple_ch_ae_flicker_freq_update_current_set args = { + .opcode = CISP_OPCODE( + CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET), + .chan = chan, + .freq = freq, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_semantic_video_enable(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_ch_semantic_video_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SEMANTIC_VIDEO_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_semantic_awb_enable(struct apple_isp *isp, u32 chan, u32 enable) +{ + struct cmd_ch_semantic_awb_enable args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_SEMANTIC_AWB_ENABLE), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_lpdp_hs_receiver_tuning_set(struct apple_isp *isp, u32 chan, u32 unk1, u32 unk2) +{ + struct cmd_ch_lpdp_hs_receiver_tuning_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_LPDP_HS_RECEIVER_TUNING_SET), + .chan = chan, + .unk1 = unk1, + .unk2 = unk2, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_property_write(struct apple_isp *isp, u32 chan, u32 prop, u32 val) +{ + struct cmd_ch_property_write args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PROPERTY_WRITE), + .chan = chan, + .prop = prop, + .val = val, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_ch_property_read(struct apple_isp *isp, u32 chan, u32 prop, u32 *val) +{ + struct cmd_ch_property_write args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_PROPERTY_READ), + .chan = chan, + .prop = prop, + .val = 0xdeadbeef, + }; + int ret = CISP_SEND_OUT(isp, &args); + + *val = args.val; + + return ret; +} diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h new file mode 100644 index 00000000000000..5a3c8cd9177e48 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_CMD_H__ +#define __ISP_CMD_H__ + +#include "isp-drv.h" + +#define CISP_CMD_START 0x0000 +#define CISP_CMD_STOP 0x0001 +#define CISP_CMD_CONFIG_GET 0x0003 +#define CISP_CMD_PRINT_ENABLE 0x0004 +#define CISP_CMD_BUILDINFO 0x0006 +#define CISP_CMD_GET_BES_PARAM 0x000f +#define CISP_CMD_POWER_DOWN 0x0010 +#define CISP_CMD_SET_ISP_PMU_BASE 0x0011 +#define CISP_CMD_PMP_CTRL_SET 0x001c +#define CISP_CMD_TRACE_ENABLE 0x001d +#define CISP_CMD_SUSPEND 0x0021 +#define CISP_CMD_FID_ENTER 0x0022 +#define CISP_CMD_FID_EXIT 0x0023 +#define CISP_CMD_FLICKER_SENSOR_SET 0x0024 +#define CISP_CMD_CH_START 0x0100 +#define CISP_CMD_CH_STOP 0x0101 +#define CISP_CMD_CH_BUFFER_RETURN 0x0104 +#define CISP_CMD_CH_CAMERA_CONFIG_CURRENT_GET 0x0105 +#define CISP_CMD_CH_CAMERA_CONFIG_GET 0x0106 +#define CISP_CMD_CH_CAMERA_CONFIG_SELECT 0x0107 +#define CISP_CMD_CH_INFO_GET 0x010d +#define CISP_CMD_CH_BUFFER_RECYCLE_MODE_SET 0x010e +#define CISP_CMD_CH_BUFFER_RECYCLE_START 0x010f +#define CISP_CMD_CH_BUFFER_RECYCLE_STOP 0x0110 +#define CISP_CMD_CH_SET_FILE_LOAD 0x0111 +#define CISP_CMD_CH_SIF_PIXEL_FORMAT_SET 0x0115 +#define CISP_CMD_CH_BUFFER_POOL_CONFIG_GET 0x0116 +#define CISP_CMD_CH_BUFFER_POOL_CONFIG_SET 0x0117 +#define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_GET 0x011a +#define CISP_CMD_CH_CAMERA_PIX_FREQUENCY_GET 0x011f +#define CISP_CMD_CH_PROPERTY_WRITE 0x0122 +#define CISP_CMD_CH_PROPERTY_READ 0x0123 +#define CISP_CMD_CH_LOCAL_RAW_BUFFER_ENABLE 0x0125 +#define CISP_CMD_CH_META_DATA_ENABLE 0x0126 +#define CISP_CMD_CH_CAMERA_MIPI_FREQUENCY_TOTAL_GET 0x0133 +#define CISP_CMD_CH_SBS_ENABLE 0x013b +#define CISP_CMD_CH_LSC_POLYNOMIAL_COEFF_GET 0x0142 +#define CISP_CMD_CH_SET_META_DATA_REQUIRED 0x014f +#define CISP_CMD_CH_BUFFER_POOL_RETURN 0x015b +#define CISP_CMD_CH_CAMERA_AGILE_FREQ_ARRAY_CURRENT_GET 0x015e +#define CISP_CMD_CH_AE_START 0x0200 +#define CISP_CMD_CH_AE_STOP 0x0201 +#define CISP_CMD_CH_AE_FRAME_RATE_MAX_GET 0x0207 +#define CISP_CMD_CH_AE_FRAME_RATE_MAX_SET 0x0208 +#define CISP_CMD_CH_AE_FRAME_RATE_MIN_GET 0x0209 +#define CISP_CMD_CH_AE_FRAME_RATE_MIN_SET 0x020a +#define CISP_CMD_CH_AE_STABILITY_SET 0x021a +#define CISP_CMD_CH_AE_STABILITY_TO_STABLE_SET 0x0229 +#define CISP_CMD_CH_SENSOR_NVM_GET 0x0501 +#define CISP_CMD_CH_SENSOR_PERMODULE_LSC_INFO_GET 0x0507 +#define CISP_CMD_CH_SENSOR_PERMODULE_LSC_GRID_GET 0x0511 +#define CISP_CMD_CH_LPDP_HS_RECEIVER_TUNING_SET 0x051b +#define CISP_CMD_CH_FOCUS_LIMITS_GET 0x0701 +#define CISP_CMD_CH_CROP_GET 0x0800 +#define CISP_CMD_CH_CROP_SET 0x0801 +#define CISP_CMD_CH_SCALER_CROP_SET 0x080a +#define CISP_CMD_CH_CROP_SCL1_GET 0x080b +#define CISP_CMD_CH_CROP_SCL1_SET 0x080c +#define CISP_CMD_CH_SCALER_CROP_SCL1_SET 0x080d +#define CISP_CMD_CH_ALS_ENABLE 0x0a1c +#define CISP_CMD_CH_ALS_DISABLE 0x0a1d +#define CISP_CMD_CH_CNR_START 0x0a2f +#define CISP_CMD_CH_MBNR_ENABLE 0x0a3a +#define CISP_CMD_CH_OUTPUT_CONFIG_SET 0x0b01 +#define CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET 0x0b09 +#define CISP_CMD_CH_PREVIEW_STREAM_SET 0x0b0d +#define CISP_CMD_CH_SEMANTIC_VIDEO_ENABLE 0x0b17 +#define CISP_CMD_CH_SEMANTIC_AWB_ENABLE 0x0b18 +#define CISP_CMD_CH_FACE_DETECTION_START 0x0d00 +#define CISP_CMD_CH_FACE_DETECTION_STOP 0x0d01 +#define CISP_CMD_CH_FACE_DETECTION_CONFIG_GET 0x0d02 +#define CISP_CMD_CH_FACE_DETECTION_CONFIG_SET 0x0d03 +#define CISP_CMD_CH_FACE_DETECTION_DISABLE 0x0d04 +#define CISP_CMD_CH_FACE_DETECTION_ENABLE 0x0d05 +#define CISP_CMD_CH_FID_START 0x3000 +#define CISP_CMD_CH_FID_STOP 0x3001 +#define CISP_CMD_IPC_ENDPOINT_SET2 0x300c +#define CISP_CMD_IPC_ENDPOINT_UNSET2 0x300d +#define CISP_CMD_SET_DSID_CLR_REG_BASE2 0x3204 +#define CISP_CMD_SET_DSID_CLR_REG_BASE 0x3205 +#define CISP_CMD_APPLE_CH_AE_METERING_MODE_SET 0x8206 +#define CISP_CMD_APPLE_CH_AE_FD_SCENE_METERING_CONFIG_SET 0x820e +#define CISP_CMD_APPLE_CH_AE_FLICKER_FREQ_UPDATE_CURRENT_SET 0x8212 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START 0xc100 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_STOP 0xc101 +#define CISP_CMD_APPLE_CH_MOTION_HISTORY_START 0xc102 +#define CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP 0xc103 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE 0xc113 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE 0xc114 + +#define CISP_POOL_TYPE_META 0x0 +#define CISP_POOL_TYPE_RENDERED 0x1 +#define CISP_POOL_TYPE_FD 0x2 +#define CISP_POOL_TYPE_RAW 0x3 +#define CISP_POOL_TYPE_STAT 0x4 +#define CISP_POOL_TYPE_RAW_AUX 0x5 +#define CISP_POOL_TYPE_YCC 0x6 +#define CISP_POOL_TYPE_CAPTURE_FULL_RES 0x7 +#define CISP_POOL_TYPE_META_CAPTURE 0x8 +#define CISP_POOL_TYPE_RENDERED_SCL1 0x9 +#define CISP_POOL_TYPE_STAT_PIXELOUTPUT 0x11 +#define CISP_POOL_TYPE_FSCL 0x12 +#define CISP_POOL_TYPE_CAPTURE_FULL_RES_YCC 0x13 +#define CISP_POOL_TYPE_RENDERED_RAW 0x14 +#define CISP_POOL_TYPE_CAPTURE_PDC_RAW 0x16 +#define CISP_POOL_TYPE_FPC_DATA 0x17 +#define CISP_POOL_TYPE_AICAM_SEG 0x19 +#define CISP_POOL_TYPE_SPD 0x1a +#define CISP_POOL_TYPE_META_DEPTH 0x1c +#define CISP_POOL_TYPE_JASPER_DEPTH 0x1d +#define CISP_POOL_TYPE_RAW_SIFR 0x1f +#define CISP_POOL_TYPE_FEP_THUMBNAIL_DYNAMIC_POOL_RAW 0x21 + +#define CISP_COLORSPACE_REC709 0x1 +#define CISP_OUTPUT_FORMAT_YUV_2PLANE 0x0 +#define CISP_OUTPUT_FORMAT_YUV_1PLANE 0x1 +#define CISP_OUTPUT_FORMAT_RGB 0x2 +#define CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY 0x1 + +struct cmd_start { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_start) == 0xc); + +struct cmd_stop { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_stop) == 0xc); + +struct cmd_power_down { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_power_down) == 0x8); + +struct cmd_suspend { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_suspend) == 0x8); + +struct cmd_print_enable { + u64 opcode; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_print_enable) == 0xc); + +struct cmd_trace_enable { + u64 opcode; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_trace_enable) == 0xc); + +struct cmd_config_get { + u64 opcode; + u32 timestamp_freq; + u32 num_channels; + u32 unk_10; + u32 unk_14; + u32 unk_18; +} __packed; +static_assert(sizeof(struct cmd_config_get) == 0x1c); + +struct cmd_set_isp_pmu_base { + u64 opcode; + u64 pmu_base; +} __packed; +static_assert(sizeof(struct cmd_set_isp_pmu_base) == 0x10); + +struct cmd_set_dsid_clr_req_base2 { + u64 opcode; + u64 dsid_clr_base0; + u64 dsid_clr_base1; + u64 dsid_clr_base2; + u64 dsid_clr_base3; + u32 dsid_clr_range0; + u32 dsid_clr_range1; + u32 dsid_clr_range2; + u32 dsid_clr_range3; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_req_base2) == 0x38); + +struct cmd_set_dsid_clr_req_base { + u64 opcode; + u64 dsid_clr_base; + u32 dsid_clr_range; +} __packed; +static_assert(sizeof(struct cmd_set_dsid_clr_req_base) == 0x14); + +struct cmd_pmp_ctrl_set { + u64 opcode; + u64 clock_scratch; + u64 clock_base; + u8 clock_bit; + u8 clock_size; + u16 clock_pad; + u64 bandwidth_scratch; + u64 bandwidth_base; + u8 bandwidth_bit; + u8 bandwidth_size; + u16 bandwidth_pad; +} __packed; +static_assert(sizeof(struct cmd_pmp_ctrl_set) == 0x30); + +struct cmd_fid_enter { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_fid_enter) == 0x8); + +struct cmd_fid_exit { + u64 opcode; +} __packed; +static_assert(sizeof(struct cmd_fid_exit) == 0x8); + +struct cmd_ipc_endpoint_set2 { + u64 opcode; + u32 unk; + u64 addr1; + u32 size1; + u64 addr2; + u32 size2; + u64 regs; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ipc_endpoint_set2) == 0x30); + +struct cmd_flicker_sensor_set { + u64 opcode; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_flicker_sensor_set) == 0xc); + +int isp_cmd_start(struct apple_isp *isp, u32 mode); +int isp_cmd_stop(struct apple_isp *isp, u32 mode); +int isp_cmd_power_down(struct apple_isp *isp); +int isp_cmd_suspend(struct apple_isp *isp); +int isp_cmd_print_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_trace_enable(struct apple_isp *isp, u32 enable); +int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); +int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); +int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, + u32 dsid_clr_range); +int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, + u64 dsid_clr_base1, u64 dsid_clr_base2, + u64 dsid_clr_base3, u32 dsid_clr_range0, + u32 dsid_clr_range1, u32 dsid_clr_range2, + u32 dsid_clr_range3); +int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, + u64 clock_base, u8 clock_bit, u8 clock_size, + u64 bandwidth_scratch, u64 bandwidth_base, + u8 bandwidth_bit, u8 bandwidth_size); +int isp_cmd_fid_enter(struct apple_isp *isp); +int isp_cmd_fid_exit(struct apple_isp *isp); +int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode); + +struct cmd_ch_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_start) == 0xc); + +struct cmd_ch_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_stop) == 0xc); + +struct cmd_ch_info { + u64 opcode; + u32 chan; + u32 unk_c; // 0x7da0001, 0x7db0001 + u32 unk_10; // 0x300ac, 0x5006d + u32 unk_14; // 0x40007, 0x10007 + u32 unk_18; // 0x5, 0x2 + u32 unk_1c; // 0x1, 0x1 + u32 version; + u32 unk_24; // 0x7, 0x9 + u32 unk_28; // 0x1, 0x1410 + u32 unk_2c; // 0x7, 0x2 + u32 pad_30[7]; + u32 unk_4c; // 0x10000, 0x50000 + u32 unk_50; // 0x1, 0x1 + u32 unk_54; // 0x0, 0x0 + u32 unk_58; // 0x4, 0x4 + u32 unk_5c; // 0x10, 0x20 + u32 num_presets; + u32 unk_64; // 0x0, 0x0 + u32 unk_68; // 0x44c0, 0x4680 + u32 unk_6c; // 0x40, 0x40 + u32 unk_70; // 0x1, 0x1 + u32 unk_74; // 0x2, 0x2 + u32 unk_78; // 0x4000, 0x4000 + u32 unk_7c; // 0x40, 0x40 + u32 unk_80; // 0x1, 0x1 + u32 pad_84[2]; + u32 unk_8c; // 0x36, 0x36 + u32 pad_90[2]; + u32 timestamp_freq; + u16 pad_9c; + char module_sn[20]; + u16 pad_b0; + u32 unk_b4; // 0x8, 0x8 + u32 pad_b8[2]; + u32 unk_c0; // 0x4, 0x1 + u32 unk_c4; // 0x0, 0x0 + u32 unk_c8; // 0x0, 0x100 + u32 pad_cc[4]; + u32 unk_dc; // 0xff0000, 0xff0000 + u32 unk_e0; // 0xc00, 0xc00 + u32 unk_e4; // 0x0, 0x0 + u32 unk_e8; // 0x1c, 0x1c + u32 unk_ec; // 0x640, 0x680 + u32 unk_f0; // 0x4, 0x4 + u32 unk_f4; // 0x4, 0x4 + u32 pad_f8[6]; + u32 unk_110; // 0x0, 0x7800000 + u32 unk_114; // 0x0, 0x780 +} __packed; +static_assert(sizeof(struct cmd_ch_info) == 0x118); + +struct cmd_ch_camera_config { + u64 opcode; + u32 chan; + u32 preset; + u16 in_width; + u16 in_height; + u16 out_width; + u16 out_height; + u32 unk_28; + u32 unk_2c; + u32 unk_30[16]; + u32 sensor_clk; + u32 unk_64[4]; + u32 timestamp_freq; + u32 unk_78[2]; + u32 unk_80[16]; + u32 in_width2; // repeated in u32?? + u32 in_height2; + u32 unk_c8[3]; + u32 out_width2; + u32 out_height2; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config) == 0xdc); + +struct cmd_ch_camera_config_select { + u64 opcode; + u32 chan; + u32 preset; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config_select) == 0x10); + +struct cmd_ch_set_file_load { + u64 opcode; + u32 chan; + u32 addr; + u32 size; +} __packed; +static_assert(sizeof(struct cmd_ch_set_file_load) == 0x14); + +struct cmd_ch_set_file_load64 { + u64 opcode; + u32 chan; + u64 addr; + u32 size; +} __packed; +static_assert(sizeof(struct cmd_ch_set_file_load64) == 0x18); + +struct cmd_ch_buffer_return { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_return) == 0xc); + +struct cmd_ch_sbs_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_sbs_enable) == 0x10); + +struct cmd_ch_crop_set { + u64 opcode; + u32 chan; + u32 x1; + u32 y1; + u32 x2; + u32 y2; +} __packed; +static_assert(sizeof(struct cmd_ch_crop_set) == 0x1c); + +struct cmd_ch_output_config_set { + u64 opcode; + u32 chan; + u32 width; + u32 height; + u32 colorspace; + u32 format; + u32 strides[3]; + u32 padding_rows; + u32 unk_h0; + u32 compress; + u32 unk_w2; +} __packed; +static_assert(sizeof(struct cmd_ch_output_config_set) == 0x38); + +struct cmd_ch_preview_stream_set { + u64 opcode; + u32 chan; + u32 stream; +} __packed; +static_assert(sizeof(struct cmd_ch_preview_stream_set) == 0x10); + +struct cmd_ch_als_disable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_als_disable) == 0xc); + +struct cmd_ch_cnr_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_cnr_start) == 0xc); + +struct cmd_ch_mbnr_enable { + u64 opcode; + u32 chan; + u32 use_case; + u32 mode; + u32 enable_chroma; +} __packed; +static_assert(sizeof(struct cmd_ch_mbnr_enable) == 0x18); + +struct cmd_ch_sif_pixel_format_set { + u64 opcode; + u32 chan; + u8 format; + u8 type; + u16 compress; + u32 unk_10; +} __packed; +static_assert(sizeof(struct cmd_ch_sif_pixel_format_set) == 0x14); + +struct cmd_ch_lpdp_hs_receiver_tuning_set { + u64 opcode; + u32 chan; + u32 unk1; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ch_lpdp_hs_receiver_tuning_set) == 0x14); + +struct cmd_ch_property_write { + u64 opcode; + u32 chan; + u32 prop; + u32 val; + u32 unk1; + u32 unk2; +} __packed; +static_assert(sizeof(struct cmd_ch_property_write) == 0x1c); + +int isp_cmd_ch_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_info *args); +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 preset, + struct cmd_ch_camera_config *args); +int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_camera_config *args); +int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, + u32 preset); +int isp_cmd_ch_set_file_load(struct apple_isp *isp, u32 chan, u64 addr, + u32 size); +int isp_cmd_ch_buffer_return(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_sbs_enable(struct apple_isp *isp, u32 chan, u32 enable); +int isp_cmd_ch_crop_set(struct apple_isp *isp, u32 chan, u32 x1, u32 y1, u32 x2, + u32 y2); +int isp_cmd_ch_output_config_set(struct apple_isp *isp, u32 chan, u32 width, + u32 height, u32 strides[3], u32 colorspace, u32 format); +int isp_cmd_ch_preview_stream_set(struct apple_isp *isp, u32 chan, u32 stream); +int isp_cmd_ch_als_disable(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_cnr_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_mbnr_enable(struct apple_isp *isp, u32 chan, u32 use_case, + u32 mode, u32 enable_chroma); +int isp_cmd_ch_sif_pixel_format_set(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_lpdp_hs_receiver_tuning_set(struct apple_isp *isp, u32 chan, u32 unk1, u32 unk2); + +int isp_cmd_ch_property_read(struct apple_isp *isp, u32 chan, u32 prop, u32 *val); +int isp_cmd_ch_property_write(struct apple_isp *isp, u32 chan, u32 prop, u32 val); + +enum isp_mbnr_mode { + ISP_MBNR_MODE_DISABLE = 0, + ISP_MBNR_MODE_ENABLE = 1, + ISP_MBNR_MODE_BYPASS = 2, +}; + +struct cmd_ch_buffer_recycle_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_recycle_mode_set) == 0x10); + +struct cmd_ch_buffer_recycle_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_recycle_start) == 0xc); + +struct cmd_ch_buffer_pool_config_set { + u64 opcode; + u32 chan; + u16 type; + u16 count; + u32 meta_size0; + u32 meta_size1; + u64 unk0; + u64 unk1; + u64 unk2; + u32 zero[0x19]; + u32 data_blocks; + u32 compress; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_config_set) == 0x9c); + +struct cmd_ch_buffer_pool_return { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_ch_buffer_pool_return) == 0xc); + +int isp_cmd_ch_buffer_recycle_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); +int isp_cmd_ch_buffer_recycle_start(struct apple_isp *isp, u32 chan); +int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, + u16 type); +int isp_cmd_ch_buffer_pool_config_get(struct apple_isp *isp, u32 chan, + u16 type); +int isp_cmd_ch_buffer_pool_return(struct apple_isp *isp, u32 chan); + +struct cmd_apple_ch_temporal_filter_start { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start) == 0x14); + +struct cmd_apple_ch_temporal_filter_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_stop) == 0xc); + +struct cmd_apple_ch_motion_history_start { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_motion_history_start) == 0xc); + +struct cmd_apple_ch_motion_history_stop { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_motion_history_stop) == 0xc); + +struct cmd_apple_ch_temporal_filter_enable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_enable) == 0xc); + +struct cmd_apple_ch_temporal_filter_disable { + u64 opcode; + u32 chan; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_disable) == 0xc); + +int isp_cmd_apple_ch_temporal_filter_start(struct apple_isp *isp, u32 chan, u32 arg); +int isp_cmd_apple_ch_temporal_filter_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan); + +struct cmd_ch_ae_stability_set { + u64 opcode; + u32 chan; + u32 stability; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_stability_set) == 0x10); + +struct cmd_ch_ae_stability_to_stable_set { + u64 opcode; + u32 chan; + u32 stability; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_stability_to_stable_set) == 0x10); + +struct cmd_ch_ae_frame_rate_max_get { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_max_get) == 0x10); + +struct cmd_ch_ae_frame_rate_max_set { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_max_set) == 0x10); + +struct cmd_ch_ae_frame_rate_min_set { + u64 opcode; + u32 chan; + u32 framerate; +} __packed; +static_assert(sizeof(struct cmd_ch_ae_frame_rate_min_set) == 0x10); + +struct cmd_apple_ch_ae_fd_scene_metering_config_set { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; + u32 unk_14; + u32 unk_18; + u32 unk_1c; + u32 unk_20; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_fd_scene_metering_config_set) == + 0x24); + +struct cmd_apple_ch_ae_metering_mode_set { + u64 opcode; + u32 chan; + u32 mode; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_metering_mode_set) == 0x10); + +struct cmd_apple_ch_ae_flicker_freq_update_current_set { + u64 opcode; + u32 chan; + u32 freq; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_ae_flicker_freq_update_current_set) == + 0x10); + +int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability); +int isp_cmd_ch_ae_stability_to_stable_set(struct apple_isp *isp, u32 chan, + u32 stability); +int isp_cmd_ch_ae_frame_rate_max_get(struct apple_isp *isp, u32 chan, + struct cmd_ch_ae_frame_rate_max_get *args); +int isp_cmd_ch_ae_frame_rate_max_set(struct apple_isp *isp, u32 chan, + u32 framerate); +int isp_cmd_ch_ae_frame_rate_min_set(struct apple_isp *isp, u32 chan, + u32 framerate); +int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, + u32 chan); +int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, + u32 mode); +int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, + u32 chan, u32 freq); + +struct cmd_ch_semantic_video_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_semantic_video_enable) == 0x10); + +struct cmd_ch_semantic_awb_enable { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_ch_semantic_awb_enable) == 0x10); + +int isp_cmd_ch_semantic_video_enable(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_ch_semantic_awb_enable(struct apple_isp *isp, u32 chan, u32 enable); + +#endif /* __ISP_CMD_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c new file mode 100644 index 00000000000000..848f7abd535a7f --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Apple Image Signal Processor driver + * + * Copyright (C) 2023 The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "isp-cam.h" +#include "isp-fw.h" +#include "isp-iommu.h" +#include "isp-v4l2.h" + +static void apple_isp_detach_genpd(struct apple_isp *isp) +{ + if (isp->pd_count <= 1) + return; + + for (int i = isp->pd_count - 1; i >= 0; i--) { + if (isp->pd_link[i]) + device_link_del(isp->pd_link[i]); + if (!IS_ERR_OR_NULL(isp->pd_dev[i])) + dev_pm_domain_detach(isp->pd_dev[i], true); + } + + return; +} + +static int apple_isp_attach_genpd(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + + isp->pd_count = of_count_phandle_with_args( + dev->of_node, "power-domains", "#power-domain-cells"); + if (isp->pd_count <= 1) + return 0; + + isp->pd_dev = devm_kcalloc(dev, isp->pd_count, sizeof(*isp->pd_dev), + GFP_KERNEL); + if (!isp->pd_dev) + return -ENOMEM; + + isp->pd_link = devm_kcalloc(dev, isp->pd_count, sizeof(*isp->pd_link), + GFP_KERNEL); + if (!isp->pd_link) + return -ENOMEM; + + for (int i = 0; i < isp->pd_count; i++) { + int flags = DL_FLAG_STATELESS; + + /* Primary power domain uses RPM integration */ + if (i == 0) + flags |= DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE; + + isp->pd_dev[i] = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR(isp->pd_dev[i])) { + apple_isp_detach_genpd(isp); + return PTR_ERR(isp->pd_dev[i]); + } + + isp->pd_link[i] = + device_link_add(dev, isp->pd_dev[i], flags); + + if (!isp->pd_link[i]) { + apple_isp_detach_genpd(isp); + return -EINVAL; + } + } + + return 0; +} + +static int apple_isp_init_iommu(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + phys_addr_t heap_base; + size_t heap_size; + u64 vm_size; + int err; + int idx; + int size; + struct device_node *mem_node; + const __be32 *maps, *end; + + isp->domain = iommu_get_domain_for_dev(isp->dev); + if (!isp->domain) + return -ENODEV; + isp->shift = __ffs(isp->domain->pgsize_bitmap); + + idx = of_property_match_string(dev->of_node, "memory-region-names", + "heap"); + mem_node = of_parse_phandle(dev->of_node, "memory-region", idx); + if (!mem_node) { + dev_err(dev, "No memory-region found for heap\n"); + return -ENODEV; + } + + maps = of_get_property(mem_node, "iommu-addresses", &size); + if (!maps || !size) { + dev_err(dev, "No valid iommu-addresses found for heap\n"); + return -ENODEV; + } + + end = maps + size / sizeof(__be32); + + while (maps < end) { + maps++; + maps = of_translate_dma_region(dev->of_node, maps, &heap_base, + &heap_size); + } + + isp->fw.heap_top = heap_base + heap_size; + + err = of_property_read_u64(dev->of_node, "apple,dart-vm-size", + &vm_size); + if (err) { + dev_err(dev, "failed to read 'apple,dart-vm-size': %d\n", err); + return err; + } + + // FIXME: refactor this, maybe use regular iova stuff? + drm_mm_init(&isp->iovad, isp->fw.heap_top, + vm_size - (heap_base & 0xffffffff)); + + return 0; +} + +static void apple_isp_free_iommu(struct apple_isp *isp) +{ + drm_mm_takedown(&isp->iovad); +} + +static int isp_of_read_coord(struct device *dev, struct device_node *np, + const char *prop, struct coord *val) +{ + u32 xy[2]; + int ret; + + ret = of_property_read_u32_array(np, prop, xy, 2); + if (ret) { + dev_err(dev, "failed to read '%s' property\n", prop); + return ret; + } + + val->x = xy[0]; + val->y = xy[1]; + return 0; +} + +static int apple_isp_init_presets(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + struct isp_preset *preset; + int err = 0; + + struct device_node *np __free(device_node) = + of_get_child_by_name(dev->of_node, "sensor-presets"); + if (!np) { + dev_err(dev, "failed to get DT node 'presets'\n"); + return -EINVAL; + } + + isp->num_presets = of_get_child_count(np); + if (!isp->num_presets) { + dev_err(dev, "no sensor presets found\n"); + return -EINVAL; + } + + isp->presets = devm_kzalloc( + dev, sizeof(*isp->presets) * isp->num_presets, GFP_KERNEL); + if (!isp->presets) + return -ENOMEM; + + preset = isp->presets; + for_each_child_of_node_scoped(np, child) { + u32 xywh[4]; + + err = of_property_read_u32(child, "apple,config-index", + &preset->index); + if (err) { + dev_err(dev, "no apple,config-index property\n"); + return err; + } + + err = isp_of_read_coord(dev, child, "apple,input-size", + &preset->input_dim); + if (err) + return err; + err = isp_of_read_coord(dev, child, "apple,output-size", + &preset->output_dim); + if (err) + return err; + + err = of_property_read_u32_array(child, "apple,crop", xywh, 4); + if (err) { + dev_err(dev, "failed to read 'apple,crop' property\n"); + return err; + } + preset->crop_offset.x = xywh[0]; + preset->crop_offset.y = xywh[1]; + preset->crop_size.x = xywh[2]; + preset->crop_size.y = xywh[3]; + + preset++; + } + + return 0; +} + +static const char * isp_fw2str(enum isp_firmware_version version) +{ + switch (version) { + case ISP_FIRMWARE_V_12_3: + return "12.3"; + case ISP_FIRMWARE_V_12_4: + return "12.4"; + case ISP_FIRMWARE_V_13_5: + return "13.5"; + default: + return "unknown"; + } +} + +#define ISP_FW_VERSION_MIN_LEN 3 +#define ISP_FW_VERSION_MAX_LEN 5 + +static enum isp_firmware_version isp_read_fw_version(struct device *dev, + const char *name) +{ + u32 ver[ISP_FW_VERSION_MAX_LEN]; + int len = of_property_read_variable_u32_array(dev->of_node, name, ver, + ISP_FW_VERSION_MIN_LEN, + ISP_FW_VERSION_MAX_LEN); + + switch (len) { + case 3: + if (ver[0] == 12 && ver[1] == 3 && ver[2] <= 1) + return ISP_FIRMWARE_V_12_3; + else if (ver[0] == 12 && ver[1] == 4 && ver[2] == 0) + return ISP_FIRMWARE_V_12_4; + else if (ver[0] == 13 && ver[1] == 5 && ver[2] == 0) + return ISP_FIRMWARE_V_13_5; + + dev_warn(dev, "unknown %s: %d.%d.%d\n", name, ver[0], ver[1], ver[2]); + break; + case 4: + dev_warn(dev, "unknown %s: %d.%d.%d.%d\n", name, ver[0], ver[1], + ver[2], ver[3]); + break; + case 5: + dev_warn(dev, "unknown %s: %d.%d.%d.%d.%d\n", name, ver[0], + ver[1], ver[2], ver[3], ver[4]); + break; + default: + dev_warn(dev, "could not parse %s: %d\n", name, len); + break; + } + + return ISP_FIRMWARE_V_UNKNOWN; +} + +static enum isp_firmware_version isp_check_firmware_version(struct device *dev) +{ + enum isp_firmware_version version, compat; + + /* firmware version is just informative */ + version = isp_read_fw_version(dev, "apple,firmware-version"); + compat = isp_read_fw_version(dev, "apple,firmware-compat"); + + dev_info(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), + isp_fw2str(version)); + + return compat; +} + +static int apple_isp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_isp *isp; + int err; + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (err) + return err; + + isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); + if (!isp) + return -ENOMEM; + + isp->dev = dev; + isp->hw = of_device_get_match_data(dev); + platform_set_drvdata(pdev, isp); + dev_set_drvdata(dev, isp); + + /* Differences between firmware versions are rather minor so try to work + * with unknown firmware. + */ + isp->fw_compat = isp_check_firmware_version(dev); + + err = of_property_read_u32(dev->of_node, "apple,platform-id", + &isp->platform_id); + if (err) { + dev_err(dev, "failed to get 'apple,platform-id' property: %d\n", + err); + return err; + } + + err = of_property_read_u32(dev->of_node, "apple,temporal-filter", + &isp->temporal_filter); + if (err) + isp->temporal_filter = 0; + + err = apple_isp_init_presets(isp); + if (err) { + dev_err(dev, "failed to initialize presets\n"); + return err; + } + + err = apple_isp_attach_genpd(isp); + if (err) { + dev_err(dev, "failed to attatch power domains\n"); + return err; + } + + isp->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(isp->coproc)) { + err = PTR_ERR(isp->coproc); + goto detach_genpd; + } + + isp->mbox = devm_platform_ioremap_resource_byname(pdev, "mbox"); + if (IS_ERR(isp->mbox)) { + err = PTR_ERR(isp->mbox); + goto detach_genpd; + } + + isp->gpio = devm_platform_ioremap_resource_byname(pdev, "gpio"); + if (IS_ERR(isp->gpio)) { + err = PTR_ERR(isp->gpio); + goto detach_genpd; + } + + isp->mbox2 = devm_platform_ioremap_resource_byname(pdev, "mbox2"); + if (IS_ERR(isp->mbox2)) { + err = PTR_ERR(isp->mbox2); + goto detach_genpd; + } + + isp->irq = platform_get_irq(pdev, 0); + if (isp->irq < 0) { + err = isp->irq; + goto detach_genpd; + } + if (!isp->irq) { + err = -ENODEV; + goto detach_genpd; + } + + mutex_init(&isp->iovad_lock); + mutex_init(&isp->video_lock); + spin_lock_init(&isp->buf_lock); + init_waitqueue_head(&isp->wait); + INIT_LIST_HEAD(&isp->gc); + INIT_LIST_HEAD(&isp->bufs_pending); + INIT_LIST_HEAD(&isp->bufs_submitted); + isp->wq = alloc_workqueue("apple-isp-wq", WQ_UNBOUND, 0); + if (!isp->wq) { + dev_err(dev, "failed to create workqueue\n"); + err = -ENOMEM; + goto detach_genpd; + } + + err = apple_isp_init_iommu(isp); + if (err) { + dev_err(dev, "failed to init iommu: %d\n", err); + goto destroy_wq; + } + + err = apple_isp_alloc_firmware_surface(isp); + if (err) { + dev_err(dev, "failed to alloc firmware surface: %d\n", err); + goto free_iommu; + } + + pm_runtime_enable(dev); + + err = apple_isp_detect_camera(isp); + if (err) { + dev_err(dev, "failed to detect camera: %d\n", err); + goto free_surface; + } + + err = apple_isp_setup_video(isp); + if (err) { + dev_err(dev, "failed to register video device: %d\n", err); + goto free_surface; + } + + dev_info(dev, "apple-isp probe!\n"); + + return 0; + +free_surface: + pm_runtime_disable(dev); + apple_isp_free_firmware_surface(isp); +free_iommu: + apple_isp_free_iommu(isp); +destroy_wq: + destroy_workqueue(isp->wq); +detach_genpd: + apple_isp_detach_genpd(isp); + return err; +} + +static void apple_isp_remove(struct platform_device *pdev) +{ + struct apple_isp *isp = platform_get_drvdata(pdev); + + apple_isp_remove_video(isp); + pm_runtime_disable(isp->dev); + apple_isp_free_firmware_surface(isp); + apple_isp_free_iommu(isp); + destroy_workqueue(isp->wq); + apple_isp_detach_genpd(isp); +} + +static const struct apple_isp_hw apple_isp_hw_t8103 = { + .gen = ISP_GEN_T8103, + .pmu_base = 0x23b704000, + + .dsid_count = 4, + .dsid_clr_base0 = 0x200014000, + .dsid_clr_base1 = 0x200054000, + .dsid_clr_base2 = 0x200094000, + .dsid_clr_base3 = 0x2000d4000, + .dsid_clr_range0 = 0x1000, + .dsid_clr_range1 = 0x1000, + .dsid_clr_range2 = 0x1000, + .dsid_clr_range3 = 0x1000, + + .clock_scratch = 0x23b738010, + .clock_base = 0x23bc3c000, + .clock_bit = 0x1, + .clock_size = 0x4, + .bandwidth_scratch = 0x23b73800c, + .bandwidth_base = 0x23bc3c000, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x4, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8103, +}; + +static const struct apple_isp_hw apple_isp_hw_t6000 = { + .gen = ISP_GEN_T8103, + .pmu_base = 0x28e584000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200014000, + .dsid_clr_base1 = 0x200054000, + .dsid_clr_base2 = 0x200094000, + .dsid_clr_base3 = 0x2000d4000, + .dsid_clr_range0 = 0x1000, + .dsid_clr_range1 = 0x1000, + .dsid_clr_range2 = 0x1000, + .dsid_clr_range3 = 0x1000, + + .clock_scratch = 0x28e3d0868, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x28e3d0980, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8103, +}; + +static const struct apple_isp_hw apple_isp_hw_t8112 = { + .gen = ISP_GEN_T8112, + .pmu_base = 0x23b704000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200f14000, + .dsid_clr_range0 = 0x1000, + + .clock_scratch = 0x23b3d0560, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x23b3d05d0, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = false, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8112, +}; + +static const struct apple_isp_hw apple_isp_hw_t6020 = { + .gen = ISP_GEN_T8112, + .pmu_base = 0x290284000, + + .dsid_count = 1, + .dsid_clr_base0 = 0x200f14000, + .dsid_clr_range0 = 0x1000, + + .clock_scratch = 0x28e3d10a8, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x28e3d1200, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + + .scl1 = true, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8112, +}; + +static const struct of_device_id apple_isp_of_match[] = { + { .compatible = "apple,t8103-isp", .data = &apple_isp_hw_t8103 }, + { .compatible = "apple,t8112-isp", .data = &apple_isp_hw_t8112 }, + { .compatible = "apple,t6000-isp", .data = &apple_isp_hw_t6000 }, + { .compatible = "apple,t6020-isp", .data = &apple_isp_hw_t6020 }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_isp_of_match); + +static __maybe_unused int apple_isp_runtime_suspend(struct device *dev) +{ + /* RPM sleep is called when the V4L2 file handle is closed */ + return 0; +} + +static __maybe_unused int apple_isp_runtime_resume(struct device *dev) +{ + return 0; +} + +static __maybe_unused int apple_isp_suspend(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + /* We must restore V4L2 context on system resume. If we were streaming + * before, we (essentially) stop streaming and start streaming again. + */ + apple_isp_video_suspend(isp); + + return 0; +} + +static __maybe_unused int apple_isp_resume(struct device *dev) +{ + struct apple_isp *isp = dev_get_drvdata(dev); + + apple_isp_video_resume(isp); + + return 0; +} + +static const struct dev_pm_ops apple_isp_pm_ops = { + SYSTEM_SLEEP_PM_OPS(apple_isp_suspend, apple_isp_resume) + RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) +}; + +static struct platform_driver apple_isp_driver = { + .driver = { + .name = "apple-isp", + .of_match_table = apple_isp_of_match, + .pm = pm_ptr(&apple_isp_pm_ops), + }, + .probe = apple_isp_probe, + .remove = apple_isp_remove, +}; +module_platform_driver(apple_isp_driver); + +MODULE_AUTHOR("Eileen Yoon "); +MODULE_DESCRIPTION("Apple ISP driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h new file mode 100644 index 00000000000000..96a1d0b39f860d --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_DRV_H__ +#define __ISP_DRV_H__ + +#include +#include +#include +#include + +#include +#include +#include +#include + +/* #define APPLE_ISP_DEBUG */ +#define APPLE_ISP_DEVICE_NAME "apple-isp" +#define APPLE_ISP_CARD_NAME "FaceTime HD Camera" + +#define ISP_MAX_CHANNELS 6 +#define ISP_IPC_MESSAGE_SIZE 64 +#define ISP_IPC_FLAG_ACK 0x1 +#define ISP_META_SIZE_T8103 0x4640 +#define ISP_META_SIZE_T8112 0x4840 + +/* used to limit the user space buffers to the buffer_pool_config */ +#define ISP_MAX_BUFFERS 16 + +enum isp_generation { + ISP_GEN_T8103, + ISP_GEN_T8112, +}; + +enum isp_firmware_version { + ISP_FIRMWARE_V_UNKNOWN, + ISP_FIRMWARE_V_12_3, + ISP_FIRMWARE_V_12_4, + ISP_FIRMWARE_V_13_5, +}; + +struct isp_surf { + struct drm_mm_node *mm; + struct list_head head; + u64 size; + u64 type; + u32 num_pages; + struct page **pages; + struct sg_table sgt; + dma_addr_t iova; + void *virt; + refcount_t refcount; + bool gc; + bool submitted; +}; + +struct isp_message { + u64 arg0; + u64 arg1; + u64 arg2; + u64 arg3; + u64 arg4; + u64 arg5; + u64 arg6; + u64 arg7; +} __packed; +static_assert(sizeof(struct isp_message) == ISP_IPC_MESSAGE_SIZE); + +struct isp_channel { + char *name; + u32 type; + u32 src; + u32 num; + u64 size; + dma_addr_t iova; + void *virt; + u32 doorbell; + u32 cursor; + struct mutex lock; + struct isp_message req; + struct isp_message rsp; + const struct isp_chan_ops *ops; +}; + +struct coord { + u32 x; + u32 y; +}; + +struct isp_preset { + u32 index; + struct coord input_dim; + struct coord output_dim; + struct coord crop_offset; + struct coord crop_size; +}; + +struct apple_isp_hw { + enum isp_generation gen; + u64 pmu_base; + + int dsid_count; + u64 dsid_clr_base0; + u64 dsid_clr_base1; + u64 dsid_clr_base2; + u64 dsid_clr_base3; + u32 dsid_clr_range0; + u32 dsid_clr_range1; + u32 dsid_clr_range2; + u32 dsid_clr_range3; + + u64 clock_scratch; + u64 clock_base; + u8 clock_bit; + u8 clock_size; + u64 bandwidth_scratch; + u64 bandwidth_base; + u8 bandwidth_bit; + u8 bandwidth_size; + + u32 meta_size; + bool scl1; + bool lpdp; +}; + +enum isp_sensor_id { + ISP_IMX248_1820_01, + ISP_IMX248_1822_02, + ISP_IMX343_5221_02, + ISP_IMX354_9251_02, + ISP_IMX356_4820_01, + ISP_IMX356_4820_02, + ISP_IMX364_8720_01, + ISP_IMX364_8723_01, + ISP_IMX372_3820_01, + ISP_IMX372_3820_02, + ISP_IMX372_3820_11, + ISP_IMX372_3820_12, + ISP_IMX405_9720_01, + ISP_IMX405_9721_01, + ISP_IMX405_9723_01, + ISP_IMX414_2520_01, + ISP_IMX503_7820_01, + ISP_IMX503_7820_02, + ISP_IMX505_3921_01, + ISP_IMX514_2820_01, + ISP_IMX514_2820_02, + ISP_IMX514_2820_03, + ISP_IMX514_2820_04, + ISP_IMX558_1921_01, + ISP_IMX558_1922_02, + ISP_IMX603_7920_01, + ISP_IMX603_7920_02, + ISP_IMX603_7921_01, + ISP_IMX613_4920_01, + ISP_IMX613_4920_02, + ISP_IMX614_2921_01, + ISP_IMX614_2921_02, + ISP_IMX614_2922_02, + ISP_IMX633_3622_01, + ISP_IMX703_7721_01, + ISP_IMX703_7722_01, + ISP_IMX713_4721_01, + ISP_IMX713_4722_01, + ISP_IMX714_2022_01, + ISP_IMX772_3721_01, + ISP_IMX772_3721_11, + ISP_IMX772_3722_01, + ISP_IMX772_3723_01, + ISP_IMX814_2123_01, + ISP_IMX853_7622_01, + ISP_IMX913_7523_01, + ISP_VD56G0_6221_01, + ISP_VD56G0_6222_01, +}; + +struct isp_format { + enum isp_sensor_id id; + u32 version; + struct isp_preset *preset; + unsigned int num_planes; + u32 strides[VB2_MAX_PLANES]; + size_t plane_size[VB2_MAX_PLANES]; + size_t total_size; +}; + +struct apple_isp { + struct device *dev; + const struct apple_isp_hw *hw; + enum isp_firmware_version fw_compat; + u32 platform_id; + u32 temporal_filter; + struct isp_preset *presets; + int num_presets; + + int num_channels; + struct isp_format fmts[ISP_MAX_CHANNELS]; + unsigned int current_ch; + + struct video_device vdev; + struct media_device mdev; + struct v4l2_device v4l2_dev; + struct vb2_queue vbq; + struct mutex video_lock; + unsigned int sequence; + bool multiplanar; + + int pd_count; + struct device **pd_dev; + struct device_link **pd_link; + bool pds_active; + + int irq; + + void __iomem *coproc; + void __iomem *mbox; + void __iomem *gpio; + void __iomem *mbox2; + + struct iommu_domain *domain; + unsigned long shift; + struct drm_mm iovad; /* TODO iova.c can't allocate bottom-up */ + struct mutex iovad_lock; + + struct isp_firmware { + u64 heap_top; + } fw; + + struct isp_surf *ipc_surf; + struct isp_surf *extra_surf; + struct isp_surf *data_surf; + struct isp_surf *log_surf; + struct isp_surf *bt_surf; + struct isp_surf *meta_surfs[ISP_MAX_BUFFERS]; + struct list_head gc; + struct workqueue_struct *wq; + + int num_ipc_chans; + struct isp_channel **ipc_chans; + struct isp_channel *chan_tm; /* TERMINAL */ + struct isp_channel *chan_io; /* IO */ + struct isp_channel *chan_dg; /* DEBUG */ + struct isp_channel *chan_bh; /* BUF_H2T */ + struct isp_channel *chan_bt; /* BUF_T2H */ + struct isp_channel *chan_sm; /* SHAREDMALLOC */ + struct isp_channel *chan_it; /* IO_T2H */ + + wait_queue_head_t wait; + dma_addr_t cmd_iova; + void *cmd_virt; + + unsigned long state; + spinlock_t buf_lock; + struct list_head bufs_pending; + struct list_head bufs_submitted; +}; + +struct isp_chan_ops { + int (*handle)(struct apple_isp *isp, struct isp_channel *chan); +}; + +struct isp_buffer { + struct vb2_v4l2_buffer vb; + struct list_head link; + struct isp_surf surfs[VB2_MAX_PLANES]; +}; + +#define to_isp_buffer(x) container_of((x), struct isp_buffer, vb) + +enum { + ISP_STATE_STREAMING, + ISP_STATE_LOGGING, + ISP_STATE_SLEEPING, +}; + +#ifdef APPLE_ISP_DEBUG +#define isp_dbg(isp, fmt, ...) \ + dev_info((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +#else +#define isp_dbg(isp, fmt, ...) \ + dev_dbg((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) +#endif + +#define isp_err(isp, fmt, ...) \ + dev_err((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) + +#define isp_get_format(isp, ch) (&(isp)->fmts[(ch)]) +#define isp_get_current_format(isp) (isp_get_format(isp, isp->current_ch)) + +#endif /* __ISP_DRV_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c new file mode 100644 index 00000000000000..962f99d2d4d433 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -0,0 +1,793 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-fw.h" + +#include +#include +#include +#include + +#include "isp-cmd.h" +#include "isp-fw.h" +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-regs.h" +#include "isp-v4l2.h" + +#define ISP_FIRMWARE_MDELAY 1 +#define ISP_FIRMWARE_MAX_TRIES 1000 + +#define ISP_FIRMWARE_IPC_SIZE 0x1c000 +#define ISP_FIRMWARE_DATA_SIZE 0x28000 + +#define ISP_COPROC_IN_WFI 0x3 + +static inline u32 isp_coproc_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->coproc + reg); +} + +static inline void isp_coproc_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->coproc + reg); +} + +static inline u32 isp_gpio_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->gpio + reg); +} + +static inline void isp_gpio_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->gpio + reg); +} + +static int apple_isp_power_up_domains(struct apple_isp *isp) +{ + int ret; + + if (isp->pds_active) + return 0; + + for (int i = 1; i < isp->pd_count; i++) { + ret = pm_runtime_get_sync(isp->pd_dev[i]); + if (ret < 0) { + dev_err(isp->dev, + "Failed to power up power domain %d: %d\n", i, ret); + while (--i != 1) + pm_runtime_put_sync(isp->pd_dev[i]); + return ret; + } + } + + isp->pds_active = true; + + return 0; +} + +static void apple_isp_power_down_domains(struct apple_isp *isp) +{ + int ret; + + if (!isp->pds_active) + return; + + for (int i = isp->pd_count - 1; i >= 1; i--) { + ret = pm_runtime_put_sync(isp->pd_dev[i]); + if (ret < 0) + dev_err(isp->dev, + "Failed to power up power domain %d: %d\n", i, ret); + } + + isp->pds_active = false; +} + +void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, + dma_addr_t iova, size_t size) +{ + dma_addr_t end = iova + size; + if (!surf) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No surface\n", + (long long)iova, size); + return NULL; + } + + if (end < iova || iova < surf->iova || + end > (surf->iova + surf->size)) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): Out of bounds\n", + (long long)iova, size); + return NULL; + } + + if (!surf->virt) { + dev_err(isp->dev, + "Failed to translate IPC iova 0x%llx (0x%zx): No VMap\n", + (long long)iova, size); + return NULL; + } + + return surf->virt + (iova - surf->iova); +} + +struct isp_firmware_bootargs { + u32 pad_0[2]; + u64 ipc_iova; + u64 shared_base; + u64 shared_size; + u64 extra_iova; + u64 extra_size; + u32 platform_id; + u32 pad_40; + u64 logbuf_addr; + u64 logbuf_size; + u64 logbuf_entsize; + u32 ipc_size; + u32 pad_60[5]; + u32 unk5; + u32 pad_7c[13]; + u32 pad_b0; + u32 unk7; + u32 pad_b8[5]; + u32 unk_iova1; + u32 pad_c0[47]; + u32 unk9; +} __packed; +static_assert(sizeof(struct isp_firmware_bootargs) == 0x180); + +struct isp_chan_desc { + char name[64]; + u32 type; + u32 src; + u32 num; + u32 pad; + u64 iova; + u32 padding[0x2a]; +} __packed; +static_assert(sizeof(struct isp_chan_desc) == 0x100); + +static const struct isp_chan_ops tm_ops = { + .handle = ipc_tm_handle, +}; + +static const struct isp_chan_ops sm_ops = { + .handle = ipc_sm_handle, +}; + +static const struct isp_chan_ops bt_ops = { + .handle = ipc_bt_handle, +}; + +static irqreturn_t apple_isp_isr(int irq, void *dev) +{ + struct apple_isp *isp = dev; + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_ACK, + isp_mbox_read32(isp, ISP_MBOX_IRQ_INTERRUPT)); + + return IRQ_WAKE_THREAD; +} + +static irqreturn_t apple_isp_isr_thread(int irq, void *dev) +{ + struct apple_isp *isp = dev; + + wake_up_all(&isp->wait); + + ipc_chan_handle(isp, isp->chan_sm); + wake_up_all(&isp->wait); /* Some commands depend on sm */ + + ipc_chan_handle(isp, isp->chan_tm); + + ipc_chan_handle(isp, isp->chan_bt); + wake_up_all(&isp->wait); + + return IRQ_HANDLED; +} + +static void isp_disable_irq(struct apple_isp *isp) +{ + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0x0); + free_irq(isp->irq, isp); + isp_gpio_write32(isp, ISP_GPIO_1, 0xfeedbabe); /* real funny */ +} + +static int isp_enable_irq(struct apple_isp *isp) +{ + int err; + + err = request_threaded_irq(isp->irq, apple_isp_isr, + apple_isp_isr_thread, 0, "apple-isp", isp); + if (err < 0) { + isp_err(isp, "failed to request IRQ#%u (%d)\n", isp->irq, err); + return err; + } + + isp_dbg(isp, "about to enable interrupts...\n"); + + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0xf); + + return 0; +} + +static int isp_reset_coproc(struct apple_isp *isp) +{ + int retries; + u32 status; + u32 val; + + isp_coproc_write32(isp, ISP_COPROC_EDPRCR, 0x2); + + isp_coproc_write32(isp, ISP_COPROC_FABRIC_0, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_1, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_2, 0xff00ff); + isp_coproc_write32(isp, ISP_COPROC_FABRIC_3, 0xff00ff); + + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_0, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_1, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_2, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_3, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_4, 0xffffffff); + isp_coproc_write32(isp, ISP_COPROC_IRQ_MASK_5, 0xffffffff); + + for (retries = 0; retries < 128; retries++) { + val = isp_coproc_read32(isp, 0x818); + if (val == 0) + break; + } + + for (retries = 0; retries < 128; retries++) { + val = isp_coproc_read32(isp, 0x81c); + if (val == 0) + break; + } + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + status = isp_coproc_read32(isp, ISP_COPROC_STATUS); + if (status & ISP_COPROC_IN_WFI) { + isp_dbg(isp, "%d: coproc in WFI (status: 0x%x)\n", + retries, status); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, "coproc NOT in WFI (status: 0x%x)\n", status); + return -ENODEV; + } + + return 0; +} + +static void isp_firmware_shutdown_stage1(struct apple_isp *isp) +{ + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + + apple_isp_power_down_domains(isp); +} + +static int isp_firmware_boot_stage1(struct apple_isp *isp) +{ + int err, retries; + // u32 val; + + err = apple_isp_power_up_domains(isp); + if (err < 0) + return err; + + + isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); + +#if 0 + /* This doesn't work well with system sleep */ + val = isp_gpio_read32(isp, ISP_GPIO_1); + if (val == 0xfeedbabe) { + err = isp_reset_coproc(isp); + if (err < 0) + return err; + } +#endif + + err = isp_reset_coproc(isp); + if (err < 0) + return err; + + isp_gpio_write32(isp, ISP_GPIO_0, 0x0); + isp_gpio_write32(isp, ISP_GPIO_1, 0x0); + isp_gpio_write32(isp, ISP_GPIO_2, 0x0); + isp_gpio_write32(isp, ISP_GPIO_3, 0x0); + isp_gpio_write32(isp, ISP_GPIO_4, 0x0); + isp_gpio_write32(isp, ISP_GPIO_5, 0x0); + isp_gpio_write32(isp, ISP_GPIO_6, 0x0); + isp_gpio_write32(isp, ISP_GPIO_7, 0x0); + + isp_mbox_write32(isp, ISP_MBOX_IRQ_ENABLE, 0x0); + + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x0); + isp_coproc_write32(isp, ISP_COPROC_CONTROL, 0x10); + + /* Wait for ISP_GPIO_7 to 0x0 -> 0x8042006 */ + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_7); + if (val == 0x8042006) { + isp_dbg(isp, + "got first magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received first magic number from firmware\n"); + return -ENODEV; + } + + return 0; +} + +int apple_isp_alloc_firmware_surface(struct apple_isp *isp) +{ + /* These are static, so let's do it once and for all */ + isp->ipc_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_IPC_SIZE); + if (!isp->ipc_surf) { + isp_err(isp, "failed to alloc shared surface for ipc\n"); + return -ENOMEM; + } + dev_info(isp->dev, "IPC surface iova: 0x%llx\n", + (long long)isp->ipc_surf->iova); + + isp->data_surf = isp_alloc_surface_vmap(isp, ISP_FIRMWARE_DATA_SIZE); + if (!isp->data_surf) { + isp_err(isp, "failed to alloc shared surface for data files\n"); + isp_free_surface(isp, isp->ipc_surf); + return -ENOMEM; + } + dev_info(isp->dev, "Data surface iova: 0x%llx\n", + (long long)isp->data_surf->iova); + + return 0; +} + +void apple_isp_free_firmware_surface(struct apple_isp *isp) +{ + isp_free_surface(isp, isp->data_surf); + isp_free_surface(isp, isp->ipc_surf); +} + +static void isp_firmware_shutdown_stage2(struct apple_isp *isp) +{ + isp_free_surface(isp, isp->extra_surf); +} + +static int isp_firmware_boot_stage2(struct apple_isp *isp) +{ + struct isp_firmware_bootargs args; + dma_addr_t args_iova; + void *args_virt; + int err, retries; + + u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); + u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); + u32 extra_size = isp_gpio_read32(isp, ISP_GPIO_3); + isp->num_ipc_chans = num_ipc_chans; + + if (!isp->num_ipc_chans) { + dev_err(isp->dev, "No IPC channels found\n"); + return -ENODEV; + } + + if (isp->num_ipc_chans != 7) + dev_warn(isp->dev, "unexpected channel count (%d)\n", + num_ipc_chans); + + isp->extra_surf = isp_alloc_surface_vmap(isp, extra_size); + if (!isp->extra_surf) { + isp_err(isp, "failed to alloc surface for extra heap\n"); + return -ENOMEM; + } + + args_iova = isp->ipc_surf->iova + args_offset + 0x40; + args_virt = isp->ipc_surf->virt + args_offset + 0x40; + isp->cmd_iova = args_iova + sizeof(args) + 0x40; + isp->cmd_virt = args_virt + sizeof(args) + 0x40; + + memset(&args, 0, sizeof(args)); + args.ipc_iova = isp->ipc_surf->iova; + args.ipc_size = isp->ipc_surf->size; + args.shared_base = isp->fw.heap_top & 0xffffffff; + args.shared_size = 0x10000000UL - args.shared_base; + args.extra_iova = isp->extra_surf->iova; + args.extra_size = isp->extra_surf->size; + args.platform_id = isp->platform_id; + args.unk5 = 0x40; + args.unk7 = 0x1; // 0? + args.unk_iova1 = args_iova + sizeof(args) - 0xc; + args.unk9 = 0x3; + memcpy(args_virt, &args, sizeof(args)); + + isp_gpio_write32(isp, ISP_GPIO_0, args_iova); + /* TODO: handle this via Kconfig depends? hardware is only present on + * 64-bit SoCs. + */ + if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT)) + isp_gpio_write32(isp, ISP_GPIO_1, args_iova >> 32); + dma_wmb(); + + /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ + isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_7); + if (val == 0x8042006) { + isp_dbg(isp, + "got second magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received second magic number from firmware\n"); + err = -ENODEV; + goto free_extra; + } + + return 0; + +free_extra: + isp_free_surface(isp, isp->extra_surf); + return err; +} + +static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, + const char *name) +{ + for (int i = 0; i < isp->num_ipc_chans; i++) { + if (!strcasecmp(isp->ipc_chans[i]->name, name)) + return isp->ipc_chans[i]; + } + return NULL; +} + +static void isp_free_channel_info(struct apple_isp *isp) +{ + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_channel *chan = isp->ipc_chans[i]; + if (!chan) + continue; + kfree(chan->name); + kfree(chan); + isp->ipc_chans[i] = NULL; + } + kfree(isp->ipc_chans); + isp->ipc_chans = NULL; +} + +static int isp_fill_channel_info(struct apple_isp *isp) +{ + u64 table_iova = isp_gpio_read32(isp, ISP_GPIO_0) | + ((u64)isp_gpio_read32(isp, ISP_GPIO_1)) << 32; + void *table_virt = apple_isp_ipc_translate( + isp, table_iova, + sizeof(struct isp_chan_desc) * isp->num_ipc_chans); + + if (!table_virt) { + dev_err(isp->dev, "Failed to find channel table\n"); + return -EIO; + } + + isp->ipc_chans = kcalloc(isp->num_ipc_chans, + sizeof(struct isp_channel *), GFP_KERNEL); + if (!isp->ipc_chans) + goto out; + + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_chan_desc desc; + void *desc_virt = table_virt + (i * sizeof(desc)); + struct isp_channel *chan = + kzalloc(sizeof(struct isp_channel), GFP_KERNEL); + if (!chan) + goto out; + isp->ipc_chans[i] = chan; + + memcpy(&desc, desc_virt, sizeof(desc)); + chan->name = kstrdup(desc.name, GFP_KERNEL); + chan->type = desc.type; + chan->src = desc.src; + chan->doorbell = 1 << chan->src; + chan->num = desc.num; + chan->size = desc.num * ISP_IPC_MESSAGE_SIZE; + chan->iova = desc.iova; + chan->virt = + apple_isp_ipc_translate(isp, desc.iova, chan->size); + chan->cursor = 0; + mutex_init(&chan->lock); + + if (!chan->virt) { + dev_err(isp->dev, "Failed to find channel buffer\n"); + goto out; + } + + if ((chan->type != ISP_IPC_CHAN_TYPE_COMMAND) && + (chan->type != ISP_IPC_CHAN_TYPE_REPLY) && + (chan->type != ISP_IPC_CHAN_TYPE_REPORT)) { + isp_err(isp, "invalid ipc chan type (%d)\n", + chan->type); + goto out; + } + + isp_dbg(isp, "chan: %s type: %d src: %d num: %d iova: %pad\n", + chan->name, chan->type, chan->src, chan->num, + &chan->iova); + } + + isp->chan_tm = isp_get_chan_index(isp, "TERMINAL"); + isp->chan_io = isp_get_chan_index(isp, "IO"); + isp->chan_dg = isp_get_chan_index(isp, "DEBUG"); + isp->chan_bh = isp_get_chan_index(isp, "BUF_H2T"); + isp->chan_bt = isp_get_chan_index(isp, "BUF_T2H"); + isp->chan_sm = isp_get_chan_index(isp, "SHAREDMALLOC"); + isp->chan_it = isp_get_chan_index(isp, "IO_T2H"); + + if (!isp->chan_tm || !isp->chan_io || !isp->chan_dg || !isp->chan_bh || + !isp->chan_bt || !isp->chan_sm || !isp->chan_it) { + isp_err(isp, "did not find all of the required ipc chans\n"); + goto out; + } + + isp->chan_tm->ops = &tm_ops; + isp->chan_sm->ops = &sm_ops; + isp->chan_bt->ops = &bt_ops; + + return 0; +out: + isp_free_channel_info(isp); + return -ENOMEM; +} + +static void isp_firmware_shutdown_stage3(struct apple_isp *isp) +{ + isp_free_channel_info(isp); +} + +static int isp_firmware_boot_stage3(struct apple_isp *isp) +{ + int err, retries; + + err = isp_fill_channel_info(isp); + if (err < 0) + return err; + + /* Mask the command channels to prepare for submission */ + for (int i = 0; i < isp->num_ipc_chans; i++) { + struct isp_channel *chan = isp->ipc_chans[i]; + if (chan->type != ISP_IPC_CHAN_TYPE_COMMAND) + continue; + for (int j = 0; j < chan->num; j++) { + struct isp_message msg; + void *msg_virt = chan->virt + (j * sizeof(msg)); + + memset(&msg, 0, sizeof(msg)); + msg.arg0 = ISP_IPC_FLAG_ACK; + memcpy(msg_virt, &msg, sizeof(msg)); + } + } + dma_wmb(); + + /* Wait for ISP_GPIO_3 to 0x8042006 -> 0x0 */ + isp_gpio_write32(isp, ISP_GPIO_3, 0x8042006); + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_3); + if (val == 0x0) { + isp_dbg(isp, + "got third magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, + "never received third magic number from firmware\n"); + isp_free_channel_info(isp); + return -ENODEV; + } + + isp_dbg(isp, "firmware booted!\n"); + + return 0; +} + +static int isp_stop_command_processor(struct apple_isp *isp) +{ + int retries; + +#if 0 + int res = isp_cmd_stop(isp, 0); + if (res) { + isp_err(isp, "isp_cmd_stop() failed\n"); + return res; + } + + /* Wait for ISP_GPIO_0 to 0xf7fbdff9 -> 0x8042006 */ + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + isp_cmd_power_down(isp); +#else + isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); + + int res = isp_cmd_suspend(isp); + if (res) { + isp_err(isp, "isp_cmd_suspend() failed\n"); + return res; + } +#endif + + for (retries = 0; retries < ISP_FIRMWARE_MAX_TRIES; retries++) { + u32 val = isp_gpio_read32(isp, ISP_GPIO_0); + if (val == 0x8042006) { + isp_dbg(isp, "got magic number (0x%x) from firmware\n", + val); + break; + } + mdelay(ISP_FIRMWARE_MDELAY); + } + if (retries >= ISP_FIRMWARE_MAX_TRIES) { + isp_err(isp, "never received magic number from firmware\n"); + return -ENODEV; + } + + return 0; +} + +static int isp_start_command_processor(struct apple_isp *isp) +{ + int err; + + err = isp_cmd_print_enable(isp, 1); + if (err) + return err; + + err = isp_cmd_set_isp_pmu_base(isp, isp->hw->pmu_base); + if (err) + return err; + + if (isp->hw->dsid_count == 1) { + err = isp_cmd_set_dsid_clr_req_base( + isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_range0); + if (err) + return err; + } else { + err = isp_cmd_set_dsid_clr_req_base2( + isp, isp->hw->dsid_clr_base0, isp->hw->dsid_clr_base1, + isp->hw->dsid_clr_base2, isp->hw->dsid_clr_base3, + isp->hw->dsid_clr_range0, isp->hw->dsid_clr_range1, + isp->hw->dsid_clr_range2, isp->hw->dsid_clr_range3); + if (err) + return err; + } + + err = isp_cmd_pmp_ctrl_set( + isp, isp->hw->clock_scratch, isp->hw->clock_base, + isp->hw->clock_bit, isp->hw->clock_size, + isp->hw->bandwidth_scratch, isp->hw->bandwidth_base, + isp->hw->bandwidth_bit, isp->hw->bandwidth_size); + if (err) + return err; + + err = isp_cmd_start(isp, 0); + if (err) + return err; + + /* Now we can access CISP_CMD_CH_* commands */ + + return 0; +} + +static void isp_collect_gc_surface(struct apple_isp *isp) +{ + struct isp_surf *tmp, *surf; + + isp->log_surf = NULL; + isp->bt_surf = NULL; + + list_for_each_entry_safe_reverse(surf, tmp, &isp->gc, head) { + isp_dbg(isp, "freeing iova: %pad size: 0x%llx virt: %pS\n", + &surf->iova, surf->size, (void *)surf->virt); + isp_free_surface(isp, surf); + } +} + +static int isp_firmware_boot(struct apple_isp *isp) +{ + int err; + + err = isp_firmware_boot_stage1(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 1: %d\n", err); + goto garbage_collect; + } + + err = isp_firmware_boot_stage2(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 2: %d\n", err); + goto shutdown_stage1; + } + + err = isp_firmware_boot_stage3(isp); + if (err < 0) { + isp_err(isp, "failed firmware boot stage 3: %d\n", err); + goto shutdown_stage2; + } + + err = isp_enable_irq(isp); + if (err < 0) { + isp_err(isp, "failed to enable interrupts: %d\n", err); + goto shutdown_stage3; + } + + err = isp_start_command_processor(isp); + if (err < 0) { + isp_err(isp, "failed to start command processor: %d\n", err); + goto disable_irqs; + } + + flush_workqueue(isp->wq); + + return 0; + +disable_irqs: + isp_disable_irq(isp); +shutdown_stage3: + isp_firmware_shutdown_stage3(isp); +shutdown_stage2: + isp_firmware_shutdown_stage2(isp); +shutdown_stage1: + isp_firmware_shutdown_stage1(isp); +garbage_collect: + isp_collect_gc_surface(isp); + return err; +} + +static void isp_firmware_shutdown(struct apple_isp *isp) +{ + flush_workqueue(isp->wq); + isp_stop_command_processor(isp); + isp_disable_irq(isp); + isp_firmware_shutdown_stage3(isp); + isp_firmware_shutdown_stage2(isp); + isp_firmware_shutdown_stage1(isp); + isp_collect_gc_surface(isp); +} + +int apple_isp_firmware_boot(struct apple_isp *isp) +{ + int err; + + /* Needs to be power cycled for IOMMU to behave correctly */ + err = pm_runtime_resume_and_get(isp->dev); + if (err < 0) { + dev_err(isp->dev, "failed to enable power: %d\n", err); + return err; + } + + err = isp_firmware_boot(isp); + if (err) { + dev_err(isp->dev, "failed to boot firmware: %d\n", err); + pm_runtime_put_sync(isp->dev); + return err; + } + + return 0; +} + +void apple_isp_firmware_shutdown(struct apple_isp *isp) +{ + isp_firmware_shutdown(isp); + pm_runtime_put_sync(isp->dev); +} diff --git a/drivers/media/platform/apple/isp/isp-fw.h b/drivers/media/platform/apple/isp/isp-fw.h new file mode 100644 index 00000000000000..974216f0989f91 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-fw.h @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_FW_H__ +#define __ISP_FW_H__ + +#include "isp-drv.h" + +int apple_isp_alloc_firmware_surface(struct apple_isp *isp); +void apple_isp_free_firmware_surface(struct apple_isp *isp); + +int apple_isp_firmware_boot(struct apple_isp *isp); +void apple_isp_firmware_shutdown(struct apple_isp *isp); + +void *apple_isp_translate(struct apple_isp *isp, struct isp_surf *surf, + dma_addr_t iova, size_t size); + +static inline void *apple_isp_ipc_translate(struct apple_isp *isp, + dma_addr_t iova, size_t size) +{ + return apple_isp_translate(isp, isp->ipc_surf, iova, size); +} + +#endif /* __ISP_FW_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c new file mode 100644 index 00000000000000..9e64b25de5c994 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include +#include + +#include "isp-iommu.h" + +static void isp_surf_free_pages(struct isp_surf *surf) +{ + for (u32 i = 0; i < surf->num_pages && surf->pages[i] != NULL; i++) { + __free_page(surf->pages[i]); + } + kvfree(surf->pages); +} + +static int isp_surf_alloc_pages(struct isp_surf *surf) +{ + surf->pages = kvmalloc_array(surf->num_pages, sizeof(*surf->pages), + GFP_KERNEL); + if (!surf->pages) + return -ENOMEM; + + for (u32 i = 0; i < surf->num_pages; i++) { + surf->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (surf->pages[i] == NULL) + goto free_pages; + } + + return 0; + +free_pages: + isp_surf_free_pages(surf); + return -ENOMEM; +} + +int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf) +{ + surf->virt = vmap(surf->pages, surf->num_pages, VM_MAP, + pgprot_writecombine(PAGE_KERNEL)); + if (surf->virt == NULL) { + dev_err(isp->dev, "failed to vmap size 0x%llx\n", surf->size); + return -EINVAL; + } + + return 0; +} + +static void isp_surf_vunmap(struct apple_isp *isp, struct isp_surf *surf) +{ + if (surf->virt) + vunmap(surf->virt); + surf->virt = NULL; +} + +static void isp_surf_unreserve_iova(struct apple_isp *isp, + struct isp_surf *surf) +{ + if (surf->mm) { + mutex_lock(&isp->iovad_lock); + drm_mm_remove_node(surf->mm); + mutex_unlock(&isp->iovad_lock); + kfree(surf->mm); + } + surf->mm = NULL; +} + +static int isp_surf_reserve_iova(struct apple_isp *isp, struct isp_surf *surf) +{ + int err; + + surf->mm = kzalloc(sizeof(*surf->mm), GFP_KERNEL); + if (!surf->mm) + return -ENOMEM; + + mutex_lock(&isp->iovad_lock); + err = drm_mm_insert_node_generic(&isp->iovad, surf->mm, + ALIGN(surf->size, 1UL << isp->shift), + 1UL << isp->shift, 0, 0); + mutex_unlock(&isp->iovad_lock); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + goto mm_free; + } + + surf->iova = surf->mm->start; + + return 0; +mm_free: + kfree(surf->mm); + surf->mm = NULL; + return err; +} + +static void isp_surf_iommu_unmap(struct apple_isp *isp, struct isp_surf *surf) +{ + iommu_unmap(isp->domain, surf->iova, surf->size); + sg_free_table(&surf->sgt); +} + +static int isp_surf_iommu_map(struct apple_isp *isp, struct isp_surf *surf) +{ + unsigned long size; + int err; + + err = sg_alloc_table_from_pages(&surf->sgt, surf->pages, + surf->num_pages, 0, surf->size, + GFP_KERNEL); + if (err < 0) { + dev_err(isp->dev, "failed to alloc sgt from pages\n"); + return err; + } + + size = iommu_map_sgtable(isp->domain, surf->iova, &surf->sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (size < surf->size) { + dev_err(isp->dev, "failed to iommu_map sgt to iova %pad\n", + &surf->iova); + sg_free_table(&surf->sgt); + return -ENXIO; + } + + return 0; +} + +static void __isp_surf_init(struct apple_isp *isp, struct isp_surf *surf, + u64 size, bool gc) +{ + surf->mm = NULL; + surf->virt = NULL; + surf->size = ALIGN(size, 1UL << isp->shift); + surf->num_pages = surf->size >> isp->shift; + surf->gc = gc; +} + +struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc) +{ + int err; + + struct isp_surf *surf = kzalloc(sizeof(struct isp_surf), GFP_KERNEL); + if (!surf) + return NULL; + + __isp_surf_init(isp, surf, size, gc); + + err = isp_surf_alloc_pages(surf); + if (err < 0) { + dev_err(isp->dev, "failed to allocate %d pages\n", + surf->num_pages); + goto free_surf; + } + + err = isp_surf_reserve_iova(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + goto free_pages; + } + + err = isp_surf_iommu_map(isp, surf); + if (err < 0) { + dev_err(isp->dev, + "failed to iommu_map size 0x%llx to iova %pad\n", + surf->size, &surf->iova); + goto unreserve_iova; + } + + refcount_set(&surf->refcount, 1); + if (surf->gc) + list_add_tail(&surf->head, &isp->gc); + + return surf; + +unreserve_iova: + isp_surf_unreserve_iova(isp, surf); +free_pages: + isp_surf_free_pages(surf); +free_surf: + kfree(surf); + return NULL; +} + +struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size) +{ + int err; + + struct isp_surf *surf = __isp_alloc_surface(isp, size, false); + if (!surf) + return NULL; + + err = isp_surf_vmap(isp, surf); + if (err < 0) { + dma_addr_t iova_end = surf->iova + surf->size; + dev_err(isp->dev, "failed to vmap iova %pad - %pad\n", + &surf->iova, &iova_end); + isp_free_surface(isp, surf); + return NULL; + } + + return surf; +} + +void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf) +{ + if (refcount_dec_and_test(&surf->refcount)) { + isp_surf_vunmap(isp, surf); + isp_surf_iommu_unmap(isp, surf); + isp_surf_unreserve_iova(isp, surf); + isp_surf_free_pages(surf); + if (surf->gc) + list_del(&surf->head); + kfree(surf); + } +} + +int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, + struct sg_table *sgt, u64 size) +{ + int err; + ssize_t mapped; + + // TODO userptr sends unaligned sizes + surf->mm = NULL; + surf->size = size; + + err = isp_surf_reserve_iova(isp, surf); + if (err < 0) { + dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", + surf->size); + return err; + } + + mapped = iommu_map_sgtable(isp->domain, surf->iova, sgt, + IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE); + if (mapped < surf->size) { + dev_err(isp->dev, "failed to iommu_map sgt to iova %pad\n", + &surf->iova); + isp_surf_unreserve_iova(isp, surf); + return -ENXIO; + } + surf->size = mapped; + + return 0; +} + +void apple_isp_iommu_unmap_sgt(struct apple_isp *isp, struct isp_surf *surf) +{ + iommu_unmap(isp->domain, surf->iova, surf->size); + isp_surf_unreserve_iova(isp, surf); +} diff --git a/drivers/media/platform/apple/isp/isp-iommu.h b/drivers/media/platform/apple/isp/isp-iommu.h new file mode 100644 index 00000000000000..b99a182e284b72 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-iommu.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_IOMMU_H__ +#define __ISP_IOMMU_H__ + +#include "isp-drv.h" + +struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc); +#define isp_alloc_surface(isp, size) (__isp_alloc_surface(isp, size, false)) +#define isp_alloc_surface_gc(isp, size) (__isp_alloc_surface(isp, size, true)) +struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size); +int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf); +void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf); + +int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, + struct sg_table *sgt, u64 size); +void apple_isp_iommu_unmap_sgt(struct apple_isp *isp, struct isp_surf *surf); + +#endif /* __ISP_IOMMU_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c new file mode 100644 index 00000000000000..a1948717a31968 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-regs.h" +#include "isp-fw.h" + +#define ISP_IPC_FLAG_TERMINAL_ACK 0x3 +#define ISP_IPC_BUFEXC_STAT_META_OFFSET 0x10 + +struct isp_sm_deferred_work { + struct work_struct work; + struct apple_isp *isp; + struct isp_surf *surf; +}; + +struct isp_bufexc_stat { + u64 unk_0; // 2 + u64 unk_8; // 2 + + u64 meta_iova; + u64 pad_20[3]; + u64 meta_size; // 0x4640 + u64 unk_38; + + u32 unk_40; // 1 + u32 unk_44; + u64 unk_48; + + u64 iova0; + u64 iova1; + u64 iova2; + u64 iova3; + u32 pad_70[4]; + + u32 unk_80; // 2 + u32 unk_84; // 1 + u32 unk_88; // 0x10 || 0x13 + u32 unk_8c; + u32 pad_90[96]; + + u32 unk_210; // 0x28 + u32 unk_214; + u32 index; + u16 bes_width; // 1296, 0x510 + u16 bes_height; // 736, 0x2e0 + + u32 unk_220; // 0x0 || 0x1 + u32 pad_224[3]; + u32 unk_230; // 0xf7ed38 + u32 unk_234; // 3 + u32 pad_238[2]; + u32 pad_240[16]; +} __packed; +static_assert(sizeof(struct isp_bufexc_stat) == ISP_IPC_BUFEXC_STAT_SIZE); + +static inline void *chan_msg_virt(struct isp_channel *chan, u32 index) +{ + return chan->virt + (index * ISP_IPC_MESSAGE_SIZE); +} + +static inline void chan_read_msg_index(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg, u32 index) +{ + memcpy(msg, chan_msg_virt(chan, index), sizeof(*msg)); +} + +static inline void chan_read_msg(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg) +{ + chan_read_msg_index(isp, chan, msg, chan->cursor); +} + +static inline void chan_write_msg_index(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg, u32 index) +{ + u64 *p0 = chan_msg_virt(chan, index); + memcpy(p0 + 1, &msg->arg1, sizeof(*msg) - 8); + + /* Make sure we write arg0 last, since that indicates message validity. */ + + dma_wmb(); + *p0 = msg->arg0; + dma_wmb(); +} + +static inline void chan_write_msg(struct apple_isp *isp, + struct isp_channel *chan, + struct isp_message *msg) +{ + chan_write_msg_index(isp, chan, msg, chan->cursor); +} + +static inline void chan_update_cursor(struct isp_channel *chan) +{ + if (chan->cursor >= (chan->num - 1)) { + chan->cursor = 0; + } else { + chan->cursor += 1; + } +} + +static int chan_handle_once(struct apple_isp *isp, struct isp_channel *chan) +{ + int err; + + lockdep_assert_held(&chan->lock); + + err = chan->ops->handle(isp, chan); + if (err < 0) { + dev_err(isp->dev, "%s: handler failed: %d)\n", chan->name, err); + return err; + } + + chan_write_msg(isp, chan, &chan->rsp); + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + + chan_update_cursor(chan); + + return 0; +} + +static inline bool chan_rx_done(struct apple_isp *isp, struct isp_channel *chan) +{ + if (((chan->req.arg0 & 0xf) == ISP_IPC_FLAG_ACK) || + ((chan->req.arg0 & 0xf) == ISP_IPC_FLAG_TERMINAL_ACK)) { + return true; + } + return false; +} + +int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + int err = 0; + + mutex_lock(&chan->lock); + while (1) { + chan_read_msg(isp, chan, &chan->req); + if (chan_rx_done(isp, chan)) { + err = 0; + break; + } + err = chan_handle_once(isp, chan); + if (err < 0) { + break; + } + } + mutex_unlock(&chan->lock); + + return err; +} + +static inline bool chan_tx_done(struct apple_isp *isp, struct isp_channel *chan) +{ + dma_rmb(); + + chan_read_msg(isp, chan, &chan->rsp); + if ((chan->rsp.arg0) == (chan->req.arg0 | ISP_IPC_FLAG_ACK)) { + chan_update_cursor(chan); + return true; + } + return false; +} + +int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout) +{ + long t; + + chan_write_msg(isp, chan, &chan->req); + dma_wmb(); + + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + + if (!timeout) + return 0; + + t = wait_event_timeout(isp->wait, chan_tx_done(isp, chan), timeout); + if (t == 0) { + dev_err(isp->dev, + "%s: timed out on request [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, chan->req.arg0, chan->req.arg1, + chan->req.arg2); + return -ETIME; + } + + isp_dbg(isp, "%s: request success (%ld)\n", chan->name, t); + + return 0; +} + +int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *rsp = &chan->rsp; + +#ifdef APPLE_ISP_DEBUG + struct isp_message *req = &chan->req; + char buf[512]; + dma_addr_t iova = req->arg0 & ~ISP_IPC_FLAG_TERMINAL_ACK; + u32 size = req->arg1; + if (iova && size && size < sizeof(buf) && + isp->log_surf) { + void *p = apple_isp_translate(isp, isp->log_surf, iova, size); + if (p) { + size = min_t(u32, size, 512); + memcpy(buf, p, size); + isp_dbg(isp, "ISPASC: %.*s", size, buf); + } + } +#endif + + rsp->arg0 = ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; + + return 0; +} + +int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *req = &chan->req, *rsp = &chan->rsp; + int err; + + if (req->arg0 == 0x0) { + struct isp_surf *surf; + + surf = isp_alloc_surface_gc(isp, req->arg1); + if (!surf) { + isp_err(isp, "failed to alloc requested size 0x%llx\n", + req->arg1); + return -ENOMEM; + } + surf->type = req->arg2; + + rsp->arg0 = surf->iova | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; /* macOS uses this to index surfaces */ + + switch (surf->type) { + case 0x4c4f47: /* "LOG" */ + isp->log_surf = surf; + break; + case 0x4d495343: /* "MISC" */ + /* Hacky... maybe there's a better way to identify this surface? */ + if (surf->size == 0xc000) + isp->bt_surf = surf; + break; + default: + // skip vmap + return 0; + } + + err = isp_surf_vmap(isp, surf); + if (err < 0) { + isp_err(isp, "failed to vmap iova=0x%llx size=0x%llx\n", + surf->iova, surf->size); + } + } else { + /* This should be the shared surface free request, but + * 1) The fw doesn't request to free all of what it requested + * 2) The fw continues to access the surface after + * So we link it to the gc, which runs after fw shutdown + */ + rsp->arg0 = req->arg0 | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = 0x0; + } + + return 0; +} diff --git a/drivers/media/platform/apple/isp/isp-ipc.h b/drivers/media/platform/apple/isp/isp-ipc.h new file mode 100644 index 00000000000000..0c1d681835c72f --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-ipc.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_IPC_H__ +#define __ISP_IPC_H__ + +#include "isp-drv.h" + +#define ISP_IPC_CHAN_TYPE_COMMAND 0 +#define ISP_IPC_CHAN_TYPE_REPLY 1 +#define ISP_IPC_CHAN_TYPE_REPORT 2 + +#define ISP_IPC_BUFEXC_STAT_SIZE 0x280 +#define ISP_IPC_BUFEXC_FLAG_RENDER 0x10000000 +#define ISP_IPC_BUFEXC_FLAG_COMMAND 0x30000000 +#define ISP_IPC_BUFEXC_FLAG_ACK 0x80000000 + +int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan); +int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout); + +int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan); +int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan); + +#endif /* __ISP_IPC_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h new file mode 100644 index 00000000000000..7357fa10fa5483 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_REGS_H__ +#define __ISP_REGS_H__ + +#include "isp-drv.h" + +#define ISP_COPROC_FABRIC_0 0x738 +#define ISP_COPROC_FABRIC_1 0x798 +#define ISP_COPROC_FABRIC_2 0x7f8 +#define ISP_COPROC_FABRIC_3 0x858 + +#define ISP_COPROC_RVBAR 0x1050000 +#define ISP_COPROC_EDPRCR 0x1010310 +#define ISP_COPROC_CONTROL 0x1400044 +#define ISP_COPROC_STATUS 0x1400048 + +#define ISP_COPROC_IRQ_MASK_0 0x1400a00 +#define ISP_COPROC_IRQ_MASK_1 0x1400a04 +#define ISP_COPROC_IRQ_MASK_2 0x1400a08 +#define ISP_COPROC_IRQ_MASK_3 0x1400a0c +#define ISP_COPROC_IRQ_MASK_4 0x1400a10 +#define ISP_COPROC_IRQ_MASK_5 0x1400a14 + +#define ISP_MBOX_IRQ_INTERRUPT 0x00 +#define ISP_MBOX_IRQ_ENABLE 0x04 +#define ISP_MBOX2_IRQ_DOORBELL 0x00 +#define ISP_MBOX2_IRQ_ACK 0x0c + +#define ISP_GPIO_0 0x00 +#define ISP_GPIO_1 0x04 +#define ISP_GPIO_2 0x08 +#define ISP_GPIO_3 0x0c +#define ISP_GPIO_4 0x10 +#define ISP_GPIO_5 0x14 +#define ISP_GPIO_6 0x18 +#define ISP_GPIO_7 0x1c +#define ISP_GPIO_CLOCK_EN 0x20 + +static inline u32 isp_mbox_read32(struct apple_isp *isp, u32 reg) +{ + return readl(isp->mbox + reg); +} + +static inline void isp_mbox_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->mbox + reg); +} + +static inline void isp_mbox2_write32(struct apple_isp *isp, u32 reg, u32 val) +{ + writel(val, isp->mbox2 + reg); +} + +#endif /* __ISP_REGS_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c new file mode 100644 index 00000000000000..8e287cfc8a29e9 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -0,0 +1,908 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#include + +#include +#include +#include +#include +#include + +#include "isp-cam.h" +#include "isp-cmd.h" +#include "isp-iommu.h" +#include "isp-ipc.h" +#include "isp-fw.h" +#include "isp-v4l2.h" + +#define ISP_MIN_FRAMES 2 +#define ISP_MAX_PLANES 4 +#define ISP_MAX_PIX_FORMATS 2 +#define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) +#define ISP_STRIDE_ALIGNMENT 64 + +static bool multiplanar = false; +module_param(multiplanar, bool, 0644); +MODULE_PARM_DESC(multiplanar, "Enable multiplanar API"); + +struct isp_buflist_buffer { + u64 iovas[ISP_MAX_PLANES]; + u32 flags[ISP_MAX_PLANES]; + u32 num_planes; + u32 pool_type; + u32 tag; + u32 pad; +} __packed; +static_assert(sizeof(struct isp_buflist_buffer) == 0x40); + +struct isp_buflist { + u64 type; + u64 num_buffers; + struct isp_buflist_buffer buffers[]; +}; + +int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) +{ + struct isp_message *req = &chan->req, *rsp = &chan->rsp; + struct isp_buffer *tmp, *buf; + struct isp_buflist *bl; + u32 count; + int err = 0; + + /* printk("H2T: 0x%llx 0x%llx 0x%llx\n", (long long)req->arg0, + (long long)req->arg1, (long long)req->arg2); */ + + if (req->arg1 < sizeof(struct isp_buflist)) { + dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, + req->arg1); + return -EIO; + } + + bl = apple_isp_translate(isp, isp->bt_surf, req->arg0, req->arg1); + + count = bl->num_buffers; + if (count > (req->arg1 - sizeof(struct isp_buffer)) / + sizeof(struct isp_buflist_buffer)) { + dev_err(isp->dev, "%s: Bad length 0x%llx\n", chan->name, + req->arg1); + return -EIO; + } + + spin_lock(&isp->buf_lock); + for (int i = 0; i < count; i++) { + struct isp_buflist_buffer *bufd = &bl->buffers[i]; + + /* printk("Return: 0x%llx (%d)\n", bufd->iovas[0], + bufd->pool_type); */ + + if (bufd->pool_type == 0) { + for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + struct isp_surf *meta = isp->meta_surfs[j]; + if ((u32)bufd->iovas[0] == (u32)meta->iova) { + WARN_ON(!meta->submitted); + meta->submitted = false; + } + } + } else { + list_for_each_entry_safe_reverse( + buf, tmp, &isp->bufs_submitted, link) { + if ((u32)buf->surfs[0].iova == + (u32)bufd->iovas[0]) { + enum vb2_buffer_state state = + VB2_BUF_STATE_ERROR; + + buf->vb.vb2_buf.timestamp = + ktime_get_ns(); + buf->vb.sequence = isp->sequence++; + buf->vb.field = V4L2_FIELD_NONE; + if (req->arg2 == + ISP_IPC_BUFEXC_FLAG_RENDER) + state = VB2_BUF_STATE_DONE; + vb2_buffer_done(&buf->vb.vb2_buf, + state); + list_del(&buf->link); + } + } + } + } + spin_unlock(&isp->buf_lock); + + rsp->arg0 = req->arg0 | ISP_IPC_FLAG_ACK; + rsp->arg1 = 0x0; + rsp->arg2 = ISP_IPC_BUFEXC_FLAG_ACK; + + return err; +} + +static int isp_submit_buffers(struct apple_isp *isp) +{ + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_channel *chan = isp->chan_bh; + struct isp_message *req = &chan->req; + struct isp_buffer *buf, *tmp; + unsigned long flags; + size_t offset; + int err; + + struct isp_buflist *bl = isp->cmd_virt; + struct isp_buflist_buffer *bufd = &bl->buffers[0]; + + bl->type = 1; + bl->num_buffers = 0; + + spin_lock_irqsave(&isp->buf_lock, flags); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + struct isp_surf *meta = isp->meta_surfs[i]; + + if (meta->submitted) + continue; + + /* printk("Submit: 0x%llx .. 0x%llx (meta)\n", meta->iova, + meta->iova + meta->size); */ + + bufd->num_planes = 1; + bufd->pool_type = 0; + bufd->iovas[0] = meta->iova; + bufd->flags[0] = 0x40000000; + bufd++; + bl->num_buffers++; + + meta->submitted = true; + } + + while ((buf = list_first_entry_or_null(&isp->bufs_pending, + struct isp_buffer, link))) { + memset(bufd, 0, sizeof(*bufd)); + + bufd->num_planes = fmt->num_planes; + bufd->pool_type = isp->hw->scl1 ? CISP_POOL_TYPE_RENDERED_SCL1 : + CISP_POOL_TYPE_RENDERED; + offset = 0; + for (int j = 0; j < fmt->num_planes; j++) { + bufd->iovas[j] = buf->surfs[0].iova + offset; + bufd->flags[j] = 0x40000000; + offset += fmt->plane_size[j]; + } + + /* printk("Submit: 0x%llx .. 0x%llx (render)\n", + buf->surfs[0].iova, + buf->surfs[0].iova + buf->surfs[0].size); */ + bufd++; + bl->num_buffers++; + + /* + * Queue the buffer as submitted and release the lock for now. + * We need to do this before actually submitting to avoid a + * race with the buffer return codepath. + */ + list_move_tail(&buf->link, &isp->bufs_submitted); + } + + spin_unlock_irqrestore(&isp->buf_lock, flags); + + req->arg0 = isp->cmd_iova; + req->arg1 = max_t(u64, ISP_IPC_BUFEXC_STAT_SIZE, + ((uintptr_t)bufd - (uintptr_t)bl)); + req->arg2 = ISP_IPC_BUFEXC_FLAG_COMMAND; + + err = ipc_chan_send(isp, chan, ISP_BUFFER_TIMEOUT); + if (err) { + /* If we fail, consider the buffer not submitted. */ + dev_err(isp->dev, + "%s: failed to send bufs: [0x%llx, 0x%llx, 0x%llx]\n", + chan->name, req->arg0, req->arg1, req->arg2); + + /* + * Try to find the buffer in the list, and if it's + * still there, move it back to the pending list. + */ + spin_lock_irqsave(&isp->buf_lock, flags); + + bufd = &bl->buffers[0]; + for (int i = 0; i < bl->num_buffers; i++, bufd++) { + list_for_each_entry_safe_reverse( + buf, tmp, &isp->bufs_submitted, link) { + if (bufd->iovas[0] == buf->surfs[0].iova) { + list_move_tail(&buf->link, + &isp->bufs_pending); + } + } + for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + struct isp_surf *meta = isp->meta_surfs[j]; + if (bufd->iovas[0] == meta->iova) { + meta->submitted = false; + } + } + } + + spin_unlock_irqrestore(&isp->buf_lock, flags); + } + + return err; +} + +/* + * Videobuf2 section + */ +static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, + unsigned int *num_planes, unsigned int sizes[], + struct device *alloc_devs[]) +{ + struct apple_isp *isp = vb2_get_drv_priv(vq); + struct isp_format *fmt = isp_get_current_format(isp); + + /* This is not strictly neccessary but makes it easy to enforce that + * at most 16 buffers are submitted at once. ISP on t6001 (FW 12.3) + * times out if more buffers are submitted than set in the buffer pool + * config before streaming is started. + */ + *nbuffers = min_t(unsigned int, *nbuffers, ISP_MAX_BUFFERS); + + if (*num_planes) { + if (sizes[0] < fmt->total_size) + return -EINVAL; + + return 0; + } + + *num_planes = 1; + sizes[0] = fmt->total_size; + + return 0; +} + +static void __isp_vb2_buf_cleanup(struct vb2_buffer *vb, unsigned int i) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + + while (i--) + apple_isp_iommu_unmap_sgt(isp, &buf->surfs[i]); +} + +static void isp_vb2_buf_cleanup(struct vb2_buffer *vb) +{ + __isp_vb2_buf_cleanup(vb, vb->num_planes); +} + +static int isp_vb2_buf_init(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + unsigned int i; + int err; + + for (i = 0; i < vb->num_planes; i++) { + struct sg_table *sgt = vb2_dma_sg_plane_desc(vb, i); + err = apple_isp_iommu_map_sgt(isp, &buf->surfs[i], sgt, + vb2_plane_size(vb, i)); + if (err) + goto cleanup; + } + + return 0; + +cleanup: + __isp_vb2_buf_cleanup(vb, i); + return err; +} + +static int isp_vb2_buf_prepare(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_format *fmt = isp_get_current_format(isp); + + if (vb2_plane_size(vb, 0) < fmt->total_size) + return -EINVAL; + + vb2_set_plane_payload(vb, 0, fmt->total_size); + + return 0; +} + +static void isp_vb2_release_buffers(struct apple_isp *isp, + enum vb2_buffer_state state) +{ + struct isp_buffer *buf; + unsigned long flags; + + spin_lock_irqsave(&isp->buf_lock, flags); + list_for_each_entry(buf, &isp->bufs_submitted, link) + vb2_buffer_done(&buf->vb.vb2_buf, state); + INIT_LIST_HEAD(&isp->bufs_submitted); + list_for_each_entry(buf, &isp->bufs_pending, link) + vb2_buffer_done(&buf->vb.vb2_buf, state); + INIT_LIST_HEAD(&isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); +} + +static void isp_vb2_buf_queue(struct vb2_buffer *vb) +{ + struct apple_isp *isp = vb2_get_drv_priv(vb->vb2_queue); + struct isp_buffer *buf = + container_of(vb, struct isp_buffer, vb.vb2_buf); + unsigned long flags; + bool empty; + + spin_lock_irqsave(&isp->buf_lock, flags); + empty = list_empty(&isp->bufs_pending) && + list_empty(&isp->bufs_submitted); + list_add_tail(&buf->link, &isp->bufs_pending); + spin_unlock_irqrestore(&isp->buf_lock, flags); + + if (test_bit(ISP_STATE_STREAMING, &isp->state) && !empty) + isp_submit_buffers(isp); +} + +static int apple_isp_start_streaming(struct apple_isp *isp) +{ + int err; + + err = apple_isp_start_camera(isp); + if (err) { + dev_err(isp->dev, "failed to start camera: %d\n", err); + goto release_buffers; + } + + err = isp_submit_buffers(isp); + if (err) { + dev_err(isp->dev, "failed to send initial batch: %d\n", err); + goto stop_camera; + } + + err = apple_isp_start_capture(isp); + if (err) { + dev_err(isp->dev, "failed to start capture: %d\n", err); + goto stop_camera; + } + + set_bit(ISP_STATE_STREAMING, &isp->state); + + return 0; + +stop_camera: + apple_isp_stop_camera(isp); +release_buffers: + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + return err; +} + +static void apple_isp_stop_streaming(struct apple_isp *isp) +{ + clear_bit(ISP_STATE_STREAMING, &isp->state); + apple_isp_stop_capture(isp); + apple_isp_stop_camera(isp); +} + +static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) +{ + struct apple_isp *isp = vb2_get_drv_priv(q); + + isp->sequence = 0; + + return apple_isp_start_streaming(isp); +} + +static void isp_vb2_stop_streaming(struct vb2_queue *q) +{ + struct apple_isp *isp = vb2_get_drv_priv(q); + + apple_isp_stop_streaming(isp); + isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); +} + +int apple_isp_video_suspend(struct apple_isp *isp) +{ + /* Swap into STATE_SLEEPING as isp_vb2_buf_queue() submits on + * STATE_STREAMING. + */ + if (test_bit(ISP_STATE_STREAMING, &isp->state)) { + /* Signal buffers to be recycled for clean shutdown */ + isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); + apple_isp_stop_streaming(isp); + set_bit(ISP_STATE_SLEEPING, &isp->state); + } + + return 0; +} + +int apple_isp_video_resume(struct apple_isp *isp) +{ + if (test_bit(ISP_STATE_SLEEPING, &isp->state)) { + clear_bit(ISP_STATE_SLEEPING, &isp->state); + apple_isp_start_streaming(isp); + } + + return 0; +} + +static const struct vb2_ops isp_vb2_ops = { + .queue_setup = isp_vb2_queue_setup, + .buf_init = isp_vb2_buf_init, + .buf_cleanup = isp_vb2_buf_cleanup, + .buf_prepare = isp_vb2_buf_prepare, + .buf_queue = isp_vb2_buf_queue, + .start_streaming = isp_vb2_start_streaming, + .stop_streaming = isp_vb2_stop_streaming, +}; + +static int isp_set_preset(struct apple_isp *isp, struct isp_format *fmt, + struct isp_preset *preset) +{ + int i; + size_t total_size; + + fmt->preset = preset; + + /* I really fucking hope they all use NV12. */ + fmt->num_planes = 2; + fmt->strides[0] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + /* UV subsampled interleaved */ + fmt->strides[1] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + fmt->plane_size[0] = fmt->strides[0] * preset->output_dim.y; + fmt->plane_size[1] = fmt->strides[1] * preset->output_dim.y / 2; + + total_size = 0; + for (i = 0; i < fmt->num_planes; i++) + total_size += fmt->plane_size[i]; + fmt->total_size = total_size; + + return 0; +} + +static struct isp_preset *isp_select_preset(struct apple_isp *isp, u32 width, + u32 height) +{ + struct isp_preset *preset, *best = &isp->presets[0]; + int i, score, best_score = INT_MAX; + + /* Default if no dimensions */ + if (width == 0 || height == 0) + return &isp->presets[0]; + + for (i = 0; i < isp->num_presets; i++) { + preset = &isp->presets[i]; + score = abs((int)preset->output_dim.x - (int)width) + + abs((int)preset->output_dim.y - (int)height); + if (score < best_score) { + best = preset; + best_score = score; + } + } + + return best; +} + +/* + * V4L2 ioctl section + */ +static int isp_vidioc_querycap(struct file *file, void *priv, + struct v4l2_capability *cap) +{ + strscpy(cap->card, APPLE_ISP_CARD_NAME, sizeof(cap->card)); + strscpy(cap->driver, APPLE_ISP_DEVICE_NAME, sizeof(cap->driver)); + + return 0; +} + +static int isp_vidioc_enum_format(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + struct apple_isp *isp = video_drvdata(file); + + if (f->index >= ISP_MAX_PIX_FORMATS) + return -EINVAL; + + switch (f->index) { + case 0: + f->pixelformat = V4L2_PIX_FMT_NV12; + break; + case 1: + if (!isp->multiplanar) + return -EINVAL; + f->pixelformat = V4L2_PIX_FMT_NV12M; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int isp_vidioc_enum_framesizes(struct file *file, void *fh, + struct v4l2_frmsizeenum *f) +{ + struct apple_isp *isp = video_drvdata(file); + + if (f->index >= isp->num_presets) + return -EINVAL; + + if ((f->pixel_format != V4L2_PIX_FMT_NV12) && + (f->pixel_format != V4L2_PIX_FMT_NV12M)) + return -EINVAL; + + f->discrete.width = isp->presets[f->index].output_dim.x; + f->discrete.height = isp->presets[f->index].output_dim.y; + f->type = V4L2_FRMSIZE_TYPE_DISCRETE; + + return 0; +} + +static int isp_vidioc_enum_frameintervals(struct file *filp, void *priv, + struct v4l2_frmivalenum *interval) +{ + if (interval->index != 0) + return -EINVAL; + + interval->type = V4L2_FRMIVAL_TYPE_DISCRETE; + interval->discrete.numerator = 1; + interval->discrete.denominator = 30; + return 0; +} + +static inline void isp_get_sp_pix_format(struct apple_isp *isp, + struct v4l2_format *f, + struct isp_format *fmt) +{ + f->fmt.pix.width = fmt->preset->output_dim.x; + f->fmt.pix.height = fmt->preset->output_dim.y; + f->fmt.pix.bytesperline = fmt->strides[0]; + f->fmt.pix.sizeimage = fmt->total_size; + + f->fmt.pix.field = V4L2_FIELD_NONE; + f->fmt.pix.pixelformat = V4L2_PIX_FMT_NV12; + f->fmt.pix.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix.ycbcr_enc = V4L2_YCBCR_ENC_709; + f->fmt.pix.xfer_func = V4L2_XFER_FUNC_709; +} + +static inline void isp_get_mp_pix_format(struct apple_isp *isp, + struct v4l2_format *f, + struct isp_format *fmt) +{ + f->fmt.pix_mp.width = fmt->preset->output_dim.x; + f->fmt.pix_mp.height = fmt->preset->output_dim.y; + f->fmt.pix_mp.num_planes = fmt->num_planes; + for (int i = 0; i < fmt->num_planes; i++) { + f->fmt.pix_mp.plane_fmt[i].sizeimage = fmt->plane_size[i]; + f->fmt.pix_mp.plane_fmt[i].bytesperline = fmt->strides[i]; + } + + f->fmt.pix_mp.field = V4L2_FIELD_NONE; + f->fmt.pix_mp.pixelformat = V4L2_PIX_FMT_NV12M; + f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709; + f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_709; + f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_709; +} + +static int isp_vidioc_get_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + + isp_get_sp_pix_format(isp, f, fmt); + + return 0; +} + +static int isp_vidioc_set_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); + err = isp_set_preset(isp, fmt, preset); + if (err) + return err; + + isp_get_sp_pix_format(isp, f, fmt); + + isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + return 0; +} + +static int isp_vidioc_try_format(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format fmt = *isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); + err = isp_set_preset(isp, &fmt, preset); + if (err) + return err; + + isp_get_sp_pix_format(isp, f, &fmt); + + return 0; +} + +static int isp_vidioc_get_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + + if (!isp->multiplanar) + return -ENOTTY; + + isp_get_mp_pix_format(isp, f, fmt); + + return 0; +} + +static int isp_vidioc_set_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + if (!isp->multiplanar) + return -ENOTTY; + + preset = isp_select_preset(isp, f->fmt.pix_mp.width, + f->fmt.pix_mp.height); + err = isp_set_preset(isp, fmt, preset); + if (err) + return err; + + isp_get_mp_pix_format(isp, f, fmt); + + isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + + return 0; +} + +static int isp_vidioc_try_format_mplane(struct file *file, void *fh, + struct v4l2_format *f) +{ + struct apple_isp *isp = video_drvdata(file); + struct isp_format fmt = *isp_get_current_format(isp); + struct isp_preset *preset; + int err; + + if (!isp->multiplanar) + return -ENOTTY; + + preset = isp_select_preset(isp, f->fmt.pix_mp.width, + f->fmt.pix_mp.height); + err = isp_set_preset(isp, &fmt, preset); + if (err) + return err; + + isp_get_mp_pix_format(isp, f, &fmt); + + return 0; +} + +static int isp_vidioc_enum_input(struct file *file, void *fh, + struct v4l2_input *inp) +{ + if (inp->index) + return -EINVAL; + + strscpy(inp->name, APPLE_ISP_DEVICE_NAME, sizeof(inp->name)); + inp->type = V4L2_INPUT_TYPE_CAMERA; + + return 0; +} + +static int isp_vidioc_get_input(struct file *file, void *fh, unsigned int *i) +{ + *i = 0; + + return 0; +} + +static int isp_vidioc_set_input(struct file *file, void *fh, unsigned int i) +{ + if (i) + return -EINVAL; + + return 0; +} + +static int isp_vidioc_get_param(struct file *file, void *fh, + struct v4l2_streamparm *a) +{ + struct apple_isp *isp = video_drvdata(file); + + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && + (!isp->multiplanar || + a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + return -EINVAL; + + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; + a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + + return 0; +} + +static int isp_vidioc_set_param(struct file *file, void *fh, + struct v4l2_streamparm *a) +{ + struct apple_isp *isp = video_drvdata(file); + + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && + (!isp->multiplanar || + a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + return -EINVAL; + + /* Not supporting frame rate sets. No use. Plus floats. */ + a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; + a->parm.capture.readbuffers = ISP_MIN_FRAMES; + a->parm.capture.timeperframe.numerator = ISP_FRAME_RATE_NUM; + a->parm.capture.timeperframe.denominator = ISP_FRAME_RATE_DEN; + + return 0; +} + +static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { + .vidioc_querycap = isp_vidioc_querycap, + + .vidioc_enum_fmt_vid_cap = isp_vidioc_enum_format, + .vidioc_g_fmt_vid_cap = isp_vidioc_get_format, + .vidioc_s_fmt_vid_cap = isp_vidioc_set_format, + .vidioc_try_fmt_vid_cap = isp_vidioc_try_format, + .vidioc_g_fmt_vid_cap_mplane = isp_vidioc_get_format_mplane, + .vidioc_s_fmt_vid_cap_mplane = isp_vidioc_set_format_mplane, + .vidioc_try_fmt_vid_cap_mplane = isp_vidioc_try_format_mplane, + + .vidioc_enum_framesizes = isp_vidioc_enum_framesizes, + .vidioc_enum_frameintervals = isp_vidioc_enum_frameintervals, + .vidioc_enum_input = isp_vidioc_enum_input, + .vidioc_g_input = isp_vidioc_get_input, + .vidioc_s_input = isp_vidioc_set_input, + .vidioc_g_parm = isp_vidioc_get_param, + .vidioc_s_parm = isp_vidioc_set_param, + + .vidioc_reqbufs = vb2_ioctl_reqbufs, + .vidioc_querybuf = vb2_ioctl_querybuf, + .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_qbuf = vb2_ioctl_qbuf, + .vidioc_expbuf = vb2_ioctl_expbuf, + .vidioc_dqbuf = vb2_ioctl_dqbuf, + .vidioc_prepare_buf = vb2_ioctl_prepare_buf, + .vidioc_streamon = vb2_ioctl_streamon, + .vidioc_streamoff = vb2_ioctl_streamoff, +}; + +static const struct v4l2_file_operations isp_v4l2_fops = { + .owner = THIS_MODULE, + .open = v4l2_fh_open, + .release = vb2_fop_release, + .read = vb2_fop_read, + .poll = vb2_fop_poll, + .mmap = vb2_fop_mmap, + .unlocked_ioctl = video_ioctl2, +}; + +static const struct media_device_ops isp_media_device_ops = { + .link_notify = v4l2_pipeline_link_notify, +}; + +int apple_isp_setup_video(struct apple_isp *isp) +{ + struct video_device *vdev = &isp->vdev; + struct vb2_queue *vbq = &isp->vbq; + struct isp_format *fmt = isp_get_current_format(isp); + int err; + + err = isp_set_preset(isp, fmt, &isp->presets[0]); + if (err) { + dev_err(isp->dev, "failed to set default preset: %d\n", err); + return err; + } + + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + isp->meta_surfs[i] = + isp_alloc_surface_vmap(isp, isp->hw->meta_size); + if (!isp->meta_surfs[i]) { + isp_err(isp, "failed to alloc meta surface\n"); + err = -ENOMEM; + goto surf_cleanup; + } + } + + media_device_init(&isp->mdev); + isp->v4l2_dev.mdev = &isp->mdev; + isp->mdev.ops = &isp_media_device_ops; + isp->mdev.dev = isp->dev; + strscpy(isp->mdev.model, APPLE_ISP_DEVICE_NAME, + sizeof(isp->mdev.model)); + + err = media_device_register(&isp->mdev); + if (err) { + dev_err(isp->dev, "failed to register media device: %d\n", err); + goto media_cleanup; + } + + isp->multiplanar = multiplanar; + + err = v4l2_device_register(isp->dev, &isp->v4l2_dev); + if (err) { + dev_err(isp->dev, "failed to register v4l2 device: %d\n", err); + goto media_unregister; + } + + vbq->drv_priv = isp; + vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + vbq->io_modes = VB2_MMAP; + vbq->dev = isp->dev; + vbq->ops = &isp_vb2_ops; + vbq->mem_ops = &vb2_dma_sg_memops; + vbq->buf_struct_size = sizeof(struct isp_buffer); + vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vbq->min_queued_buffers = ISP_MIN_FRAMES; + vbq->lock = &isp->video_lock; + + err = vb2_queue_init(vbq); + if (err) { + dev_err(isp->dev, "failed to init vb2 queue: %d\n", err); + goto v4l2_unregister; + } + + vdev->queue = vbq; + vdev->fops = &isp_v4l2_fops; + vdev->ioctl_ops = &isp_v4l2_ioctl_ops; + vdev->device_caps = V4L2_BUF_TYPE_VIDEO_CAPTURE | V4L2_CAP_STREAMING; + if (isp->multiplanar) + vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; + vdev->v4l2_dev = &isp->v4l2_dev; + vdev->vfl_type = VFL_TYPE_VIDEO; + vdev->vfl_dir = VFL_DIR_RX; + vdev->release = video_device_release_empty; + vdev->lock = &isp->video_lock; + strscpy(vdev->name, APPLE_ISP_DEVICE_NAME, sizeof(vdev->name)); + video_set_drvdata(vdev, isp); + + err = video_register_device(vdev, VFL_TYPE_VIDEO, 0); + if (err) { + dev_err(isp->dev, "failed to register video device: %d\n", err); + goto v4l2_unregister; + } + + return 0; + +v4l2_unregister: + v4l2_device_unregister(&isp->v4l2_dev); +media_unregister: + media_device_unregister(&isp->mdev); +media_cleanup: + media_device_cleanup(&isp->mdev); +surf_cleanup: + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } + + return err; +} + +void apple_isp_remove_video(struct apple_isp *isp) +{ + vb2_video_unregister_device(&isp->vdev); + v4l2_device_unregister(&isp->v4l2_dev); + media_device_unregister(&isp->mdev); + media_device_cleanup(&isp->mdev); + for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + if (isp->meta_surfs[i]) + isp_free_surface(isp, isp->meta_surfs[i]); + isp->meta_surfs[i] = NULL; + } +} diff --git a/drivers/media/platform/apple/isp/isp-v4l2.h b/drivers/media/platform/apple/isp/isp-v4l2.h new file mode 100644 index 00000000000000..4d47deeb83b055 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright 2023 Eileen Yoon */ + +#ifndef __ISP_V4L2_H__ +#define __ISP_V4L2_H__ + +#include "isp-drv.h" + +int apple_isp_setup_video(struct apple_isp *isp); +void apple_isp_remove_video(struct apple_isp *isp); +int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan); + +int apple_isp_video_suspend(struct apple_isp *isp); +int apple_isp_video_resume(struct apple_isp *isp); + +#endif /* __ISP_V4L2_H__ */ diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-common.h b/drivers/media/platform/arm/mali-c55/mali-c55-common.h index 31c1deaca14661..13a3e9dc424311 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-common.h +++ b/drivers/media/platform/arm/mali-c55/mali-c55-common.h @@ -306,5 +306,7 @@ bool mali_c55_pipeline_ready(struct mali_c55 *mali_c55); void mali_c55_stats_fill_buffer(struct mali_c55 *mali_c55, enum mali_c55_config_spaces cfg_space); void mali_c55_params_write_config(struct mali_c55 *mali_c55); +void mali_c55_params_init_isp_config(struct mali_c55 *mali_c55, + const struct v4l2_subdev_state *state); #endif /* _MALI_C55_COMMON_H */ diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-core.c b/drivers/media/platform/arm/mali-c55/mali-c55-core.c index 43b834459ccf3e..c1a562cd214ec5 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-core.c +++ b/drivers/media/platform/arm/mali-c55/mali-c55-core.c @@ -663,41 +663,6 @@ static int mali_c55_init_context(struct mali_c55 *mali_c55, mali_c55->base + config_space_addrs[MALI_C55_CONFIG_PING], MALI_C55_CONFIG_SPACE_SIZE); - /* - * Some features of the ISP need to be disabled by default and only - * enabled at the same time as they're configured by a parameters buffer - */ - - /* Bypass the sqrt and square compression and expansion modules */ - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_1, - MALI_C55_REG_BYPASS_1_FE_SQRT, - MALI_C55_REG_BYPASS_1_FE_SQRT); - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_3, - MALI_C55_REG_BYPASS_3_SQUARE_BE, - MALI_C55_REG_BYPASS_3_SQUARE_BE); - - /* Bypass the temper module */ - mali_c55_ctx_write(mali_c55, MALI_C55_REG_BYPASS_2, - MALI_C55_REG_BYPASS_2_TEMPER); - - /* Disable the temper module's DMA read/write */ - mali_c55_ctx_write(mali_c55, MALI_C55_REG_TEMPER_DMA_IO, 0x0); - - /* Bypass the colour noise reduction */ - mali_c55_ctx_write(mali_c55, MALI_C55_REG_BYPASS_4, - MALI_C55_REG_BYPASS_4_CNR); - - /* Disable the sinter module */ - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_SINTER_CONFIG, - MALI_C55_SINTER_ENABLE_MASK, 0); - - /* Disable the RGB Gamma module for each output */ - mali_c55_ctx_write(mali_c55, MALI_C55_REG_FR_GAMMA_RGB_ENABLE, 0); - mali_c55_ctx_write(mali_c55, MALI_C55_REG_DS_GAMMA_RGB_ENABLE, 0); - - /* Disable the colour correction matrix */ - mali_c55_ctx_write(mali_c55, MALI_C55_REG_CCM_ENABLE, 0); - return 0; } diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-isp.c b/drivers/media/platform/arm/mali-c55/mali-c55-isp.c index 497f25fbdd13b0..4c0fd1ec741c3b 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-isp.c +++ b/drivers/media/platform/arm/mali-c55/mali-c55-isp.c @@ -112,9 +112,6 @@ static int mali_c55_isp_start(struct mali_c55 *mali_c55, const struct v4l2_subdev_state *state) { struct mali_c55_context *ctx = mali_c55_get_active_context(mali_c55); - const struct mali_c55_isp_format_info *cfg; - const struct v4l2_mbus_framefmt *format; - const struct v4l2_rect *crop; u32 val; int ret; @@ -122,35 +119,11 @@ static int mali_c55_isp_start(struct mali_c55 *mali_c55, MALI_C55_REG_MCU_CONFIG_WRITE_MASK, MALI_C55_REG_MCU_CONFIG_WRITE_PING); - /* Apply input windowing */ - crop = v4l2_subdev_state_get_crop(state, MALI_C55_ISP_PAD_SINK_VIDEO); - format = v4l2_subdev_state_get_format(state, - MALI_C55_ISP_PAD_SINK_VIDEO); - cfg = mali_c55_isp_get_mbus_config_by_code(format->code); - - mali_c55_write(mali_c55, MALI_C55_REG_HC_START, - MALI_C55_HC_START(crop->left)); - mali_c55_write(mali_c55, MALI_C55_REG_HC_SIZE, - MALI_C55_HC_SIZE(crop->width)); - mali_c55_write(mali_c55, MALI_C55_REG_VC_START_SIZE, - MALI_C55_VC_START(crop->top) | - MALI_C55_VC_SIZE(crop->height)); - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BASE_ADDR, - MALI_C55_REG_ACTIVE_WIDTH_MASK, format->width); - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BASE_ADDR, - MALI_C55_REG_ACTIVE_HEIGHT_MASK, - format->height << 16); - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BAYER_ORDER, - MALI_C55_BAYER_ORDER_MASK, cfg->order); - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_INPUT_WIDTH, - MALI_C55_INPUT_WIDTH_MASK, - MALI_C55_INPUT_WIDTH_20BIT); - - mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_ISP_RAW_BYPASS, - MALI_C55_ISP_RAW_BYPASS_BYPASS_MASK, - cfg->bypass ? MALI_C55_ISP_RAW_BYPASS_BYPASS_MASK : - 0x00); - + /* + * Apply default ISP configuration and the apply configurations from + * the first available parameters buffer. + */ + mali_c55_params_init_isp_config(mali_c55, state); mali_c55_params_write_config(mali_c55); ret = mali_c55_config_write(ctx, MALI_C55_CONFIG_PING, true); if (ret) { diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-params.c b/drivers/media/platform/arm/mali-c55/mali-c55-params.c index be0e909bcf29f6..f6f742a939c765 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-params.c +++ b/drivers/media/platform/arm/mali-c55/mali-c55-params.c @@ -732,6 +732,128 @@ void mali_c55_params_write_config(struct mali_c55 *mali_c55) vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE); } +void mali_c55_params_init_isp_config(struct mali_c55 *mali_c55, + const struct v4l2_subdev_state *state) +{ + const struct mali_c55_isp_format_info *cfg; + const struct v4l2_mbus_framefmt *format; + const struct v4l2_rect *crop; + + /* Apply input windowing */ + crop = v4l2_subdev_state_get_crop(state, MALI_C55_ISP_PAD_SINK_VIDEO); + format = v4l2_subdev_state_get_format(state, + MALI_C55_ISP_PAD_SINK_VIDEO); + cfg = mali_c55_isp_get_mbus_config_by_code(format->code); + + mali_c55_write(mali_c55, MALI_C55_REG_HC_START, + MALI_C55_HC_START(crop->left)); + mali_c55_write(mali_c55, MALI_C55_REG_HC_SIZE, + MALI_C55_HC_SIZE(crop->width)); + mali_c55_write(mali_c55, MALI_C55_REG_VC_START_SIZE, + MALI_C55_VC_START(crop->top) | + MALI_C55_VC_SIZE(crop->height)); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BASE_ADDR, + MALI_C55_REG_ACTIVE_WIDTH_MASK, format->width); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BASE_ADDR, + MALI_C55_REG_ACTIVE_HEIGHT_MASK, + format->height << 16); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BAYER_ORDER, + MALI_C55_BAYER_ORDER_MASK, cfg->order); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_INPUT_WIDTH, + MALI_C55_INPUT_WIDTH_MASK, + MALI_C55_INPUT_WIDTH_20BIT); + + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_ISP_RAW_BYPASS, + MALI_C55_ISP_RAW_BYPASS_BYPASS_MASK, + cfg->bypass ? MALI_C55_ISP_RAW_BYPASS_BYPASS_MASK : + 0x00); + + /* + * Some features of the ISP need to be disabled by default and only + * enabled at the same time as they're configured by a parameters buffer + */ + + /* Bypass the sqrt and square compression and expansion modules */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_1, + MALI_C55_REG_BYPASS_1_FE_SQRT, + MALI_C55_REG_BYPASS_1_FE_SQRT); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_3, + MALI_C55_REG_BYPASS_3_SQUARE_BE, + MALI_C55_REG_BYPASS_3_SQUARE_BE); + + /* Bypass the sensor offset correction (BLS) module */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_BYPASS_3, + MALI_C55_REG_BYPASS_3_SENSOR_OFFSET_PRE_SH, + MALI_C55_REG_BYPASS_3_SENSOR_OFFSET_PRE_SH); + + /* Configure 1x digital gain. */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_DIGITAL_GAIN, + MALI_C55_DIGITAL_GAIN_MASK, 256); + + /* Set all AWB gains to 1x. at both AWB configuration points*/ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS1, + MALI_C55_AWB_GAIN00_MASK, 256); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS1, + MALI_C55_AWB_GAIN01_MASK, + MALI_C55_AWB_GAIN01(256)); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS2, + MALI_C55_AWB_GAIN10_MASK, 256); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS2, + MALI_C55_AWB_GAIN11_MASK, + MALI_C55_AWB_GAIN11(256)); + + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS1_AEXP, + MALI_C55_AWB_GAIN00_MASK, 256); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS1_AEXP, + MALI_C55_AWB_GAIN01_MASK, + MALI_C55_AWB_GAIN01(256)); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS2_AEXP, + MALI_C55_AWB_GAIN10_MASK, 256); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_AWB_GAINS2_AEXP, + MALI_C55_AWB_GAIN11_MASK, + MALI_C55_AWB_GAIN11(256)); + + /* Bypass mesh shading corrections (LSC). */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_MESH_SHADING_CONFIG, + MALI_C55_MESH_SHADING_ENABLE_MASK, + false); + + /* Bypass the temper module */ + mali_c55_ctx_write(mali_c55, MALI_C55_REG_BYPASS_2, + MALI_C55_REG_BYPASS_2_TEMPER); + + /* Disable the temper module's DMA read/write */ + mali_c55_ctx_write(mali_c55, MALI_C55_REG_TEMPER_DMA_IO, 0x0); + + /* Bypass the colour noise reduction */ + mali_c55_ctx_write(mali_c55, MALI_C55_REG_BYPASS_4, + MALI_C55_REG_BYPASS_4_CNR); + + /* Disable the sinter module */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_SINTER_CONFIG, + MALI_C55_SINTER_ENABLE_MASK, 0); + + /* Disable the RGB Gamma module for each output */ + mali_c55_ctx_write(mali_c55, MALI_C55_REG_FR_GAMMA_RGB_ENABLE, 0); + mali_c55_ctx_write(mali_c55, MALI_C55_REG_DS_GAMMA_RGB_ENABLE, 0); + + /* Disable the colour correction matrix */ + mali_c55_ctx_write(mali_c55, MALI_C55_REG_CCM_ENABLE, 0); + + /* Disable AWB stats. */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_METERING_CONFIG, + MALI_C55_AWB_DISABLE_MASK, + MALI_C55_AWB_DISABLE_MASK); + + /* Disable auto-exposure 1024-bin histograms at both tap points. */ + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_METERING_CONFIG, + MALI_C55_AEXP_HIST_DISABLE_MASK, + MALI_C55_AEXP_HIST_DISABLE); + mali_c55_ctx_update_bits(mali_c55, MALI_C55_REG_METERING_CONFIG, + MALI_C55_AEXP_IHIST_DISABLE_MASK, + MALI_C55_AEXP_IHIST_DISABLE); +} + void mali_c55_unregister_params(struct mali_c55 *mali_c55) { struct mali_c55_params *params = &mali_c55->params; diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-registers.h b/drivers/media/platform/arm/mali-c55/mali-c55-registers.h index f5a148add1c811..f098effde7b497 100644 --- a/drivers/media/platform/arm/mali-c55/mali-c55-registers.h +++ b/drivers/media/platform/arm/mali-c55/mali-c55-registers.h @@ -128,8 +128,8 @@ enum mali_c55_interrupts { #define MALI_C55_REG_BYPASS_3_SENSOR_OFFSET_PRE_SH BIT(1) #define MALI_C55_REG_BYPASS_3_MESH_SHADING BIT(3) #define MALI_C55_REG_BYPASS_3_WHITE_BALANCE BIT(4) -#define MALI_C55_REG_BYPASS_3_IRIDIX BIT(5) -#define MALI_C55_REG_BYPASS_3_IRIDIX_GAIN BIT(6) +#define MALI_C55_REG_BYPASS_3_IRIDIX_GAIN BIT(5) +#define MALI_C55_REG_BYPASS_3_IRIDIX BIT(6) #define MALI_C55_REG_BYPASS_4 0x18ec0 #define MALI_C55_REG_BYPASS_4_DEMOSAIC_RGB BIT(1) #define MALI_C55_REG_BYPASS_4_PF_CORRECTION BIT(3) diff --git a/drivers/media/platform/chips-media/wave5/wave5-vdi.c b/drivers/media/platform/chips-media/wave5/wave5-vdi.c index bb13267ced38aa..8f71920a8a35cf 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vdi.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vdi.c @@ -49,6 +49,7 @@ int wave5_vdi_init(struct device *dev) if (!PRODUCT_CODE_W_SERIES(vpu_dev->product_code)) { WARN_ONCE(1, "unsupported product code: 0x%x\n", vpu_dev->product_code); + wave5_vdi_free_dma_memory(vpu_dev, &vpu_dev->common_mem); return -EOPNOTSUPP; } diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index 80e1831a42e02f..d419076d7052c6 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -1303,13 +1303,17 @@ static void wave5_vpu_dec_buf_queue_dst(struct vb2_buffer *vb) if (vb2_is_streaming(vb->vb2_queue) && v4l2_m2m_dst_buf_is_last(m2m_ctx)) { unsigned int i; + unsigned long flags; for (i = 0; i < vb->num_planes; i++) vb2_set_plane_payload(vb, i, 0); vbuf->field = V4L2_FIELD_NONE; + spin_lock_irqsave(&inst->state_spinlock, flags); send_eos_event(inst); + spin_unlock_irqrestore(&inst->state_spinlock, flags); + v4l2_m2m_last_buffer_done(m2m_ctx, vbuf); } else { v4l2_m2m_buf_queue(m2m_ctx, vbuf); @@ -1462,8 +1466,13 @@ static int streamoff_output(struct vb2_queue *q) inst->codec_info->dec_info.stream_rd_ptr = new_rd_ptr; inst->codec_info->dec_info.stream_wr_ptr = new_rd_ptr; - if (v4l2_m2m_has_stopped(m2m_ctx)) + if (v4l2_m2m_has_stopped(m2m_ctx)) { + unsigned long flags; + + spin_lock_irqsave(&inst->state_spinlock, flags); send_eos_event(inst); + spin_unlock_irqrestore(&inst->state_spinlock, flags); + } /* streamoff on output cancels any draining operation */ inst->eos = false; @@ -1584,6 +1593,7 @@ static int initialize_sequence(struct vpu_instance *inst) { struct dec_initial_info initial_info; int ret = 0; + unsigned long flags; memset(&initial_info, 0, sizeof(struct dec_initial_info)); @@ -1605,7 +1615,9 @@ static int initialize_sequence(struct vpu_instance *inst) return ret; } + spin_lock_irqsave(&inst->state_spinlock, flags); handle_dynamic_resolution_change(inst); + spin_unlock_irqrestore(&inst->state_spinlock, flags); return 0; } diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c index c01124a349f61c..8c684756d5fc25 100644 --- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c +++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c @@ -1202,6 +1202,7 @@ static int mtk_jpeg_release(struct file *file) struct mtk_jpeg_dev *jpeg = video_drvdata(file); struct mtk_jpeg_ctx *ctx = mtk_jpeg_file_to_ctx(file); + cancel_work_sync(&ctx->jpeg_work); mutex_lock(&jpeg->lock); v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c index c977ed0c09b6b3..4e4541b2fc8eae 100644 --- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c +++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc_drv.c @@ -215,6 +215,15 @@ static int fops_vcodec_release(struct file *file) v4l2_fh_exit(&ctx->fh); v4l2_ctrl_handler_free(&ctx->ctrl_hdl); + /* + * Cancel any pending encode work before freeing the context. + * Although v4l2_m2m_ctx_release() waits for m2m job completion, + * the workqueue handler (mtk_venc_worker) may still be accessing + * the context after v4l2_m2m_job_finish() returns. Without this, + * a use-after-free occurs when the worker accesses ctx after kfree. + */ + cancel_work_sync(&ctx->encode_work); + spin_lock_irqsave(&dev->dev_ctx_lock, flags); list_del_init(&ctx->list); spin_unlock_irqrestore(&dev->dev_ctx_lock, flags); diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c index 13682bf6e9f889..1be3a728f32f89 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c @@ -1410,7 +1410,7 @@ int mxc_isi_video_register(struct mxc_isi_pipe *pipe, q->mem_ops = &vb2_dma_contig_memops; q->buf_struct_size = sizeof(struct mxc_isi_buffer); q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; - q->min_queued_buffers = 2; + q->min_queued_buffers = 0; q->lock = &video->lock; q->dev = pipe->isi->dev; diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen3.c b/drivers/media/platform/qcom/camss/camss-csid-gen3.c index 664245cf6eb0ca..bd059243790ede 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-gen3.c +++ b/drivers/media/platform/qcom/camss/camss-csid-gen3.c @@ -48,9 +48,9 @@ #define IS_CSID_690(csid) ((csid->camss->res->version == CAMSS_8775P) \ || (csid->camss->res->version == CAMSS_8300)) #define CSID_BUF_DONE_IRQ_STATUS 0x8C -#define BUF_DONE_IRQ_STATUS_RDI_OFFSET (csid_is_lite(csid) ?\ - 1 : (IS_CSID_690(csid) ?\ - 13 : 14)) +#define BUF_DONE_IRQ_STATUS_RDI_OFFSET (csid_is_lite(csid) ? \ + ((IS_CSID_690(csid) ? 0 : 1)) : \ + ((IS_CSID_690(csid) ? 13 : 14))) #define CSID_BUF_DONE_IRQ_MASK 0x90 #define CSID_BUF_DONE_IRQ_CLEAR 0x94 #define CSID_BUF_DONE_IRQ_SET 0x98 diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c index 00b87fd9afbd89..9335636d7c4dfc 100644 --- a/drivers/media/platform/qcom/camss/camss.c +++ b/drivers/media/platform/qcom/camss/camss.c @@ -3598,12 +3598,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID2 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite0" }, .interrupt = { "csid_lite0" }, @@ -3617,12 +3615,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID3 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite1" }, .interrupt = { "csid_lite1" }, @@ -3636,12 +3632,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID4 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite2" }, .interrupt = { "csid_lite2" }, @@ -3655,12 +3649,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID5 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite3" }, .interrupt = { "csid_lite3" }, @@ -3674,12 +3666,10 @@ static const struct camss_subdev_resources csid_res_8775p[] = { /* CSID6 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", - "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + .clock = { "vfe_lite_csid", "vfe_lite_cphy_rx" }, .clock_rate = { - { 0, 0, 400000000, 400000000, 0}, - { 0, 0, 400000000, 480000000, 0} + { 400000000, 480000000 }, + { 400000000, 480000000 } }, .reg = { "csid_lite4" }, .interrupt = { "csid_lite4" }, @@ -3752,15 +3742,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE2 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite0" }, .interrupt = { "vfe_lite0" }, @@ -3775,15 +3767,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE3 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite1" }, .interrupt = { "vfe_lite1" }, @@ -3798,15 +3792,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE4 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite2" }, .interrupt = { "vfe_lite2" }, @@ -3821,15 +3817,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE5 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite3" }, .interrupt = { "vfe_lite3" }, @@ -3844,15 +3842,17 @@ static const struct camss_subdev_resources vfe_res_8775p[] = { /* VFE6 (lite) */ { .regulators = {}, - .clock = { "cpas_vfe_lite", "vfe_lite_ahb", + .clock = { "cpas_ahb", "cpas_vfe_lite", "vfe_lite_ahb", "vfe_lite_csid", "vfe_lite_cphy_rx", - "vfe_lite"}, + "vfe_lite", "camnoc_axi"}, .clock_rate = { - { 0, 0, 0, 0 }, + { 0 }, + { 0 }, { 300000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 400000000, 400000000, 400000000, 400000000 }, { 480000000, 600000000, 600000000, 600000000 }, + { 400000000 }, }, .reg = { "vfe_lite4" }, .interrupt = { "vfe_lite4" }, diff --git a/drivers/media/platform/qcom/iris/Kconfig b/drivers/media/platform/qcom/iris/Kconfig index 3c803a05305a80..5498f48362d15c 100644 --- a/drivers/media/platform/qcom/iris/Kconfig +++ b/drivers/media/platform/qcom/iris/Kconfig @@ -3,7 +3,7 @@ config VIDEO_QCOM_IRIS depends on VIDEO_DEV depends on ARCH_QCOM || COMPILE_TEST select V4L2_MEM2MEM_DEV - select QCOM_MDT_LOADER if ARCH_QCOM + select QCOM_MDT_LOADER select QCOM_SCM select VIDEOBUF2_DMA_CONTIG help diff --git a/drivers/media/platform/qcom/iris/iris_buffer.c b/drivers/media/platform/qcom/iris/iris_buffer.c index 9151f43bc6b9c2..1d53c7414b754b 100644 --- a/drivers/media/platform/qcom/iris/iris_buffer.c +++ b/drivers/media/platform/qcom/iris/iris_buffer.c @@ -582,10 +582,12 @@ static int iris_release_internal_buffers(struct iris_inst *inst, continue; if (!(buffer->attr & BUF_ATTR_QUEUED)) continue; + buffer->attr |= BUF_ATTR_PENDING_RELEASE; ret = hfi_ops->session_release_buf(inst, buffer); - if (ret) + if (ret) { + buffer->attr &= ~BUF_ATTR_PENDING_RELEASE; return ret; - buffer->attr |= BUF_ATTR_PENDING_RELEASE; + } } return 0; diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c index 8406c48d635b6e..dbaac01eb15a0e 100644 --- a/drivers/media/platform/qcom/iris/iris_core.c +++ b/drivers/media/platform/qcom/iris/iris_core.c @@ -75,6 +75,10 @@ int iris_core_init(struct iris_core *core) if (ret) goto error_unload_fw; + ret = iris_vpu_switch_to_hwmode(core); + if (ret) + goto error_unload_fw; + ret = iris_hfi_core_init(core); if (ret) goto error_unload_fw; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_common.c b/drivers/media/platform/qcom/iris/iris_hfi_common.c index 92112eb16c1104..621c66593d88d4 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_common.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_common.c @@ -159,6 +159,10 @@ int iris_hfi_pm_resume(struct iris_core *core) if (ret) goto err_suspend_hw; + ret = iris_vpu_switch_to_hwmode(core); + if (ret) + goto err_suspend_hw; + ret = ops->sys_interframe_powercollapse(core); if (ret) goto err_suspend_hw; diff --git a/drivers/media/platform/qcom/iris/iris_hfi_queue.c b/drivers/media/platform/qcom/iris/iris_hfi_queue.c index b3ed06297953b9..bf6db23b53e210 100644 --- a/drivers/media/platform/qcom/iris/iris_hfi_queue.c +++ b/drivers/media/platform/qcom/iris/iris_hfi_queue.c @@ -263,7 +263,7 @@ int iris_hfi_queues_init(struct iris_core *core) GFP_KERNEL, DMA_ATTR_WRITE_COMBINE); if (!core->sfr_vaddr) { dev_err(core->dev, "sfr alloc and map failed\n"); - dma_free_attrs(core->dev, sizeof(*q_tbl_hdr), core->iface_q_table_vaddr, + dma_free_attrs(core->dev, queue_size, core->iface_q_table_vaddr, core->iface_q_table_daddr, DMA_ATTR_WRITE_COMBINE); return -ENOMEM; } diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c index 719217399a304e..99d544e2af4f98 100644 --- a/drivers/media/platform/qcom/iris/iris_vdec.c +++ b/drivers/media/platform/qcom/iris/iris_vdec.c @@ -61,12 +61,6 @@ int iris_vdec_inst_init(struct iris_inst *inst) return iris_ctrls_init(inst); } -void iris_vdec_inst_deinit(struct iris_inst *inst) -{ - kfree(inst->fmt_dst); - kfree(inst->fmt_src); -} - static const struct iris_fmt iris_vdec_formats_cap[] = { [IRIS_FMT_NV12] = { .pixfmt = V4L2_PIX_FMT_NV12, diff --git a/drivers/media/platform/qcom/iris/iris_vdec.h b/drivers/media/platform/qcom/iris/iris_vdec.h index ec1ce55d1375fd..5123d2a340e15f 100644 --- a/drivers/media/platform/qcom/iris/iris_vdec.h +++ b/drivers/media/platform/qcom/iris/iris_vdec.h @@ -9,7 +9,6 @@ struct iris_inst; int iris_vdec_inst_init(struct iris_inst *inst); -void iris_vdec_inst_deinit(struct iris_inst *inst); int iris_vdec_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f); int iris_vdec_try_fmt(struct iris_inst *inst, struct v4l2_format *f); int iris_vdec_s_fmt(struct iris_inst *inst, struct v4l2_format *f); diff --git a/drivers/media/platform/qcom/iris/iris_venc.c b/drivers/media/platform/qcom/iris/iris_venc.c index aa27b22704eb9e..4d886769d958b9 100644 --- a/drivers/media/platform/qcom/iris/iris_venc.c +++ b/drivers/media/platform/qcom/iris/iris_venc.c @@ -79,12 +79,6 @@ int iris_venc_inst_init(struct iris_inst *inst) return iris_ctrls_init(inst); } -void iris_venc_inst_deinit(struct iris_inst *inst) -{ - kfree(inst->fmt_dst); - kfree(inst->fmt_src); -} - static const struct iris_fmt iris_venc_formats_cap[] = { [IRIS_FMT_H264] = { .pixfmt = V4L2_PIX_FMT_H264, diff --git a/drivers/media/platform/qcom/iris/iris_venc.h b/drivers/media/platform/qcom/iris/iris_venc.h index c4db7433da5375..00c1716b2747c7 100644 --- a/drivers/media/platform/qcom/iris/iris_venc.h +++ b/drivers/media/platform/qcom/iris/iris_venc.h @@ -9,7 +9,6 @@ struct iris_inst; int iris_venc_inst_init(struct iris_inst *inst); -void iris_venc_inst_deinit(struct iris_inst *inst); int iris_venc_enum_fmt(struct iris_inst *inst, struct v4l2_fmtdesc *f); int iris_venc_try_fmt(struct iris_inst *inst, struct v4l2_format *f); int iris_venc_s_fmt(struct iris_inst *inst, struct v4l2_format *f); diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c index bd38d84c9cc79d..5eb1786b07371d 100644 --- a/drivers/media/platform/qcom/iris/iris_vidc.c +++ b/drivers/media/platform/qcom/iris/iris_vidc.c @@ -289,10 +289,6 @@ int iris_close(struct file *filp) v4l2_m2m_ctx_release(inst->m2m_ctx); v4l2_m2m_release(inst->m2m_dev); mutex_lock(&inst->lock); - if (inst->domain == DECODER) - iris_vdec_inst_deinit(inst); - else if (inst->domain == ENCODER) - iris_venc_inst_deinit(inst); iris_session_close(inst); iris_inst_change_state(inst, IRIS_INST_DEINIT); iris_v4l2_fh_deinit(inst, filp); @@ -304,6 +300,8 @@ int iris_close(struct file *filp) mutex_unlock(&inst->lock); mutex_destroy(&inst->ctx_q_lock); mutex_destroy(&inst->lock); + kfree(inst->fmt_src); + kfree(inst->fmt_dst); kfree(inst); return 0; diff --git a/drivers/media/platform/qcom/iris/iris_vpu2.c b/drivers/media/platform/qcom/iris/iris_vpu2.c index 9c103a2e4e4eaf..01ef40f3895743 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu2.c +++ b/drivers/media/platform/qcom/iris/iris_vpu2.c @@ -44,4 +44,5 @@ const struct vpu_ops iris_vpu2_ops = { .power_off_controller = iris_vpu_power_off_controller, .power_on_controller = iris_vpu_power_on_controller, .calc_freq = iris_vpu2_calc_freq, + .set_hwmode = iris_vpu_set_hwmode, }; diff --git a/drivers/media/platform/qcom/iris/iris_vpu3x.c b/drivers/media/platform/qcom/iris/iris_vpu3x.c index fe4423b951b1e9..3dad47be78b58f 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu3x.c +++ b/drivers/media/platform/qcom/iris/iris_vpu3x.c @@ -234,14 +234,8 @@ static int iris_vpu35_power_on_hw(struct iris_core *core) if (ret) goto err_disable_hw_free_clk; - ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true); - if (ret) - goto err_disable_hw_clk; - return 0; -err_disable_hw_clk: - iris_disable_unprepare_clock(core, IRIS_HW_CLK); err_disable_hw_free_clk: iris_disable_unprepare_clock(core, IRIS_HW_FREERUN_CLK); err_disable_axi_clk: @@ -266,6 +260,7 @@ const struct vpu_ops iris_vpu3_ops = { .power_off_controller = iris_vpu_power_off_controller, .power_on_controller = iris_vpu_power_on_controller, .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, + .set_hwmode = iris_vpu_set_hwmode, }; const struct vpu_ops iris_vpu33_ops = { @@ -274,6 +269,7 @@ const struct vpu_ops iris_vpu33_ops = { .power_off_controller = iris_vpu33_power_off_controller, .power_on_controller = iris_vpu_power_on_controller, .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, + .set_hwmode = iris_vpu_set_hwmode, }; const struct vpu_ops iris_vpu35_ops = { @@ -283,4 +279,5 @@ const struct vpu_ops iris_vpu35_ops = { .power_on_controller = iris_vpu35_vpu4x_power_on_controller, .program_bootup_registers = iris_vpu35_vpu4x_program_bootup_registers, .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, + .set_hwmode = iris_vpu_set_hwmode, }; diff --git a/drivers/media/platform/qcom/iris/iris_vpu4x.c b/drivers/media/platform/qcom/iris/iris_vpu4x.c index a8db02ce5c5ec5..02e100a4045fce 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu4x.c +++ b/drivers/media/platform/qcom/iris/iris_vpu4x.c @@ -252,21 +252,10 @@ static int iris_vpu4x_power_on_hardware(struct iris_core *core) ret = iris_vpu4x_power_on_apv(core); if (ret) goto disable_hw_clocks; - - iris_vpu4x_ahb_sync_reset_apv(core); } - iris_vpu4x_ahb_sync_reset_hardware(core); - - ret = iris_vpu4x_genpd_set_hwmode(core, true, efuse_value); - if (ret) - goto disable_apv_power_domain; - return 0; -disable_apv_power_domain: - if (!(efuse_value & DISABLE_VIDEO_APV_BIT)) - iris_vpu4x_power_off_apv(core); disable_hw_clocks: iris_vpu4x_disable_hardware_clocks(core, efuse_value); disable_vpp1_power_domain: @@ -359,6 +348,18 @@ static void iris_vpu4x_power_off_hardware(struct iris_core *core) iris_disable_power_domains(core, core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN]); } +static int iris_vpu4x_set_hwmode(struct iris_core *core) +{ + u32 efuse_value = readl(core->reg_base + WRAPPER_EFUSE_MONITOR); + + if (!(efuse_value & DISABLE_VIDEO_APV_BIT)) + iris_vpu4x_ahb_sync_reset_apv(core); + + iris_vpu4x_ahb_sync_reset_hardware(core); + + return iris_vpu4x_genpd_set_hwmode(core, true, efuse_value); +} + const struct vpu_ops iris_vpu4x_ops = { .power_off_hw = iris_vpu4x_power_off_hardware, .power_on_hw = iris_vpu4x_power_on_hardware, @@ -366,4 +367,5 @@ const struct vpu_ops iris_vpu4x_ops = { .power_on_controller = iris_vpu35_vpu4x_power_on_controller, .program_bootup_registers = iris_vpu35_vpu4x_program_bootup_registers, .calc_freq = iris_vpu3x_vpu4x_calculate_frequency, + .set_hwmode = iris_vpu4x_set_hwmode, }; diff --git a/drivers/media/platform/qcom/iris/iris_vpu_buffer.h b/drivers/media/platform/qcom/iris/iris_vpu_buffer.h index 12640eb5ed8c45..8c0d6b7b5de85f 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_buffer.h +++ b/drivers/media/platform/qcom/iris/iris_vpu_buffer.h @@ -67,7 +67,7 @@ struct iris_inst; #define SIZE_DOLBY_RPU_METADATA (41 * 1024) #define H264_CABAC_HDR_RATIO_HD_TOT 1 #define H264_CABAC_RES_RATIO_HD_TOT 3 -#define H265D_MAX_SLICE 1200 +#define H265D_MAX_SLICE 3600 #define SIZE_H265D_HW_PIC_T SIZE_H264D_HW_PIC_T #define H265_CABAC_HDR_RATIO_HD_TOT 2 #define H265_CABAC_RES_RATIO_HD_TOT 2 diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.c b/drivers/media/platform/qcom/iris/iris_vpu_common.c index 548e5f1727fdb7..69e6126dc4d95e 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_common.c +++ b/drivers/media/platform/qcom/iris/iris_vpu_common.c @@ -292,14 +292,8 @@ int iris_vpu_power_on_hw(struct iris_core *core) if (ret && ret != -ENOENT) goto err_disable_hw_clock; - ret = dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true); - if (ret) - goto err_disable_hw_ahb_clock; - return 0; -err_disable_hw_ahb_clock: - iris_disable_unprepare_clock(core, IRIS_HW_AHB_CLK); err_disable_hw_clock: iris_disable_unprepare_clock(core, IRIS_HW_CLK); err_disable_power: @@ -308,6 +302,16 @@ int iris_vpu_power_on_hw(struct iris_core *core) return ret; } +int iris_vpu_set_hwmode(struct iris_core *core) +{ + return dev_pm_genpd_set_hwmode(core->pmdomain_tbl->pd_devs[IRIS_HW_POWER_DOMAIN], true); +} + +int iris_vpu_switch_to_hwmode(struct iris_core *core) +{ + return core->iris_platform_data->vpu_ops->set_hwmode(core); +} + int iris_vpu35_vpu4x_power_off_controller(struct iris_core *core) { u32 clk_rst_tbl_size = core->iris_platform_data->clk_rst_tbl_size; diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.h b/drivers/media/platform/qcom/iris/iris_vpu_common.h index f6dffc613b8223..dee3b1349c5e86 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_common.h +++ b/drivers/media/platform/qcom/iris/iris_vpu_common.h @@ -21,6 +21,7 @@ struct vpu_ops { int (*power_on_controller)(struct iris_core *core); void (*program_bootup_registers)(struct iris_core *core); u64 (*calc_freq)(struct iris_inst *inst, size_t data_size); + int (*set_hwmode)(struct iris_core *core); }; int iris_vpu_boot_firmware(struct iris_core *core); @@ -30,6 +31,8 @@ int iris_vpu_watchdog(struct iris_core *core, u32 intr_status); int iris_vpu_prepare_pc(struct iris_core *core); int iris_vpu_power_on_controller(struct iris_core *core); int iris_vpu_power_on_hw(struct iris_core *core); +int iris_vpu_set_hwmode(struct iris_core *core); +int iris_vpu_switch_to_hwmode(struct iris_core *core); int iris_vpu_power_on(struct iris_core *core); int iris_vpu_power_off_controller(struct iris_core *core); void iris_vpu_power_off_hw(struct iris_core *core); diff --git a/drivers/media/platform/qcom/venus/Kconfig b/drivers/media/platform/qcom/venus/Kconfig index ffb731ecd48c90..63ee8c78dc6d75 100644 --- a/drivers/media/platform/qcom/venus/Kconfig +++ b/drivers/media/platform/qcom/venus/Kconfig @@ -4,7 +4,7 @@ config VIDEO_QCOM_VENUS depends on VIDEO_DEV && QCOM_SMEM depends on (ARCH_QCOM && ARM64 && IOMMU_API) || COMPILE_TEST select OF_DYNAMIC if ARCH_QCOM - select QCOM_MDT_LOADER if ARCH_QCOM + select QCOM_MDT_LOADER select QCOM_SCM select VIDEOBUF2_DMA_CONTIG select V4L2_MEM2MEM_DEV diff --git a/drivers/media/platform/renesas/rcar-vin/rcar-dma.c b/drivers/media/platform/renesas/rcar-vin/rcar-dma.c index b619d1436a4151..f9af9177e02ffa 100644 --- a/drivers/media/platform/renesas/rcar-vin/rcar-dma.c +++ b/drivers/media/platform/renesas/rcar-vin/rcar-dma.c @@ -676,8 +676,30 @@ void rvin_crop_scale_comp(struct rvin_dev *vin) if (vin->scaler) vin->scaler(vin); + /* + * VNIS_REG has four lowest bits always 0, i.e. the stride has to be + * aligned to 16 bytes. This is done in rvin_format_bytesperline(). + */ + fmt = rvin_format_from_pixel(vin, vin->format.pixelformat); stride = vin->format.bytesperline / fmt->bpp; + + /* + * RAW8 format bpp is 1, but the hardware process RAW8 format in 2 pixel + * units, so we need to divide the stride by 2. + */ + switch (vin->format.pixelformat) { + case V4L2_PIX_FMT_SBGGR8: + case V4L2_PIX_FMT_SGBRG8: + case V4L2_PIX_FMT_SGRBG8: + case V4L2_PIX_FMT_SRGGB8: + case V4L2_PIX_FMT_GREY: + stride /= 2; + break; + default: + break; + } + rvin_write(vin, stride, VNIS_REG); } diff --git a/drivers/media/platform/renesas/rcar-vin/rcar-v4l2.c b/drivers/media/platform/renesas/rcar-vin/rcar-v4l2.c index 079dbaf016c251..9d45e11898c13d 100644 --- a/drivers/media/platform/renesas/rcar-vin/rcar-v4l2.c +++ b/drivers/media/platform/renesas/rcar-vin/rcar-v4l2.c @@ -155,6 +155,18 @@ static u32 rvin_format_bytesperline(struct rvin_dev *vin, case V4L2_PIX_FMT_NV16: align = 0x20; break; + case V4L2_PIX_FMT_SBGGR8: + case V4L2_PIX_FMT_SGBRG8: + case V4L2_PIX_FMT_SGRBG8: + case V4L2_PIX_FMT_SRGGB8: + case V4L2_PIX_FMT_GREY: + /* + * RAW8 format bpp is 1, but the hardware process RAW8 format in + * 2 pixel units, and we need to align to 32 bytes. See + * rvin_crop_scale_comp(). + */ + align = 0x20; + break; default: align = 0x10; break; diff --git a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c index 799453250b855b..3580a57738a67f 100644 --- a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c +++ b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c @@ -7,6 +7,7 @@ #include "rzv2h-ivc.h" +#include #include #include #include @@ -24,7 +25,7 @@ #include #define RZV2H_IVC_FIXED_HBLANK 0x20 -#define RZV2H_IVC_MIN_VBLANK(hts) max(0x1b, 15 + (120501 / (hts))) +#define RZV2H_IVC_MIN_VBLANK(hts) max(0x1b, 70100 / (hts)) struct rzv2h_ivc_buf { struct vb2_v4l2_buffer vb; @@ -148,24 +149,26 @@ static void rzv2h_ivc_transfer_buffer(struct work_struct *work) buffers.work); struct rzv2h_ivc_buf *buf; + guard(spinlock_irqsave)(&ivc->spinlock); + + if (ivc->vvalid_ifp) + return; + /* Setup buffers */ scoped_guard(spinlock_irqsave, &ivc->buffers.lock) { buf = list_first_entry_or_null(&ivc->buffers.queue, struct rzv2h_ivc_buf, queue); - } - - if (!buf) - return; + if (!buf) + return; - list_del(&buf->queue); + list_del(&buf->queue); + ivc->buffers.curr = buf; + } - ivc->buffers.curr = buf; buf->addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0); rzv2h_ivc_write(ivc, RZV2H_IVC_REG_AXIRX_SADDL_P0, buf->addr); - scoped_guard(spinlock_irqsave, &ivc->spinlock) { - ivc->vvalid_ifp = 2; - } + ivc->vvalid_ifp = 2; rzv2h_ivc_write(ivc, RZV2H_IVC_REG_FM_FRCON, 0x1); } @@ -200,7 +203,7 @@ static void rzv2h_ivc_buf_queue(struct vb2_buffer *vb) } scoped_guard(spinlock_irq, &ivc->spinlock) { - if (vb2_is_streaming(vb->vb2_queue) && !ivc->vvalid_ifp) + if (vb2_is_streaming(vb->vb2_queue)) queue_work(ivc->buffers.async_wq, &ivc->buffers.work); } } @@ -214,10 +217,10 @@ static void rzv2h_ivc_format_configure(struct rzv2h_ivc *ivc) /* Currently only CRU packed pixel formats are supported */ rzv2h_ivc_write(ivc, RZV2H_IVC_REG_AXIRX_PXFMT, - RZV2H_IVC_INPUT_FMT_CRU_PACKED); - - rzv2h_ivc_update_bits(ivc, RZV2H_IVC_REG_AXIRX_PXFMT, - RZV2H_IVC_PXFMT_DTYPE, fmt->dtype); + FIELD_PREP(RZV2H_IVC_AXIRX_PXFMT_FIELD_DTYPE, + fmt->dtype) | + FIELD_PREP(RZV2H_IVC_AXIRX_PXFMT_FIELD_CLFMT, + RZV2H_IVC_CLFMT_CRU_PACKED)); rzv2h_ivc_write(ivc, RZV2H_IVC_REG_AXIRX_HSIZE, pix->width); rzv2h_ivc_write(ivc, RZV2H_IVC_REG_AXIRX_VSIZE, pix->height); @@ -235,8 +238,10 @@ static void rzv2h_ivc_format_configure(struct rzv2h_ivc *ivc) hts = pix->width + RZV2H_IVC_FIXED_HBLANK; vblank = RZV2H_IVC_MIN_VBLANK(hts); - rzv2h_ivc_write(ivc, RZV2H_IVC_REG_AXIRX_BLANK, - RZV2H_IVC_VBLANK(vblank)); + rzv2h_ivc_update_bits(ivc, RZV2H_IVC_REG_AXIRX_BLANK, + RZV2H_IVC_AXIRX_BLANK_FIELD_VBLANK, + FIELD_PREP(RZV2H_IVC_AXIRX_BLANK_FIELD_VBLANK, + vblank)); } static void rzv2h_ivc_return_buffers(struct rzv2h_ivc *ivc, @@ -294,9 +299,10 @@ static void rzv2h_ivc_stop_streaming(struct vb2_queue *q) struct rzv2h_ivc *ivc = vb2_get_drv_priv(q); u32 val = 0; - rzv2h_ivc_write(ivc, RZV2H_IVC_REG_FM_STOP, 0x1); + rzv2h_ivc_write(ivc, RZV2H_IVC_REG_FM_STOP, RZV2H_IVC_REG_FM_STOP_FSTOP); readl_poll_timeout(ivc->base + RZV2H_IVC_REG_FM_STOP, - val, !val, 10 * USEC_PER_MSEC, 250 * USEC_PER_MSEC); + val, !(val & RZV2H_IVC_REG_FM_STOP_FSTOP), + 10 * USEC_PER_MSEC, 250 * USEC_PER_MSEC); rzv2h_ivc_return_buffers(ivc, VB2_BUF_STATE_ERROR); video_device_pipeline_stop(&ivc->vdev.dev); diff --git a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc.h b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc.h index 3bcaab990b0f8d..049f223200e395 100644 --- a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc.h +++ b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc.h @@ -24,9 +24,10 @@ #define RZV2H_IVC_ONE_EXPOSURE 0x00 #define RZV2H_IVC_TWO_EXPOSURE 0x01 #define RZV2H_IVC_REG_AXIRX_PXFMT 0x0004 -#define RZV2H_IVC_INPUT_FMT_MIPI (0 << 16) -#define RZV2H_IVC_INPUT_FMT_CRU_PACKED BIT(16) -#define RZV2H_IVC_PXFMT_DTYPE GENMASK(7, 0) +#define RZV2H_IVC_AXIRX_PXFMT_FIELD_CLFMT GENMASK(17, 16) +#define RZV2H_IVC_CLFMT_MIPI 0 +#define RZV2H_IVC_CLFMT_CRU_PACKED 1 +#define RZV2H_IVC_AXIRX_PXFMT_FIELD_DTYPE GENMASK(7, 0) #define RZV2H_IVC_REG_AXIRX_SADDL_P0 0x0010 #define RZV2H_IVC_REG_AXIRX_SADDH_P0 0x0014 #define RZV2H_IVC_REG_AXIRX_SADDL_P1 0x0018 @@ -34,7 +35,7 @@ #define RZV2H_IVC_REG_AXIRX_HSIZE 0x0020 #define RZV2H_IVC_REG_AXIRX_VSIZE 0x0024 #define RZV2H_IVC_REG_AXIRX_BLANK 0x0028 -#define RZV2H_IVC_VBLANK(x) ((x) << 16) +#define RZV2H_IVC_AXIRX_BLANK_FIELD_VBLANK GENMASK(25, 16) #define RZV2H_IVC_REG_AXIRX_STRD 0x0030 #define RZV2H_IVC_REG_AXIRX_ISSU 0x0040 #define RZV2H_IVC_REG_AXIRX_ERACT 0x0048 @@ -45,6 +46,7 @@ #define RZV2H_IVC_REG_FM_MCON 0x0104 #define RZV2H_IVC_REG_FM_FRCON 0x0108 #define RZV2H_IVC_REG_FM_STOP 0x010c +#define RZV2H_IVC_REG_FM_STOP_FSTOP BIT(20) #define RZV2H_IVC_REG_FM_INT_EN 0x0120 #define RZV2H_IVC_VVAL_IFPE BIT(0) #define RZV2H_IVC_REG_FM_INT_STA 0x0124 diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drv.c b/drivers/media/platform/renesas/vsp1/vsp1_drv.c index 2de515c497eb85..627b5046fa8006 100644 --- a/drivers/media/platform/renesas/vsp1/vsp1_drv.c +++ b/drivers/media/platform/renesas/vsp1/vsp1_drv.c @@ -240,8 +240,12 @@ static void vsp1_destroy_entities(struct vsp1_device *vsp1) media_device_unregister(&vsp1->media_dev); media_device_cleanup(&vsp1->media_dev); - if (!vsp1->info->uapi) - vsp1_drm_cleanup(vsp1); + if (!vsp1->info->uapi) { + if (vsp1->info->version == VI6_IP_VERSION_MODEL_VSPX_GEN4) + vsp1_vspx_cleanup(vsp1); + else + vsp1_drm_cleanup(vsp1); + } } static int vsp1_create_entities(struct vsp1_device *vsp1) diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c b/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c index 1b81bcc067efd6..9e67160a16e468 100644 --- a/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c +++ b/drivers/media/platform/rockchip/rkcif/rkcif-capture-mipi.c @@ -489,8 +489,8 @@ static inline unsigned int rkcif_mipi_get_reg(struct rkcif_interface *interface, block = interface->index - RKCIF_MIPI_BASE; - if (WARN_ON_ONCE(block > RKCIF_MIPI_MAX - RKCIF_MIPI_BASE) || - WARN_ON_ONCE(index > RKCIF_MIPI_REGISTER_MAX)) + if (WARN_ON_ONCE(block >= ARRAY_SIZE(rkcif->match_data->mipi->blocks)) || + WARN_ON_ONCE(index >= ARRAY_SIZE(rkcif->match_data->mipi->regs))) return RKCIF_REGISTER_NOTSUPPORTED; offset = rkcif->match_data->mipi->blocks[block].offset; @@ -510,9 +510,9 @@ static inline unsigned int rkcif_mipi_id_get_reg(struct rkcif_stream *stream, block = stream->interface->index - RKCIF_MIPI_BASE; id = stream->id; - if (WARN_ON_ONCE(block > RKCIF_MIPI_MAX - RKCIF_MIPI_BASE) || - WARN_ON_ONCE(id > RKCIF_ID_MAX) || - WARN_ON_ONCE(index > RKCIF_MIPI_ID_REGISTER_MAX)) + if (WARN_ON_ONCE(block >= ARRAY_SIZE(rkcif->match_data->mipi->blocks)) || + WARN_ON_ONCE(id >= ARRAY_SIZE(rkcif->match_data->mipi->regs_id)) || + WARN_ON_ONCE(index >= ARRAY_SIZE(rkcif->match_data->mipi->regs_id[id]))) return RKCIF_REGISTER_NOTSUPPORTED; offset = rkcif->match_data->mipi->blocks[block].offset; diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-interface.c b/drivers/media/platform/rockchip/rkcif/rkcif-interface.c index 523103872b7a19..414a9980cf2e55 100644 --- a/drivers/media/platform/rockchip/rkcif/rkcif-interface.c +++ b/drivers/media/platform/rockchip/rkcif/rkcif-interface.c @@ -378,7 +378,8 @@ int rkcif_interface_register(struct rkcif_device *rkcif, snprintf(sd->name, sizeof(sd->name), "rkcif-mipi%d", interface->index - RKCIF_MIPI_BASE); - pads[RKCIF_IF_PAD_SINK].flags = MEDIA_PAD_FL_SINK; + pads[RKCIF_IF_PAD_SINK].flags = MEDIA_PAD_FL_SINK | + MEDIA_PAD_FL_MUST_CONNECT; pads[RKCIF_IF_PAD_SRC].flags = MEDIA_PAD_FL_SOURCE; ret = media_entity_pads_init(&sd->entity, RKCIF_IF_PAD_MAX, pads); if (ret) diff --git a/drivers/media/platform/rockchip/rkcif/rkcif-stream.c b/drivers/media/platform/rockchip/rkcif/rkcif-stream.c index e00010a91e8be4..3130d420ad559f 100644 --- a/drivers/media/platform/rockchip/rkcif/rkcif-stream.c +++ b/drivers/media/platform/rockchip/rkcif/rkcif-stream.c @@ -106,42 +106,42 @@ static int rkcif_stream_init_buffers(struct rkcif_stream *stream) { struct v4l2_pix_format_mplane *pix = &stream->pix; - stream->buffers[0] = rkcif_stream_pop_buffer(stream); - if (!stream->buffers[0]) - goto err_buff_0; - - stream->buffers[1] = rkcif_stream_pop_buffer(stream); - if (!stream->buffers[1]) - goto err_buff_1; - - if (stream->queue_buffer) { - stream->queue_buffer(stream, 0); - stream->queue_buffer(stream, 1); - } - stream->dummy.size = pix->num_planes * pix->plane_fmt[0].sizeimage; stream->dummy.vaddr = dma_alloc_attrs(stream->rkcif->dev, stream->dummy.size, &stream->dummy.buffer.buff_addr[0], GFP_KERNEL, DMA_ATTR_NO_KERNEL_MAPPING); if (!stream->dummy.vaddr) - goto err_dummy; + return -ENOMEM; for (unsigned int i = 1; i < pix->num_planes; i++) stream->dummy.buffer.buff_addr[i] = stream->dummy.buffer.buff_addr[i - 1] + pix->plane_fmt[i - 1].bytesperline * pix->height; - return 0; + stream->buffers[0] = rkcif_stream_pop_buffer(stream); + if (!stream->buffers[0]) + goto err_dummy_free; + + stream->buffers[1] = rkcif_stream_pop_buffer(stream); + if (!stream->buffers[1]) { + stream->buffers[1] = &stream->dummy.buffer; + stream->buffers[1]->is_dummy = true; + } -err_dummy: - rkcif_stream_return_buffer(stream->buffers[1], VB2_BUF_STATE_QUEUED); - stream->buffers[1] = NULL; + if (stream->queue_buffer) { + stream->queue_buffer(stream, 0); + stream->queue_buffer(stream, 1); + } + + return 0; -err_buff_1: - rkcif_stream_return_buffer(stream->buffers[0], VB2_BUF_STATE_QUEUED); - stream->buffers[0] = NULL; -err_buff_0: +err_dummy_free: + dma_free_attrs(stream->rkcif->dev, stream->dummy.size, + stream->dummy.vaddr, + stream->dummy.buffer.buff_addr[0], + DMA_ATTR_NO_KERNEL_MAPPING); + stream->dummy.vaddr = NULL; return -EINVAL; } @@ -555,7 +555,7 @@ int rkcif_stream_register(struct rkcif_device *rkcif, vdev->vfl_dir = VFL_DIR_RX; video_set_drvdata(vdev, stream); - stream->pad.flags = MEDIA_PAD_FL_SINK; + stream->pad.flags = MEDIA_PAD_FL_SINK | MEDIA_PAD_FL_MUST_CONNECT; stream->pix.height = CIF_MIN_HEIGHT; stream->pix.width = CIF_MIN_WIDTH; diff --git a/drivers/media/platform/synopsys/Kconfig b/drivers/media/platform/synopsys/Kconfig index bf2ac092fbb39d..b109de2c8111cf 100644 --- a/drivers/media/platform/synopsys/Kconfig +++ b/drivers/media/platform/synopsys/Kconfig @@ -4,6 +4,7 @@ source "drivers/media/platform/synopsys/hdmirx/Kconfig" config VIDEO_DW_MIPI_CSI2RX tristate "Synopsys DesignWare MIPI CSI-2 Receiver" + depends on ARCH_ROCKCHIP || COMPILE_TEST depends on VIDEO_DEV depends on V4L_PLATFORM_DRIVERS depends on PM && COMMON_CLK diff --git a/drivers/media/platform/ti/omap3isp/ispvideo.c b/drivers/media/platform/ti/omap3isp/ispvideo.c index 64e76e3576a868..b946c8087c7780 100644 --- a/drivers/media/platform/ti/omap3isp/ispvideo.c +++ b/drivers/media/platform/ti/omap3isp/ispvideo.c @@ -1403,6 +1403,7 @@ static int isp_video_open(struct file *file) ret = vb2_queue_init(&handle->queue); if (ret < 0) { + v4l2_pipeline_pm_put(&video->video.entity); omap3isp_put(video->isp); goto done; } diff --git a/drivers/media/platform/ti/vpe/vip.c b/drivers/media/platform/ti/vpe/vip.c index a4b616a5ece755..0e91e87bda9b9c 100644 --- a/drivers/media/platform/ti/vpe/vip.c +++ b/drivers/media/platform/ti/vpe/vip.c @@ -3641,6 +3641,7 @@ static void vip_remove(struct platform_device *pdev) } v4l2_ctrl_handler_free(&shared->ctrl_handler); + v4l2_device_unregister(&shared->v4l2_dev); pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/media/rc/igorplugusb.c b/drivers/media/rc/igorplugusb.c index e034c93d57cf03..6938d9a90c58a4 100644 --- a/drivers/media/rc/igorplugusb.c +++ b/drivers/media/rc/igorplugusb.c @@ -34,7 +34,7 @@ struct igorplugusb { struct device *dev; struct urb *urb; - struct usb_ctrlrequest request; + struct usb_ctrlrequest *request; struct timer_list timer; @@ -122,7 +122,7 @@ static void igorplugusb_cmd(struct igorplugusb *ir, int cmd) { int ret; - ir->request.bRequest = cmd; + ir->request->bRequest = cmd; ir->urb->transfer_flags = 0; ret = usb_submit_urb(ir->urb, GFP_ATOMIC); if (ret && ret != -EPERM) @@ -164,13 +164,17 @@ static int igorplugusb_probe(struct usb_interface *intf, if (!ir) return -ENOMEM; + ir->request = kzalloc_obj(*ir->request, GFP_KERNEL); + if (!ir->request) + goto fail; + ir->dev = &intf->dev; timer_setup(&ir->timer, igorplugusb_timer, 0); - ir->request.bRequest = GET_INFRACODE; - ir->request.bRequestType = USB_TYPE_VENDOR | USB_DIR_IN; - ir->request.wLength = cpu_to_le16(MAX_PACKET); + ir->request->bRequest = GET_INFRACODE; + ir->request->bRequestType = USB_TYPE_VENDOR | USB_DIR_IN; + ir->request->wLength = cpu_to_le16(MAX_PACKET); ir->urb = usb_alloc_urb(0, GFP_KERNEL); if (!ir->urb) @@ -228,6 +232,7 @@ static int igorplugusb_probe(struct usb_interface *intf, usb_free_urb(ir->urb); rc_free_device(ir->rc); kfree(ir->buf_in); + kfree(ir->request); return ret; } @@ -243,6 +248,7 @@ static void igorplugusb_disconnect(struct usb_interface *intf) usb_unpoison_urb(ir->urb); usb_free_urb(ir->urb); kfree(ir->buf_in); + kfree(ir->request); } static const struct usb_device_id igorplugusb_table[] = { diff --git a/drivers/media/rc/streamzap.c b/drivers/media/rc/streamzap.c index 5a18603f9a95ce..6587c64a9d93a7 100644 --- a/drivers/media/rc/streamzap.c +++ b/drivers/media/rc/streamzap.c @@ -219,9 +219,8 @@ static void streamzap_callback(struct urb *urb) case -ESHUTDOWN: /* * this urb is terminated, clean up. - * sz might already be invalid at this point */ - dev_err(sz->dev, "urb terminated, status: %d\n", urb->status); + dev_dbg(sz->dev, "urb terminated, status: %d\n", urb->status); return; default: break; @@ -358,11 +357,16 @@ static int streamzap_probe(struct usb_interface *intf, usb_set_intfdata(intf, sz); - if (usb_submit_urb(sz->urb_in, GFP_ATOMIC)) + retval = usb_submit_urb(sz->urb_in, GFP_ATOMIC); + if (retval < 0) { dev_err(sz->dev, "urb submit failed\n"); + goto rc_submit_fail; + } return 0; - +rc_submit_fail: + rc_free_device(sz->rdev); + usb_set_intfdata(intf, NULL); rc_dev_fail: usb_free_urb(sz->urb_in); free_buf_in: diff --git a/drivers/media/rc/xbox_remote.c b/drivers/media/rc/xbox_remote.c index 3e3da70cf8da05..e2ed2e2c2723e0 100644 --- a/drivers/media/rc/xbox_remote.c +++ b/drivers/media/rc/xbox_remote.c @@ -55,7 +55,7 @@ struct xbox_remote { struct usb_interface *interface; struct urb *irq_urb; - unsigned char inbuf[DATA_BUFSIZE] __aligned(sizeof(u16)); + u8 *inbuf; char rc_name[NAME_BUFSIZE]; char rc_phys[NAME_BUFSIZE]; @@ -218,6 +218,10 @@ static int xbox_remote_probe(struct usb_interface *interface, if (!xbox_remote || !rc_dev) goto exit_free_dev_rdev; + xbox_remote->inbuf = kzalloc(DATA_BUFSIZE, GFP_KERNEL); + if (!xbox_remote->inbuf) + goto exit_free_inbuf; + /* Allocate URB buffer */ xbox_remote->irq_urb = usb_alloc_urb(0, GFP_KERNEL); if (!xbox_remote->irq_urb) @@ -262,6 +266,8 @@ static int xbox_remote_probe(struct usb_interface *interface, usb_kill_urb(xbox_remote->irq_urb); exit_free_buffers: usb_free_urb(xbox_remote->irq_urb); +exit_free_inbuf: + kfree(xbox_remote->inbuf); exit_free_dev_rdev: rc_free_device(rc_dev); kfree(xbox_remote); @@ -286,6 +292,7 @@ static void xbox_remote_disconnect(struct usb_interface *interface) usb_kill_urb(xbox_remote->irq_urb); rc_unregister_device(xbox_remote->rdev); usb_free_urb(xbox_remote->irq_urb); + kfree(xbox_remote->inbuf); kfree(xbox_remote); } diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c index b6203e10e37aa1..a8a76434989c2c 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c +++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c @@ -237,8 +237,10 @@ static int vidtv_start_feed(struct dvb_demux_feed *feed) if (dvb->nfeeds == 1) { ret = vidtv_start_streaming(dvb); - if (ret < 0) + if (ret < 0) { + dvb->nfeeds--; rc = ret; + } } mutex_unlock(&dvb->feed_lock); diff --git a/drivers/media/test-drivers/vidtv/vidtv_channel.c b/drivers/media/test-drivers/vidtv/vidtv_channel.c index da20657adc7472..5f8c3af8717111 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_channel.c +++ b/drivers/media/test-drivers/vidtv/vidtv_channel.c @@ -341,6 +341,10 @@ vidtv_channel_pmt_match_sections(struct vidtv_channel *channels, tail = vidtv_psi_pmt_stream_init(tail, s->type, e_pid); + if (!tail) { + vidtv_psi_pmt_stream_destroy(head); + return; + } if (!head) head = tail; diff --git a/drivers/media/test-drivers/vidtv/vidtv_mux.c b/drivers/media/test-drivers/vidtv/vidtv_mux.c index 403fbedb866360..f0134e38a1fb8d 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_mux.c +++ b/drivers/media/test-drivers/vidtv/vidtv_mux.c @@ -233,7 +233,7 @@ static u32 vidtv_mux_push_pcr(struct vidtv_mux *m) /* the 27Mhz clock will feed both parts of the PCR bitfield */ args.pcr = m->timing.clk; - nbytes += vidtv_ts_pcr_write_into(args); + nbytes += vidtv_ts_pcr_write_into(&args); m->mux_buf_offset += nbytes; m->num_streamed_pcr++; @@ -363,7 +363,7 @@ static u32 vidtv_mux_pad_with_nulls(struct vidtv_mux *m, u32 npkts) args.continuity_counter = &ctx->cc; for (i = 0; i < npkts; ++i) { - m->mux_buf_offset += vidtv_ts_null_write_into(args); + m->mux_buf_offset += vidtv_ts_null_write_into(&args); args.dest_offset = m->mux_buf_offset; } diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.c b/drivers/media/test-drivers/vidtv/vidtv_ts.c index ca4bb9c40b78ef..cbe9aff9ffb564 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_ts.c +++ b/drivers/media/test-drivers/vidtv/vidtv_ts.c @@ -48,7 +48,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter) *continuity_counter = 0; } -u32 vidtv_ts_null_write_into(struct null_packet_write_args args) +u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args) { u32 nbytes = 0; struct vidtv_mpeg_ts ts_header = {}; @@ -56,21 +56,21 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args) ts_header.sync_byte = TS_SYNC_BYTE; ts_header.bitfield = cpu_to_be16(TS_NULL_PACKET_PID); ts_header.payload = 1; - ts_header.continuity_counter = *args.continuity_counter; + ts_header.continuity_counter = *args->continuity_counter; /* copy TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_header, sizeof(ts_header)); - vidtv_ts_inc_cc(args.continuity_counter); + vidtv_ts_inc_cc(args->continuity_counter); /* fill the rest with empty data */ - nbytes += vidtv_memset(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memset(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, TS_FILL_BYTE, TS_PACKET_LEN - nbytes); @@ -83,17 +83,17 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args) return nbytes; } -u32 vidtv_ts_pcr_write_into(struct pcr_write_args args) +u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args) { u32 nbytes = 0; struct vidtv_mpeg_ts ts_header = {}; struct vidtv_mpeg_ts_adaption ts_adap = {}; ts_header.sync_byte = TS_SYNC_BYTE; - ts_header.bitfield = cpu_to_be16(args.pid); + ts_header.bitfield = cpu_to_be16(args->pid); ts_header.scrambling = 0; /* cc is not incremented, but it is needed. see 13818-1 clause 2.4.3.3 */ - ts_header.continuity_counter = *args.continuity_counter; + ts_header.continuity_counter = *args->continuity_counter; ts_header.payload = 0; ts_header.adaptation_field = 1; @@ -102,27 +102,27 @@ u32 vidtv_ts_pcr_write_into(struct pcr_write_args args) ts_adap.PCR = 1; /* copy TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_header, sizeof(ts_header)); /* write the adap after the TS header */ - nbytes += vidtv_memcpy(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memcpy(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, &ts_adap, sizeof(ts_adap)); /* write the PCR optional */ - nbytes += vidtv_ts_write_pcr_bits(args.dest_buf, - args.dest_offset + nbytes, - args.pcr); + nbytes += vidtv_ts_write_pcr_bits(args->dest_buf, + args->dest_offset + nbytes, + args->pcr); - nbytes += vidtv_memset(args.dest_buf, - args.dest_offset + nbytes, - args.buf_sz, + nbytes += vidtv_memset(args->dest_buf, + args->dest_offset + nbytes, + args->buf_sz, TS_FILL_BYTE, TS_PACKET_LEN - nbytes); diff --git a/drivers/media/test-drivers/vidtv/vidtv_ts.h b/drivers/media/test-drivers/vidtv/vidtv_ts.h index 09b4ffd0282921..3606398e160d9f 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_ts.h +++ b/drivers/media/test-drivers/vidtv/vidtv_ts.h @@ -90,7 +90,7 @@ void vidtv_ts_inc_cc(u8 *continuity_counter); * * Return: The number of bytes written into the buffer. */ -u32 vidtv_ts_null_write_into(struct null_packet_write_args args); +u32 vidtv_ts_null_write_into(const struct null_packet_write_args *args); /** * vidtv_ts_pcr_write_into - Write a PCR packet into a buffer. @@ -101,6 +101,6 @@ u32 vidtv_ts_null_write_into(struct null_packet_write_args args); * * Return: The number of bytes written into the buffer. */ -u32 vidtv_ts_pcr_write_into(struct pcr_write_args args); +u32 vidtv_ts_pcr_write_into(const struct pcr_write_args *args); #endif //VIDTV_TS_H diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c index 8e480ab78f9b2b..a11024451cebdc 100644 --- a/drivers/media/usb/as102/as102_usb_drv.c +++ b/drivers/media/usb/as102/as102_usb_drv.c @@ -403,7 +403,9 @@ static int as102_usb_probe(struct usb_interface *intf, failed_dvb: as102_free_usb_stream_buffer(as102_dev); failed_stream: + usb_set_intfdata(intf, NULL); usb_deregister_dev(intf, &as102_usb_class_driver); + return ret; failed: usb_put_dev(as102_dev->bus_adap.usb_dev); usb_set_intfdata(intf, NULL); diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c index b0c184f237a721..5f13f63fbdee2d 100644 --- a/drivers/media/usb/em28xx/em28xx-video.c +++ b/drivers/media/usb/em28xx/em28xx-video.c @@ -2126,7 +2126,7 @@ static int em28xx_v4l2_open(struct file *filp) { struct video_device *vdev = video_devdata(filp); struct em28xx *dev = video_drvdata(filp); - struct em28xx_v4l2 *v4l2 = dev->v4l2; + struct em28xx_v4l2 *v4l2; enum v4l2_buf_type fh_type = 0; int ret; @@ -2143,13 +2143,19 @@ static int em28xx_v4l2_open(struct file *filp) return -EINVAL; } + if (mutex_lock_interruptible(&dev->lock)) + return -ERESTARTSYS; + + v4l2 = dev->v4l2; + if (!v4l2) { + mutex_unlock(&dev->lock); + return -ENODEV; + } + em28xx_videodbg("open dev=%s type=%s users=%d\n", video_device_node_name(vdev), v4l2_type_names[fh_type], v4l2->users); - if (mutex_lock_interruptible(&dev->lock)) - return -ERESTARTSYS; - ret = v4l2_fh_open(filp); if (ret) { dev_err(&dev->intf->dev, diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c index 94d356fba6122d..a15829a60e8833 100644 --- a/drivers/media/usb/hackrf/hackrf.c +++ b/drivers/media/usb/hackrf/hackrf.c @@ -1485,7 +1485,7 @@ static int hackrf_probe(struct usb_interface *intf, if (ret) { dev_err(dev->dev, "Failed to register as video device (%d)\n", ret); - goto err_v4l2_device_unregister; + goto err_v4l2_device_put; } dev_info(dev->dev, "Registered as %s\n", video_device_node_name(&dev->rx_vdev)); @@ -1513,8 +1513,9 @@ static int hackrf_probe(struct usb_interface *intf, return 0; err_video_unregister_device_rx: video_unregister_device(&dev->rx_vdev); -err_v4l2_device_unregister: - v4l2_device_unregister(&dev->v4l2_dev); +err_v4l2_device_put: + v4l2_device_put(&dev->v4l2_dev); + return ret; err_v4l2_ctrl_handler_free_tx: v4l2_ctrl_handler_free(&dev->tx_ctrl_handler); err_v4l2_ctrl_handler_free_rx: diff --git a/drivers/media/usb/uvc/uvc_queue.c b/drivers/media/usb/uvc/uvc_queue.c index 8b8f44b4a0451e..0eddd4f872ca76 100644 --- a/drivers/media/usb/uvc/uvc_queue.c +++ b/drivers/media/usb/uvc/uvc_queue.c @@ -243,7 +243,7 @@ int uvc_queue_init(struct uvc_video_queue *queue, enum v4l2_buf_type type) int ret; queue->queue.type = type; - queue->queue.io_modes = VB2_MMAP | VB2_USERPTR; + queue->queue.io_modes = VB2_MMAP | VB2_USERPTR | VB2_DMABUF; queue->queue.drv_priv = queue; queue->queue.buf_struct_size = sizeof(struct uvc_buffer); queue->queue.mem_ops = &vb2_vmalloc_memops; @@ -256,7 +256,6 @@ int uvc_queue_init(struct uvc_video_queue *queue, enum v4l2_buf_type type) queue->queue.ops = &uvc_meta_queue_qops; break; default: - queue->queue.io_modes |= VB2_DMABUF; queue->queue.ops = &uvc_queue_qops; break; } diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c index ff26815e51f17f..5cfbc169c5f988 100644 --- a/drivers/memory/tegra/tegra124-emc.c +++ b/drivers/memory/tegra/tegra124-emc.c @@ -608,7 +608,7 @@ static int tegra124_emc_prepare_timing_change(struct tegra_emc *emc, if ((last->emc_mode_1 & 0x1) == (timing->emc_mode_1 & 0x1)) dll_change = DLL_CHANGE_NONE; - else if (timing->emc_mode_1 & 0x1) + else if (!(timing->emc_mode_1 & 0x1)) dll_change = DLL_CHANGE_ON; else dll_change = DLL_CHANGE_OFF; diff --git a/drivers/memory/tegra/tegra30-emc.c b/drivers/memory/tegra/tegra30-emc.c index 606106dd2b32ef..5812c8cd6ce4fc 100644 --- a/drivers/memory/tegra/tegra30-emc.c +++ b/drivers/memory/tegra/tegra30-emc.c @@ -554,14 +554,14 @@ static int emc_prepare_timing_change(struct tegra_emc *emc, unsigned long rate) emc->emc_cfg = readl_relaxed(emc->regs + EMC_CFG); emc_dbg = readl_relaxed(emc->regs + EMC_DBG); - if (emc->dll_on == !!(timing->emc_mode_1 & 0x1)) + if (emc->dll_on == !(timing->emc_mode_1 & 0x1)) dll_change = DLL_CHANGE_NONE; - else if (timing->emc_mode_1 & 0x1) + else if (!(timing->emc_mode_1 & 0x1)) dll_change = DLL_CHANGE_ON; else dll_change = DLL_CHANGE_OFF; - emc->dll_on = !!(timing->emc_mode_1 & 0x1); + emc->dll_on = !(timing->emc_mode_1 & 0x1); if (timing->data[80] && !readl_relaxed(emc->regs + EMC_ZCAL_INTERVAL)) emc->zcal_long = true; diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 1b7e7b3e785f27..358feec2d088fc 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -46,6 +46,7 @@ static const struct mfd_cell apple_smc_devs[] = { MFD_CELL_NAME("macsmc-input"), + MFD_CELL_NAME("macsmc-power"), MFD_CELL_OF("macsmc-gpio", NULL, NULL, 0, 0, "apple,smc-gpio"), MFD_CELL_OF("macsmc-hwmon", NULL, NULL, 0, 0, "apple,smc-hwmon"), MFD_CELL_OF("macsmc-reboot", NULL, NULL, 0, 0, "apple,smc-reboot"), diff --git a/drivers/mfd/mc13xxx-core.c b/drivers/mfd/mc13xxx-core.c index 920797b806ced8..786eab3b2d03c6 100644 --- a/drivers/mfd/mc13xxx-core.c +++ b/drivers/mfd/mc13xxx-core.c @@ -377,7 +377,7 @@ static int mc13xxx_add_subdevice_pdata(struct mc13xxx *mc13xxx, if (snprintf(buf, sizeof(buf), format, name) > sizeof(buf)) return -E2BIG; - cell.name = kmemdup(buf, strlen(buf) + 1, GFP_KERNEL); + cell.name = devm_kmemdup(mc13xxx->dev, buf, strlen(buf) + 1, GFP_KERNEL); if (!cell.name) return -ENOMEM; diff --git a/drivers/mfd/mfd-core.c b/drivers/mfd/mfd-core.c index 6be58eb5a746d8..7aa32b90cf1eb7 100644 --- a/drivers/mfd/mfd-core.c +++ b/drivers/mfd/mfd-core.c @@ -88,7 +88,17 @@ static void mfd_acpi_add_device(const struct mfd_cell *cell, } } - device_set_node(&pdev->dev, acpi_fwnode_handle(adev ?: parent)); + /* + * NOTE: The fwnode design doesn't allow proper stacking/sharing. This + * should eventually turn into a device fwnode API call that will allow + * prepending to a list of fwnodes (with ACPI taking precedence). + * + * set_primary_fwnode() is used here, instead of device_set_node(), as + * device_set_node() will overwrite the existing fwnode, which may be an + * OF node that was populated earlier. To support a use case where ACPI + * and OF is used in conjunction, we call set_primary_fwnode() instead. + */ + set_primary_fwnode(&pdev->dev, acpi_fwnode_handle(adev ?: parent)); } #else static inline void mfd_acpi_add_device(const struct mfd_cell *cell, diff --git a/drivers/mfd/sec-acpm.c b/drivers/mfd/sec-acpm.c index 537ea65685bfeb..0e23b9d9f7ee72 100644 --- a/drivers/mfd/sec-acpm.c +++ b/drivers/mfd/sec-acpm.c @@ -367,7 +367,7 @@ static const struct regmap_config s2mpg11_regmap_config_meter = { }; struct sec_pmic_acpm_shared_bus_context { - const struct acpm_handle *acpm; + struct acpm_handle *acpm; unsigned int acpm_chan_id; u8 speedy_channel; }; @@ -390,7 +390,7 @@ static int sec_pmic_acpm_bus_write(void *context, const void *data, size_t count) { struct sec_pmic_acpm_bus_context *ctx = context; - const struct acpm_handle *acpm = ctx->shared->acpm; + struct acpm_handle *acpm = ctx->shared->acpm; const struct acpm_pmic_ops *pmic_ops = &acpm->ops.pmic_ops; size_t val_count = count - BITS_TO_BYTES(ACPM_ADDR_BITS); const u8 *d = data; @@ -410,7 +410,7 @@ static int sec_pmic_acpm_bus_read(void *context, const void *reg_buf, size_t reg void *val_buf, size_t val_size) { struct sec_pmic_acpm_bus_context *ctx = context; - const struct acpm_handle *acpm = ctx->shared->acpm; + struct acpm_handle *acpm = ctx->shared->acpm; const struct acpm_pmic_ops *pmic_ops = &acpm->ops.pmic_ops; const u8 *r = reg_buf; u8 reg; @@ -429,7 +429,7 @@ static int sec_pmic_acpm_bus_reg_update_bits(void *context, unsigned int reg, un unsigned int val) { struct sec_pmic_acpm_bus_context *ctx = context; - const struct acpm_handle *acpm = ctx->shared->acpm; + struct acpm_handle *acpm = ctx->shared->acpm; const struct acpm_pmic_ops *pmic_ops = &acpm->ops.pmic_ops; return pmic_ops->update_reg(acpm, ctx->shared->acpm_chan_id, ctx->type, reg & 0xff, @@ -480,7 +480,7 @@ static int sec_pmic_acpm_probe(struct platform_device *pdev) struct regmap *regmap_common, *regmap_pmic, *regmap; const struct sec_pmic_acpm_platform_data *pdata; struct sec_pmic_acpm_shared_bus_context *shared_ctx; - const struct acpm_handle *acpm; + struct acpm_handle *acpm; struct device *dev = &pdev->dev; int ret, irq; diff --git a/drivers/mfd/stpmic1.c b/drivers/mfd/stpmic1.c index 081827bc05961c..7c677b0344c604 100644 --- a/drivers/mfd/stpmic1.c +++ b/drivers/mfd/stpmic1.c @@ -16,6 +16,8 @@ #include +#define STPMIC1_MAX_RETRIES 2 + #define STPMIC1_MAIN_IRQ 0 static const struct regmap_range stpmic1_readable_ranges[] = { @@ -121,9 +123,23 @@ static const struct regmap_irq_chip stpmic1_regmap_irq_chip = { static int stpmic1_power_off(struct sys_off_data *data) { struct stpmic1 *ddata = data->cb_data; + int ret; + + /* + * Attempt to shut down again, in case the first attempt failed. + * The STPMIC1 might get confused and the first regmap_update_bits() + * returns with -ETIMEDOUT / -110 . If that or similar transient + * failure occurs, try to shut down again. If the second attempt + * fails, there is some bigger problem, report it to user. + */ + for (int retries = 0; retries < STPMIC1_MAX_RETRIES; retries++) { + ret = regmap_update_bits(ddata->regmap, MAIN_CR, SOFTWARE_SWITCH_OFF, + SOFTWARE_SWITCH_OFF); + if (!ret) + return NOTIFY_DONE; + } - regmap_update_bits(ddata->regmap, MAIN_CR, - SOFTWARE_SWITCH_OFF, SOFTWARE_SWITCH_OFF); + dev_err(ddata->dev, "Failed to access PMIC I2C bus (%d)\n", ret); return NOTIFY_DONE; } diff --git a/drivers/misc/ibmasm/ibmasmfs.c b/drivers/misc/ibmasm/ibmasmfs.c index f68a8957b98f87..dfdfa9ba474799 100644 --- a/drivers/misc/ibmasm/ibmasmfs.c +++ b/drivers/misc/ibmasm/ibmasmfs.c @@ -303,6 +303,8 @@ static ssize_t command_file_write(struct file *file, const char __user *ubuff, s return -EINVAL; if (count == 0 || count > IBMASM_CMD_MAX_BUFFER_SIZE) return 0; + if (count < sizeof(struct dot_command_header)) + return -EINVAL; if (*offset != 0) return 0; @@ -319,6 +321,11 @@ static ssize_t command_file_write(struct file *file, const char __user *ubuff, s return -EFAULT; } + if (count < get_dot_command_size(cmd->buffer)) { + command_put(cmd); + return -EINVAL; + } + spin_lock_irqsave(&command_data->sp->lock, flags); if (command_data->command) { spin_unlock_irqrestore(&command_data->sp->lock, flags); diff --git a/drivers/misc/ibmasm/lowlevel.c b/drivers/misc/ibmasm/lowlevel.c index 6922dc6c10db08..5313230f36ad4d 100644 --- a/drivers/misc/ibmasm/lowlevel.c +++ b/drivers/misc/ibmasm/lowlevel.c @@ -19,17 +19,21 @@ static struct i2o_header header = I2O_HEADER_TEMPLATE; int ibmasm_send_i2o_message(struct service_processor *sp) { u32 mfa; - unsigned int command_size; + size_t command_size; struct i2o_message *message; struct command *command = sp->current_command; + command_size = get_dot_command_size(command->buffer); + if (command_size > command->buffer_size) + return 1; + if (command_size > I2O_COMMAND_SIZE) + command_size = I2O_COMMAND_SIZE; + mfa = get_mfa_inbound(sp->base_address); if (!mfa) return 1; - command_size = get_dot_command_size(command->buffer); - header.message_size = outgoing_message_size(command_size); - + header.message_size = outgoing_message_size((unsigned int)command_size); message = get_i2o_message(sp->base_address, mfa); memcpy_toio(&message->header, &header, sizeof(struct i2o_header)); diff --git a/drivers/misc/ibmasm/remote.c b/drivers/misc/ibmasm/remote.c index ec816d3b38cbd4..521531738c9af3 100644 --- a/drivers/misc/ibmasm/remote.c +++ b/drivers/misc/ibmasm/remote.c @@ -177,6 +177,11 @@ void ibmasm_handle_mouse_interrupt(struct service_processor *sp) writer = get_queue_writer(sp); while (reader != writer) { + if (reader >= REMOTE_QUEUE_SIZE || writer >= REMOTE_QUEUE_SIZE) { + set_queue_reader(sp, 0); + break; + } + memcpy_fromio(&input, get_queue_entry(sp, reader), sizeof(struct remote_input)); diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c index e6a1d3534663ac..bea7a47d216e06 100644 --- a/drivers/misc/mei/bus-fixup.c +++ b/drivers/misc/mei/bus-fixup.c @@ -303,9 +303,9 @@ static void mei_wd(struct mei_cl_device *cldev) { struct pci_dev *pdev = to_pci_dev(cldev->dev.parent); - if (pdev->device == MEI_DEV_ID_WPT_LP || - pdev->device == MEI_DEV_ID_SPT || - pdev->device == MEI_DEV_ID_SPT_H) + if (pdev->device == PCI_DEVICE_ID_INTEL_MEI_WPT_LP || + pdev->device == PCI_DEVICE_ID_INTEL_MEI_SPT || + pdev->device == PCI_DEVICE_ID_INTEL_MEI_SPT_H) cldev->me_cl->props.protocol_version = 0x2; cldev->do_match = 1; diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h index fa30899a5fa260..5967f95891a1fc 100644 --- a/drivers/misc/mei/hw-me-regs.h +++ b/drivers/misc/mei/hw-me-regs.h @@ -9,120 +9,121 @@ /* * MEI device IDs */ -#define MEI_DEV_ID_82946GZ 0x2974 /* 82946GZ/GL */ -#define MEI_DEV_ID_82G35 0x2984 /* 82G35 Express */ -#define MEI_DEV_ID_82Q965 0x2994 /* 82Q963/Q965 */ -#define MEI_DEV_ID_82G965 0x29A4 /* 82P965/G965 */ +#define PCI_DEVICE_ID_INTEL_MEI_82946GZ 0x2974 /* 82946GZ/GL */ +#define PCI_DEVICE_ID_INTEL_MEI_82G35 0x2984 /* 82G35 Express */ +#define PCI_DEVICE_ID_INTEL_MEI_82Q965 0x2994 /* 82Q963/Q965 */ +#define PCI_DEVICE_ID_INTEL_MEI_82G965 0x29A4 /* 82P965/G965 */ -#define MEI_DEV_ID_82GM965 0x2A04 /* Mobile PM965/GM965 */ -#define MEI_DEV_ID_82GME965 0x2A14 /* Mobile GME965/GLE960 */ +#define PCI_DEVICE_ID_INTEL_MEI_82GM965 0x2A04 /* Mobile PM965/GM965 */ +#define PCI_DEVICE_ID_INTEL_MEI_82GME965 0x2A14 /* Mobile GME965/GLE960 */ -#define MEI_DEV_ID_ICH9_82Q35 0x29B4 /* 82Q35 Express */ -#define MEI_DEV_ID_ICH9_82G33 0x29C4 /* 82G33/G31/P35/P31 Express */ -#define MEI_DEV_ID_ICH9_82Q33 0x29D4 /* 82Q33 Express */ -#define MEI_DEV_ID_ICH9_82X38 0x29E4 /* 82X38/X48 Express */ -#define MEI_DEV_ID_ICH9_3200 0x29F4 /* 3200/3210 Server */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_82Q35 0x29B4 /* 82Q35 Express */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_82G33 0x29C4 /* 82G33/G31/P35/P31 Express */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_82Q33 0x29D4 /* 82Q33 Express */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_82X38 0x29E4 /* 82X38/X48 Express */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_3200 0x29F4 /* 3200/3210 Server */ -#define MEI_DEV_ID_ICH9_6 0x28B4 /* Bearlake */ -#define MEI_DEV_ID_ICH9_7 0x28C4 /* Bearlake */ -#define MEI_DEV_ID_ICH9_8 0x28D4 /* Bearlake */ -#define MEI_DEV_ID_ICH9_9 0x28E4 /* Bearlake */ -#define MEI_DEV_ID_ICH9_10 0x28F4 /* Bearlake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_6 0x28B4 /* Bearlake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_7 0x28C4 /* Bearlake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_8 0x28D4 /* Bearlake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_9 0x28E4 /* Bearlake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9_10 0x28F4 /* Bearlake */ -#define MEI_DEV_ID_ICH9M_1 0x2A44 /* Cantiga */ -#define MEI_DEV_ID_ICH9M_2 0x2A54 /* Cantiga */ -#define MEI_DEV_ID_ICH9M_3 0x2A64 /* Cantiga */ -#define MEI_DEV_ID_ICH9M_4 0x2A74 /* Cantiga */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9M_1 0x2A44 /* Cantiga */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9M_2 0x2A54 /* Cantiga */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9M_3 0x2A64 /* Cantiga */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH9M_4 0x2A74 /* Cantiga */ -#define MEI_DEV_ID_ICH10_1 0x2E04 /* Eaglelake */ -#define MEI_DEV_ID_ICH10_2 0x2E14 /* Eaglelake */ -#define MEI_DEV_ID_ICH10_3 0x2E24 /* Eaglelake */ -#define MEI_DEV_ID_ICH10_4 0x2E34 /* Eaglelake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH10_1 0x2E04 /* Eaglelake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH10_2 0x2E14 /* Eaglelake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH10_3 0x2E24 /* Eaglelake */ +#define PCI_DEVICE_ID_INTEL_MEI_ICH10_4 0x2E34 /* Eaglelake */ -#define MEI_DEV_ID_IBXPK_1 0x3B64 /* Calpella */ -#define MEI_DEV_ID_IBXPK_2 0x3B65 /* Calpella */ +#define PCI_DEVICE_ID_INTEL_MEI_IBXPK_1 0x3B64 /* Calpella */ +#define PCI_DEVICE_ID_INTEL_MEI_IBXPK_2 0x3B65 /* Calpella */ -#define MEI_DEV_ID_CPT_1 0x1C3A /* Couger Point */ -#define MEI_DEV_ID_PBG_1 0x1D3A /* C600/X79 Patsburg */ +#define PCI_DEVICE_ID_INTEL_MEI_CPT_1 0x1C3A /* Couger Point */ +#define PCI_DEVICE_ID_INTEL_MEI_PBG_1 0x1D3A /* C600/X79 Patsburg */ -#define MEI_DEV_ID_PPT_1 0x1E3A /* Panther Point */ -#define MEI_DEV_ID_PPT_2 0x1CBA /* Panther Point */ -#define MEI_DEV_ID_PPT_3 0x1DBA /* Panther Point */ +#define PCI_DEVICE_ID_INTEL_MEI_PPT_1 0x1E3A /* Panther Point */ +#define PCI_DEVICE_ID_INTEL_MEI_PPT_2 0x1CBA /* Panther Point */ +#define PCI_DEVICE_ID_INTEL_MEI_PPT_3 0x1DBA /* Panther Point */ -#define MEI_DEV_ID_LPT_H 0x8C3A /* Lynx Point H */ -#define MEI_DEV_ID_LPT_W 0x8D3A /* Lynx Point - Wellsburg */ -#define MEI_DEV_ID_LPT_LP 0x9C3A /* Lynx Point LP */ -#define MEI_DEV_ID_LPT_HR 0x8CBA /* Lynx Point H Refresh */ +#define PCI_DEVICE_ID_INTEL_MEI_LPT_H 0x8C3A /* Lynx Point H */ +#define PCI_DEVICE_ID_INTEL_MEI_LPT_W 0x8D3A /* Lynx Point - Wellsburg */ +#define PCI_DEVICE_ID_INTEL_MEI_LPT_LP 0x9C3A /* Lynx Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_LPT_HR 0x8CBA /* Lynx Point H Refresh */ -#define MEI_DEV_ID_WPT_LP 0x9CBA /* Wildcat Point LP */ -#define MEI_DEV_ID_WPT_LP_2 0x9CBB /* Wildcat Point LP 2 */ +#define PCI_DEVICE_ID_INTEL_MEI_WPT_LP 0x9CBA /* Wildcat Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_WPT_LP_2 0x9CBB /* Wildcat Point LP 2 */ -#define MEI_DEV_ID_SPT 0x9D3A /* Sunrise Point */ -#define MEI_DEV_ID_SPT_2 0x9D3B /* Sunrise Point 2 */ -#define MEI_DEV_ID_SPT_3 0x9D3E /* Sunrise Point 3 (iToutch) */ -#define MEI_DEV_ID_SPT_H 0xA13A /* Sunrise Point H */ -#define MEI_DEV_ID_SPT_H_2 0xA13B /* Sunrise Point H 2 */ +#define PCI_DEVICE_ID_INTEL_MEI_SPT 0x9D3A /* Sunrise Point */ +#define PCI_DEVICE_ID_INTEL_MEI_SPT_2 0x9D3B /* Sunrise Point 2 */ +#define PCI_DEVICE_ID_INTEL_MEI_SPT_3 0x9D3E /* Sunrise Point 3 (iToutch) */ +#define PCI_DEVICE_ID_INTEL_MEI_SPT_H 0xA13A /* Sunrise Point H */ +#define PCI_DEVICE_ID_INTEL_MEI_SPT_H_2 0xA13B /* Sunrise Point H 2 */ -#define MEI_DEV_ID_LBG 0xA1BA /* Lewisburg (SPT) */ +#define PCI_DEVICE_ID_INTEL_MEI_LBG 0xA1BA /* Lewisburg (SPT) */ -#define MEI_DEV_ID_BXT_M 0x1A9A /* Broxton M */ -#define MEI_DEV_ID_APL_I 0x5A9A /* Apollo Lake I */ +#define PCI_DEVICE_ID_INTEL_MEI_BXT_M 0x1A9A /* Broxton M */ +#define PCI_DEVICE_ID_INTEL_MEI_APL_I 0x5A9A /* Apollo Lake I */ -#define MEI_DEV_ID_DNV_IE 0x19E5 /* Denverton IE */ +#define PCI_DEVICE_ID_INTEL_MEI_DNV_IE 0x19E5 /* Denverton IE */ -#define MEI_DEV_ID_GLK 0x319A /* Gemini Lake */ +#define PCI_DEVICE_ID_INTEL_MEI_GLK 0x319A /* Gemini Lake */ -#define MEI_DEV_ID_KBP 0xA2BA /* Kaby Point */ -#define MEI_DEV_ID_KBP_2 0xA2BB /* Kaby Point 2 */ -#define MEI_DEV_ID_KBP_3 0xA2BE /* Kaby Point 3 (iTouch) */ +#define PCI_DEVICE_ID_INTEL_MEI_KBP 0xA2BA /* Kaby Point */ +#define PCI_DEVICE_ID_INTEL_MEI_KBP_2 0xA2BB /* Kaby Point 2 */ +#define PCI_DEVICE_ID_INTEL_MEI_KBP_3 0xA2BE /* Kaby Point 3 (iTouch) */ -#define MEI_DEV_ID_CNP_LP 0x9DE0 /* Cannon Point LP */ -#define MEI_DEV_ID_CNP_LP_3 0x9DE4 /* Cannon Point LP 3 (iTouch) */ -#define MEI_DEV_ID_CNP_H 0xA360 /* Cannon Point H */ -#define MEI_DEV_ID_CNP_H_3 0xA364 /* Cannon Point H 3 (iTouch) */ +#define PCI_DEVICE_ID_INTEL_MEI_CNP_LP 0x9DE0 /* Cannon Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_CNP_LP_3 0x9DE4 /* Cannon Point LP 3 (iTouch) */ +#define PCI_DEVICE_ID_INTEL_MEI_CNP_H 0xA360 /* Cannon Point H */ +#define PCI_DEVICE_ID_INTEL_MEI_CNP_H_3 0xA364 /* Cannon Point H 3 (iTouch) */ -#define MEI_DEV_ID_CMP_LP 0x02e0 /* Comet Point LP */ -#define MEI_DEV_ID_CMP_LP_3 0x02e4 /* Comet Point LP 3 (iTouch) */ +#define PCI_DEVICE_ID_INTEL_MEI_CMP_LP 0x02e0 /* Comet Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_CMP_LP_3 0x02e4 /* Comet Point LP 3 (iTouch) */ -#define MEI_DEV_ID_CMP_V 0xA3BA /* Comet Point Lake V */ +#define PCI_DEVICE_ID_INTEL_MEI_CMP_V 0xA3BA /* Comet Point Lake V */ -#define MEI_DEV_ID_CMP_H 0x06e0 /* Comet Lake H */ -#define MEI_DEV_ID_CMP_H_3 0x06e4 /* Comet Lake H 3 (iTouch) */ +#define PCI_DEVICE_ID_INTEL_MEI_CMP_H 0x06e0 /* Comet Lake H */ +#define PCI_DEVICE_ID_INTEL_MEI_CMP_H_3 0x06e4 /* Comet Lake H 3 (iTouch) */ -#define MEI_DEV_ID_CDF 0x18D3 /* Cedar Fork */ +#define PCI_DEVICE_ID_INTEL_MEI_CDF 0x18D3 /* Cedar Fork */ -#define MEI_DEV_ID_ICP_LP 0x34E0 /* Ice Lake Point LP */ -#define MEI_DEV_ID_ICP_N 0x38E0 /* Ice Lake Point N */ +#define PCI_DEVICE_ID_INTEL_MEI_ICP_LP 0x34E0 /* Ice Lake Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_ICP_N 0x38E0 /* Ice Lake Point N */ -#define MEI_DEV_ID_JSP_N 0x4DE0 /* Jasper Lake Point N */ +#define PCI_DEVICE_ID_INTEL_MEI_JSP_N 0x4DE0 /* Jasper Lake Point N */ -#define MEI_DEV_ID_TGP_LP 0xA0E0 /* Tiger Lake Point LP */ -#define MEI_DEV_ID_TGP_H 0x43E0 /* Tiger Lake Point H */ +#define PCI_DEVICE_ID_INTEL_MEI_TGP_LP 0xA0E0 /* Tiger Lake Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_TGP_H 0x43E0 /* Tiger Lake Point H */ -#define MEI_DEV_ID_MCC 0x4B70 /* Mule Creek Canyon (EHL) */ -#define MEI_DEV_ID_MCC_4 0x4B75 /* Mule Creek Canyon 4 (EHL) */ +#define PCI_DEVICE_ID_INTEL_MEI_MCC 0x4B70 /* Mule Creek Canyon (EHL) */ +#define PCI_DEVICE_ID_INTEL_MEI_MCC_4 0x4B75 /* Mule Creek Canyon 4 (EHL) */ -#define MEI_DEV_ID_EBG 0x1BE0 /* Emmitsburg WS */ +#define PCI_DEVICE_ID_INTEL_MEI_EBG 0x1BE0 /* Emmitsburg WS */ -#define MEI_DEV_ID_ADP_S 0x7AE8 /* Alder Lake Point S */ -#define MEI_DEV_ID_ADP_LP 0x7A60 /* Alder Lake Point LP */ -#define MEI_DEV_ID_ADP_P 0x51E0 /* Alder Lake Point P */ -#define MEI_DEV_ID_ADP_N 0x54E0 /* Alder Lake Point N */ +#define PCI_DEVICE_ID_INTEL_MEI_ADP_S 0x7AE8 /* Alder Lake Point S */ +#define PCI_DEVICE_ID_INTEL_MEI_ADP_LP 0x7A60 /* Alder Lake Point LP */ +#define PCI_DEVICE_ID_INTEL_MEI_ADP_P 0x51E0 /* Alder Lake Point P */ +#define PCI_DEVICE_ID_INTEL_MEI_ADP_N 0x54E0 /* Alder Lake Point N */ -#define MEI_DEV_ID_RPL_S 0x7A68 /* Raptor Lake Point S */ +#define PCI_DEVICE_ID_INTEL_MEI_RPL_S 0x7A68 /* Raptor Lake Point S */ -#define MEI_DEV_ID_MTL_M 0x7E70 /* Meteor Lake Point M */ -#define MEI_DEV_ID_ARL_S 0x7F68 /* Arrow Lake Point S */ -#define MEI_DEV_ID_ARL_H 0x7770 /* Arrow Lake Point H */ +#define PCI_DEVICE_ID_INTEL_MEI_MTL_M 0x7E70 /* Meteor Lake Point M */ +#define PCI_DEVICE_ID_INTEL_MEI_ARL_S 0x7F68 /* Arrow Lake Point S */ +#define PCI_DEVICE_ID_INTEL_MEI_ARL_H 0x7770 /* Arrow Lake Point H */ -#define MEI_DEV_ID_LNL_M 0xA870 /* Lunar Lake Point M */ +#define PCI_DEVICE_ID_INTEL_MEI_LNL_M 0xA870 /* Lunar Lake Point M */ -#define MEI_DEV_ID_PTL_H 0xE370 /* Panther Lake H */ -#define MEI_DEV_ID_PTL_P 0xE470 /* Panther Lake P */ +#define PCI_DEVICE_ID_INTEL_MEI_PTL_H 0xE370 /* Panther Lake H */ +#define PCI_DEVICE_ID_INTEL_MEI_PTL_P 0xE470 /* Panther Lake P */ -#define MEI_DEV_ID_WCL_P 0x4D70 /* Wildcat Lake P */ +#define PCI_DEVICE_ID_INTEL_MEI_WCL_P 0x4D70 /* Wildcat Lake P */ -#define MEI_DEV_ID_NVL_S 0x6E68 /* Nova Lake Point S */ +#define PCI_DEVICE_ID_INTEL_MEI_NVL_S 0x6E68 /* Nova Lake Point S */ +#define PCI_DEVICE_ID_INTEL_MEI_NVL_H 0xD370 /* Nova Lake Point H */ /* * MEI HW Section diff --git a/drivers/misc/mei/hw-me.h b/drivers/misc/mei/hw-me.h index 204b92af6c4785..843ec2497b523a 100644 --- a/drivers/misc/mei/hw-me.h +++ b/drivers/misc/mei/hw-me.h @@ -33,12 +33,6 @@ struct mei_cfg { u32 hw_trc_supported:1; }; - -#define MEI_PCI_DEVICE(dev, cfg) \ - .vendor = PCI_VENDOR_ID_INTEL, .device = (dev), \ - .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, \ - .driver_data = (kernel_ulong_t)(cfg), - #define MEI_ME_RPM_TIMEOUT 500 /* ms */ /** diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index 2a6e569558b943..5b6aaa4d3a1cb9 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -26,110 +26,111 @@ /* mei_pci_tbl - PCI Device ID Table */ static const struct pci_device_id mei_me_pci_tbl[] = { - {MEI_PCI_DEVICE(MEI_DEV_ID_82946GZ, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_82G35, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_82Q965, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_82G965, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_82GM965, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_82GME965, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82Q35, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82G33, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82Q33, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_82X38, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_3200, MEI_ME_ICH_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_6, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_7, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_8, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_9, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9_10, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_1, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_2, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_3, MEI_ME_ICH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH9M_4, MEI_ME_ICH_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_1, MEI_ME_ICH10_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_2, MEI_ME_ICH10_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_3, MEI_ME_ICH10_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICH10_4, MEI_ME_ICH10_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_IBXPK_1, MEI_ME_PCH6_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_IBXPK_2, MEI_ME_PCH6_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CPT_1, MEI_ME_PCH_CPT_PBG_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_PBG_1, MEI_ME_PCH_CPT_PBG_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_PPT_1, MEI_ME_PCH7_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_PPT_2, MEI_ME_PCH7_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_PPT_3, MEI_ME_PCH7_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_LPT_H, MEI_ME_PCH8_SPS_4_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_LPT_W, MEI_ME_PCH8_SPS_4_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_LPT_LP, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_LPT_HR, MEI_ME_PCH8_SPS_4_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_WPT_LP, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_WPT_LP_2, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_SPT, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_2, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_3, MEI_ME_PCH8_ITOUCH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H, MEI_ME_PCH8_SPS_4_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_SPT_H_2, MEI_ME_PCH8_SPS_4_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_LBG, MEI_ME_PCH12_SPS_4_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_BXT_M, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_APL_I, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_DNV_IE, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_GLK, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_KBP, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_KBP_2, MEI_ME_PCH8_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_KBP_3, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_CNP_LP, MEI_ME_PCH12_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CNP_LP_3, MEI_ME_PCH8_ITOUCH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CNP_H, MEI_ME_PCH12_SPS_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CNP_H_3, MEI_ME_PCH12_SPS_ITOUCH_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_CMP_LP, MEI_ME_PCH12_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CMP_LP_3, MEI_ME_PCH8_ITOUCH_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CMP_V, MEI_ME_PCH12_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CMP_H, MEI_ME_PCH12_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_CMP_H_3, MEI_ME_PCH8_ITOUCH_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_ICP_LP, MEI_ME_PCH12_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ICP_N, MEI_ME_PCH12_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_TGP_LP, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_TGP_H, MEI_ME_PCH15_SPS_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_JSP_N, MEI_ME_PCH15_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_MCC, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_MCC_4, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_CDF, MEI_ME_PCH8_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_EBG, MEI_ME_PCH15_SPS_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_S, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_LP, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_P, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ADP_N, MEI_ME_PCH15_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_RPL_S, MEI_ME_PCH15_SPS_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_MTL_M, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ARL_S, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_ARL_H, MEI_ME_PCH15_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_LNL_M, MEI_ME_PCH15_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_PTL_H, MEI_ME_PCH15_CFG)}, - {MEI_PCI_DEVICE(MEI_DEV_ID_PTL_P, MEI_ME_PCH15_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_WCL_P, MEI_ME_PCH15_CFG)}, - - {MEI_PCI_DEVICE(MEI_DEV_ID_NVL_S, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_82946GZ, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_82G35, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_82Q965, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_82G965, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_82GM965, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_82GME965, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_82Q35, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_82G33, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_82Q33, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_82X38, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_3200, MEI_ME_ICH_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_6, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_7, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_8, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_9, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9_10, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9M_1, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9M_2, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9M_3, MEI_ME_ICH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH9M_4, MEI_ME_ICH_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_ICH10_1, MEI_ME_ICH10_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH10_2, MEI_ME_ICH10_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH10_3, MEI_ME_ICH10_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICH10_4, MEI_ME_ICH10_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_IBXPK_1, MEI_ME_PCH6_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_IBXPK_2, MEI_ME_PCH6_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CPT_1, MEI_ME_PCH_CPT_PBG_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_PBG_1, MEI_ME_PCH_CPT_PBG_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_PPT_1, MEI_ME_PCH7_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_PPT_2, MEI_ME_PCH7_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_PPT_3, MEI_ME_PCH7_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_LPT_H, MEI_ME_PCH8_SPS_4_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_LPT_W, MEI_ME_PCH8_SPS_4_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_LPT_LP, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_LPT_HR, MEI_ME_PCH8_SPS_4_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_WPT_LP, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_WPT_LP_2, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_SPT, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_SPT_2, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_SPT_3, MEI_ME_PCH8_ITOUCH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_SPT_H, MEI_ME_PCH8_SPS_4_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_SPT_H_2, MEI_ME_PCH8_SPS_4_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_LBG, MEI_ME_PCH12_SPS_4_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_BXT_M, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_APL_I, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_DNV_IE, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_GLK, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_KBP, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_KBP_2, MEI_ME_PCH8_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_KBP_3, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_CNP_LP, MEI_ME_PCH12_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CNP_LP_3, MEI_ME_PCH8_ITOUCH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CNP_H, MEI_ME_PCH12_SPS_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CNP_H_3, MEI_ME_PCH12_SPS_ITOUCH_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_CMP_LP, MEI_ME_PCH12_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CMP_LP_3, MEI_ME_PCH8_ITOUCH_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CMP_V, MEI_ME_PCH12_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CMP_H, MEI_ME_PCH12_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_CMP_H_3, MEI_ME_PCH8_ITOUCH_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_ICP_LP, MEI_ME_PCH12_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ICP_N, MEI_ME_PCH12_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_TGP_LP, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_TGP_H, MEI_ME_PCH15_SPS_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_JSP_N, MEI_ME_PCH15_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_MCC, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_MCC_4, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_CDF, MEI_ME_PCH8_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_EBG, MEI_ME_PCH15_SPS_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_ADP_S, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ADP_LP, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ADP_P, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ADP_N, MEI_ME_PCH15_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_RPL_S, MEI_ME_PCH15_SPS_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_MTL_M, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ARL_S, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_ARL_H, MEI_ME_PCH15_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_LNL_M, MEI_ME_PCH15_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_PTL_H, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_PTL_P, MEI_ME_PCH15_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_WCL_P, MEI_ME_PCH15_CFG)}, + + {PCI_DEVICE_DATA(INTEL, MEI_NVL_S, MEI_ME_PCH15_CFG)}, + {PCI_DEVICE_DATA(INTEL, MEI_NVL_H, MEI_ME_PCH15_CFG)}, /* required last entry */ {0, } diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 05ee76cb0a08e9..db8c99c73a6173 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -1401,6 +1401,9 @@ static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq, rq_data_dir(req) == WRITE && (md->flags & MMC_BLK_REL_WR); + if (mqrq->flags & MQRQ_XFER_SINGLE_BLOCK) + recovery_mode = 1; + memset(brq, 0, sizeof(struct mmc_blk_request)); mmc_crypto_prepare_req(mqrq); @@ -1540,10 +1543,13 @@ static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req) err = 0; if (err) { - if (mqrq->retries++ < MMC_CQE_RETRIES) + if (mqrq->retries++ < MMC_CQE_RETRIES) { + if (rq_data_dir(req) == WRITE) + mqrq->flags |= MQRQ_XFER_SINGLE_BLOCK; blk_mq_requeue_request(req, true); - else + } else { blk_mq_end_request(req, BLK_STS_IOERR); + } } else if (mrq->data) { if (blk_update_request(req, BLK_STS_OK, mrq->data->bytes_xfered)) blk_mq_requeue_request(req, true); @@ -2085,6 +2091,8 @@ static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req) } else if (!blk_rq_bytes(req)) { __blk_mq_end_request(req, BLK_STS_IOERR); } else if (mqrq->retries++ < MMC_MAX_RETRIES) { + if (rq_data_dir(req) == WRITE) + mqrq->flags |= MQRQ_XFER_SINGLE_BLOCK; blk_mq_requeue_request(req, true); } else { if (mmc_card_removed(mq->card)) diff --git a/drivers/mmc/core/card.h b/drivers/mmc/core/card.h index 1200951bab08c2..a7c364d0030ad5 100644 --- a/drivers/mmc/core/card.h +++ b/drivers/mmc/core/card.h @@ -89,6 +89,7 @@ struct mmc_fixup { #define CID_MANFID_MICRON 0x13 #define CID_MANFID_SAMSUNG 0x15 #define CID_MANFID_APACER 0x27 +#define CID_MANFID_SANDISK_MMC 0x45 #define CID_MANFID_SWISSBIT 0x5D #define CID_MANFID_KINGSTON 0x70 #define CID_MANFID_HYNIX 0x90 @@ -305,4 +306,14 @@ static inline int mmc_card_no_uhs_ddr50_tuning(const struct mmc_card *c) return c->quirks & MMC_QUIRK_NO_UHS_DDR50_TUNING; } +static inline int mmc_card_broken_mdt(const struct mmc_card *c) +{ + return c->quirks & MMC_QUIRK_BROKEN_MDT; +} + +static inline int mmc_card_fixed_secure_erase_trim_time(const struct mmc_card *c) +{ + return c->quirks & MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME; +} + #endif diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c index 7c86efb1044a37..8846550a8892a5 100644 --- a/drivers/mmc/core/mmc.c +++ b/drivers/mmc/core/mmc.c @@ -671,7 +671,19 @@ static int mmc_decode_ext_csd(struct mmc_card *card, u8 *ext_csd) card->ext_csd.enhanced_rpmb_supported = (card->ext_csd.rel_param & EXT_CSD_WR_REL_PARAM_EN_RPMB_REL_WR); + + if (card->ext_csd.rev >= 9) { + /* Adjust production date as per JEDEC JESD84-B51B September 2025 */ + if (card->cid.year < 2023) + card->cid.year += 16; + } else { + /* Handle vendors with broken MDT reporting */ + if (mmc_card_broken_mdt(card) && card->cid.year >= 2010 && + card->cid.year <= 2012) + card->cid.year += 16; + } } + out: return err; } diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index 13000fc57e2e12..39fcb662c43fc0 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -184,8 +184,13 @@ static void mmc_queue_setup_discard(struct mmc_card *card, return; lim->max_hw_discard_sectors = max_discard; - if (mmc_card_can_secure_erase_trim(card)) - lim->max_secure_erase_sectors = max_discard; + if (mmc_card_can_secure_erase_trim(card)) { + if (mmc_card_fixed_secure_erase_trim_time(card)) + lim->max_secure_erase_sectors = UINT_MAX >> card->erase_shift; + else + lim->max_secure_erase_sectors = max_discard; + } + if (mmc_card_can_trim(card) && card->erased_byte == 0) lim->max_write_zeroes_sectors = max_discard; diff --git a/drivers/mmc/core/queue.h b/drivers/mmc/core/queue.h index 1498840a4ea008..c254e6580afd62 100644 --- a/drivers/mmc/core/queue.h +++ b/drivers/mmc/core/queue.h @@ -61,6 +61,8 @@ enum mmc_drv_op { MMC_DRV_OP_GET_EXT_CSD, }; +#define MQRQ_XFER_SINGLE_BLOCK BIT(0) + struct mmc_queue_req { struct mmc_blk_request brq; struct scatterlist *sg; @@ -69,6 +71,7 @@ struct mmc_queue_req { void *drv_op_data; unsigned int ioc_count; int retries; + u32 flags; }; struct mmc_queue { diff --git a/drivers/mmc/core/quirks.h b/drivers/mmc/core/quirks.h index c417ed34c05767..6f727b4a60a527 100644 --- a/drivers/mmc/core/quirks.h +++ b/drivers/mmc/core/quirks.h @@ -153,6 +153,15 @@ static const struct mmc_fixup __maybe_unused mmc_blk_fixups[] = { MMC_FIXUP("M62704", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc, MMC_QUIRK_TRIM_BROKEN), + /* + * On Some Kingston eMMCs, secure erase/trim time is independent + * of erase size, fixed at approximately 2 seconds. + */ + MMC_FIXUP("IY2964", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc, + MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME), + MMC_FIXUP("IB2932", CID_MANFID_KINGSTON, 0x0100, add_quirk_mmc, + MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME), + END_FIXUP }; @@ -170,6 +179,9 @@ static const struct mmc_fixup __maybe_unused mmc_ext_csd_fixups[] = { MMC_FIXUP_EXT_CSD_REV(CID_NAME_ANY, CID_MANFID_NUMONYX, 0x014e, add_quirk, MMC_QUIRK_BROKEN_HPI, 6), + MMC_FIXUP(CID_NAME_ANY, CID_MANFID_SANDISK_MMC, CID_OEMID_ANY, add_quirk_mmc, + MMC_QUIRK_BROKEN_MDT), + END_FIXUP }; diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index 2b75a36c096b8f..bf2cb49ddd8196 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -738,12 +738,15 @@ static void dwcmshc_rk3568_set_clock(struct sdhci_host *host, unsigned int clock extra |= BIT(4); sdhci_writel(host, extra, reg); + /* Disable clock while config DLL */ + sdhci_writew(host, 0, SDHCI_CLOCK_CONTROL); + if (clock <= 52000000) { if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200 || host->mmc->ios.timing == MMC_TIMING_MMC_HS400) { dev_err(mmc_dev(host->mmc), "Can't reduce the clock below 52MHz in HS200/HS400 mode"); - return; + goto enable_clk; } /* @@ -763,7 +766,7 @@ static void dwcmshc_rk3568_set_clock(struct sdhci_host *host, unsigned int clock DLL_STRBIN_DELAY_NUM_SEL | DLL_STRBIN_DELAY_NUM_DEFAULT << DLL_STRBIN_DELAY_NUM_OFFSET; sdhci_writel(host, extra, DWCMSHC_EMMC_DLL_STRBIN); - return; + goto enable_clk; } /* Reset DLL */ @@ -790,7 +793,7 @@ static void dwcmshc_rk3568_set_clock(struct sdhci_host *host, unsigned int clock 500 * USEC_PER_MSEC); if (err) { dev_err(mmc_dev(host->mmc), "DLL lock timeout!\n"); - return; + goto enable_clk; } extra = 0x1 << 16 | /* tune clock stop en */ @@ -823,6 +826,16 @@ static void dwcmshc_rk3568_set_clock(struct sdhci_host *host, unsigned int clock DLL_STRBIN_TAPNUM_DEFAULT | DLL_STRBIN_TAPNUM_FROM_SW; sdhci_writel(host, extra, DWCMSHC_EMMC_DLL_STRBIN); + +enable_clk: + /* + * The sdclk frequency select bits in SDHCI_CLOCK_CONTROL are not functional + * on Rockchip's SDHCI implementation. Instead, the clock frequency is fully + * controlled via external clk provider by calling clk_set_rate(). Consequently, + * passing 0 to sdhci_enable_clk() only re-enables the already-configured clock, + * which matches the hardware's actual behavior. + */ + sdhci_enable_clk(host, 0); } static void rk35xx_sdhci_reset(struct sdhci_host *host, u8 mask) diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 47a0a738862b58..93122133202367 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -2129,6 +2130,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( struct sdhci_host *host; int ret, bar = first_bar + slotno; size_t priv_size = chip->fixes ? chip->fixes->priv_size : 0; + u32 cd_debounce_delay_ms; if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "BAR %d is not iomem. Aborting.\n", bar); @@ -2195,6 +2197,10 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( if (host->mmc->caps & MMC_CAP_CD_WAKE) device_init_wakeup(&pdev->dev, true); + if (device_property_read_u32(&pdev->dev, "cd-debounce-delay-ms", + &cd_debounce_delay_ms)) + cd_debounce_delay_ms = 200; + if (slot->cd_idx >= 0) { struct gpiod_lookup_table *cd_gpio_lookup_table; @@ -2213,7 +2219,7 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( ret = mmc_gpiod_request_cd(host->mmc, NULL, slot->cd_idx, slot->cd_override_level, - 0); + cd_debounce_delay_ms * 1000); if (ret == -EPROBE_DEFER) goto remove; @@ -2221,6 +2227,16 @@ static struct sdhci_pci_slot *sdhci_pci_probe_slot( dev_warn(&pdev->dev, "failed to setup card detect gpio\n"); slot->cd_idx = -1; } + } else if (is_of_node(pdev->dev.fwnode)) { + /* Allow all OF systems to use a CD GPIO if provided */ + + ret = mmc_gpiod_request_cd(host->mmc, "cd", 0, + slot->cd_override_level, + cd_debounce_delay_ms * 1000); + if (ret == -EPROBE_DEFER) + goto remove; + else if (ret == 0) + slot->cd_idx = 0; } if (chip->fixes && chip->fixes->add_host) diff --git a/drivers/mtd/devices/docg3.c b/drivers/mtd/devices/docg3.c index 33050a2a80f79e..603fd0efc2ea2d 100644 --- a/drivers/mtd/devices/docg3.c +++ b/drivers/mtd/devices/docg3.c @@ -2049,7 +2049,6 @@ static int __init docg3_probe(struct platform_device *pdev) static void docg3_release(struct platform_device *pdev) { struct docg3_cascade *cascade = platform_get_drvdata(pdev); - struct docg3 *docg3 = cascade->floors[0]->priv; int floor; doc_unregister_sysfs(pdev, cascade); @@ -2057,7 +2056,7 @@ static void docg3_release(struct platform_device *pdev) if (cascade->floors[floor]) doc_release_device(cascade->floors[floor]); - bch_free(docg3->cascade->bch); + bch_free(cascade->bch); } #ifdef CONFIG_OF diff --git a/drivers/mtd/maps/physmap-gemini.c b/drivers/mtd/maps/physmap-gemini.c index 9d3b4bf84a1ad7..1c34b4ef77ea3f 100644 --- a/drivers/mtd/maps/physmap-gemini.c +++ b/drivers/mtd/maps/physmap-gemini.c @@ -181,7 +181,7 @@ int of_flash_probe_gemini(struct platform_device *pdev, dev_err(dev, "no enabled pin control state\n"); gf->disabled_state = pinctrl_lookup_state(gf->p, "disabled"); - if (IS_ERR(gf->enabled_state)) { + if (IS_ERR(gf->disabled_state)) { dev_err(dev, "no disabled pin control state\n"); } else { ret = pinctrl_select_state(gf->p, gf->disabled_state); diff --git a/drivers/mtd/nand/raw/sunxi_nand.c b/drivers/mtd/nand/raw/sunxi_nand.c index e66adfcca7cd63..85b869041a37fb 100644 --- a/drivers/mtd/nand/raw/sunxi_nand.c +++ b/drivers/mtd/nand/raw/sunxi_nand.c @@ -1048,9 +1048,9 @@ static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand, if (len <= 0) return; - if (!cur_off || *cur_off != offset) - nand_change_read_column_op(nand, mtd->writesize, NULL, 0, - false); + if (!cur_off || *cur_off != (offset + mtd->writesize)) + nand_change_read_column_op(nand, mtd->writesize + offset, + NULL, 0, false); if (!randomize) sunxi_nfc_read_buf(nand, oob + offset, len); diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index 8aa3753aaaa1d3..0b076790bd9df6 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -100,6 +100,17 @@ spinand_fill_page_read_op(struct spinand_device *spinand, u64 addr) return op; } +static struct spi_mem_op +spinand_fill_page_read_packed_op(struct spinand_device *spinand, u64 addr) +{ + struct spi_mem_op op = spinand->op_templates->page_read; + + op.cmd.opcode |= addr >> 16; + op.addr.val = addr & 0xFFFF; + + return op; +} + struct spi_mem_op spinand_fill_prog_exec_op(struct spinand_device *spinand, u64 addr) { @@ -453,7 +464,10 @@ static int spinand_load_page_op(struct spinand_device *spinand, { struct nand_device *nand = spinand_to_nand(spinand); unsigned int row = nanddev_pos_to_row(nand, &req->pos); - struct spi_mem_op op = SPINAND_OP(spinand, page_read, row); + bool packed = spinand->flags & SPINAND_ODTR_PACKED_PAGE_READ; + struct spi_mem_op op = packed ? + SPINAND_OP(spinand, page_read_packed, row) : + SPINAND_OP(spinand, page_read, row); return spi_mem_exec_op(spinand->spimem, &op); } @@ -1489,9 +1503,13 @@ static int spinand_init_odtr_instruction_set(struct spinand_device *spinand) if (!spi_mem_supports_op(spinand->spimem, &tmpl->blk_erase)) return -EOPNOTSUPP; - tmpl->page_read = (struct spi_mem_op)SPINAND_PAGE_READ_8D_8D_0_OP(0); - if (!spi_mem_supports_op(spinand->spimem, &tmpl->page_read)) + if (spinand->flags & SPINAND_ODTR_PACKED_PAGE_READ) + tmpl->page_read = (struct spi_mem_op)SPINAND_PAGE_READ_PACKED_8D_8D_0_OP(0); + else + tmpl->page_read = (struct spi_mem_op)SPINAND_PAGE_READ_8D_8D_0_OP(0); + if (!spi_mem_supports_op(spinand->spimem, &tmpl->page_read)) { return -EOPNOTSUPP; + } tmpl->prog_exec = (struct spi_mem_op)SPINAND_PROG_EXEC_8D_8D_0_OP(0); if (!spi_mem_supports_op(spinand->spimem, &tmpl->prog_exec)) diff --git a/drivers/mtd/nand/spi/winbond.c b/drivers/mtd/nand/spi/winbond.c index 6dfd0dcc8ee7af..7cc0f0091430c1 100644 --- a/drivers/mtd/nand/spi/winbond.c +++ b/drivers/mtd/nand/spi/winbond.c @@ -99,7 +99,7 @@ static SPINAND_OP_VARIANTS(update_cache_variants, #define SPINAND_WINBOND_WRITE_VCR_8D_8D_8D(reg, buf) \ SPI_MEM_OP(SPI_MEM_DTR_OP_RPT_CMD(0x81, 8), \ - SPI_MEM_DTR_OP_ADDR(4, reg, 8), \ + SPI_MEM_DTR_OP_ADDR(4, reg << 8, 8), \ SPI_MEM_OP_NO_DUMMY, \ SPI_MEM_DTR_OP_DATA_OUT(2, buf, 8)) @@ -337,16 +337,19 @@ static int w25n0xjw_hs_cfg(struct spinand_device *spinand, if (iface != SSDR) return -EOPNOTSUPP; + /* + * SDR dual and quad I/O operations over 104MHz require the HS bit to + * enable a few more dummy cycles. + */ op = spinand->op_templates->read_cache; if (op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr) hs = false; - else if (op->cmd.buswidth == 1 && op->addr.buswidth == 1 && - op->dummy.buswidth == 1 && op->data.buswidth == 1) + else if (op->cmd.buswidth != 1 || op->addr.buswidth == 1) hs = false; - else if (!op->max_freq) - hs = true; - else + else if (op->max_freq && op->max_freq <= 104 * HZ_PER_MHZ) hs = false; + else + hs = true; ret = spinand_read_reg_op(spinand, W25N0XJW_SR4, &sr4); if (ret) @@ -485,7 +488,7 @@ static const struct spinand_info winbond_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_dual_quad_dtr_variants, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&w25n01jw_ooblayout, NULL), SPINAND_CONFIGURE_CHIP(w25n0xjw_hs_cfg)), SPINAND_INFO("W25N01KV", /* 3.3V */ @@ -515,7 +518,7 @@ static const struct spinand_info winbond_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants, &write_cache_octal_variants, &update_cache_octal_variants), - 0, + SPINAND_ODTR_PACKED_PAGE_READ, SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops), SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL), SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)), @@ -526,7 +529,7 @@ static const struct spinand_info winbond_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants, &write_cache_octal_variants, &update_cache_octal_variants), - 0, + SPINAND_ODTR_PACKED_PAGE_READ, SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops), SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL), SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)), @@ -549,7 +552,7 @@ static const struct spinand_info winbond_spinand_table[] = { SPINAND_INFO_OP_VARIANTS(&read_cache_dual_quad_dtr_variants, &write_cache_variants, &update_cache_variants), - 0, + SPINAND_HAS_QE_BIT, SPINAND_ECCINFO(&w25m02gv_ooblayout, NULL), SPINAND_CONFIGURE_CHIP(w25n0xjw_hs_cfg)), SPINAND_INFO("W25N02KV", /* 3.3V */ diff --git a/drivers/mtd/parsers/ofpart_core.c b/drivers/mtd/parsers/ofpart_core.c index 0029bda165bde1..262c4221d23f72 100644 --- a/drivers/mtd/parsers/ofpart_core.c +++ b/drivers/mtd/parsers/ofpart_core.c @@ -75,7 +75,7 @@ static int parse_fixed_partitions(struct mtd_info *master, dedicated = false; } } else { /* Partition */ - ofpart_node = mtd_node; + ofpart_node = of_node_get(mtd_node); } of_id = of_match_node(parse_ofpart_match_table, ofpart_node); @@ -195,11 +195,11 @@ static int parse_fixed_partitions(struct mtd_info *master, ofpart_fail: pr_err("%s: error parsing ofpart partition %pOF (%pOF)\n", master->name, pp, mtd_node); + of_node_put(pp); ret = -EINVAL; ofpart_none: if (dedicated) of_node_put(ofpart_node); - of_node_put(pp); kfree(parts); return ret; } diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c index 1eee519c01e5ca..5dd0b3cb525033 100644 --- a/drivers/mtd/spi-nor/core.c +++ b/drivers/mtd/spi-nor/core.c @@ -2393,7 +2393,7 @@ static int spi_nor_spimem_check_readop(struct spi_nor *nor, /* convert the dummy cycles to the number of bytes */ op.dummy.nbytes = (read->num_mode_clocks + read->num_wait_states) * op.dummy.buswidth / 8; - if (spi_nor_protocol_is_dtr(nor->read_proto)) + if (spi_nor_protocol_is_dtr(read->proto)) op.dummy.nbytes *= 2; return spi_nor_spimem_check_read_pp_op(nor, &op); diff --git a/drivers/mtd/spi-nor/core.h b/drivers/mtd/spi-nor/core.h index 16b382d4f04f20..e838c40a25897b 100644 --- a/drivers/mtd/spi-nor/core.h +++ b/drivers/mtd/spi-nor/core.h @@ -413,7 +413,7 @@ struct spi_nor_flash_parameter { * number of dummy cycles in read register ops. * @smpt_map_id: called after map ID in SMPT table has been determined for the * case the map ID is wrong and needs to be fixed. - * @post_sfdp: called after SFDP has been parsed (is also called for SPI NORs + * @post_sfdp: called after SFDP has been parsed (is not called for SPI NORs * that do not support RDSFDP). Typically used to tweak various * parameters that could not be extracted by other means (i.e. * when information provided by the SFDP/flash_info tables are diff --git a/drivers/mtd/spi-nor/debugfs.c b/drivers/mtd/spi-nor/debugfs.c index fa6956144d2e44..14ba1680c31547 100644 --- a/drivers/mtd/spi-nor/debugfs.c +++ b/drivers/mtd/spi-nor/debugfs.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -92,7 +93,8 @@ static int spi_nor_params_show(struct seq_file *s, void *data) seq_printf(s, "address nbytes\t%u\n", nor->addr_nbytes); seq_puts(s, "flags\t\t"); - spi_nor_print_flags(s, nor->flags, snor_f_names, sizeof(snor_f_names)); + spi_nor_print_flags(s, nor->flags, snor_f_names, + ARRAY_SIZE(snor_f_names)); seq_puts(s, "\n"); seq_puts(s, "\nopcodes\n"); diff --git a/drivers/mtd/spi-nor/micron-st.c b/drivers/mtd/spi-nor/micron-st.c index 88033384a71e5d..b2b473501d0238 100644 --- a/drivers/mtd/spi-nor/micron-st.c +++ b/drivers/mtd/spi-nor/micron-st.c @@ -167,6 +167,16 @@ static int mt35xu512aba_post_sfdp_fixup(struct spi_nor *nor) 0, 20, SPINOR_OP_MT_DTR_RD, SNOR_PROTO_8_8_8_DTR); + /* + * Some batches of mt35xu512aba do not contain the OCT DTR command + * information, but do support OCT DTR mode. Add the settings for + * SNOR_CMD_PP_8_8_8_DTR here. This also makes sure the flash can switch + * to OCT DTR mode. + */ + nor->params->hwcaps.mask |= SNOR_HWCAPS_PP_8_8_8_DTR; + spi_nor_set_pp_settings(&nor->params->page_programs[SNOR_CMD_PP_8_8_8_DTR], + SPINOR_OP_PP_4B, SNOR_PROTO_8_8_8_DTR); + nor->cmd_ext_type = SPI_NOR_EXT_REPEAT; nor->params->rdsr_dummy = 8; nor->params->rdsr_addr_nbytes = 0; diff --git a/drivers/mtd/spi-nor/sst.c b/drivers/mtd/spi-nor/sst.c index 175211fe6a5ed2..db02c14ba16faf 100644 --- a/drivers/mtd/spi-nor/sst.c +++ b/drivers/mtd/spi-nor/sst.c @@ -203,6 +203,8 @@ static int sst_nor_write(struct mtd_info *mtd, loff_t to, size_t len, /* Start write from odd address. */ if (to % 2) { + bool needs_write_enable = (len > 1); + /* write one byte. */ ret = sst_nor_write_data(nor, to, 1, buf); if (ret < 0) @@ -210,6 +212,17 @@ static int sst_nor_write(struct mtd_info *mtd, loff_t to, size_t len, to++; actual++; + + /* + * Byte program clears the write enable latch. If more + * data needs to be written using the AAI sequence, + * re-enable writes. + */ + if (needs_write_enable) { + ret = spi_nor_write_enable(nor); + if (ret) + goto out; + } } /* Write out most of the data here. */ diff --git a/drivers/mtd/spi-nor/swp.c b/drivers/mtd/spi-nor/swp.c index 9b07f83aeac76d..e67a81dbb6bf6d 100644 --- a/drivers/mtd/spi-nor/swp.c +++ b/drivers/mtd/spi-nor/swp.c @@ -28,8 +28,10 @@ static u8 spi_nor_get_sr_tb_mask(struct spi_nor *nor) { if (nor->flags & SNOR_F_HAS_SR_TB_BIT6) return SR_TB_BIT6; - else + else if (nor->flags & SNOR_F_HAS_SR_TB) return SR_TB_BIT5; + else + return 0; } static u64 spi_nor_get_min_prot_length_sr(struct spi_nor *nor) diff --git a/drivers/mux/Kconfig b/drivers/mux/Kconfig index c68132e38138ef..281d3bad07448f 100644 --- a/drivers/mux/Kconfig +++ b/drivers/mux/Kconfig @@ -31,6 +31,19 @@ config MUX_ADGS1408 To compile the driver as a module, choose M here: the module will be called mux-adgs1408. +config MUX_APPLE_DPXBAR + tristate "Apple Silicon Display Crossbar" + depends on ARCH_APPLE + help + Apple Silicon Display Crossbar multiplexer. + + This drivers adds support for the display crossbar used to route + display controller streams to the three different modes + (DP AltMode, USB4 Tunnel #0/#1) of the Type-C ports. + + To compile this driver as a module, chose M here: the module will be + called mux-apple-display-crossbar. + config MUX_GPIO tristate "GPIO-controlled Multiplexer" depends on GPIOLIB || COMPILE_TEST diff --git a/drivers/mux/Makefile b/drivers/mux/Makefile index 6e9fa47daf5663..7b5b3325068010 100644 --- a/drivers/mux/Makefile +++ b/drivers/mux/Makefile @@ -8,9 +8,11 @@ mux-adg792a-objs := adg792a.o mux-adgs1408-objs := adgs1408.o mux-gpio-objs := gpio.o mux-mmio-objs := mmio.o +mux-apple-display-crossbar-objs := apple-display-crossbar.o obj-$(CONFIG_MULTIPLEXER) += mux-core.o obj-$(CONFIG_MUX_ADG792A) += mux-adg792a.o obj-$(CONFIG_MUX_ADGS1408) += mux-adgs1408.o +obj-$(CONFIG_MUX_APPLE_DPXBAR) += mux-apple-display-crossbar.o obj-$(CONFIG_MUX_GPIO) += mux-gpio.o obj-$(CONFIG_MUX_MMIO) += mux-mmio.o diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c new file mode 100644 index 00000000000000..9b17371d92c3ba --- /dev/null +++ b/drivers/mux/apple-display-crossbar.c @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Silicon Display Crossbar multiplexer driver + * + * Copyright (C) Asahi Linux Contributors + * + * Author: Sven Peter + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * T602x register interface is cleary different so most of the names below are + * probably wrong. + */ + +#define T602X_FIFO_WR_DPTX_CLK_EN 0x000 +#define T602X_FIFO_WR_N_CLK_EN 0x004 +#define T602X_FIFO_WR_UNK_EN 0x008 +#define T602X_REG_00C 0x00c +#define T602X_REG_014 0x014 +#define T602X_REG_018 0x018 +#define T602X_REG_01C 0x01c +#define T602X_FIFO_RD_PCLK2_EN 0x024 +#define T602X_FIFO_RD_N_CLK_EN 0x028 +#define T602X_FIFO_RD_UNK_EN 0x02c +#define T602X_REG_030 0x030 +#define T602X_REG_034 0x034 + +#define T602X_REG_804_STAT 0x804 // status of 0x004 +#define T602X_REG_810_STAT 0x810 // status of 0x014 +#define T602X_REG_81C_STAT 0x81c // status of 0x024 + +/* + * T8013, T600x, T8112 dp crossbar registers. + */ + +#define FIFO_WR_DPTX_CLK_EN 0x000 +#define FIFO_WR_N_CLK_EN 0x004 +#define FIFO_WR_UNK_EN 0x008 +#define FIFO_RD_PCLK1_EN 0x020 +#define FIFO_RD_PCLK2_EN 0x024 +#define FIFO_RD_N_CLK_EN 0x028 +#define FIFO_RD_UNK_EN 0x02c + +#define OUT_PCLK1_EN 0x040 +#define OUT_PCLK2_EN 0x044 +#define OUT_N_CLK_EN 0x048 +#define OUT_UNK_EN 0x04c + +#define CROSSBAR_DISPEXT_EN 0x050 +#define CROSSBAR_MUX_CTRL 0x060 +#define CROSSBAR_MUX_CTRL_DPPHY_SELECT0 GENMASK(23, 20) +#define CROSSBAR_MUX_CTRL_DPIN1_SELECT0 GENMASK(19, 16) +#define CROSSBAR_MUX_CTRL_DPIN0_SELECT0 GENMASK(15, 12) +#define CROSSBAR_MUX_CTRL_DPPHY_SELECT1 GENMASK(11, 8) +#define CROSSBAR_MUX_CTRL_DPIN1_SELECT1 GENMASK(7, 4) +#define CROSSBAR_MUX_CTRL_DPIN0_SELECT1 GENMASK(3, 0) +#define CROSSBAR_ATC_EN 0x070 + +#define FIFO_WR_DPTX_CLK_EN_STAT 0x800 +#define FIFO_WR_N_CLK_EN_STAT 0x804 +#define FIFO_RD_PCLK1_EN_STAT 0x820 +#define FIFO_RD_PCLK2_EN_STAT 0x824 +#define FIFO_RD_N_CLK_EN_STAT 0x828 + +#define OUT_PCLK1_EN_STAT 0x840 +#define OUT_PCLK2_EN_STAT 0x844 +#define OUT_N_CLK_EN_STAT 0x848 + +#define UNK_TUNABLE 0xc00 + +#define ATC_DPIN0 BIT(0) +#define ATC_DPIN1 BIT(4) +#define ATC_DPPHY BIT(8) + +enum { MUX_DPPHY = 0, MUX_DPIN0 = 1, MUX_DPIN1 = 2, MUX_MAX = 3 }; +static const char *apple_dpxbar_names[MUX_MAX] = { "dpphy", "dpin0", "dpin1" }; + +struct apple_dpxbar_hw { + unsigned int n_ufp; + u32 tunable; + const struct mux_control_ops *ops; +}; + +struct apple_dpxbar { + struct device *dev; + void __iomem *regs; + int selected_dispext[MUX_MAX]; + spinlock_t lock; +}; + +static inline void dpxbar_mask32(struct apple_dpxbar *xbar, u32 reg, u32 mask, + u32 set) +{ + u32 value = readl(xbar->regs + reg); + value &= ~mask; + value |= set; + writel(value, xbar->regs + reg); +} + +static inline void dpxbar_set32(struct apple_dpxbar *xbar, u32 reg, u32 set) +{ + dpxbar_mask32(xbar, reg, 0, set); +} + +static inline void dpxbar_clear32(struct apple_dpxbar *xbar, u32 reg, u32 clear) +{ + dpxbar_mask32(xbar, reg, clear, 0); +} + +static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) +{ + struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + unsigned int mux_state; + unsigned int dispext_bit; + unsigned int dispext_bit_en; + bool enable; + int ret = 0; + + if (state == MUX_IDLE_DISCONNECT) { + /* + * Technically this will select dispext0,0 in the mux control + * register. Practically that doesn't matter since everything + * else is disabled. + */ + mux_state = 0; + enable = false; + } else if (state >= 0 && state < 9) { + dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); + mux_state = state; + enable = true; + } else { + return -EINVAL; + } + + spin_lock_irqsave(&dpxbar->lock, flags); + + /* ensure the selected dispext isn't already used in this crossbar */ + if (enable) { + for (int i = 0; i < MUX_MAX; ++i) { + if (i == index) + continue; + if (dpxbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&dpxbar->lock, flags); + return -EBUSY; + } + } + } + + if (dpxbar->selected_dispext[index] >= 0) { + u32 prev_dispext_bit = 1 << dpxbar->selected_dispext[index]; + u32 prev_dispext_bit_en = 1 << (2 * dpxbar->selected_dispext[index]); + + dpxbar_clear32(dpxbar, T602X_FIFO_RD_UNK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, T602X_REG_00C, prev_dispext_bit_en); + + dpxbar_clear32(dpxbar, T602X_REG_01C, 0x100); + + dpxbar_clear32(dpxbar, T602X_FIFO_WR_UNK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, T602X_REG_018, prev_dispext_bit_en); + + dpxbar_clear32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + + dpxbar_set32(dpxbar, T602X_FIFO_WR_N_CLK_EN, prev_dispext_bit); + dpxbar_set32(dpxbar, T602X_REG_014, 0x4); + + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, 0x100); + + dpxbar->selected_dispext[index] = -1; + } + + if (enable) { + dpxbar_set32(dpxbar, T602X_REG_030, state << 20); + dpxbar_set32(dpxbar, T602X_REG_030, state << 8); + udelay(10); + + dpxbar_clear32(dpxbar, T602X_FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, T602X_REG_014, 0x4); + + dpxbar_clear32(dpxbar, T602X_FIFO_RD_PCLK2_EN, 0x100); + + dpxbar_set32(dpxbar, T602X_FIFO_WR_UNK_EN, dispext_bit); + dpxbar_set32(dpxbar, T602X_REG_018, dispext_bit_en); + + dpxbar_set32(dpxbar, T602X_FIFO_RD_N_CLK_EN, 0x100); + dpxbar_set32(dpxbar, T602X_FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, T602X_REG_00C, dispext_bit); + + dpxbar_set32(dpxbar, T602X_REG_01C, 0x100); + dpxbar_set32(dpxbar, T602X_REG_034, 0x100); + + dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); + + dpxbar->selected_dispext[index] = state; + } + + spin_unlock_irqrestore(&dpxbar->lock, flags); + + if (enable) + dev_info(dpxbar->dev, "Switched %s to dispext%u,%u\n", + apple_dpxbar_names[index], mux_state >> 1, + mux_state & 1); + else + dev_info(dpxbar->dev, "Switched %s to disconnected state\n", + apple_dpxbar_names[index]); + + return ret; +} + +static int apple_dpxbar_set(struct mux_control *mux, int state) +{ + struct apple_dpxbar *dpxbar = mux_chip_priv(mux->chip); + unsigned int index = mux_control_get_index(mux); + unsigned long flags; + unsigned int mux_state; + unsigned int dispext_bit; + unsigned int dispext_bit_en; + unsigned int atc_bit; + bool enable; + int ret = 0; + u32 mux_mask, mux_set; + + if (state == MUX_IDLE_DISCONNECT) { + /* + * Technically this will select dispext0,0 in the mux control + * register. Practically that doesn't matter since everything + * else is disabled. + */ + mux_state = 0; + enable = false; + } else if (state >= 0 && state < 9) { + dispext_bit = 1 << state; + dispext_bit_en = 1 << (2 * state); + mux_state = state; + enable = true; + } else { + return -EINVAL; + } + + switch (index) { + case MUX_DPPHY: + mux_mask = CROSSBAR_MUX_CTRL_DPPHY_SELECT0 | + CROSSBAR_MUX_CTRL_DPPHY_SELECT1; + mux_set = + FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPPHY_SELECT1, mux_state); + atc_bit = ATC_DPPHY; + break; + case MUX_DPIN0: + mux_mask = CROSSBAR_MUX_CTRL_DPIN0_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN0_SELECT1; + mux_set = + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN0_SELECT1, mux_state); + atc_bit = ATC_DPIN0; + break; + case MUX_DPIN1: + mux_mask = CROSSBAR_MUX_CTRL_DPIN1_SELECT0 | + CROSSBAR_MUX_CTRL_DPIN1_SELECT1; + mux_set = + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT0, mux_state) | + FIELD_PREP(CROSSBAR_MUX_CTRL_DPIN1_SELECT1, mux_state); + atc_bit = ATC_DPIN1; + break; + default: + return -EINVAL; + } + + spin_lock_irqsave(&dpxbar->lock, flags); + + /* ensure the selected dispext isn't already used in this crossbar */ + if (enable) { + for (int i = 0; i < MUX_MAX; ++i) { + if (i == index) + continue; + if (dpxbar->selected_dispext[i] == state) { + spin_unlock_irqrestore(&dpxbar->lock, flags); + return -EBUSY; + } + } + } + + dpxbar_set32(dpxbar, OUT_N_CLK_EN, atc_bit); + dpxbar_clear32(dpxbar, OUT_UNK_EN, atc_bit); + dpxbar_clear32(dpxbar, OUT_PCLK1_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + + if (dpxbar->selected_dispext[index] >= 0) { + u32 prev_dispext_bit = 1 << dpxbar->selected_dispext[index]; + u32 prev_dispext_bit_en = 1 << (2 * dpxbar->selected_dispext[index]); + + dpxbar_set32(dpxbar, FIFO_WR_N_CLK_EN, prev_dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_N_CLK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, FIFO_WR_UNK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_UNK_EN, prev_dispext_bit_en); + dpxbar_clear32(dpxbar, FIFO_WR_DPTX_CLK_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, prev_dispext_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + + dpxbar->selected_dispext[index] = -1; + } + + dpxbar_mask32(dpxbar, CROSSBAR_MUX_CTRL, mux_mask, mux_set); + + if (enable) { + dpxbar_clear32(dpxbar, FIFO_WR_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, FIFO_RD_N_CLK_EN, dispext_bit); + dpxbar_clear32(dpxbar, OUT_N_CLK_EN, atc_bit); + dpxbar_set32(dpxbar, FIFO_WR_UNK_EN, dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_UNK_EN, dispext_bit_en); + dpxbar_set32(dpxbar, OUT_UNK_EN, atc_bit); + dpxbar_set32(dpxbar, FIFO_WR_DPTX_CLK_EN, dispext_bit); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + dpxbar_set32(dpxbar, OUT_PCLK1_EN, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + + /* + * Work around some HW quirk: + * Without toggling the RD_PCLK enable here the connection + * doesn't come up. Testing has shown that a delay of about + * 5 usec is required which is doubled here to be on the + * safe side. + */ + dpxbar_clear32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + udelay(10); + dpxbar_set32(dpxbar, FIFO_RD_PCLK1_EN, dispext_bit); + + dpxbar->selected_dispext[index] = state; + } + + spin_unlock_irqrestore(&dpxbar->lock, flags); + + if (enable) + dev_info(dpxbar->dev, "Switched %s to dispext%u,%u\n", + apple_dpxbar_names[index], mux_state >> 1, + mux_state & 1); + else + dev_info(dpxbar->dev, "Switched %s to disconnected state\n", + apple_dpxbar_names[index]); + + return ret; +} + +static const struct mux_control_ops apple_dpxbar_ops = { + .set = apple_dpxbar_set, +}; + +static const struct mux_control_ops apple_dpxbar_t602x_ops = { + .set = apple_dpxbar_set_t602x, +}; + +static int apple_dpxbar_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mux_chip *mux_chip; + struct apple_dpxbar *dpxbar; + const struct apple_dpxbar_hw *hw; + int ret; + + hw = of_device_get_match_data(dev); + mux_chip = devm_mux_chip_alloc(dev, MUX_MAX, sizeof(*dpxbar)); + if (IS_ERR(mux_chip)) + return PTR_ERR(mux_chip); + + dpxbar = mux_chip_priv(mux_chip); + mux_chip->ops = hw->ops; + spin_lock_init(&dpxbar->lock); + + dpxbar->dev = dev; + dpxbar->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dpxbar->regs)) + return PTR_ERR(dpxbar->regs); + + if (!of_device_is_compatible(dev->of_node, "apple,t6020-display-crossbar")) { + readl(dpxbar->regs + UNK_TUNABLE); + writel(hw->tunable, dpxbar->regs + UNK_TUNABLE); + readl(dpxbar->regs + UNK_TUNABLE); + } + + for (unsigned int i = 0; i < MUX_MAX; ++i) { + mux_chip->mux[i].states = hw->n_ufp; + mux_chip->mux[i].idle_state = MUX_IDLE_DISCONNECT; + dpxbar->selected_dispext[i] = -1; + } + + ret = devm_mux_chip_register(dev, mux_chip); + if (ret < 0) + return ret; + + return 0; +} + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8103 = { + .n_ufp = 2, + .tunable = 0, + .ops = &apple_dpxbar_ops, +}; + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t8112 = { + .n_ufp = 4, + .tunable = 4278196325, + .ops = &apple_dpxbar_ops, +}; + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t6000 = { + .n_ufp = 9, + .tunable = 5, + .ops = &apple_dpxbar_ops, +}; + +static const struct apple_dpxbar_hw apple_dpxbar_hw_t6020 = { + .n_ufp = 9, + .ops = &apple_dpxbar_t602x_ops, +}; + +static const struct of_device_id apple_dpxbar_ids[] = { + { + .compatible = "apple,t8103-display-crossbar", + .data = &apple_dpxbar_hw_t8103, + }, + { + .compatible = "apple,t8112-display-crossbar", + .data = &apple_dpxbar_hw_t8112, + }, + { + .compatible = "apple,t6000-display-crossbar", + .data = &apple_dpxbar_hw_t6000, + }, + { + .compatible = "apple,t6020-display-crossbar", + .data = &apple_dpxbar_hw_t6020, + }, + {} +}; +MODULE_DEVICE_TABLE(of, apple_dpxbar_ids); + +static struct platform_driver apple_dpxbar_driver = { + .driver = { + .name = "apple-display-crossbar", + .of_match_table = apple_dpxbar_ids, + }, + .probe = apple_dpxbar_probe, +}; +module_platform_driver(apple_dpxbar_driver); + +MODULE_DESCRIPTION("Apple Silicon display crossbar multiplexer driver"); +MODULE_AUTHOR("Sven Peter "); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c index 0df3208783ad9e..da5866ba069995 100644 --- a/drivers/net/bareudp.c +++ b/drivers/net/bareudp.c @@ -529,6 +529,9 @@ static int bareudp_fill_metadata_dst(struct net_device *dev, struct in6_addr saddr; struct socket *sock = rcu_dereference(bareudp->sock); + if (!sock) + return -ESHUTDOWN; + dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sock, 0, &saddr, &info->key, sport, bareudp->port, info->key.tos, diff --git a/drivers/net/bonding/bond_3ad.c b/drivers/net/bonding/bond_3ad.c index af7f74cfdc086c..f0aa7d2f21717a 100644 --- a/drivers/net/bonding/bond_3ad.c +++ b/drivers/net/bonding/bond_3ad.c @@ -1029,6 +1029,7 @@ static void ad_cond_set_peer_notif(struct port *port) static void ad_mux_machine(struct port *port, bool *update_slave_arr) { struct bonding *bond = __get_bond_by_port(port); + struct aggregator *aggregator; mux_states_t last_state; /* keep current State Machine state to compare later if it was @@ -1036,6 +1037,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) */ last_state = port->sm_mux_state; + aggregator = rcu_dereference(port->aggregator); if (port->sm_vars & AD_PORT_BEGIN) { port->sm_mux_state = AD_MUX_DETACHED; } else { @@ -1055,7 +1057,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) * cycle to update ready variable, we check * READY_N and update READY here */ - __set_agg_ports_ready(port->aggregator, __agg_ports_are_ready(port->aggregator)); + __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator)); port->sm_mux_state = AD_MUX_DETACHED; break; } @@ -1070,7 +1072,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) * update ready variable, we check READY_N and update * READY here */ - __set_agg_ports_ready(port->aggregator, __agg_ports_are_ready(port->aggregator)); + __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator)); /* if the wait_while_timer expired, and the port is * in READY state, move to ATTACHED state @@ -1086,7 +1088,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) if ((port->sm_vars & AD_PORT_SELECTED) && (port->partner_oper.port_state & LACP_STATE_SYNCHRONIZATION) && !__check_agg_selection_timer(port)) { - if (port->aggregator->is_active) { + if (aggregator->is_active) { int state = AD_MUX_COLLECTING_DISTRIBUTING; if (!bond->params.coupled_control) @@ -1102,9 +1104,9 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) * cycle to update ready variable, we check * READY_N and update READY here */ - __set_agg_ports_ready(port->aggregator, __agg_ports_are_ready(port->aggregator)); + __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator)); port->sm_mux_state = AD_MUX_DETACHED; - } else if (port->aggregator->is_active) { + } else if (aggregator->is_active) { port->actor_oper_port_state |= LACP_STATE_SYNCHRONIZATION; } @@ -1115,7 +1117,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) * sure that a collecting distributing * port in an active aggregator is enabled */ - if (port->aggregator->is_active && + if (aggregator->is_active && !__port_is_collecting_distributing(port)) { __enable_port(port); *update_slave_arr = true; @@ -1134,7 +1136,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) */ struct slave *slave = port->slave; - if (port->aggregator->is_active && + if (aggregator->is_active && bond_is_slave_rx_disabled(slave)) { ad_enable_collecting(port); *update_slave_arr = true; @@ -1154,8 +1156,8 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) * sure that a collecting distributing * port in an active aggregator is enabled */ - if (port->aggregator && - port->aggregator->is_active && + if (aggregator && + aggregator->is_active && !__port_is_collecting_distributing(port)) { __enable_port(port); *update_slave_arr = true; @@ -1187,7 +1189,7 @@ static void ad_mux_machine(struct port *port, bool *update_slave_arr) port->sm_mux_timer_counter = __ad_timer_to_ticks(AD_WAIT_WHILE_TIMER, 0); break; case AD_MUX_ATTACHED: - if (port->aggregator->is_active) + if (aggregator->is_active) port->actor_oper_port_state |= LACP_STATE_SYNCHRONIZATION; else @@ -1561,9 +1563,9 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) bond = __get_bond_by_port(port); /* if the port is connected to other aggregator, detach it */ - if (port->aggregator) { + temp_aggregator = rcu_dereference(port->aggregator); + if (temp_aggregator) { /* detach the port from its former aggregator */ - temp_aggregator = port->aggregator; for (curr_port = temp_aggregator->lag_ports; curr_port; last_port = curr_port, curr_port = curr_port->next_port_in_aggregator) { @@ -1586,7 +1588,7 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) /* clear the port's relations to this * aggregator */ - port->aggregator = NULL; + RCU_INIT_POINTER(port->aggregator, NULL); port->next_port_in_aggregator = NULL; port->actor_port_aggregator_identifier = 0; @@ -1609,7 +1611,7 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) port->slave->bond->dev->name, port->slave->dev->name, port->actor_port_number, - port->aggregator->aggregator_identifier); + temp_aggregator->aggregator_identifier); } } /* search on all aggregators for a suitable aggregator for this port */ @@ -1633,15 +1635,15 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) ) ) { /* attach to the founded aggregator */ - port->aggregator = aggregator; + rcu_assign_pointer(port->aggregator, aggregator); port->actor_port_aggregator_identifier = - port->aggregator->aggregator_identifier; + aggregator->aggregator_identifier; port->next_port_in_aggregator = aggregator->lag_ports; - port->aggregator->num_of_ports++; + aggregator->num_of_ports++; aggregator->lag_ports = port; slave_dbg(bond->dev, slave->dev, "Port %d joined LAG %d (existing LAG)\n", port->actor_port_number, - port->aggregator->aggregator_identifier); + aggregator->aggregator_identifier); /* mark this port as selected */ port->sm_vars |= AD_PORT_SELECTED; @@ -1656,39 +1658,40 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) if (!found) { if (free_aggregator) { /* assign port a new aggregator */ - port->aggregator = free_aggregator; port->actor_port_aggregator_identifier = - port->aggregator->aggregator_identifier; + free_aggregator->aggregator_identifier; /* update the new aggregator's parameters * if port was responsed from the end-user */ if (port->actor_oper_port_key & AD_DUPLEX_KEY_MASKS) /* if port is full duplex */ - port->aggregator->is_individual = false; + free_aggregator->is_individual = false; else - port->aggregator->is_individual = true; + free_aggregator->is_individual = true; - port->aggregator->actor_admin_aggregator_key = + free_aggregator->actor_admin_aggregator_key = port->actor_admin_port_key; - port->aggregator->actor_oper_aggregator_key = + free_aggregator->actor_oper_aggregator_key = port->actor_oper_port_key; - port->aggregator->partner_system = + free_aggregator->partner_system = port->partner_oper.system; - port->aggregator->partner_system_priority = + free_aggregator->partner_system_priority = port->partner_oper.system_priority; - port->aggregator->partner_oper_aggregator_key = port->partner_oper.key; - port->aggregator->receive_state = 1; - port->aggregator->transmit_state = 1; - port->aggregator->lag_ports = port; - port->aggregator->num_of_ports++; + free_aggregator->partner_oper_aggregator_key = port->partner_oper.key; + free_aggregator->receive_state = 1; + free_aggregator->transmit_state = 1; + free_aggregator->lag_ports = port; + free_aggregator->num_of_ports++; + + rcu_assign_pointer(port->aggregator, free_aggregator); /* mark this port as selected */ port->sm_vars |= AD_PORT_SELECTED; slave_dbg(bond->dev, port->slave->dev, "Port %d joined LAG %d (new LAG)\n", port->actor_port_number, - port->aggregator->aggregator_identifier); + free_aggregator->aggregator_identifier); } else { slave_err(bond->dev, port->slave->dev, "Port %d did not find a suitable aggregator\n", @@ -1700,13 +1703,12 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) * in all aggregator's ports, else set ready=FALSE in all * aggregator's ports */ - __set_agg_ports_ready(port->aggregator, - __agg_ports_are_ready(port->aggregator)); + aggregator = rcu_dereference(port->aggregator); + __set_agg_ports_ready(aggregator, __agg_ports_are_ready(aggregator)); - aggregator = __get_first_agg(port); - ad_agg_selection_logic(aggregator, update_slave_arr); + ad_agg_selection_logic(__get_first_agg(port), update_slave_arr); - if (!port->aggregator->is_active) + if (!aggregator->is_active) port->actor_oper_port_state &= ~LACP_STATE_SYNCHRONIZATION; } @@ -2075,13 +2077,15 @@ static void ad_initialize_port(struct port *port, const struct bond_params *bond */ static void ad_enable_collecting(struct port *port) { - if (port->aggregator->is_active) { + struct aggregator *aggregator = rcu_dereference(port->aggregator); + + if (aggregator->is_active) { struct slave *slave = port->slave; slave_dbg(slave->bond->dev, slave->dev, "Enabling collecting on port %d (LAG %d)\n", port->actor_port_number, - port->aggregator->aggregator_identifier); + aggregator->aggregator_identifier); __enable_collecting_port(port); } } @@ -2093,11 +2097,13 @@ static void ad_enable_collecting(struct port *port) */ static void ad_disable_distributing(struct port *port, bool *update_slave_arr) { - if (port->aggregator && __agg_has_partner(port->aggregator)) { + struct aggregator *aggregator = rcu_dereference(port->aggregator); + + if (aggregator && __agg_has_partner(aggregator)) { slave_dbg(port->slave->bond->dev, port->slave->dev, "Disabling distributing on port %d (LAG %d)\n", port->actor_port_number, - port->aggregator->aggregator_identifier); + aggregator->aggregator_identifier); __disable_distributing_port(port); /* Slave array needs an update */ *update_slave_arr = true; @@ -2114,11 +2120,13 @@ static void ad_disable_distributing(struct port *port, bool *update_slave_arr) static void ad_enable_collecting_distributing(struct port *port, bool *update_slave_arr) { - if (port->aggregator->is_active) { + struct aggregator *aggregator = rcu_dereference(port->aggregator); + + if (aggregator->is_active) { slave_dbg(port->slave->bond->dev, port->slave->dev, "Enabling port %d (LAG %d)\n", port->actor_port_number, - port->aggregator->aggregator_identifier); + aggregator->aggregator_identifier); __enable_port(port); /* Slave array needs update */ *update_slave_arr = true; @@ -2135,11 +2143,13 @@ static void ad_enable_collecting_distributing(struct port *port, static void ad_disable_collecting_distributing(struct port *port, bool *update_slave_arr) { - if (port->aggregator && __agg_has_partner(port->aggregator)) { + struct aggregator *aggregator = rcu_dereference(port->aggregator); + + if (aggregator && __agg_has_partner(aggregator)) { slave_dbg(port->slave->bond->dev, port->slave->dev, "Disabling port %d (LAG %d)\n", port->actor_port_number, - port->aggregator->aggregator_identifier); + aggregator->aggregator_identifier); __disable_port(port); /* Slave array needs an update */ *update_slave_arr = true; @@ -2379,7 +2389,7 @@ void bond_3ad_unbind_slave(struct slave *slave) */ for (temp_port = aggregator->lag_ports; temp_port; temp_port = temp_port->next_port_in_aggregator) { - temp_port->aggregator = new_aggregator; + rcu_assign_pointer(temp_port->aggregator, new_aggregator); temp_port->actor_port_aggregator_identifier = new_aggregator->aggregator_identifier; } @@ -2848,15 +2858,16 @@ int bond_3ad_set_carrier(struct bonding *bond) int __bond_3ad_get_active_agg_info(struct bonding *bond, struct ad_info *ad_info) { - struct aggregator *aggregator = NULL; + struct aggregator *aggregator = NULL, *tmp; struct list_head *iter; struct slave *slave; struct port *port; bond_for_each_slave_rcu(bond, slave, iter) { port = &(SLAVE_AD_INFO(slave)->port); - if (port->aggregator && port->aggregator->is_active) { - aggregator = port->aggregator; + tmp = rcu_dereference(port->aggregator); + if (tmp && tmp->is_active) { + aggregator = tmp; break; } } diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index a5484d11553d1a..eb49ce486992de 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -1435,7 +1435,7 @@ static void bond_poll_controller(struct net_device *bond_dev) if (BOND_MODE(bond) == BOND_MODE_8023AD) { struct aggregator *agg = - SLAVE_AD_INFO(slave)->port.aggregator; + rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator); if (agg && agg->aggregator_identifier != ad_info.aggregator_id) @@ -5181,15 +5181,16 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave) spin_unlock_bh(&bond->mode_lock); agg_id = ad_info.aggregator_id; } + rcu_read_lock(); bond_for_each_slave(bond, slave, iter) { if (skipslave == slave) continue; all_slaves->arr[all_slaves->count++] = slave; if (BOND_MODE(bond) == BOND_MODE_8023AD) { - struct aggregator *agg; + const struct aggregator *agg; - agg = SLAVE_AD_INFO(slave)->port.aggregator; + agg = rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator); if (!agg || agg->aggregator_identifier != agg_id) continue; } @@ -5201,6 +5202,7 @@ int bond_update_slave_arr(struct bonding *bond, struct slave *skipslave) usable_slaves->arr[usable_slaves->count++] = slave; } + rcu_read_unlock(); bond_set_slave_arr(bond, usable_slaves, all_slaves); return ret; diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 286f11c517f762..c7d3e0602c831d 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -29,6 +29,8 @@ static size_t bond_get_slave_size(const struct net_device *bond_dev, nla_total_size(sizeof(u16)) + /* IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE */ nla_total_size(sizeof(s32)) + /* IFLA_BOND_SLAVE_PRIO */ nla_total_size(sizeof(u16)) + /* IFLA_BOND_SLAVE_ACTOR_PORT_PRIO */ + nla_total_size(sizeof(u8)) + /* IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE */ + nla_total_size(sizeof(u8)) + /* IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE */ 0; } @@ -64,20 +66,29 @@ static int bond_fill_slave_info(struct sk_buff *skb, const struct port *ad_port; ad_port = &SLAVE_AD_INFO(slave)->port; - agg = SLAVE_AD_INFO(slave)->port.aggregator; + rcu_read_lock(); + agg = rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator); if (agg) { if (nla_put_u16(skb, IFLA_BOND_SLAVE_AD_AGGREGATOR_ID, agg->aggregator_identifier)) - goto nla_put_failure; + goto nla_put_failure_rcu; if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE, ad_port->actor_oper_port_state)) - goto nla_put_failure; + goto nla_put_failure_rcu; if (nla_put_u16(skb, IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE, ad_port->partner_oper.port_state)) - goto nla_put_failure; + goto nla_put_failure_rcu; + + if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE, + ad_port->sm_churn_actor_state)) + goto nla_put_failure_rcu; + if (nla_put_u8(skb, IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE, + ad_port->sm_churn_partner_state)) + goto nla_put_failure_rcu; } + rcu_read_unlock(); if (nla_put_u16(skb, IFLA_BOND_SLAVE_ACTOR_PORT_PRIO, SLAVE_AD_INFO(slave)->port_priority)) @@ -86,6 +97,8 @@ static int bond_fill_slave_info(struct sk_buff *skb, return 0; +nla_put_failure_rcu: + rcu_read_unlock(); nla_put_failure: return -EMSGSIZE; } diff --git a/drivers/net/bonding/bond_procfs.c b/drivers/net/bonding/bond_procfs.c index 7edf72ec816abd..0c0146b7617721 100644 --- a/drivers/net/bonding/bond_procfs.c +++ b/drivers/net/bonding/bond_procfs.c @@ -187,6 +187,7 @@ static void bond_info_show_master(struct seq_file *seq) } } +/* Note: runs under rcu_read_lock() */ static void bond_info_show_slave(struct seq_file *seq, const struct slave *slave) { @@ -213,7 +214,7 @@ static void bond_info_show_slave(struct seq_file *seq, if (BOND_MODE(bond) == BOND_MODE_8023AD) { const struct port *port = &SLAVE_AD_INFO(slave)->port; - const struct aggregator *agg = port->aggregator; + const struct aggregator *agg = rcu_dereference(port->aggregator); if (agg) { seq_printf(seq, "Aggregator ID: %d\n", diff --git a/drivers/net/bonding/bond_sysfs_slave.c b/drivers/net/bonding/bond_sysfs_slave.c index 36d0e8440b5b94..fc6fe7181789d3 100644 --- a/drivers/net/bonding/bond_sysfs_slave.c +++ b/drivers/net/bonding/bond_sysfs_slave.c @@ -62,10 +62,15 @@ static ssize_t ad_aggregator_id_show(struct slave *slave, char *buf) const struct aggregator *agg; if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) { - agg = SLAVE_AD_INFO(slave)->port.aggregator; - if (agg) - return sysfs_emit(buf, "%d\n", - agg->aggregator_identifier); + rcu_read_lock(); + agg = rcu_dereference(SLAVE_AD_INFO(slave)->port.aggregator); + if (agg) { + ssize_t res = sysfs_emit(buf, "%d\n", + agg->aggregator_identifier); + rcu_read_unlock(); + return res; + } + rcu_read_unlock(); } return sysfs_emit(buf, "N/A\n"); @@ -78,7 +83,7 @@ static ssize_t ad_actor_oper_port_state_show(struct slave *slave, char *buf) if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) { ad_port = &SLAVE_AD_INFO(slave)->port; - if (ad_port->aggregator) + if (rcu_access_pointer(ad_port->aggregator)) return sysfs_emit(buf, "%u\n", ad_port->actor_oper_port_state); } @@ -93,7 +98,7 @@ static ssize_t ad_partner_oper_port_state_show(struct slave *slave, char *buf) if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) { ad_port = &SLAVE_AD_INFO(slave)->port; - if (ad_port->aggregator) + if (rcu_access_pointer(ad_port->aggregator)) return sysfs_emit(buf, "%u\n", ad_port->partner_oper.port_state); } diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 0ea0ac75e42f9a..ee3c1abbd06305 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -1397,7 +1397,7 @@ static int ucan_probe(struct usb_interface *intf, */ /* Prepare Memory for control transfers */ - ctl_msg_buffer = devm_kzalloc(&udev->dev, + ctl_msg_buffer = devm_kzalloc(&intf->dev, sizeof(union ucan_ctl_payload), GFP_KERNEL); if (!ctl_msg_buffer) { diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c index 31fa94dac627d6..c35cef01ec265e 100644 --- a/drivers/net/dsa/realtek/rtl8365mb.c +++ b/drivers/net/dsa/realtek/rtl8365mb.c @@ -216,7 +216,7 @@ (_extint) == 2 ? RTL8365MB_DIGITAL_INTERFACE_SELECT_REG1 : \ 0x0) #define RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_MASK(_extint) \ - (0xF << (((_extint) % 2))) + (0xF << (((_extint) % 2) * 4)) #define RTL8365MB_DIGITAL_INTERFACE_SELECT_MODE_OFFSET(_extint) \ (((_extint) % 2) * 4) diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 91cb63a32d9904..83882a8953d259 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -76,7 +76,7 @@ static void airoha_set_macaddr(struct airoha_gdm_port *port, const u8 *addr) struct airoha_eth *eth = port->qdma->eth; u32 val, reg; - reg = airhoa_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H + reg = airoha_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H; val = (addr[0] << 16) | (addr[1] << 8) | addr[2]; airoha_fe_wr(eth, reg, val); @@ -107,19 +107,7 @@ static int airoha_set_vip_for_gdm_port(struct airoha_gdm_port *port, struct airoha_eth *eth = port->qdma->eth; u32 vip_port; - switch (port->id) { - case AIROHA_GDM3_IDX: - /* FIXME: handle XSI_PCIE1_PORT */ - vip_port = XSI_PCIE0_VIP_PORT_MASK; - break; - case AIROHA_GDM4_IDX: - /* FIXME: handle XSI_USB_PORT */ - vip_port = XSI_ETH_VIP_PORT_MASK; - break; - default: - return 0; - } - + vip_port = eth->soc->ops.get_vip_port(port, port->nbq); if (enable) { airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port); airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port); @@ -293,16 +281,18 @@ static void airoha_fe_pse_ports_init(struct airoha_eth *eth) [FE_PSE_PORT_GDM4] = 2, [FE_PSE_PORT_CDM5] = 2, }; - u32 all_rsv; int q; - all_rsv = airoha_fe_get_pse_all_rsv(eth); if (airoha_ppe_is_enabled(eth, 1)) { + u32 all_rsv; + /* hw misses PPE2 oq rsv */ + all_rsv = airoha_fe_get_pse_all_rsv(eth); all_rsv += PSE_RSV_PAGES * pse_port_num_queues[FE_PSE_PORT_PPE2]; + airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, + FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); } - airoha_fe_set(eth, REG_FE_PSE_BUF_SET, all_rsv); /* CMD1 */ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++) @@ -584,7 +574,7 @@ static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) static int airoha_qdma_get_gdm_port(struct airoha_eth *eth, struct airoha_qdma_desc *desc) { - u32 port, sport, msg1 = le32_to_cpu(desc->msg1); + u32 port, sport, msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, msg1); switch (sport) { @@ -612,21 +602,24 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) while (done < budget) { struct airoha_queue_entry *e = &q->entry[q->tail]; struct airoha_qdma_desc *desc = &q->desc[q->tail]; - u32 hash, reason, msg1 = le32_to_cpu(desc->msg1); - struct page *page = virt_to_head_page(e->buf); - u32 desc_ctrl = le32_to_cpu(desc->ctrl); + u32 hash, reason, msg1, desc_ctrl; struct airoha_gdm_port *port; int data_len, len, p; + struct page *page; + desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); if (!(desc_ctrl & QDMA_DESC_DONE_MASK)) break; + dma_rmb(); + q->tail = (q->tail + 1) % q->ndesc; q->queued--; dma_sync_single_for_cpu(eth->dev, e->dma_addr, SKB_WITH_OVERHEAD(q->buf_size), dir); + page = virt_to_head_page(e->buf); len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl); data_len = q->skb ? q->buf_size : SKB_WITH_OVERHEAD(q->buf_size); @@ -670,8 +663,8 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) * DMA descriptor. Report DSA tag to the DSA stack * via skb dst info. */ - u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, - le32_to_cpu(desc->msg0)); + u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0)); + u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0); if (sptag < ARRAY_SIZE(port->dsa_meta) && port->dsa_meta[sptag]) @@ -679,6 +672,7 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) &port->dsa_meta[sptag]->dst); } + msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1); if (hash != AIROHA_RXD4_FOE_ENTRY) skb_set_hash(q->skb, jhash_1word(hash, 0), @@ -751,14 +745,18 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q, dma_addr_t dma_addr; q->buf_size = PAGE_SIZE / 2; - q->ndesc = ndesc; q->qdma = qdma; - q->entry = devm_kzalloc(eth->dev, q->ndesc * sizeof(*q->entry), + q->entry = devm_kzalloc(eth->dev, ndesc * sizeof(*q->entry), GFP_KERNEL); if (!q->entry) return -ENOMEM; + q->desc = dmam_alloc_coherent(eth->dev, ndesc * sizeof(*q->desc), + &dma_addr, GFP_KERNEL); + if (!q->desc) + return -ENOMEM; + q->page_pool = page_pool_create(&pp_params); if (IS_ERR(q->page_pool)) { int err = PTR_ERR(q->page_pool); @@ -767,11 +765,7 @@ static int airoha_qdma_init_rx_queue(struct airoha_queue *q, return err; } - q->desc = dmam_alloc_coherent(eth->dev, q->ndesc * sizeof(*q->desc), - &dma_addr, GFP_KERNEL); - if (!q->desc) - return -ENOMEM; - + q->ndesc = ndesc; netif_napi_add(eth->napi_dev, &q->napi, airoha_qdma_rx_napi_poll); airoha_qdma_wr(qdma, REG_RX_RING_BASE(qid), dma_addr); @@ -819,6 +813,11 @@ static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) } q->head = q->tail; + /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is + * empty. + */ + airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK, + FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); } @@ -844,6 +843,32 @@ static int airoha_qdma_init_rx(struct airoha_qdma *qdma) return 0; } +static void airoha_qdma_wake_netdev_txqs(struct airoha_queue *q) +{ + struct airoha_qdma *qdma = q->qdma; + struct airoha_eth *eth = qdma->eth; + int i, qid = q - &qdma->q_tx[0]; + + for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { + struct airoha_gdm_port *port = eth->ports[i]; + int j; + + if (!port) + continue; + + if (port->qdma != qdma) + continue; + + for (j = 0; j < port->dev->num_tx_queues; j++) { + if (airoha_qdma_get_txq(qdma, j) != qid) + continue; + + netif_wake_subqueue(port->dev, j); + } + } + q->txq_stopped = false; +} + static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget) { struct airoha_tx_irq_queue *irq_q; @@ -915,17 +940,25 @@ static int airoha_qdma_tx_napi_poll(struct napi_struct *napi, int budget) q->queued--; if (skb) { - u16 queue = skb_get_queue_mapping(skb); struct netdev_queue *txq; - txq = netdev_get_tx_queue(skb->dev, queue); + txq = skb_get_tx_queue(skb->dev, skb); netdev_tx_completed_queue(txq, 1, skb->len); - if (netif_tx_queue_stopped(txq) && - q->ndesc - q->queued >= q->free_thr) - netif_tx_wake_queue(txq); - dev_kfree_skb_any(skb); } + + if (q->txq_stopped && q->ndesc - q->queued >= q->free_thr) { + /* Since multiple net_device TX queues can share the + * same hw QDMA TX queue, there is no guarantee we have + * inflight packets queued in hw belonging to a + * net_device TX queue stopped in the xmit path. + * In order to avoid any potential net_device TX queue + * stall, we need to wake all the net_device TX queues + * feeding the same hw QDMA TX queue. + */ + airoha_qdma_wake_netdev_txqs(q); + } + unlock: spin_unlock_bh(&q->lock); } @@ -955,27 +988,27 @@ static int airoha_qdma_init_tx_queue(struct airoha_queue *q, dma_addr_t dma_addr; spin_lock_init(&q->lock); - q->ndesc = size; q->qdma = qdma; q->free_thr = 1 + MAX_SKB_FRAGS; INIT_LIST_HEAD(&q->tx_list); - q->entry = devm_kzalloc(eth->dev, q->ndesc * sizeof(*q->entry), + q->entry = devm_kzalloc(eth->dev, size * sizeof(*q->entry), GFP_KERNEL); if (!q->entry) return -ENOMEM; - q->desc = dmam_alloc_coherent(eth->dev, q->ndesc * sizeof(*q->desc), + q->desc = dmam_alloc_coherent(eth->dev, size * sizeof(*q->desc), &dma_addr, GFP_KERNEL); if (!q->desc) return -ENOMEM; - for (i = 0; i < q->ndesc; i++) { + for (i = 0; i < size; i++) { u32 val = FIELD_PREP(QDMA_DESC_DONE_MASK, 1); list_add_tail(&q->entry[i].list, &q->tx_list); WRITE_ONCE(q->desc[i].ctrl, cpu_to_le32(val)); } + q->ndesc = size; /* xmit ring drop default setting */ airoha_qdma_set(qdma, REG_TX_RING_BLOCKING(qid), @@ -997,8 +1030,6 @@ static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q, struct airoha_eth *eth = qdma->eth; dma_addr_t dma_addr; - netif_napi_add_tx(eth->napi_dev, &irq_q->napi, - airoha_qdma_tx_napi_poll); irq_q->q = dmam_alloc_coherent(eth->dev, size * sizeof(u32), &dma_addr, GFP_KERNEL); if (!irq_q->q) @@ -1008,6 +1039,9 @@ static int airoha_qdma_tx_irq_init(struct airoha_tx_irq_queue *irq_q, irq_q->size = size; irq_q->qdma = qdma; + netif_napi_add_tx(eth->napi_dev, &irq_q->napi, + airoha_qdma_tx_napi_poll); + airoha_qdma_wr(qdma, REG_TX_IRQ_BASE(id), dma_addr); airoha_qdma_rmw(qdma, REG_TX_IRQ_CFG(id), TX_IRQ_DEPTH_MASK, FIELD_PREP(TX_IRQ_DEPTH_MASK, size)); @@ -1040,12 +1074,15 @@ static int airoha_qdma_init_tx(struct airoha_qdma *qdma) static void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q) { - struct airoha_eth *eth = q->qdma->eth; - int i; + struct airoha_qdma *qdma = q->qdma; + struct airoha_eth *eth = qdma->eth; + int i, qid = q - &qdma->q_tx[0]; + u16 index = 0; spin_lock_bh(&q->lock); for (i = 0; i < q->ndesc; i++) { struct airoha_queue_entry *e = &q->entry[i]; + struct airoha_qdma_desc *desc = &q->desc[i]; if (!e->dma_addr) continue; @@ -1056,8 +1093,33 @@ static void airoha_qdma_cleanup_tx_queue(struct airoha_queue *q) e->dma_addr = 0; e->skb = NULL; list_add_tail(&e->list, &q->tx_list); + + /* Reset DMA descriptor */ + WRITE_ONCE(desc->ctrl, 0); + WRITE_ONCE(desc->addr, 0); + WRITE_ONCE(desc->data, 0); + WRITE_ONCE(desc->msg0, 0); + WRITE_ONCE(desc->msg1, 0); + WRITE_ONCE(desc->msg2, 0); + q->queued--; } + + if (!list_empty(&q->tx_list)) { + struct airoha_queue_entry *e; + + e = list_first_entry(&q->tx_list, struct airoha_queue_entry, + list); + index = e - q->entry; + } + /* Set TX_DMA_IDX to TX_CPU_IDX to notify the hw the QDMA TX ring is + * empty. + */ + airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK, + FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); + airoha_qdma_rmw(qdma, REG_TX_DMA_IDX(qid), TX_RING_DMA_IDX_MASK, + FIELD_PREP(TX_RING_DMA_IDX_MASK, index)); + spin_unlock_bh(&q->lock); } @@ -1383,6 +1445,37 @@ static int airoha_qdma_init(struct platform_device *pdev, return airoha_qdma_hw_init(qdma); } +static void airoha_qdma_cleanup(struct airoha_qdma *qdma) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { + if (!qdma->q_rx[i].ndesc) + continue; + + netif_napi_del(&qdma->q_rx[i].napi); + airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); + if (qdma->q_rx[i].page_pool) { + page_pool_destroy(qdma->q_rx[i].page_pool); + qdma->q_rx[i].page_pool = NULL; + } + } + + for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) { + if (!qdma->q_tx_irq[i].size) + continue; + + netif_napi_del(&qdma->q_tx_irq[i].napi); + } + + for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { + if (!qdma->q_tx[i].ndesc) + continue; + + airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]); + } +} + static int airoha_hw_init(struct platform_device *pdev, struct airoha_eth *eth) { @@ -1410,41 +1503,30 @@ static int airoha_hw_init(struct platform_device *pdev, for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { err = airoha_qdma_init(pdev, eth, ð->qdma[i]); if (err) - return err; + goto error; } err = airoha_ppe_init(eth); if (err) - return err; + goto error; set_bit(DEV_STATE_INITIALIZED, ð->state); return 0; +error: + for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) + airoha_qdma_cleanup(ð->qdma[i]); + + return err; } -static void airoha_hw_cleanup(struct airoha_qdma *qdma) +static void airoha_hw_cleanup(struct airoha_eth *eth) { int i; - for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { - if (!qdma->q_rx[i].ndesc) - continue; - - netif_napi_del(&qdma->q_rx[i].napi); - airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); - if (qdma->q_rx[i].page_pool) - page_pool_destroy(qdma->q_rx[i].page_pool); - } - - for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) - netif_napi_del(&qdma->q_tx_irq[i].napi); - - for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { - if (!qdma->q_tx[i].ndesc) - continue; - - airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]); - } + for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) + airoha_qdma_cleanup(ð->qdma[i]); + airoha_ppe_deinit(eth); } static void airoha_qdma_start_napi(struct airoha_qdma *qdma) @@ -1656,14 +1738,11 @@ static int airoha_dev_stop(struct net_device *dev) { struct airoha_gdm_port *port = netdev_priv(dev); struct airoha_qdma *qdma = port->qdma; - int i, err; + int i; netif_tx_disable(dev); - err = airoha_set_vip_for_gdm_port(port, false); - if (err) - return err; - - for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) + airoha_set_vip_for_gdm_port(port, false); + for (i = 0; i < dev->num_tx_queues; i++) netdev_tx_reset_subqueue(dev, i); if (atomic_dec_and_test(&qdma->users)) { @@ -1699,7 +1778,7 @@ static int airoha_dev_set_macaddr(struct net_device *dev, void *p) static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port) { struct airoha_eth *eth = port->qdma->eth; - u32 val, pse_port, chan, nbq; + u32 val, pse_port, chan; int src_port; /* Forward the traffic to the proper GDM port */ @@ -1729,9 +1808,7 @@ static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port) airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX)); airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX)); - /* XXX: handle XSI_USB_PORT and XSI_PCE1_PORT */ - nbq = port->id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; - src_port = eth->soc->ops.get_src_port_id(port, nbq); + src_port = eth->soc->ops.get_src_port_id(port, port->nbq); if (src_port < 0) return src_port; @@ -1925,12 +2002,12 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, struct netdev_queue *txq; struct airoha_queue *q; LIST_HEAD(tx_list); + int i = 0, qid; void *data; - int i, qid; u16 index; u8 fport; - qid = skb_get_queue_mapping(skb) % ARRAY_SIZE(qdma->q_tx); + qid = airoha_qdma_get_txq(qdma, skb_get_queue_mapping(skb)); tag = airoha_get_dsa_tag(skb, dev); msg0 = FIELD_PREP(QDMA_ETH_TXMSG_CHAN_MASK, @@ -1967,12 +2044,13 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, spin_lock_bh(&q->lock); - txq = netdev_get_tx_queue(dev, qid); + txq = skb_get_tx_queue(dev, skb); nr_frags = 1 + skb_shinfo(skb)->nr_frags; if (q->queued + nr_frags >= q->ndesc) { /* not enough space in the queue */ netif_tx_stop_queue(txq); + q->txq_stopped = true; spin_unlock_bh(&q->lock); return NETDEV_TX_BUSY; } @@ -1984,7 +2062,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, list); index = e - q->entry; - for (i = 0; i < nr_frags; i++) { + while (true) { struct airoha_qdma_desc *desc = &q->desc[index]; skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; dma_addr_t addr; @@ -1996,7 +2074,7 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, goto error_unmap; list_move_tail(&e->list, &tx_list); - e->skb = i ? NULL : skb; + e->skb = i == nr_frags - 1 ? skb : NULL; e->dma_addr = addr; e->dma_len = len; @@ -2015,6 +2093,9 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, WRITE_ONCE(desc->msg1, cpu_to_le32(msg1)); WRITE_ONCE(desc->msg2, cpu_to_le32(0xffff)); + if (++i == nr_frags) + break; + data = skb_frag_address(frag); len = skb_frag_size(frag); } @@ -2022,15 +2103,16 @@ static netdev_tx_t airoha_dev_xmit(struct sk_buff *skb, skb_tx_timestamp(skb); netdev_tx_sent_queue(txq, skb->len); + if (q->ndesc - q->queued < q->free_thr) { + netif_tx_stop_queue(txq); + q->txq_stopped = true; + } if (netif_xmit_stopped(txq) || !netdev_xmit_more()) airoha_qdma_rmw(qdma, REG_TX_CPU_IDX(qid), TX_RING_CPU_IDX_MASK, FIELD_PREP(TX_RING_CPU_IDX_MASK, index)); - if (q->ndesc - q->queued < q->free_thr) - netif_tx_stop_queue(txq); - spin_unlock_bh(&q->lock); return NETDEV_TX_OK; @@ -2940,6 +3022,8 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth, port->qdma = qdma; port->dev = dev; port->id = id; + /* XXX: Read nbq from DTS */ + port->nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; eth->ports[p] = port; return airoha_metadata_dst_alloc(port); @@ -3034,7 +3118,7 @@ static int airoha_probe(struct platform_device *pdev) err = airoha_hw_init(pdev, eth); if (err) - goto error_hw_cleanup; + goto error_netdev_free; for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) airoha_qdma_start_napi(ð->qdma[i]); @@ -3063,10 +3147,6 @@ static int airoha_probe(struct platform_device *pdev) error_napi_stop: for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) airoha_qdma_stop_napi(ð->qdma[i]); - airoha_ppe_deinit(eth); -error_hw_cleanup: - for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) - airoha_hw_cleanup(ð->qdma[i]); for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { struct airoha_gdm_port *port = eth->ports[i]; @@ -3078,6 +3158,8 @@ static int airoha_probe(struct platform_device *pdev) unregister_netdev(port->dev); airoha_metadata_dst_free(port); } + airoha_hw_cleanup(eth); +error_netdev_free: free_netdev(eth->napi_dev); platform_set_drvdata(pdev, NULL); @@ -3089,10 +3171,8 @@ static void airoha_remove(struct platform_device *pdev) struct airoha_eth *eth = platform_get_drvdata(pdev); int i; - for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { + for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) airoha_qdma_stop_napi(ð->qdma[i]); - airoha_hw_cleanup(ð->qdma[i]); - } for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { struct airoha_gdm_port *port = eth->ports[i]; @@ -3103,9 +3183,9 @@ static void airoha_remove(struct platform_device *pdev) unregister_netdev(port->dev); airoha_metadata_dst_free(port); } - free_netdev(eth->napi_dev); + airoha_hw_cleanup(eth); - airoha_ppe_deinit(eth); + free_netdev(eth->napi_dev); platform_set_drvdata(pdev, NULL); } @@ -3141,6 +3221,28 @@ static int airoha_en7581_get_src_port_id(struct airoha_gdm_port *port, int nbq) return -EINVAL; } +static u32 airoha_en7581_get_vip_port(struct airoha_gdm_port *port, int nbq) +{ + switch (port->id) { + case AIROHA_GDM3_IDX: + if (nbq == 4) + return XSI_PCIE0_VIP_PORT_MASK; + if (nbq == 5) + return XSI_PCIE1_VIP_PORT_MASK; + break; + case AIROHA_GDM4_IDX: + if (!nbq) + return XSI_ETH_VIP_PORT_MASK; + if (nbq == 1) + return XSI_USB_VIP_PORT_MASK; + break; + default: + break; + } + + return 0; +} + static const char * const an7583_xsi_rsts_names[] = { "xsi-mac", "hsi0-mac", @@ -3170,6 +3272,26 @@ static int airoha_an7583_get_src_port_id(struct airoha_gdm_port *port, int nbq) return -EINVAL; } +static u32 airoha_an7583_get_vip_port(struct airoha_gdm_port *port, int nbq) +{ + switch (port->id) { + case AIROHA_GDM3_IDX: + if (!nbq) + return XSI_ETH_VIP_PORT_MASK; + break; + case AIROHA_GDM4_IDX: + if (!nbq) + return XSI_PCIE0_VIP_PORT_MASK; + if (nbq == 1) + return XSI_USB_VIP_PORT_MASK; + break; + default: + break; + } + + return 0; +} + static const struct airoha_eth_soc_data en7581_soc_data = { .version = 0x7581, .xsi_rsts_names = en7581_xsi_rsts_names, @@ -3177,6 +3299,7 @@ static const struct airoha_eth_soc_data en7581_soc_data = { .num_ppe = 2, .ops = { .get_src_port_id = airoha_en7581_get_src_port_id, + .get_vip_port = airoha_en7581_get_vip_port, }, }; @@ -3187,6 +3310,7 @@ static const struct airoha_eth_soc_data an7583_soc_data = { .num_ppe = 1, .ops = { .get_src_port_id = airoha_an7583_get_src_port_id, + .get_vip_port = airoha_an7583_get_vip_port, }, }; diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h index a97903569335ff..7098e95f0067a4 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.h +++ b/drivers/net/ethernet/airoha/airoha_eth.h @@ -193,6 +193,7 @@ struct airoha_queue { int ndesc; int free_thr; int buf_size; + bool txq_stopped; struct napi_struct napi; struct page_pool *page_pool; @@ -536,6 +537,7 @@ struct airoha_gdm_port { struct airoha_qdma *qdma; struct net_device *dev; int id; + int nbq; struct airoha_hw_stats stats; @@ -576,6 +578,7 @@ struct airoha_eth_soc_data { int num_ppe; struct { int (*get_src_port_id)(struct airoha_gdm_port *port, int nbq); + u32 (*get_vip_port)(struct airoha_gdm_port *port, int nbq); } ops; }; @@ -627,7 +630,12 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val); #define airoha_qdma_clear(qdma, offset, val) \ airoha_rmw((qdma)->regs, (offset), (val), 0) -static inline bool airhoa_is_lan_gdm_port(struct airoha_gdm_port *port) +static inline u16 airoha_qdma_get_txq(struct airoha_qdma *qdma, u16 qid) +{ + return qid % ARRAY_SIZE(qdma->q_tx); +} + +static inline bool airoha_is_lan_gdm_port(struct airoha_gdm_port *port) { /* GDM1 port on EN7581 SoC is connected to the lan dsa switch. * GDM{2,3,4} can be used as wan port connected to an external diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index c2c32b6833df94..f2af45f7d66d4d 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -111,13 +111,13 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i), PPE_BIND_AGE0_DELTA_NON_L4 | PPE_BIND_AGE0_DELTA_UDP, - FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 1) | - FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 12)); + FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 60) | + FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 60)); airoha_fe_rmw(eth, REG_PPE_BND_AGE1(i), PPE_BIND_AGE1_DELTA_TCP_FIN | PPE_BIND_AGE1_DELTA_TCP, FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | - FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 7)); + FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 60)); airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i), PPE_SRAM_TABLE_EN_MASK | @@ -145,7 +145,15 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, dram_num_entries)); + airoha_fe_rmw(eth, REG_PPE_BIND_RATE(i), + PPE_BIND_RATE_L2B_BIND_MASK | + PPE_BIND_RATE_BIND_MASK, + FIELD_PREP(PPE_BIND_RATE_L2B_BIND_MASK, 0x1e) | + FIELD_PREP(PPE_BIND_RATE_BIND_MASK, 0x1e)); + airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED); + airoha_fe_clear(eth, REG_PPE_PPE_FLOW_CFG(i), + PPE_FLOW_CFG_IP6_6RD_MASK); for (p = 0; p < ARRAY_SIZE(eth->ports); p++) airoha_fe_rmw(eth, REG_PPE_MTU(i, p), @@ -323,7 +331,7 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth, /* For downlink traffic consume SRAM memory for hw * forwarding descriptors queue. */ - if (airhoa_is_lan_gdm_port(port)) + if (airoha_is_lan_gdm_port(port)) val |= AIROHA_FOE_IB2_FAST_PATH; if (dsa_port >= 0) val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, @@ -1327,6 +1335,29 @@ static struct airoha_npu *airoha_ppe_npu_get(struct airoha_eth *eth) return npu; } +static int airoha_ppe_wait_for_npu_init(struct airoha_eth *eth) +{ + int err; + u32 val; + + /* PPE_FLOW_CFG default register value is 0. Since we reset FE + * during the device probe we can just check the configured value + * is not 0 here. + */ + err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC, + 100 * USEC_PER_MSEC, false, eth, + REG_PPE_PPE_FLOW_CFG(0)); + if (err) + return err; + + if (airoha_ppe_is_enabled(eth, 1)) + err = read_poll_timeout(airoha_fe_rr, val, val, USEC_PER_MSEC, + 100 * USEC_PER_MSEC, false, eth, + REG_PPE_PPE_FLOW_CFG(1)); + + return err; +} + static int airoha_ppe_offload_setup(struct airoha_eth *eth) { struct airoha_npu *npu = airoha_ppe_npu_get(eth); @@ -1340,6 +1371,11 @@ static int airoha_ppe_offload_setup(struct airoha_eth *eth) if (err) goto error_npu_put; + /* Wait for NPU PPE configuration to complete */ + err = airoha_ppe_wait_for_npu_init(eth); + if (err) + goto error_npu_put; + ppe_num_stats_entries = airoha_ppe_get_total_num_stats_entries(ppe); if (ppe_num_stats_entries > 0) { err = npu->ops.ppe_init_stats(npu, ppe->foe_stats_dma, diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c index e67b592e569763..8c86789d867a5f 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_com.c @@ -1782,20 +1782,23 @@ void ena_com_phc_destroy(struct ena_com_dev *ena_dev) int ena_com_phc_get_timestamp(struct ena_com_dev *ena_dev, u64 *timestamp) { - volatile struct ena_admin_phc_resp *resp = ena_dev->phc.virt_addr; const ktime_t zero_system_time = ktime_set(0, 0); struct ena_com_phc_info *phc = &ena_dev->phc; + volatile struct ena_admin_phc_resp *resp; ktime_t expire_time; ktime_t block_time; unsigned long flags = 0; int ret = 0; + spin_lock_irqsave(&phc->lock, flags); + if (!phc->active) { + spin_unlock_irqrestore(&phc->lock, flags); netdev_err(ena_dev->net_device, "PHC feature is not active in the device\n"); return -EOPNOTSUPP; } - spin_lock_irqsave(&phc->lock, flags); + resp = ena_dev->phc.virt_addr; /* Check if PHC is in blocked state */ if (unlikely(ktime_compare(phc->system_time, zero_system_time))) { diff --git a/drivers/net/ethernet/amazon/ena/ena_phc.c b/drivers/net/ethernet/amazon/ena/ena_phc.c index 7867e893fd15f9..c2a3ff1ef645c5 100644 --- a/drivers/net/ethernet/amazon/ena/ena_phc.c +++ b/drivers/net/ethernet/amazon/ena/ena_phc.c @@ -46,9 +46,12 @@ static int ena_phc_gettimex64(struct ptp_clock_info *clock_info, spin_unlock_irqrestore(&phc_info->lock, flags); + if (rc) + return rc; + *ts = ns_to_timespec64(timestamp_nsec); - return rc; + return 0; } static int ena_phc_settime64(struct ptp_clock_info *clock_info, diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c index e9e38af680c34a..39e1b606a75a9d 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_pci_func.c @@ -371,7 +371,7 @@ static void aq_pci_shutdown(struct pci_dev *pdev) pci_disable_device(pdev); if (system_state == SYSTEM_POWER_OFF) { - pci_wake_from_d3(pdev, false); + pci_wake_from_d3(pdev, self->aq_hw->aq_nic_cfg->wol); pci_set_power_state(pdev, PCI_D3hot); } } diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c index b942076762ef59..67e93e17d4d9f1 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c @@ -194,6 +194,7 @@ void bnge_rdma_aux_device_add(struct bnge_dev *bd) dev_warn(bd->dev, "Failed to add auxiliary device for ROCE\n"); auxiliary_device_uninit(aux_dev); bd->flags &= ~BNGE_EN_ROCE; + return; } bd->auxr_dev->net = bd->netdev; diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_core.c b/drivers/net/ethernet/broadcom/bnge/bnge_core.c index b4090283df0f29..99d7aeeb2ddcd7 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_core.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_core.c @@ -73,6 +73,13 @@ static int bnge_func_qcaps(struct bnge_dev *bd) return rc; } + return 0; +} + +static int bnge_func_qrcaps_qcfg(struct bnge_dev *bd) +{ + int rc; + rc = bnge_hwrm_func_resc_qcaps(bd); if (rc) { dev_err(bd->dev, "query resc caps failure rc: %d\n", rc); @@ -132,23 +139,28 @@ static int bnge_fw_register_dev(struct bnge_dev *bd) bnge_hwrm_fw_set_time(bd); - rc = bnge_hwrm_func_drv_rgtr(bd); + /* Get the resources and configuration from firmware */ + rc = bnge_func_qcaps(bd); if (rc) { - dev_err(bd->dev, "Failed to rgtr with firmware rc: %d\n", rc); + dev_err(bd->dev, "Failed querying caps rc: %d\n", rc); return rc; } rc = bnge_alloc_ctx_mem(bd); if (rc) { dev_err(bd->dev, "Failed to allocate ctx mem rc: %d\n", rc); - goto err_func_unrgtr; + goto err_free_ctx_mem; } - /* Get the resources and configuration from firmware */ - rc = bnge_func_qcaps(bd); + rc = bnge_hwrm_func_drv_rgtr(bd); if (rc) { - dev_err(bd->dev, "Failed initial configuration rc: %d\n", rc); - rc = -ENODEV; + dev_err(bd->dev, "Failed to rgtr with firmware rc: %d\n", rc); + goto err_free_ctx_mem; + } + + rc = bnge_func_qrcaps_qcfg(bd); + if (rc) { + dev_err(bd->dev, "Failed querying resources rc: %d\n", rc); goto err_func_unrgtr; } @@ -157,7 +169,9 @@ static int bnge_fw_register_dev(struct bnge_dev *bd) return 0; err_func_unrgtr: - bnge_fw_unregister_dev(bd); + bnge_hwrm_func_drv_unrgtr(bd); +err_free_ctx_mem: + bnge_free_ctx_mem(bd); return rc; } diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_rmem.c b/drivers/net/ethernet/broadcom/bnge/bnge_rmem.c index 94f15e08a88c18..b066ee887a0996 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_rmem.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_rmem.c @@ -324,7 +324,6 @@ int bnge_alloc_ctx_mem(struct bnge_dev *bd) u32 l2_qps, qp1_qps, max_qps; u32 ena, entries_sp, entries; u32 srqs, max_srqs, min; - u32 num_mr, num_ah; u32 extra_srqs = 0; u32 extra_qps = 0; u32 fast_qpmd_qps; @@ -390,21 +389,6 @@ int bnge_alloc_ctx_mem(struct bnge_dev *bd) if (!bnge_is_roce_en(bd)) goto skip_rdma; - ctxm = &ctx->ctx_arr[BNGE_CTX_MRAV]; - /* 128K extra is needed to accommodate static AH context - * allocation by f/w. - */ - num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256); - num_ah = min_t(u32, num_mr, 1024 * 128); - ctxm->split_entry_cnt = BNGE_CTX_MRAV_AV_SPLIT_ENTRY + 1; - if (!ctxm->mrav_av_entries || ctxm->mrav_av_entries > num_ah) - ctxm->mrav_av_entries = num_ah; - - rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, num_mr + num_ah, 2); - if (rc) - return rc; - ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV; - ctxm = &ctx->ctx_arr[BNGE_CTX_TIM]; rc = bnge_setup_ctxm_pg_tbls(bd, ctxm, l2_qps + qp1_qps + extra_qps, 1); if (rc) diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 482a31e7b72bc3..54f71b1e85fc48 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1819,15 +1819,15 @@ static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv, { struct enet_cb *tx_cb_ptr; - tx_cb_ptr = ring->cbs; - tx_cb_ptr += ring->write_ptr - ring->cb_ptr; - /* Rewinding local write pointer */ if (ring->write_ptr == ring->cb_ptr) ring->write_ptr = ring->end_ptr; else ring->write_ptr--; + tx_cb_ptr = ring->cbs; + tx_cb_ptr += ring->write_ptr - ring->cb_ptr; + return tx_cb_ptr; } @@ -1985,6 +1985,7 @@ static unsigned int bcmgenet_tx_reclaim(struct net_device *dev, drop = (ring->prod_index - ring->c_index) & DMA_C_INDEX_MASK; released += drop; ring->prod_index = ring->c_index & DMA_C_INDEX_MASK; + ring->free_bds += drop; while (drop--) { cb_ptr = bcmgenet_put_txcb(priv, ring); skb = cb_ptr->skb; @@ -1996,6 +1997,7 @@ static unsigned int bcmgenet_tx_reclaim(struct net_device *dev, } if (skb) dev_consume_skb_any(skb); + netdev_tx_reset_queue(netdev_get_tx_queue(dev, ring->index)); bcmgenet_tdma_ring_writel(priv, ring->index, ring->prod_index, TDMA_PROD_INDEX); wr_ptr = ring->write_ptr * WORDS_PER_BD(priv); @@ -3475,27 +3477,23 @@ static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring) static void bcmgenet_timeout(struct net_device *dev, unsigned int txqueue) { struct bcmgenet_priv *priv = netdev_priv(dev); - u32 int1_enable = 0; - unsigned int q; + struct bcmgenet_tx_ring *ring = &priv->tx_rings[txqueue]; + struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n"); - for (q = 0; q <= priv->hw_params->tx_queues; q++) - bcmgenet_dump_tx_queue(&priv->tx_rings[q]); - - bcmgenet_tx_reclaim_all(dev); + bcmgenet_dump_tx_queue(ring); - for (q = 0; q <= priv->hw_params->tx_queues; q++) - int1_enable |= (1 << q); + bcmgenet_tx_reclaim(dev, ring, true); - /* Re-enable TX interrupts if disabled */ - bcmgenet_intrl2_1_writel(priv, int1_enable, INTRL2_CPU_MASK_CLEAR); + /* Re-enable the TX interrupt for this ring */ + bcmgenet_intrl2_1_writel(priv, 1 << txqueue, INTRL2_CPU_MASK_CLEAR); - netif_trans_update(dev); + txq_trans_cond_update(txq); - BCMGENET_STATS64_INC((&priv->tx_rings[txqueue].stats64), errors); + BCMGENET_STATS64_INC((&ring->stats64), errors); - netif_tx_wake_all_queues(dev); + netif_tx_wake_queue(txq); } #define MAX_MDF_FILTER 17 diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile index de7b3184223309..d0a259e47960f8 100644 --- a/drivers/net/ethernet/freescale/Makefile +++ b/drivers/net/ethernet/freescale/Makefile @@ -22,6 +22,5 @@ ucc_geth_driver-objs := ucc_geth.o ucc_geth_ethtool.o obj-$(CONFIG_FSL_FMAN) += fman/ obj-$(CONFIG_FSL_DPAA_ETH) += dpaa/ -obj-$(CONFIG_FSL_DPAA2_ETH) += dpaa2/ - +obj-y += dpaa2/ obj-y += enetc/ diff --git a/drivers/net/ethernet/freescale/dpaa2/Kconfig b/drivers/net/ethernet/freescale/dpaa2/Kconfig index d029b69c3f183e..36280e5d99e1f3 100644 --- a/drivers/net/ethernet/freescale/dpaa2/Kconfig +++ b/drivers/net/ethernet/freescale/dpaa2/Kconfig @@ -34,6 +34,10 @@ config FSL_DPAA2_SWITCH tristate "Freescale DPAA2 Ethernet Switch" depends on BRIDGE || BRIDGE=n depends on NET_SWITCHDEV + depends on FSL_MC_BUS && FSL_MC_DPIO + select PHYLINK + select PCS_LYNX + select FSL_XGMAC_MDIO help Driver for Freescale DPAA2 Ethernet Switch. This driver manages switch objects discovered on the Freeescale MC bus. diff --git a/drivers/net/ethernet/freescale/enetc/ntmp.c b/drivers/net/ethernet/freescale/enetc/ntmp.c index 0c1d343253bfb7..70bbc5d2d5d422 100644 --- a/drivers/net/ethernet/freescale/enetc/ntmp.c +++ b/drivers/net/ethernet/freescale/enetc/ntmp.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "ntmp_private.h" @@ -42,6 +43,12 @@ int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev, if (!cbdr->addr_base) return -ENOMEM; + cbdr->swcbd = vcalloc(cbd_num, sizeof(struct netc_swcbd)); + if (!cbdr->swcbd) { + dma_free_coherent(dev, size, cbdr->addr_base, cbdr->dma_base); + return -ENOMEM; + } + cbdr->dma_size = size; cbdr->bd_num = cbd_num; cbdr->regs = *regs; @@ -52,10 +59,10 @@ int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev, cbdr->addr_base_align = PTR_ALIGN(cbdr->addr_base, NTMP_BASE_ADDR_ALIGN); - spin_lock_init(&cbdr->ring_lock); + mutex_init(&cbdr->ring_lock); cbdr->next_to_use = netc_read(cbdr->regs.pir); - cbdr->next_to_clean = netc_read(cbdr->regs.cir); + cbdr->next_to_clean = netc_read(cbdr->regs.cir) & NETC_CBDRCIR_INDEX; /* Step 1: Configure the base address of the Control BD Ring */ netc_write(cbdr->regs.bar0, lower_32_bits(cbdr->dma_base_align)); @@ -71,10 +78,24 @@ int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev, } EXPORT_SYMBOL_GPL(ntmp_init_cbdr); +static void ntmp_free_data_mem(struct device *dev, struct netc_swcbd *swcbd) +{ + if (unlikely(!swcbd->buf)) + return; + + dma_free_coherent(dev, swcbd->size + NTMP_DATA_ADDR_ALIGN, + swcbd->buf, swcbd->dma); +} + void ntmp_free_cbdr(struct netc_cbdr *cbdr) { /* Disable the Control BD Ring */ netc_write(cbdr->regs.mr, 0); + + for (int i = 0; i < cbdr->bd_num; i++) + ntmp_free_data_mem(cbdr->dev, &cbdr->swcbd[i]); + + vfree(cbdr->swcbd); dma_free_coherent(cbdr->dev, cbdr->dma_size, cbdr->addr_base, cbdr->dma_base); memset(cbdr, 0, sizeof(*cbdr)); @@ -94,40 +115,59 @@ static union netc_cbd *ntmp_get_cbd(struct netc_cbdr *cbdr, int index) static void ntmp_clean_cbdr(struct netc_cbdr *cbdr) { - union netc_cbd *cbd; - int i; + int i = cbdr->next_to_clean; + + while ((netc_read(cbdr->regs.cir) & NETC_CBDRCIR_INDEX) != i) { + union netc_cbd *cbd = ntmp_get_cbd(cbdr, i); + struct netc_swcbd *swcbd = &cbdr->swcbd[i]; - i = cbdr->next_to_clean; - while (netc_read(cbdr->regs.cir) != i) { - cbd = ntmp_get_cbd(cbdr, i); + ntmp_free_data_mem(cbdr->dev, swcbd); + memset(swcbd, 0, sizeof(*swcbd)); memset(cbd, 0, sizeof(*cbd)); i = (i + 1) % cbdr->bd_num; } + dma_wmb(); cbdr->next_to_clean = i; } -static int netc_xmit_ntmp_cmd(struct ntmp_user *user, union netc_cbd *cbd) +static void ntmp_select_and_lock_cbdr(struct ntmp_user *user, + struct netc_cbdr **cbdr) +{ + /* Currently only ENETC is supported, and it has only one command + * BD ring. + */ + *cbdr = &user->ring[0]; + + mutex_lock(&(*cbdr)->ring_lock); +} + +static void ntmp_unlock_cbdr(struct netc_cbdr *cbdr) +{ + mutex_unlock(&cbdr->ring_lock); +} + +static int netc_xmit_ntmp_cmd(struct netc_cbdr *cbdr, union netc_cbd *cbd, + struct netc_swcbd *swcbd) { union netc_cbd *cur_cbd; - struct netc_cbdr *cbdr; - int i, err; + int i, err, used_bds; u16 status; u32 val; - /* Currently only i.MX95 ENETC is supported, and it only has one - * command BD ring - */ - cbdr = &user->ring[0]; - - spin_lock_bh(&cbdr->ring_lock); - - if (unlikely(!ntmp_get_free_cbd_num(cbdr))) + used_bds = cbdr->bd_num - ntmp_get_free_cbd_num(cbdr); + if (unlikely(used_bds >= NETC_CBDR_CLEAN_WORK)) { ntmp_clean_cbdr(cbdr); + if (unlikely(!ntmp_get_free_cbd_num(cbdr))) { + ntmp_free_data_mem(cbdr->dev, swcbd); + return -EBUSY; + } + } i = cbdr->next_to_use; cur_cbd = ntmp_get_cbd(cbdr, i); *cur_cbd = *cbd; + cbdr->swcbd[i] = *swcbd; dma_wmb(); /* Update producer index of both software and hardware */ @@ -135,11 +175,17 @@ static int netc_xmit_ntmp_cmd(struct ntmp_user *user, union netc_cbd *cbd) cbdr->next_to_use = i; netc_write(cbdr->regs.pir, i); - err = read_poll_timeout_atomic(netc_read, val, val == i, - NETC_CBDR_DELAY_US, NETC_CBDR_TIMEOUT, - true, cbdr->regs.cir); + err = read_poll_timeout(netc_read, val, + (val & NETC_CBDRCIR_INDEX) == i, + NETC_CBDR_DELAY_US, NETC_CBDR_TIMEOUT, + true, cbdr->regs.cir); if (unlikely(err)) - goto cbdr_unlock; + return err; + + if (unlikely(val & NETC_CBDRCIR_SBE)) { + dev_err(cbdr->dev, "Command BD system bus error\n"); + return -EIO; + } dma_rmb(); /* Get the writeback command BD, because the caller may need @@ -150,40 +196,29 @@ static int netc_xmit_ntmp_cmd(struct ntmp_user *user, union netc_cbd *cbd) /* Check the writeback error status */ status = le16_to_cpu(cbd->resp_hdr.error_rr) & NTMP_RESP_ERROR; if (unlikely(status)) { - err = -EIO; - dev_err(user->dev, "Command BD error: 0x%04x\n", status); + dev_err(cbdr->dev, "Command BD error: 0x%04x\n", status); + return -EIO; } - ntmp_clean_cbdr(cbdr); - dma_wmb(); - -cbdr_unlock: - spin_unlock_bh(&cbdr->ring_lock); - - return err; + return 0; } -static int ntmp_alloc_data_mem(struct ntmp_dma_buf *data, void **buf_align) +static int ntmp_alloc_data_mem(struct device *dev, struct netc_swcbd *swcbd, + void **buf_align) { void *buf; - buf = dma_alloc_coherent(data->dev, data->size + NTMP_DATA_ADDR_ALIGN, - &data->dma, GFP_KERNEL); + buf = dma_alloc_coherent(dev, swcbd->size + NTMP_DATA_ADDR_ALIGN, + &swcbd->dma, GFP_KERNEL); if (!buf) return -ENOMEM; - data->buf = buf; + swcbd->buf = buf; *buf_align = PTR_ALIGN(buf, NTMP_DATA_ADDR_ALIGN); return 0; } -static void ntmp_free_data_mem(struct ntmp_dma_buf *data) -{ - dma_free_coherent(data->dev, data->size + NTMP_DATA_ADDR_ALIGN, - data->buf, data->dma); -} - static void ntmp_fill_request_hdr(union netc_cbd *cbd, dma_addr_t dma, int len, int table_id, int cmd, int access_method) @@ -234,37 +269,39 @@ static int ntmp_delete_entry_by_id(struct ntmp_user *user, int tbl_id, u8 tbl_ver, u32 entry_id, u32 req_len, u32 resp_len) { - struct ntmp_dma_buf data = { - .dev = user->dev, + struct netc_swcbd swcbd = { .size = max(req_len, resp_len), }; struct ntmp_req_by_eid *req; + struct netc_cbdr *cbdr; union netc_cbd cbd; int err; - err = ntmp_alloc_data_mem(&data, (void **)&req); + err = ntmp_alloc_data_mem(user->dev, &swcbd, (void **)&req); if (err) return err; ntmp_fill_crd_eid(req, tbl_ver, 0, 0, entry_id); - ntmp_fill_request_hdr(&cbd, data.dma, NTMP_LEN(req_len, resp_len), + ntmp_fill_request_hdr(&cbd, swcbd.dma, NTMP_LEN(req_len, resp_len), tbl_id, NTMP_CMD_DELETE, NTMP_AM_ENTRY_ID); - err = netc_xmit_ntmp_cmd(user, &cbd); + ntmp_select_and_lock_cbdr(user, &cbdr); + err = netc_xmit_ntmp_cmd(cbdr, &cbd, &swcbd); if (err) dev_err(user->dev, "Failed to delete entry 0x%x of %s, err: %pe", entry_id, ntmp_table_name(tbl_id), ERR_PTR(err)); - - ntmp_free_data_mem(&data); + ntmp_unlock_cbdr(cbdr); return err; } -static int ntmp_query_entry_by_id(struct ntmp_user *user, int tbl_id, - u32 len, struct ntmp_req_by_eid *req, - dma_addr_t dma, bool compare_eid) +static int ntmp_query_entry_by_id(struct netc_cbdr *cbdr, int tbl_id, + struct ntmp_req_by_eid *req, + struct netc_swcbd *swcbd, + bool compare_eid) { + u32 len = NTMP_LEN(sizeof(*req), swcbd->size); struct ntmp_cmn_resp_query *resp; int cmd = NTMP_CMD_QUERY; union netc_cbd cbd; @@ -276,10 +313,11 @@ static int ntmp_query_entry_by_id(struct ntmp_user *user, int tbl_id, cmd = NTMP_CMD_QU; /* Request header */ - ntmp_fill_request_hdr(&cbd, dma, len, tbl_id, cmd, NTMP_AM_ENTRY_ID); - err = netc_xmit_ntmp_cmd(user, &cbd); + ntmp_fill_request_hdr(&cbd, swcbd->dma, len, tbl_id, cmd, + NTMP_AM_ENTRY_ID); + err = netc_xmit_ntmp_cmd(cbdr, &cbd, swcbd); if (err) { - dev_err(user->dev, + dev_err(cbdr->dev, "Failed to query entry 0x%x of %s, err: %pe\n", entry_id, ntmp_table_name(tbl_id), ERR_PTR(err)); return err; @@ -293,7 +331,7 @@ static int ntmp_query_entry_by_id(struct ntmp_user *user, int tbl_id, resp = (struct ntmp_cmn_resp_query *)req; if (unlikely(le32_to_cpu(resp->entry_id) != entry_id)) { - dev_err(user->dev, + dev_err(cbdr->dev, "%s: query EID 0x%x doesn't match response EID 0x%x\n", ntmp_table_name(tbl_id), entry_id, le32_to_cpu(resp->entry_id)); return -EIO; @@ -305,15 +343,15 @@ static int ntmp_query_entry_by_id(struct ntmp_user *user, int tbl_id, int ntmp_maft_add_entry(struct ntmp_user *user, u32 entry_id, struct maft_entry_data *maft) { - struct ntmp_dma_buf data = { - .dev = user->dev, + struct netc_swcbd swcbd = { .size = sizeof(struct maft_req_add), }; struct maft_req_add *req; + struct netc_cbdr *cbdr; union netc_cbd cbd; int err; - err = ntmp_alloc_data_mem(&data, (void **)&req); + err = ntmp_alloc_data_mem(user->dev, &swcbd, (void **)&req); if (err) return err; @@ -322,14 +360,15 @@ int ntmp_maft_add_entry(struct ntmp_user *user, u32 entry_id, req->keye = maft->keye; req->cfge = maft->cfge; - ntmp_fill_request_hdr(&cbd, data.dma, NTMP_LEN(data.size, 0), + ntmp_fill_request_hdr(&cbd, swcbd.dma, NTMP_LEN(swcbd.size, 0), NTMP_MAFT_ID, NTMP_CMD_ADD, NTMP_AM_ENTRY_ID); - err = netc_xmit_ntmp_cmd(user, &cbd); + + ntmp_select_and_lock_cbdr(user, &cbdr); + err = netc_xmit_ntmp_cmd(cbdr, &cbd, &swcbd); if (err) dev_err(user->dev, "Failed to add MAFT entry 0x%x, err: %pe\n", entry_id, ERR_PTR(err)); - - ntmp_free_data_mem(&data); + ntmp_unlock_cbdr(cbdr); return err; } @@ -338,31 +377,31 @@ EXPORT_SYMBOL_GPL(ntmp_maft_add_entry); int ntmp_maft_query_entry(struct ntmp_user *user, u32 entry_id, struct maft_entry_data *maft) { - struct ntmp_dma_buf data = { - .dev = user->dev, + struct netc_swcbd swcbd = { .size = sizeof(struct maft_resp_query), }; struct maft_resp_query *resp; struct ntmp_req_by_eid *req; + struct netc_cbdr *cbdr; int err; - err = ntmp_alloc_data_mem(&data, (void **)&req); + err = ntmp_alloc_data_mem(user->dev, &swcbd, (void **)&req); if (err) return err; ntmp_fill_crd_eid(req, user->tbl.maft_ver, 0, 0, entry_id); - err = ntmp_query_entry_by_id(user, NTMP_MAFT_ID, - NTMP_LEN(sizeof(*req), data.size), - req, data.dma, true); + + ntmp_select_and_lock_cbdr(user, &cbdr); + err = ntmp_query_entry_by_id(cbdr, NTMP_MAFT_ID, req, &swcbd, true); if (err) - goto end; + goto unlock_cbdr; resp = (struct maft_resp_query *)req; maft->keye = resp->keye; maft->cfge = resp->cfge; -end: - ntmp_free_data_mem(&data); +unlock_cbdr: + ntmp_unlock_cbdr(cbdr); return err; } @@ -378,8 +417,9 @@ EXPORT_SYMBOL_GPL(ntmp_maft_delete_entry); int ntmp_rsst_update_entry(struct ntmp_user *user, const u32 *table, int count) { - struct ntmp_dma_buf data = {.dev = user->dev}; struct rsst_req_update *req; + struct netc_swcbd swcbd; + struct netc_cbdr *cbdr; union netc_cbd cbd; int err, i; @@ -387,8 +427,8 @@ int ntmp_rsst_update_entry(struct ntmp_user *user, const u32 *table, /* HW only takes in a full 64 entry table */ return -EINVAL; - data.size = struct_size(req, groups, count); - err = ntmp_alloc_data_mem(&data, (void **)&req); + swcbd.size = struct_size(req, groups, count); + err = ntmp_alloc_data_mem(user->dev, &swcbd, (void **)&req); if (err) return err; @@ -398,15 +438,15 @@ int ntmp_rsst_update_entry(struct ntmp_user *user, const u32 *table, for (i = 0; i < count; i++) req->groups[i] = (u8)(table[i]); - ntmp_fill_request_hdr(&cbd, data.dma, NTMP_LEN(data.size, 0), + ntmp_fill_request_hdr(&cbd, swcbd.dma, NTMP_LEN(swcbd.size, 0), NTMP_RSST_ID, NTMP_CMD_UPDATE, NTMP_AM_ENTRY_ID); - err = netc_xmit_ntmp_cmd(user, &cbd); + ntmp_select_and_lock_cbdr(user, &cbdr); + err = netc_xmit_ntmp_cmd(cbdr, &cbd, &swcbd); if (err) dev_err(user->dev, "Failed to update RSST entry, err: %pe\n", ERR_PTR(err)); - - ntmp_free_data_mem(&data); + ntmp_unlock_cbdr(cbdr); return err; } @@ -414,8 +454,9 @@ EXPORT_SYMBOL_GPL(ntmp_rsst_update_entry); int ntmp_rsst_query_entry(struct ntmp_user *user, u32 *table, int count) { - struct ntmp_dma_buf data = {.dev = user->dev}; struct ntmp_req_by_eid *req; + struct netc_swcbd swcbd; + struct netc_cbdr *cbdr; union netc_cbd cbd; int err, i; u8 *group; @@ -424,21 +465,23 @@ int ntmp_rsst_query_entry(struct ntmp_user *user, u32 *table, int count) /* HW only takes in a full 64 entry table */ return -EINVAL; - data.size = NTMP_ENTRY_ID_SIZE + RSST_STSE_DATA_SIZE(count) + - RSST_CFGE_DATA_SIZE(count); - err = ntmp_alloc_data_mem(&data, (void **)&req); + swcbd.size = NTMP_ENTRY_ID_SIZE + RSST_STSE_DATA_SIZE(count) + + RSST_CFGE_DATA_SIZE(count); + err = ntmp_alloc_data_mem(user->dev, &swcbd, (void **)&req); if (err) return err; /* Set the request data buffer */ ntmp_fill_crd_eid(req, user->tbl.rsst_ver, 0, 0, 0); - ntmp_fill_request_hdr(&cbd, data.dma, NTMP_LEN(sizeof(*req), data.size), + ntmp_fill_request_hdr(&cbd, swcbd.dma, NTMP_LEN(sizeof(*req), swcbd.size), NTMP_RSST_ID, NTMP_CMD_QUERY, NTMP_AM_ENTRY_ID); - err = netc_xmit_ntmp_cmd(user, &cbd); + + ntmp_select_and_lock_cbdr(user, &cbdr); + err = netc_xmit_ntmp_cmd(cbdr, &cbd, &swcbd); if (err) { dev_err(user->dev, "Failed to query RSST entry, err: %pe\n", ERR_PTR(err)); - goto end; + goto unlock_cbdr; } group = (u8 *)req; @@ -446,8 +489,8 @@ int ntmp_rsst_query_entry(struct ntmp_user *user, u32 *table, int count) for (i = 0; i < count; i++) table[i] = group[i]; -end: - ntmp_free_data_mem(&data); +unlock_cbdr: + ntmp_unlock_cbdr(cbdr); return err; } diff --git a/drivers/net/ethernet/freescale/enetc/ntmp_private.h b/drivers/net/ethernet/freescale/enetc/ntmp_private.h index 34394e40fddd4d..f8dff3ba2c28a2 100644 --- a/drivers/net/ethernet/freescale/enetc/ntmp_private.h +++ b/drivers/net/ethernet/freescale/enetc/ntmp_private.h @@ -12,6 +12,9 @@ #define NTMP_EID_REQ_LEN 8 #define NETC_CBDR_BD_NUM 256 +#define NETC_CBDRCIR_INDEX GENMASK(9, 0) +#define NETC_CBDRCIR_SBE BIT(31) +#define NETC_CBDR_CLEAN_WORK 16 union netc_cbd { struct { @@ -54,13 +57,6 @@ union netc_cbd { } resp_hdr; /* NTMP Response Message Header Format */ }; -struct ntmp_dma_buf { - struct device *dev; - size_t size; - void *buf; - dma_addr_t dma; -}; - struct ntmp_cmn_req_data { __le16 update_act; u8 dbg_opt; diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 4d6b9e83e3412d..b813dd9b39ed4b 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -1756,6 +1756,27 @@ static int ibmveth_set_mac_addr(struct net_device *dev, void *p) return 0; } +static netdev_features_t ibmveth_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + /* Some physical adapters do not support segmentation offload with + * MSS < 224. Disable GSO for such packets to avoid adapter freeze. + * Note: Single-segment packets (gso_segs == 1) don't need this check + * as they bypass the LSO path and are transmitted without segmentation. + */ + if (skb_is_gso(skb)) { + if (skb_shinfo(skb)->gso_size < IBMVETH_MIN_LSO_MSS) { + netdev_warn_once(dev, + "MSS %u too small for LSO, disabling GSO\n", + skb_shinfo(skb)->gso_size); + features &= ~NETIF_F_GSO_MASK; + } + } + + return vlan_features_check(skb, features); +} + static const struct net_device_ops ibmveth_netdev_ops = { .ndo_open = ibmveth_open, .ndo_stop = ibmveth_close, @@ -1767,6 +1788,7 @@ static const struct net_device_ops ibmveth_netdev_ops = { .ndo_set_features = ibmveth_set_features, .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = ibmveth_set_mac_addr, + .ndo_features_check = ibmveth_features_check, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = ibmveth_poll_controller, #endif diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h index 068f99df133ec0..d87713668ed300 100644 --- a/drivers/net/ethernet/ibm/ibmveth.h +++ b/drivers/net/ethernet/ibm/ibmveth.h @@ -37,6 +37,7 @@ #define IBMVETH_ILLAN_IPV4_TCP_CSUM 0x0000000000000002UL #define IBMVETH_ILLAN_ACTIVE_TRUNK 0x0000000000000001UL +#define IBMVETH_MIN_LSO_MSS 224 /* Minimum MSS for LSO */ /* hcall macros */ #define h_register_logical_lan(ua, buflst, rxq, fltlst, mac) \ plpar_hcall_norets(H_REGISTER_LOGICAL_LAN, ua, buflst, rxq, fltlst, mac) diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 9befdacd673014..7ce0cc8ab8f4c7 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -7706,6 +7706,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) err_register: if (!(adapter->flags & FLAG_HAS_AMT)) e1000e_release_hw_control(adapter); + e1000e_ptp_remove(adapter); err_eeprom: if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw)) e1000_phy_hw_reset(&adapter->hw); diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index dcb50c2e1aa277..83e780919ac97f 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -1318,6 +1318,7 @@ void i40e_ptp_restore_hw_time(struct i40e_pf *pf); void i40e_ptp_init(struct i40e_pf *pf); void i40e_ptp_stop(struct i40e_pf *pf); int i40e_ptp_alloc_pins(struct i40e_pf *pf); +void i40e_ptp_free_pins(struct i40e_pf *pf); int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset); int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi); int i40e_get_partition_bw_setting(struct i40e_pf *pf); diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 926d001b215020..807ccbbf0182f4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -13783,7 +13783,6 @@ static int i40e_config_netdev(struct i40e_vsi *vsi) netdev->neigh_priv_len = sizeof(u32) * 4; netdev->priv_flags |= IFF_UNICAST_FLT; - netdev->priv_flags |= IFF_SUPP_NOFCS; /* Setup netdev TC information */ i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); @@ -16112,6 +16111,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) i40e_clear_interrupt_scheme(pf); kfree(pf->vsi); err_switch_setup: + i40e_ptp_free_pins(pf); i40e_reset_interrupt_capability(pf); timer_shutdown_sync(&pf->service_timer); err_mac_addr: diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c index 404a716db8da71..7d07c389bb2312 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c @@ -940,12 +940,13 @@ int i40e_ptp_hwtstamp_get(struct net_device *netdev, * * Release memory allocated for PTP pins. **/ -static void i40e_ptp_free_pins(struct i40e_pf *pf) +void i40e_ptp_free_pins(struct i40e_pf *pf) { if (i40e_is_ptp_pin_dev(&pf->hw)) { kfree(pf->ptp_pins); kfree(pf->ptp_caps.pin_config); pf->ptp_pins = NULL; + pf->ptp_caps.pin_config = NULL; } } diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index e9fb0a0919e376..050f8241ef5e6b 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -158,11 +158,10 @@ struct iavf_vlan { enum iavf_vlan_state_t { IAVF_VLAN_INVALID, IAVF_VLAN_ADD, /* filter needs to be added */ - IAVF_VLAN_IS_NEW, /* filter is new, wait for PF answer */ - IAVF_VLAN_ACTIVE, /* filter is accepted by PF */ - IAVF_VLAN_DISABLE, /* filter needs to be deleted by PF, then marked INACTIVE */ - IAVF_VLAN_INACTIVE, /* filter is inactive, we are in IFF_DOWN */ - IAVF_VLAN_REMOVE, /* filter needs to be removed from list */ + IAVF_VLAN_ADDING, /* ADD sent to PF, waiting for response */ + IAVF_VLAN_ACTIVE, /* PF confirmed, filter is in HW */ + IAVF_VLAN_REMOVE, /* filter queued for DEL from PF */ + IAVF_VLAN_REMOVING, /* DEL sent to PF, waiting for response */ }; struct iavf_vlan_filter { diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index dad001abc9086b..d373feee4c7e9c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -757,10 +757,10 @@ iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, adapter->num_vlan_filters++; iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER); } else if (f->state == IAVF_VLAN_REMOVE) { - /* Re-add the filter since we cannot tell whether the - * pending delete has already been processed by the PF. - * A duplicate add is harmless. - */ + /* DEL not yet sent to PF, cancel it */ + f->state = IAVF_VLAN_ACTIVE; + } else if (f->state == IAVF_VLAN_REMOVING) { + /* DEL already sent to PF, re-add after completion */ f->state = IAVF_VLAN_ADD; iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER); @@ -791,37 +791,19 @@ static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan) list_del(&f->list); kfree(f); adapter->num_vlan_filters--; - } else { + } else if (f->state != IAVF_VLAN_REMOVING) { f->state = IAVF_VLAN_REMOVE; iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_VLAN_FILTER); } + /* If REMOVING, DEL is already sent to PF; completion + * handler will free the filter when PF confirms. + */ } spin_unlock_bh(&adapter->mac_vlan_list_lock); } -/** - * iavf_restore_filters - * @adapter: board private structure - * - * Restore existing non MAC filters when VF netdev comes back up - **/ -static void iavf_restore_filters(struct iavf_adapter *adapter) -{ - struct iavf_vlan_filter *f; - - /* re-add all VLAN filters */ - spin_lock_bh(&adapter->mac_vlan_list_lock); - - list_for_each_entry(f, &adapter->vlan_filter_list, list) { - if (f->state == IAVF_VLAN_INACTIVE) - f->state = IAVF_VLAN_ADD; - } - - spin_unlock_bh(&adapter->mac_vlan_list_lock); - adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; -} /** * iavf_get_num_vlans_added - get number of VLANs added @@ -1240,13 +1222,12 @@ static void iavf_up_complete(struct iavf_adapter *adapter) } /** - * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF - * yet and mark other to be removed. + * iavf_clear_mac_filters - Remove MAC filters not sent to PF yet and mark + * others to be removed. * @adapter: board private structure **/ -static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter) +static void iavf_clear_mac_filters(struct iavf_adapter *adapter) { - struct iavf_vlan_filter *vlf, *vlftmp; struct iavf_mac_filter *f, *ftmp; spin_lock_bh(&adapter->mac_vlan_list_lock); @@ -1265,11 +1246,6 @@ static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter) } } - /* disable all VLAN filters */ - list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, - list) - vlf->state = IAVF_VLAN_DISABLE; - spin_unlock_bh(&adapter->mac_vlan_list_lock); } @@ -1365,7 +1341,7 @@ void iavf_down(struct iavf_adapter *adapter) iavf_napi_disable_all(adapter); iavf_irq_disable(adapter); - iavf_clear_mac_vlan_filters(adapter); + iavf_clear_mac_filters(adapter); iavf_clear_cloud_filters(adapter); iavf_clear_fdir_filters(adapter); iavf_clear_adv_rss_conf(adapter); @@ -1382,8 +1358,6 @@ void iavf_down(struct iavf_adapter *adapter) */ if (!list_empty(&adapter->mac_filter_list)) adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; - if (!list_empty(&adapter->vlan_filter_list)) - adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; if (!list_empty(&adapter->cloud_filter_list)) adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; if (!list_empty(&adapter->fdir_list_head)) @@ -4488,8 +4462,6 @@ static int iavf_open(struct net_device *netdev) iavf_add_filter(adapter, adapter->hw.mac.addr); spin_unlock_bh(&adapter->mac_vlan_list_lock); - /* Restore filters that were removed with IFF_DOWN */ - iavf_restore_filters(adapter); iavf_restore_fdir_filters(adapter); iavf_configure(adapter); diff --git a/drivers/net/ethernet/intel/iavf/iavf_type.h b/drivers/net/ethernet/intel/iavf/iavf_type.h index 1d8cf29cb65ac5..5bb1de1cfd33b1 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_type.h +++ b/drivers/net/ethernet/intel/iavf/iavf_type.h @@ -277,7 +277,7 @@ struct iavf_rx_desc { /* L2 Tag 2 Presence */ #define IAVF_RXD_LEGACY_L2TAG2P_M BIT(0) /* Stripped S-TAG VLAN from the receive packet */ -#define IAVF_RXD_LEGACY_L2TAG2_M GENMASK_ULL(63, 32) +#define IAVF_RXD_LEGACY_L2TAG2_M GENMASK_ULL(63, 48) /* Stripped S-TAG VLAN from the receive packet */ #define IAVF_RXD_FLEX_L2TAG2_2_M GENMASK_ULL(63, 48) /* The packet is a UDP tunneled packet */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index a52c100dcbc56d..4f2defd2331b17 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -746,7 +746,7 @@ static void iavf_vlan_add_reject(struct iavf_adapter *adapter) spin_lock_bh(&adapter->mac_vlan_list_lock); list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { - if (f->state == IAVF_VLAN_IS_NEW) { + if (f->state == IAVF_VLAN_ADDING) { list_del(&f->list); kfree(f); adapter->num_vlan_filters--; @@ -812,7 +812,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter) if (f->state == IAVF_VLAN_ADD) { vvfl->vlan_id[i] = f->vlan.vid; i++; - f->state = IAVF_VLAN_IS_NEW; + f->state = IAVF_VLAN_ADDING; if (i == count) break; } @@ -874,7 +874,7 @@ void iavf_add_vlans(struct iavf_adapter *adapter) vlan->tpid = f->vlan.tpid; i++; - f->state = IAVF_VLAN_IS_NEW; + f->state = IAVF_VLAN_ADDING; } } @@ -911,22 +911,12 @@ void iavf_del_vlans(struct iavf_adapter *adapter) spin_lock_bh(&adapter->mac_vlan_list_lock); list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { - /* since VLAN capabilities are not allowed, we dont want to send - * a VLAN delete request because it will most likely fail and - * create unnecessary errors/noise, so just free the VLAN - * filters marked for removal to enable bailing out before - * sending a virtchnl message - */ if (f->state == IAVF_VLAN_REMOVE && !VLAN_FILTERING_ALLOWED(adapter)) { list_del(&f->list); kfree(f); adapter->num_vlan_filters--; - } else if (f->state == IAVF_VLAN_DISABLE && - !VLAN_FILTERING_ALLOWED(adapter)) { - f->state = IAVF_VLAN_INACTIVE; - } else if (f->state == IAVF_VLAN_REMOVE || - f->state == IAVF_VLAN_DISABLE) { + } else if (f->state == IAVF_VLAN_REMOVE) { count++; } } @@ -958,18 +948,10 @@ void iavf_del_vlans(struct iavf_adapter *adapter) vvfl->vsi_id = adapter->vsi_res->vsi_id; vvfl->num_elements = count; - list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { - if (f->state == IAVF_VLAN_DISABLE) { - vvfl->vlan_id[i] = f->vlan.vid; - f->state = IAVF_VLAN_INACTIVE; - i++; - if (i == count) - break; - } else if (f->state == IAVF_VLAN_REMOVE) { + list_for_each_entry(f, &adapter->vlan_filter_list, list) { + if (f->state == IAVF_VLAN_REMOVE) { vvfl->vlan_id[i] = f->vlan.vid; - list_del(&f->list); - kfree(f); - adapter->num_vlan_filters--; + f->state = IAVF_VLAN_REMOVING; i++; if (i == count) break; @@ -1006,9 +988,8 @@ void iavf_del_vlans(struct iavf_adapter *adapter) vvfl_v2->vport_id = adapter->vsi_res->vsi_id; vvfl_v2->num_elements = count; - list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, list) { - if (f->state == IAVF_VLAN_DISABLE || - f->state == IAVF_VLAN_REMOVE) { + list_for_each_entry(f, &adapter->vlan_filter_list, list) { + if (f->state == IAVF_VLAN_REMOVE) { struct virtchnl_vlan_supported_caps *filtering_support = &adapter->vlan_v2_caps.filtering.filtering_support; struct virtchnl_vlan *vlan; @@ -1022,13 +1003,7 @@ void iavf_del_vlans(struct iavf_adapter *adapter) vlan->tci = f->vlan.vid; vlan->tpid = f->vlan.tpid; - if (f->state == IAVF_VLAN_DISABLE) { - f->state = IAVF_VLAN_INACTIVE; - } else { - list_del(&f->list); - kfree(f); - adapter->num_vlan_filters--; - } + f->state = IAVF_VLAN_REMOVING; i++; if (i == count) break; @@ -2391,10 +2366,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); wake_up(&adapter->vc_waitqueue); break; - case VIRTCHNL_OP_DEL_VLAN: - dev_err(&adapter->pdev->dev, "Failed to delete VLAN filter, error %s\n", - iavf_stat_str(&adapter->hw, v_retval)); - break; case VIRTCHNL_OP_DEL_ETH_ADDR: dev_err(&adapter->pdev->dev, "Failed to delete MAC filter, error %s\n", iavf_stat_str(&adapter->hw, v_retval)); @@ -2905,17 +2876,42 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, spin_unlock_bh(&adapter->adv_rss_lock); } break; + case VIRTCHNL_OP_ADD_VLAN: case VIRTCHNL_OP_ADD_VLAN_V2: { struct iavf_vlan_filter *f; + if (v_retval) + break; + spin_lock_bh(&adapter->mac_vlan_list_lock); list_for_each_entry(f, &adapter->vlan_filter_list, list) { - if (f->state == IAVF_VLAN_IS_NEW) + if (f->state == IAVF_VLAN_ADDING) f->state = IAVF_VLAN_ACTIVE; } spin_unlock_bh(&adapter->mac_vlan_list_lock); } break; + case VIRTCHNL_OP_DEL_VLAN: + case VIRTCHNL_OP_DEL_VLAN_V2: { + struct iavf_vlan_filter *f, *ftmp; + + spin_lock_bh(&adapter->mac_vlan_list_lock); + list_for_each_entry_safe(f, ftmp, &adapter->vlan_filter_list, + list) { + if (f->state == IAVF_VLAN_REMOVING) { + if (v_retval) { + /* PF rejected DEL, keep filter */ + f->state = IAVF_VLAN_ACTIVE; + } else { + list_del(&f->list); + kfree(f); + adapter->num_vlan_filters--; + } + } + } + spin_unlock_bh(&adapter->mac_vlan_list_lock); + } + break; case VIRTCHNL_OP_ENABLE_VLAN_STRIPPING: /* PF enabled vlan strip on this VF. * Update netdev->features if needed to be in sync with ethtool. diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index 6144cee8034d77..641d6e289d5ce6 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -1245,6 +1245,8 @@ static int ice_devlink_reinit_up(struct ice_pf *pf) return err; } + ice_init_dev_hw(pf); + /* load MSI-X values */ ice_set_min_max_msix(pf); diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index eb3a48330cc155..725b130dd3a2c6 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -753,7 +753,7 @@ static inline bool ice_is_xdp_ena_vsi(struct ice_vsi *vsi) static inline void ice_set_ring_xdp(struct ice_tx_ring *ring) { - ring->flags |= ICE_TX_FLAGS_RING_XDP; + set_bit(ICE_TX_RING_FLAGS_XDP, ring->flags); } /** @@ -778,7 +778,7 @@ static inline bool ice_is_txtime_ena(const struct ice_tx_ring *ring) */ static inline bool ice_is_txtime_cfg(const struct ice_tx_ring *ring) { - return !!(ring->flags & ICE_TX_FLAGS_TXTIME); + return test_bit(ICE_TX_RING_FLAGS_TXTIME, ring->flags); } /** diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 859e9c66f3e7e8..3cbb1b0582e321 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -1252,7 +1252,7 @@ struct ice_aqc_get_link_status_data { #define ICE_AQ_LINK_PWR_QSFP_CLASS_3 2 #define ICE_AQ_LINK_PWR_QSFP_CLASS_4 3 __le16 link_speed; -#define ICE_AQ_LINK_SPEED_M 0x7FF +#define ICE_AQ_LINK_SPEED_M GENMASK(11, 0) #define ICE_AQ_LINK_SPEED_10MB BIT(0) #define ICE_AQ_LINK_SPEED_100MB BIT(1) #define ICE_AQ_LINK_SPEED_1000MB BIT(2) diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index ce11fea122d03e..b617a6bff89134 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -1126,8 +1126,6 @@ int ice_init_hw(struct ice_hw *hw) if (status) goto err_unroll_fltr_mgmt_struct; - ice_init_dev_hw(hw->back); - mutex_init(&hw->tnl_lock); ice_init_chk_recipe_reuse_support(hw); diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c index bd77f1c001ee83..16aa255351523d 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c @@ -943,7 +943,7 @@ ice_tx_prepare_vlan_flags_dcb(struct ice_tx_ring *tx_ring, /* if this is not already set it means a VLAN 0 + priority needs * to be offloaded */ - if (tx_ring->flags & ICE_TX_FLAGS_RING_VLAN_L2TAG2) + if (test_bit(ICE_TX_RING_FLAGS_VLAN_L2TAG2, tx_ring->flags)) first->tx_flags |= ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN; else first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c index 62f75701d65205..27b460926baced 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.c +++ b/drivers/net/ethernet/intel/ice/ice_dpll.c @@ -1154,6 +1154,32 @@ ice_dpll_input_state_get(const struct dpll_pin *pin, void *pin_priv, extack, ICE_DPLL_PIN_TYPE_INPUT); } +/** + * ice_dpll_sw_pin_notify_peer - notify the paired SW pin after a state change + * @d: pointer to dplls struct + * @changed: the SW pin that was explicitly changed (already notified by dpll core) + * + * SMA and U.FL pins share physical signal paths in pairs (SMA1/U.FL1 and + * SMA2/U.FL2). When one pin's routing changes via the PCA9575 GPIO + * expander, the paired pin's state may also change. Send a change + * notification for the peer pin so userspace consumers monitoring the + * peer via dpll netlink learn about the update. + * + * Context: Called from dpll_pin_ops callbacks after pf->dplls.lock is + * released. Uses __dpll_pin_change_ntf() because dpll_lock is + * still held by the dpll netlink layer. + */ +static void ice_dpll_sw_pin_notify_peer(struct ice_dplls *d, + struct ice_dpll_pin *changed) +{ + struct ice_dpll_pin *peer; + + peer = (changed >= d->sma && changed < d->sma + ICE_DPLL_PIN_SW_NUM) ? + &d->ufl[changed->idx] : &d->sma[changed->idx]; + if (peer->pin) + __dpll_pin_change_ntf(peer->pin); +} + /** * ice_dpll_sma_direction_set - set direction of SMA pin * @p: pointer to a pin @@ -1171,6 +1197,8 @@ static int ice_dpll_sma_direction_set(struct ice_dpll_pin *p, enum dpll_pin_direction direction, struct netlink_ext_ack *extack) { + struct ice_dplls *d = &p->pf->dplls; + struct ice_dpll_pin *peer; u8 data; int ret; @@ -1189,8 +1217,9 @@ static int ice_dpll_sma_direction_set(struct ice_dpll_pin *p, case ICE_DPLL_PIN_SW_2_IDX: if (direction == DPLL_PIN_DIRECTION_INPUT) { data &= ~ICE_SMA2_DIR_EN; + data |= ICE_SMA2_UFL2_RX_DIS; } else { - data &= ~ICE_SMA2_TX_EN; + data &= ~(ICE_SMA2_TX_EN | ICE_SMA2_UFL2_RX_DIS); data |= ICE_SMA2_DIR_EN; } break; @@ -1202,6 +1231,34 @@ static int ice_dpll_sma_direction_set(struct ice_dpll_pin *p, ret = ice_dpll_pin_state_update(p->pf, p, ICE_DPLL_PIN_TYPE_SOFTWARE, extack); + if (ret) + return ret; + + /* When a direction change activates the paired U.FL pin, enable + * its backing CGU pin so the pin reports as connected. Without + * this the U.FL routing is correct but the CGU pin stays disabled + * and userspace sees the pin as disconnected. Do not disable the + * backing pin when U.FL becomes inactive because the SMA pin may + * still be using it. + */ + peer = &d->ufl[p->idx]; + if (peer->active) { + struct ice_dpll_pin *target; + enum ice_dpll_pin_type type; + + if (peer->output) { + target = peer->output; + type = ICE_DPLL_PIN_TYPE_OUTPUT; + } else { + target = peer->input; + type = ICE_DPLL_PIN_TYPE_INPUT; + } + ret = ice_dpll_pin_enable(&p->pf->hw, target, + d->eec.dpll_idx, type, extack); + if (!ret) + ret = ice_dpll_pin_state_update(p->pf, target, + type, extack); + } return ret; } @@ -1253,6 +1310,14 @@ ice_dpll_ufl_pin_state_set(const struct dpll_pin *pin, void *pin_priv, data &= ~ICE_SMA1_MASK; enable = true; } else if (state == DPLL_PIN_STATE_DISCONNECTED) { + /* Skip if U.FL1 is not active, setting TX_EN + * while DIR_EN is set would also deactivate + * the paired SMA1 output. + */ + if (data & (ICE_SMA1_DIR_EN | ICE_SMA1_TX_EN)) { + ret = 0; + goto unlock; + } data |= ICE_SMA1_TX_EN; enable = false; } else { @@ -1267,6 +1332,15 @@ ice_dpll_ufl_pin_state_set(const struct dpll_pin *pin, void *pin_priv, data &= ~ICE_SMA2_UFL2_RX_DIS; enable = true; } else if (state == DPLL_PIN_STATE_DISCONNECTED) { + /* Skip if U.FL2 is not active, setting + * UFL2_RX_DIS could also disable the paired + * SMA2 input. + */ + if (!(data & ICE_SMA2_DIR_EN) || + (data & ICE_SMA2_UFL2_RX_DIS)) { + ret = 0; + goto unlock; + } data |= ICE_SMA2_UFL2_RX_DIS; enable = false; } else { @@ -1296,6 +1370,8 @@ ice_dpll_ufl_pin_state_set(const struct dpll_pin *pin, void *pin_priv, unlock: mutex_unlock(&pf->dplls.lock); + if (!ret) + ice_dpll_sw_pin_notify_peer(&pf->dplls, p); return ret; } @@ -1414,6 +1490,8 @@ ice_dpll_sma_pin_state_set(const struct dpll_pin *pin, void *pin_priv, unlock: mutex_unlock(&pf->dplls.lock); + if (!ret) + ice_dpll_sw_pin_notify_peer(&pf->dplls, sma); return ret; } @@ -1609,6 +1687,8 @@ ice_dpll_pin_sma_direction_set(const struct dpll_pin *pin, void *pin_priv, mutex_lock(&pf->dplls.lock); ret = ice_dpll_sma_direction_set(p, direction, extack); mutex_unlock(&pf->dplls.lock); + if (!ret) + ice_dpll_sw_pin_notify_peer(&pf->dplls, p); return ret; } @@ -1915,7 +1995,10 @@ ice_dpll_phase_offset_get(const struct dpll_pin *pin, void *pin_priv, d->active_input == p->input->pin)) *phase_offset = d->phase_offset * ICE_DPLL_PHASE_OFFSET_FACTOR; else if (d->phase_offset_monitor_period) - *phase_offset = p->phase_offset * ICE_DPLL_PHASE_OFFSET_FACTOR; + *phase_offset = (p->input && + p->direction == DPLL_PIN_DIRECTION_INPUT ? + p->input->phase_offset : + p->phase_offset) * ICE_DPLL_PHASE_OFFSET_FACTOR; else *phase_offset = 0; mutex_unlock(&pf->dplls.lock); @@ -2609,6 +2692,27 @@ static u64 ice_generate_clock_id(struct ice_pf *pf) return pci_get_dsn(pf->pdev); } +/** + * ice_dpll_pin_ntf - notify pin change including any SW pin wrappers + * @dplls: pointer to dplls struct + * @pin: the dpll_pin that changed + * + * Send a change notification for @pin and for any registered SMA/U.FL pin + * whose backing CGU input matches @pin. + */ +static void ice_dpll_pin_ntf(struct ice_dplls *dplls, struct dpll_pin *pin) +{ + dpll_pin_change_ntf(pin); + for (int i = 0; i < ICE_DPLL_PIN_SW_NUM; i++) { + if (dplls->sma[i].pin && dplls->sma[i].input && + dplls->sma[i].input->pin == pin) + dpll_pin_change_ntf(dplls->sma[i].pin); + if (dplls->ufl[i].pin && dplls->ufl[i].input && + dplls->ufl[i].input->pin == pin) + dpll_pin_change_ntf(dplls->ufl[i].pin); + } +} + /** * ice_dpll_notify_changes - notify dpll subsystem about changes * @d: pointer do dpll @@ -2617,6 +2721,7 @@ static u64 ice_generate_clock_id(struct ice_pf *pf) */ static void ice_dpll_notify_changes(struct ice_dpll *d) { + struct ice_dplls *dplls = &d->pf->dplls; bool pin_notified = false; if (d->prev_dpll_state != d->dpll_state) { @@ -2625,17 +2730,17 @@ static void ice_dpll_notify_changes(struct ice_dpll *d) } if (d->prev_input != d->active_input) { if (d->prev_input) - dpll_pin_change_ntf(d->prev_input); + ice_dpll_pin_ntf(dplls, d->prev_input); d->prev_input = d->active_input; if (d->active_input) { - dpll_pin_change_ntf(d->active_input); + ice_dpll_pin_ntf(dplls, d->active_input); pin_notified = true; } } if (d->prev_phase_offset != d->phase_offset) { d->prev_phase_offset = d->phase_offset; if (!pin_notified && d->active_input) - dpll_pin_change_ntf(d->active_input); + ice_dpll_pin_ntf(dplls, d->active_input); } } @@ -2664,6 +2769,7 @@ static bool ice_dpll_is_pps_phase_monitor(struct ice_pf *pf) /** * ice_dpll_pins_notify_mask - notify dpll subsystem about bulk pin changes + * @dplls: pointer to dplls struct * @pins: array of ice_dpll_pin pointers registered within dpll subsystem * @pin_num: number of pins * @phase_offset_ntf_mask: bitmask of pin indexes to notify @@ -2673,15 +2779,14 @@ static bool ice_dpll_is_pps_phase_monitor(struct ice_pf *pf) * * Context: Must be called while pf->dplls.lock is released. */ -static void ice_dpll_pins_notify_mask(struct ice_dpll_pin *pins, +static void ice_dpll_pins_notify_mask(struct ice_dplls *dplls, + struct ice_dpll_pin *pins, u8 pin_num, u32 phase_offset_ntf_mask) { - int i = 0; - - for (i = 0; i < pin_num; i++) - if (phase_offset_ntf_mask & (1 << i)) - dpll_pin_change_ntf(pins[i].pin); + for (int i = 0; i < pin_num; i++) + if (phase_offset_ntf_mask & BIT(i)) + ice_dpll_pin_ntf(dplls, pins[i].pin); } /** @@ -2857,7 +2962,7 @@ static void ice_dpll_periodic_work(struct kthread_work *work) ice_dpll_notify_changes(de); ice_dpll_notify_changes(dp); if (phase_offset_ntf) - ice_dpll_pins_notify_mask(d->inputs, d->num_inputs, + ice_dpll_pins_notify_mask(d, d->inputs, d->num_inputs, phase_offset_ntf); resched: @@ -4014,6 +4119,7 @@ static int ice_dpll_init_info_sw_pins(struct ice_pf *pf) struct ice_dpll_pin *pin; u32 phase_adj_max, caps; int i, ret; + u8 data; if (pf->hw.device_id == ICE_DEV_ID_E810C_QSFP) input_idx_offset = ICE_E810_RCLK_PINS_NUM; @@ -4073,6 +4179,22 @@ static int ice_dpll_init_info_sw_pins(struct ice_pf *pf) } ice_dpll_phase_range_set(&pin->prop.phase_range, phase_adj_max); } + + /* Initialize the SMA control register to a known-good default state. + * Without this write the PCA9575 GPIO expander retains its power-on + * default (all outputs high) which makes all SW pins appear inactive. + * Set SMA1 and SMA2 as active inputs, disable U.FL1 output and + * U.FL2 input. + */ + ret = ice_read_sma_ctrl(&pf->hw, &data); + if (ret) + return ret; + data &= ~ICE_ALL_SMA_MASK; + data |= ICE_SMA1_TX_EN | ICE_SMA2_TX_EN | ICE_SMA2_UFL2_RX_DIS; + ret = ice_write_sma_ctrl(&pf->hw, data); + if (ret) + return ret; + ret = ice_dpll_pin_state_update(pf, pin, ICE_DPLL_PIN_TYPE_SOFTWARE, NULL); if (ret) diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index e6a20af6f63de5..f28416a707d77c 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3290,6 +3290,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, tx_rings[i].desc = NULL; tx_rings[i].tx_buf = NULL; tx_rings[i].tstamp_ring = NULL; + clear_bit(ICE_TX_RING_FLAGS_TXTIME, tx_rings[i].flags); tx_rings[i].tx_tstamps = &pf->ptp.port.tx; err = ice_setup_tx_ring(&tx_rings[i]); if (err) { diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 689c6025ea82e5..837b71b7b2b7c6 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -1412,9 +1412,9 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi) ring->count = vsi->num_tx_desc; ring->txq_teid = ICE_INVAL_TEID; if (dvm_ena) - ring->flags |= ICE_TX_FLAGS_RING_VLAN_L2TAG2; + set_bit(ICE_TX_RING_FLAGS_VLAN_L2TAG2, ring->flags); else - ring->flags |= ICE_TX_FLAGS_RING_VLAN_L2TAG1; + set_bit(ICE_TX_RING_FLAGS_VLAN_L2TAG1, ring->flags); WRITE_ONCE(vsi->tx_rings[i], ring); } diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 3c36e3641b9e99..055968485af6c4 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -1922,82 +1922,6 @@ static void ice_handle_mdd_event(struct ice_pf *pf) ice_print_vfs_mdd_events(pf); } -/** - * ice_force_phys_link_state - Force the physical link state - * @vsi: VSI to force the physical link state to up/down - * @link_up: true/false indicates to set the physical link to up/down - * - * Force the physical link state by getting the current PHY capabilities from - * hardware and setting the PHY config based on the determined capabilities. If - * link changes a link event will be triggered because both the Enable Automatic - * Link Update and LESM Enable bits are set when setting the PHY capabilities. - * - * Returns 0 on success, negative on failure - */ -static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) -{ - struct ice_aqc_get_phy_caps_data *pcaps; - struct ice_aqc_set_phy_cfg_data *cfg; - struct ice_port_info *pi; - struct device *dev; - int retcode; - - if (!vsi || !vsi->port_info || !vsi->back) - return -EINVAL; - if (vsi->type != ICE_VSI_PF) - return 0; - - dev = ice_pf_to_dev(vsi->back); - - pi = vsi->port_info; - - pcaps = kzalloc_obj(*pcaps); - if (!pcaps) - return -ENOMEM; - - retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, - NULL); - if (retcode) { - dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", - vsi->vsi_num, retcode); - retcode = -EIO; - goto out; - } - - /* No change in link */ - if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && - link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) - goto out; - - /* Use the current user PHY configuration. The current user PHY - * configuration is initialized during probe from PHY capabilities - * software mode, and updated on set PHY configuration. - */ - cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); - if (!cfg) { - retcode = -ENOMEM; - goto out; - } - - cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; - if (link_up) - cfg->caps |= ICE_AQ_PHY_ENA_LINK; - else - cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; - - retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); - if (retcode) { - dev_err(dev, "Failed to set phy config, VSI %d error %d\n", - vsi->vsi_num, retcode); - retcode = -EIO; - } - - kfree(cfg); -out: - kfree(pcaps); - return retcode; -} - /** * ice_init_nvm_phy_type - Initialize the NVM PHY type * @pi: port info structure @@ -2066,7 +1990,7 @@ static void ice_init_link_dflt_override(struct ice_port_info *pi) * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state * is used to indicate that the user PHY cfg default override is initialized * and the PHY has not been configured with the default override settings. The - * state is set here, and cleared in ice_configure_phy the first time the PHY is + * state is set here, and cleared in ice_phy_cfg the first time the PHY is * configured. * * This function should be called only if the FW doesn't support default @@ -2172,14 +2096,18 @@ static int ice_init_phy_user_cfg(struct ice_port_info *pi) } /** - * ice_configure_phy - configure PHY + * ice_phy_cfg - configure PHY * @vsi: VSI of PHY + * @link_en: true/false indicates to set link to enable/disable * * Set the PHY configuration. If the current PHY configuration is the same as - * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise - * configure the based get PHY capabilities for topology with media. + * the curr_user_phy_cfg and link_en hasn't changed, then do nothing to avoid + * link flap. Otherwise configure the PHY based get PHY capabilities for + * topology with media and link_en. + * + * Return: 0 on success, negative on failure */ -static int ice_configure_phy(struct ice_vsi *vsi) +static int ice_phy_cfg(struct ice_vsi *vsi, bool link_en) { struct device *dev = ice_pf_to_dev(vsi->back); struct ice_port_info *pi = vsi->port_info; @@ -2199,9 +2127,6 @@ static int ice_configure_phy(struct ice_vsi *vsi) phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) return -EPERM; - if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) - return ice_force_phys_link_state(vsi, true); - pcaps = kzalloc_obj(*pcaps); if (!pcaps) return -ENOMEM; @@ -2215,10 +2140,8 @@ static int ice_configure_phy(struct ice_vsi *vsi) goto done; } - /* If PHY enable link is configured and configuration has not changed, - * there's nothing to do - */ - if (pcaps->caps & ICE_AQC_PHY_EN_LINK && + /* Configuration has not changed. There's nothing to do. */ + if (link_en == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg)) goto done; @@ -2282,8 +2205,12 @@ static int ice_configure_phy(struct ice_vsi *vsi) */ ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req); - /* Enable link and link update */ - cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; + /* Enable/Disable link and link update */ + cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; + if (link_en) + cfg->caps |= ICE_AQ_PHY_ENA_LINK; + else + cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL); if (err) @@ -2336,7 +2263,7 @@ static void ice_check_media_subtask(struct ice_pf *pf) test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) return; - err = ice_configure_phy(vsi); + err = ice_phy_cfg(vsi, true); if (!err) clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); @@ -4892,9 +4819,15 @@ static int ice_init_link(struct ice_pf *pf) if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { struct ice_vsi *vsi = ice_get_main_vsi(pf); + struct ice_link_default_override_tlv *ldo; + bool link_en; + + ldo = &pf->link_dflt_override; + link_en = !(ldo->options & + ICE_LINK_OVERRIDE_AUTO_LINK_DIS); if (vsi) - ice_configure_phy(vsi); + ice_phy_cfg(vsi, link_en); } } else { set_bit(ICE_FLAG_NO_MEDIA, pf->flags); @@ -5312,6 +5245,8 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) return err; } + ice_init_dev_hw(pf); + adapter = ice_adapter_get(pdev); if (IS_ERR(adapter)) { err = PTR_ERR(adapter); @@ -9707,7 +9642,7 @@ int ice_open_internal(struct net_device *netdev) } } - err = ice_configure_phy(vsi); + err = ice_phy_cfg(vsi, true); if (err) { netdev_err(netdev, "Failed to set physical link up, error %d\n", err); @@ -9748,7 +9683,7 @@ int ice_stop(struct net_device *netdev) } if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { - int link_err = ice_force_phys_link_state(vsi, false); + int link_err = ice_phy_cfg(vsi, false); if (link_err) { if (link_err == -ENOMEDIUM) diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 6cb0cf7a98912d..36df742c326c77 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -2710,7 +2710,7 @@ static bool ice_any_port_has_timestamps(struct ice_pf *pf) bool ice_ptp_tx_tstamps_pending(struct ice_pf *pf) { struct ice_hw *hw = &pf->hw; - unsigned int i; + int ret; /* Check software indicator */ switch (pf->ptp.tx_interrupt_mode) { @@ -2731,16 +2731,19 @@ bool ice_ptp_tx_tstamps_pending(struct ice_pf *pf) } /* Check hardware indicator */ - for (i = 0; i < ICE_GET_QUAD_NUM(hw->ptp.num_lports); i++) { - u64 tstamp_ready = 0; - int err; - - err = ice_get_phy_tx_tstamp_ready(&pf->hw, i, &tstamp_ready); - if (err || tstamp_ready) - return true; + ret = ice_check_phy_tx_tstamp_ready(hw); + if (ret < 0) { + dev_dbg(ice_pf_to_dev(pf), "Unable to read PHY Tx timestamp ready bitmap, err %d\n", + ret); + /* Stop triggering IRQs if we're unable to read PHY */ + return false; } - return false; + /* ice_check_phy_tx_tstamp_ready() returns 1 if there are timestamps + * available, 0 if there are no waiting timestamps, and a negative + * value if there was an error (which we checked for above). + */ + return ret > 0; } /** @@ -2824,8 +2827,7 @@ static void ice_ptp_maybe_trigger_tx_interrupt(struct ice_pf *pf) { struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; - bool trigger_oicr = false; - unsigned int i; + int ret; if (!pf->ptp.port.tx.has_ready_bitmap) return; @@ -2833,21 +2835,11 @@ static void ice_ptp_maybe_trigger_tx_interrupt(struct ice_pf *pf) if (!ice_pf_src_tmr_owned(pf)) return; - for (i = 0; i < ICE_GET_QUAD_NUM(hw->ptp.num_lports); i++) { - u64 tstamp_ready; - int err; - - err = ice_get_phy_tx_tstamp_ready(&pf->hw, i, &tstamp_ready); - if (!err && tstamp_ready) { - trigger_oicr = true; - break; - } - } - - if (trigger_oicr) { - /* Trigger a software interrupt, to ensure this data - * gets processed. - */ + ret = ice_check_phy_tx_tstamp_ready(hw); + if (ret < 0) { + dev_dbg(dev, "PTP periodic task unable to read PHY timestamp ready bitmap, err %d\n", + ret); + } else if (ret) { dev_dbg(dev, "PTP periodic task detected waiting timestamps. Triggering Tx timestamp interrupt now.\n"); wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M); diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_consts.h b/drivers/net/ethernet/intel/ice/ice_ptp_consts.h index 19dddd9b53dddb..4d298c27bfb273 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_consts.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp_consts.h @@ -78,14 +78,14 @@ struct ice_eth56g_mac_reg_cfg eth56g_mac_cfg[NUM_ICE_ETH56G_LNK_SPD] = { .blktime = 0x666, /* 3.2 */ .tx_offset = { .serdes = 0x234c, /* 17.6484848 */ - .no_fec = 0x8e80, /* 71.25 */ + .no_fec = 0x93d9, /* 73 */ .fc = 0xb4a4, /* 90.32 */ .sfd = 0x4a4, /* 2.32 */ .onestep = 0x4ccd /* 38.4 */ }, .rx_offset = { .serdes = 0xffffeb27, /* -10.42424 */ - .no_fec = 0xffffcccd, /* -25.6 */ + .no_fec = 0xffffc7b6, /* -28 */ .fc = 0xfffc557b, /* -469.26 */ .sfd = 0x4a4, /* 2.32 */ .bs_ds = 0x32 /* 0.0969697 */ @@ -118,17 +118,17 @@ struct ice_eth56g_mac_reg_cfg eth56g_mac_cfg[NUM_ICE_ETH56G_LNK_SPD] = { .mktime = 0x147b, /* 10.24, only if RS-FEC enabled */ .tx_offset = { .serdes = 0xe1e, /* 7.0593939 */ - .no_fec = 0x3857, /* 28.17 */ + .no_fec = 0x4266, /* 33 */ .fc = 0x48c3, /* 36.38 */ - .rs = 0x8100, /* 64.5 */ + .rs = 0x8a00, /* 69 */ .sfd = 0x1dc, /* 0.93 */ .onestep = 0x1eb8 /* 15.36 */ }, .rx_offset = { .serdes = 0xfffff7a9, /* -4.1697 */ - .no_fec = 0xffffe71a, /* -12.45 */ + .no_fec = 0xffffe700, /* -12 */ .fc = 0xfffe894d, /* -187.35 */ - .rs = 0xfffff8cd, /* -3.6 */ + .rs = 0xfffff8cc, /* -3 */ .sfd = 0x1dc, /* 0.93 */ .bs_ds = 0x14 /* 0.0387879, RS-FEC 0 */ } diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c index 61c0a0d93ea892..24fb7a3e14d63e 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -377,6 +377,31 @@ static void ice_ptp_cfg_sync_delay(const struct ice_hw *hw, u32 delay) * The following functions operate on devices with the ETH 56G PHY. */ +/** + * ice_ptp_init_phc_e825c - Perform E825C specific PHC initialization + * @hw: pointer to HW struct + * + * Perform E825C-specific PTP hardware clock initialization steps. + * + * Return: 0 on success, or a negative error value on failure. + */ +static int ice_ptp_init_phc_e825c(struct ice_hw *hw) +{ + int err; + + /* Soft reset all ports, to ensure everything is at a clean state */ + for (int port = 0; port < hw->ptp.num_lports; port++) { + err = ice_ptp_phy_soft_reset_eth56g(hw, port); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to soft reset port %d, err %d\n", + port, err); + return err; + } + } + + return 0; +} + /** * ice_ptp_get_dest_dev_e825 - get destination PHY for given port number * @hw: pointer to the HW struct @@ -1847,6 +1872,8 @@ static int ice_phy_cfg_mac_eth56g(struct ice_hw *hw, u8 port) * @ena: enable or disable interrupt * @threshold: interrupt threshold * + * The threshold cannot be 0 while the interrupt is enabled. + * * Configure TX timestamp interrupt for the specified port * * Return: @@ -1858,19 +1885,45 @@ int ice_phy_cfg_intr_eth56g(struct ice_hw *hw, u8 port, bool ena, u8 threshold) int err; u32 val; + if (ena && !threshold) + return -EINVAL; + err = ice_read_ptp_reg_eth56g(hw, port, PHY_REG_TS_INT_CONFIG, &val); if (err) return err; + val &= ~PHY_TS_INT_CONFIG_ENA_M; if (ena) { - val |= PHY_TS_INT_CONFIG_ENA_M; val &= ~PHY_TS_INT_CONFIG_THRESHOLD_M; val |= FIELD_PREP(PHY_TS_INT_CONFIG_THRESHOLD_M, threshold); - } else { - val &= ~PHY_TS_INT_CONFIG_ENA_M; + err = ice_write_ptp_reg_eth56g(hw, port, PHY_REG_TS_INT_CONFIG, + val); + if (err) { + ice_debug(hw, ICE_DBG_PTP, + "Failed to update 'threshold' PHY_REG_TS_INT_CONFIG port=%u ena=%u threshold=%u\n", + port, !!ena, threshold); + return err; + } + val |= PHY_TS_INT_CONFIG_ENA_M; + } + + err = ice_write_ptp_reg_eth56g(hw, port, PHY_REG_TS_INT_CONFIG, val); + if (err) { + ice_debug(hw, ICE_DBG_PTP, + "Failed to update 'ena' PHY_REG_TS_INT_CONFIG port=%u ena=%u threshold=%u\n", + port, !!ena, threshold); + return err; } - return ice_write_ptp_reg_eth56g(hw, port, PHY_REG_TS_INT_CONFIG, val); + err = ice_read_ptp_reg_eth56g(hw, port, PHY_REG_TS_INT_CONFIG, &val); + if (err) { + ice_debug(hw, ICE_DBG_PTP, + "Failed to read PHY_REG_TS_INT_CONFIG port=%u ena=%u threshold=%u\n", + port, !!ena, threshold); + return err; + } + + return 0; } /** @@ -2115,6 +2168,35 @@ int ice_start_phy_timer_eth56g(struct ice_hw *hw, u8 port) return 0; } +/** + * ice_check_phy_tx_tstamp_ready_eth56g - Check Tx memory status for all ports + * @hw: pointer to the HW struct + * + * Check the PHY_REG_TX_MEMORY_STATUS for all ports. A set bit indicates + * a waiting timestamp. + * + * Return: 1 if any port has at least one timestamp ready bit set, + * 0 otherwise, and a negative error code if unable to read the bitmap. + */ +static int ice_check_phy_tx_tstamp_ready_eth56g(struct ice_hw *hw) +{ + int port; + + for (port = 0; port < hw->ptp.num_lports; port++) { + u64 tstamp_ready; + int err; + + err = ice_get_phy_tx_tstamp_ready(hw, port, &tstamp_ready); + if (err) + return err; + + if (tstamp_ready) + return 1; + } + + return 0; +} + /** * ice_ptp_read_tx_hwtstamp_status_eth56g - Get TX timestamp status * @hw: pointer to the HW struct @@ -2137,13 +2219,19 @@ int ice_ptp_read_tx_hwtstamp_status_eth56g(struct ice_hw *hw, u32 *ts_status) *ts_status = 0; for (phy = 0; phy < params->num_phys; phy++) { + u8 port; int err; - err = ice_read_phy_eth56g(hw, phy, PHY_PTP_INT_STATUS, &status); + /* ice_read_phy_eth56g expects a port index, so use the first + * port of the PHY + */ + port = phy * hw->ptp.ports_per_phy; + + err = ice_read_phy_eth56g(hw, port, PHY_PTP_INT_STATUS, &status); if (err) return err; - *ts_status |= (status & mask) << (phy * hw->ptp.ports_per_phy); + *ts_status |= (status & mask) << port; } ice_debug(hw, ICE_DBG_PTP, "PHY interrupt err: %x\n", *ts_status); @@ -2151,6 +2239,69 @@ int ice_ptp_read_tx_hwtstamp_status_eth56g(struct ice_hw *hw, u32 *ts_status) return 0; } +/** + * ice_ptp_phy_soft_reset_eth56g - Perform a PHY soft reset on ETH56G + * @hw: pointer to the HW structure + * @port: PHY port number + * + * Trigger a soft reset of the ETH56G PHY by toggling the soft reset + * bit in the PHY global register. The reset sequence consists of: + * 1. Clearing the soft reset bit + * 2. Asserting the soft reset bit + * 3. Clearing the soft reset bit again + * + * Short delays are inserted between each step to allow the hardware + * to settle. This provides a controlled way to reinitialize the PHY + * without requiring a full device reset. + * + * Return: 0 on success, or a negative error code on failure when + * reading or writing the PHY register. + */ +int ice_ptp_phy_soft_reset_eth56g(struct ice_hw *hw, u8 port) +{ + u32 global_val; + int err; + + err = ice_read_ptp_reg_eth56g(hw, port, PHY_REG_GLOBAL, &global_val); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to read PHY_REG_GLOBAL for port %d, err %d\n", + port, err); + return err; + } + + global_val &= ~PHY_REG_GLOBAL_SOFT_RESET_M; + ice_debug(hw, ICE_DBG_PTP, "Clearing soft reset bit for port %d, val: 0x%x\n", + port, global_val); + err = ice_write_ptp_reg_eth56g(hw, port, PHY_REG_GLOBAL, global_val); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write PHY_REG_GLOBAL for port %d, err %d\n", + port, err); + return err; + } + + usleep_range(5000, 6000); + + global_val |= PHY_REG_GLOBAL_SOFT_RESET_M; + ice_debug(hw, ICE_DBG_PTP, "Set soft reset bit for port %d, val: 0x%x\n", + port, global_val); + err = ice_write_ptp_reg_eth56g(hw, port, PHY_REG_GLOBAL, global_val); + if (err) { + ice_debug(hw, ICE_DBG_PTP, "Failed to write PHY_REG_GLOBAL for port %d, err %d\n", + port, err); + return err; + } + usleep_range(5000, 6000); + + global_val &= ~PHY_REG_GLOBAL_SOFT_RESET_M; + ice_debug(hw, ICE_DBG_PTP, "Clear soft reset bit for port %d, val: 0x%x\n", + port, global_val); + err = ice_write_ptp_reg_eth56g(hw, port, PHY_REG_GLOBAL, global_val); + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to write PHY_REG_GLOBAL for port %d, err %d\n", + port, err); + return err; +} + /** * ice_get_phy_tx_tstamp_ready_eth56g - Read the Tx memory status register * @hw: pointer to the HW struct @@ -4202,6 +4353,35 @@ ice_get_phy_tx_tstamp_ready_e82x(struct ice_hw *hw, u8 quad, u64 *tstamp_ready) return 0; } +/** + * ice_check_phy_tx_tstamp_ready_e82x - Check Tx memory status for all quads + * @hw: pointer to the HW struct + * + * Check the Q_REG_TX_MEMORY_STATUS for all quads. A set bit indicates + * a waiting timestamp. + * + * Return: 1 if any quad has at least one timestamp ready bit set, + * 0 otherwise, and a negative error value if unable to read the bitmap. + */ +static int ice_check_phy_tx_tstamp_ready_e82x(struct ice_hw *hw) +{ + int quad; + + for (quad = 0; quad < ICE_GET_QUAD_NUM(hw->ptp.num_lports); quad++) { + u64 tstamp_ready; + int err; + + err = ice_get_phy_tx_tstamp_ready(hw, quad, &tstamp_ready); + if (err) + return err; + + if (tstamp_ready) + return 1; + } + + return 0; +} + /** * ice_phy_cfg_intr_e82x - Configure TX timestamp interrupt * @hw: pointer to the HW struct @@ -4755,6 +4935,23 @@ ice_get_phy_tx_tstamp_ready_e810(struct ice_hw *hw, u8 port, u64 *tstamp_ready) return 0; } +/** + * ice_check_phy_tx_tstamp_ready_e810 - Check Tx memory status register + * @hw: pointer to the HW struct + * + * The E810 devices do not have a Tx memory status register. Note this is + * intentionally different behavior from ice_get_phy_tx_tstamp_ready_e810 + * which always says that all bits are ready. This function is called in cases + * where code will trigger interrupts if timestamps are waiting, and should + * not be called for E810 hardware. + * + * Return: 0. + */ +static int ice_check_phy_tx_tstamp_ready_e810(struct ice_hw *hw) +{ + return 0; +} + /* E810 SMA functions * * The following functions operate specifically on E810 hardware and are used @@ -5009,6 +5206,21 @@ static void ice_get_phy_tx_tstamp_ready_e830(const struct ice_hw *hw, u8 port, *tstamp_ready |= rd32(hw, E830_PRTMAC_TS_TX_MEM_VALID_L); } +/** + * ice_check_phy_tx_tstamp_ready_e830 - Check Tx memory status register + * @hw: pointer to the HW struct + * + * Return: 1 if the device has waiting timestamps, 0 otherwise. + */ +static int ice_check_phy_tx_tstamp_ready_e830(struct ice_hw *hw) +{ + u64 tstamp_ready; + + ice_get_phy_tx_tstamp_ready_e830(hw, 0, &tstamp_ready); + + return !!tstamp_ready; +} + /** * ice_ptp_init_phy_e830 - initialize PHY parameters * @ptp: pointer to the PTP HW struct @@ -5381,8 +5593,8 @@ int ice_ptp_write_incval_locked(struct ice_hw *hw, u64 incval) */ int ice_ptp_adj_clock(struct ice_hw *hw, s32 adj) { + int err = 0; u8 tmr_idx; - int err; tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; @@ -5399,8 +5611,8 @@ int ice_ptp_adj_clock(struct ice_hw *hw, s32 adj) err = ice_ptp_prep_phy_adj_e810(hw, adj); break; case ICE_MAC_E830: - /* E830 sync PHYs automatically after setting GLTSYN_SHADJ */ - return 0; + /* E830 sync PHYs automatically after setting cmd register */ + break; case ICE_MAC_GENERIC: err = ice_ptp_prep_phy_adj_e82x(hw, adj); break; @@ -5564,7 +5776,7 @@ int ice_ptp_init_phc(struct ice_hw *hw) case ICE_MAC_GENERIC: return ice_ptp_init_phc_e82x(hw); case ICE_MAC_GENERIC_3K_E825: - return 0; + return ice_ptp_init_phc_e825c(hw); default: return -EOPNOTSUPP; } @@ -5601,6 +5813,33 @@ int ice_get_phy_tx_tstamp_ready(struct ice_hw *hw, u8 block, u64 *tstamp_ready) } } +/** + * ice_check_phy_tx_tstamp_ready - Check PHY Tx timestamp memory status + * @hw: pointer to the HW struct + * + * Check the PHY for Tx timestamp memory status on all ports. If you need to + * see individual timestamp status for each index, use + * ice_get_phy_tx_tstamp_ready() instead. + * + * Return: 1 if any port has timestamps available, 0 if there are no timestamps + * available, and a negative error code on failure. + */ +int ice_check_phy_tx_tstamp_ready(struct ice_hw *hw) +{ + switch (hw->mac_type) { + case ICE_MAC_E810: + return ice_check_phy_tx_tstamp_ready_e810(hw); + case ICE_MAC_E830: + return ice_check_phy_tx_tstamp_ready_e830(hw); + case ICE_MAC_GENERIC: + return ice_check_phy_tx_tstamp_ready_e82x(hw); + case ICE_MAC_GENERIC_3K_E825: + return ice_check_phy_tx_tstamp_ready_eth56g(hw); + default: + return -EOPNOTSUPP; + } +} + /** * ice_cgu_get_pin_desc_e823 - get pin description array * @hw: pointer to the hw struct diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h index 5896b346e57901..1b58b054f4a5b2 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h @@ -300,6 +300,7 @@ void ice_ptp_reset_ts_memory(struct ice_hw *hw); int ice_ptp_init_phc(struct ice_hw *hw); void ice_ptp_init_hw(struct ice_hw *hw); int ice_get_phy_tx_tstamp_ready(struct ice_hw *hw, u8 block, u64 *tstamp_ready); +int ice_check_phy_tx_tstamp_ready(struct ice_hw *hw); int ice_ptp_one_port_cmd(struct ice_hw *hw, u8 configured_port, enum ice_ptp_tmr_cmd configured_cmd); @@ -374,6 +375,7 @@ int ice_stop_phy_timer_eth56g(struct ice_hw *hw, u8 port, bool soft_reset); int ice_start_phy_timer_eth56g(struct ice_hw *hw, u8 port); int ice_phy_cfg_intr_eth56g(struct ice_hw *hw, u8 port, bool ena, u8 threshold); int ice_phy_cfg_ptp_1step_eth56g(struct ice_hw *hw, u8 port); +int ice_ptp_phy_soft_reset_eth56g(struct ice_hw *hw, u8 port); #define ICE_ETH56G_NOMINAL_INCVAL 0x140000000ULL #define ICE_ETH56G_NOMINAL_PCS_REF_TUS 0x100000000ULL @@ -676,6 +678,9 @@ static inline u64 ice_get_base_incval(struct ice_hw *hw) #define ICE_P0_GNSS_PRSNT_N BIT(4) /* ETH56G PHY register addresses */ +#define PHY_REG_GLOBAL 0x0 +#define PHY_REG_GLOBAL_SOFT_RESET_M BIT(11) + /* Timestamp PHY incval registers */ #define PHY_REG_TIMETUS_L 0x8 #define PHY_REG_TIMETUS_U 0xC diff --git a/drivers/net/ethernet/intel/ice/ice_sf_eth.c b/drivers/net/ethernet/intel/ice/ice_sf_eth.c index 2cf04bc6edceb1..a730aa368c92f6 100644 --- a/drivers/net/ethernet/intel/ice/ice_sf_eth.c +++ b/drivers/net/ethernet/intel/ice/ice_sf_eth.c @@ -305,6 +305,8 @@ ice_sf_eth_activate(struct ice_dynamic_port *dyn_port, aux_dev_uninit: auxiliary_device_uninit(&sf_dev->adev); + return err; + sf_dev_free: kfree(sf_dev); xa_erase: diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index a2cd4cf3773482..4ca1a0602307d0 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -190,9 +190,10 @@ void ice_free_tstamp_ring(struct ice_tx_ring *tx_ring) void ice_free_tx_tstamp_ring(struct ice_tx_ring *tx_ring) { ice_free_tstamp_ring(tx_ring); + clear_bit(ICE_TX_RING_FLAGS_TXTIME, tx_ring->flags); + smp_wmb(); /* order flag clear before pointer NULL */ kfree_rcu(tx_ring->tstamp_ring, rcu); - tx_ring->tstamp_ring = NULL; - tx_ring->flags &= ~ICE_TX_FLAGS_TXTIME; + WRITE_ONCE(tx_ring->tstamp_ring, NULL); } /** @@ -405,7 +406,7 @@ static int ice_alloc_tstamp_ring(struct ice_tx_ring *tx_ring) tx_ring->tstamp_ring = tstamp_ring; tstamp_ring->desc = NULL; tstamp_ring->count = ice_calc_ts_ring_count(tx_ring); - tx_ring->flags |= ICE_TX_FLAGS_TXTIME; + set_bit(ICE_TX_RING_FLAGS_TXTIME, tx_ring->flags); return 0; } @@ -1521,13 +1522,20 @@ ice_tx_map(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first, return; if (ice_is_txtime_cfg(tx_ring)) { - struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; - u32 tstamp_count = tstamp_ring->count; - u32 j = tstamp_ring->next_to_use; + struct ice_tstamp_ring *tstamp_ring; + u32 tstamp_count, j; struct ice_ts_desc *ts_desc; struct timespec64 ts; u32 tstamp; + smp_rmb(); /* order flag read before pointer read */ + tstamp_ring = READ_ONCE(tx_ring->tstamp_ring); + if (unlikely(!tstamp_ring)) + goto ring_kick; + + tstamp_count = tstamp_ring->count; + j = tstamp_ring->next_to_use; + ts = ktime_to_timespec64(first->skb->tstamp); tstamp = ts.tv_nsec >> ICE_TXTIME_CTX_RESOLUTION_128NS; @@ -1555,6 +1563,7 @@ ice_tx_map(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first, tstamp_ring->next_to_use = j; writel_relaxed(j, tstamp_ring->tail); } else { +ring_kick: writel_relaxed(i, tx_ring->tail); } return; @@ -1814,7 +1823,7 @@ ice_tx_prepare_vlan_flags(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first) */ if (skb_vlan_tag_present(skb)) { first->vid = skb_vlan_tag_get(skb); - if (tx_ring->flags & ICE_TX_FLAGS_RING_VLAN_L2TAG2) + if (test_bit(ICE_TX_RING_FLAGS_VLAN_L2TAG2, tx_ring->flags)) first->tx_flags |= ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN; else first->tx_flags |= ICE_TX_FLAGS_HW_VLAN; @@ -2158,6 +2167,9 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_tx_ring *tx_ring) ice_trace(xmit_frame_ring, tx_ring, skb); + /* record the location of the first descriptor for this packet */ + first = &tx_ring->tx_buf[tx_ring->next_to_use]; + count = ice_xmit_desc_count(skb); if (ice_chk_linearize(skb, count)) { if (__skb_linearize(skb)) @@ -2183,8 +2195,6 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_tx_ring *tx_ring) offload.tx_ring = tx_ring; - /* record the location of the first descriptor for this packet */ - first = &tx_ring->tx_buf[tx_ring->next_to_use]; first->skb = skb; first->type = ICE_TX_BUF_SKB; first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); @@ -2249,6 +2259,7 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_tx_ring *tx_ring) out_drop: ice_trace(xmit_frame_ring_drop, tx_ring, skb); dev_kfree_skb_any(skb); + first->type = ICE_TX_BUF_EMPTY; return NETDEV_TX_OK; } diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index b6547e1b7c4235..5e517f2193798f 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -212,6 +212,14 @@ enum ice_rx_dtype { ICE_RX_DTYPE_SPLIT_ALWAYS = 2, }; +enum ice_tx_ring_flags { + ICE_TX_RING_FLAGS_XDP, + ICE_TX_RING_FLAGS_VLAN_L2TAG1, + ICE_TX_RING_FLAGS_VLAN_L2TAG2, + ICE_TX_RING_FLAGS_TXTIME, + ICE_TX_RING_FLAGS_NBITS, +}; + struct ice_pkt_ctx { u64 cached_phctime; __be16 vlan_proto; @@ -352,11 +360,7 @@ struct ice_tx_ring { u16 count; /* Number of descriptors */ u16 q_index; /* Queue number of ring */ - u8 flags; -#define ICE_TX_FLAGS_RING_XDP BIT(0) -#define ICE_TX_FLAGS_RING_VLAN_L2TAG1 BIT(1) -#define ICE_TX_FLAGS_RING_VLAN_L2TAG2 BIT(2) -#define ICE_TX_FLAGS_TXTIME BIT(3) + DECLARE_BITMAP(flags, ICE_TX_RING_FLAGS_NBITS); struct xsk_buff_pool *xsk_pool; @@ -398,7 +402,7 @@ static inline bool ice_ring_ch_enabled(struct ice_tx_ring *ring) static inline bool ice_ring_is_xdp(struct ice_tx_ring *ring) { - return !!(ring->flags & ICE_TX_FLAGS_RING_XDP); + return test_bit(ICE_TX_RING_FLAGS_XDP, ring->flags); } enum ice_container_type { diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c index c8bc952f05cdb5..51259a4fdda4b9 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c @@ -804,7 +804,12 @@ void ice_reset_all_vfs(struct ice_pf *pf) ice_vf_ctrl_invalidate_vsi(vf); ice_vf_pre_vsi_rebuild(vf); - ice_vf_rebuild_vsi(vf); + if (ice_vf_rebuild_vsi(vf)) { + dev_err(dev, "VF %u VSI rebuild failed, leaving VF disabled\n", + vf->vf_id); + mutex_unlock(&vf->cfg_lock); + continue; + } ice_vf_post_vsi_rebuild(vf); ice_eswitch_attach_vf(pf, vf); diff --git a/drivers/net/ethernet/intel/idpf/idpf_idc.c b/drivers/net/ethernet/intel/idpf/idpf_idc.c index 7e4f4ac9265377..b7d6b08fc89e89 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_idc.c +++ b/drivers/net/ethernet/intel/idpf/idpf_idc.c @@ -90,7 +90,10 @@ static int idpf_plug_vport_aux_dev(struct iidc_rdma_core_dev_info *cdev_info, return 0; err_aux_dev_add: + ida_free(&idpf_idc_ida, adev->id); + vdev_info->adev = NULL; auxiliary_device_uninit(adev); + return ret; err_aux_dev_init: ida_free(&idpf_idc_ida, adev->id); err_ida_alloc: @@ -228,7 +231,10 @@ static int idpf_plug_core_aux_dev(struct iidc_rdma_core_dev_info *cdev_info) return 0; err_aux_dev_add: + ida_free(&idpf_idc_ida, adev->id); + cdev_info->adev = NULL; auxiliary_device_uninit(adev); + return ret; err_aux_dev_init: ida_free(&idpf_idc_ida, adev->id); err_ida_alloc: diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c index b579d5b545c46d..8347e696937cdb 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c @@ -409,10 +409,17 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, data_offset = OCTEP_VF_OQ_RESP_HW_SIZE; rx_ol_flags = 0; } - rx_bytes += buff_info->len; - if (buff_info->len <= oq->max_single_buffer_size) { skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); + if (!skb) { + oq->stats->alloc_failures++; + desc_used++; + read_idx++; + if (read_idx == oq->max_count) + read_idx = 0; + continue; + } + rx_bytes += buff_info->len; skb_reserve(skb, data_offset); skb_put(skb, buff_info->len); read_idx++; @@ -424,6 +431,31 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, u16 data_len; skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); + if (!skb) { + oq->stats->alloc_failures++; + desc_used++; + read_idx++; + if (read_idx == oq->max_count) + read_idx = 0; + data_len = buff_info->len - oq->max_single_buffer_size; + while (data_len) { + dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, + PAGE_SIZE, DMA_FROM_DEVICE); + buff_info = (struct octep_vf_rx_buffer *) + &oq->buff_info[read_idx]; + buff_info->page = NULL; + if (data_len < oq->buffer_size) + data_len = 0; + else + data_len -= oq->buffer_size; + desc_used++; + read_idx++; + if (read_idx == oq->max_count) + read_idx = 0; + } + continue; + } + rx_bytes += buff_info->len; skb_reserve(skb, data_offset); /* Head fragment includes response header(s); * subsequent fragments contains only data. diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index ddc321a02fdae0..796f79088f3662 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -3566,12 +3566,23 @@ static int mtk_device_event(struct notifier_block *n, unsigned long event, void return NOTIFY_DONE; } +static int mtk_max_gmac_mtu(struct mtk_eth *eth) +{ + int i, max_mtu = ETH_DATA_LEN; + + for (i = 0; i < ARRAY_SIZE(eth->netdev); i++) + if (eth->netdev[i] && eth->netdev[i]->mtu > max_mtu) + max_mtu = eth->netdev[i]->mtu; + + return max_mtu; +} + static int mtk_open(struct net_device *dev) { struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; struct mtk_mac *target_mac; - int i, err, ppe_num; + int i, err, ppe_num, mtu; ppe_num = eth->soc->ppe_num; @@ -3618,6 +3629,10 @@ static int mtk_open(struct net_device *dev) mtk_gdm_config(eth, target_mac->id, gdm_config); } + mtu = mtk_max_gmac_mtu(eth); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_update_mtu(eth->ppe[i], mtu); + napi_enable(ð->tx_napi); napi_enable(ð->rx_napi); mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); @@ -4311,6 +4326,7 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) int length = new_mtu + MTK_RX_ETH_HLEN; struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; + int max_mtu, i; if (rcu_access_pointer(eth->prog) && length > MTK_PP_MAX_BUF_SIZE) { @@ -4321,6 +4337,10 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) mtk_set_mcr_max_rx(mac, length); WRITE_ONCE(dev->mtu, new_mtu); + max_mtu = mtk_max_gmac_mtu(eth); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_update_mtu(eth->ppe[i], max_mtu); + return 0; } diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index 75f7728fc79623..18279e2a7022ef 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -973,6 +973,36 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) } } +void mtk_ppe_update_mtu(struct mtk_ppe *ppe, int mtu) +{ + int base; + u32 val; + + if (!ppe) + return; + + /* The PPE checks output frame size against per-tag-layer MTU limits, + * treating PPPoE and DSA tags just like 802.1Q VLAN tags. The Linux + * device MTU already accounts for PPPoE (PPPOE_SES_HLEN) and DSA tag + * overhead, but 802.1Q VLAN tags are handled transparently without + * being reflected by the lower device MTU being increased by 4. + * Use the maximum MTU across all GMAC interfaces so that PPE output + * frame limits are sufficiently high regardless of which port a flow + * egresses through. + */ + base = ETH_HLEN + mtu; + + val = FIELD_PREP(MTK_PPE_VLAN_MTU0_NONE, base) | + FIELD_PREP(MTK_PPE_VLAN_MTU0_1TAG, base + VLAN_HLEN); + ppe_w32(ppe, MTK_PPE_VLAN_MTU0, val); + + val = FIELD_PREP(MTK_PPE_VLAN_MTU1_2TAG, + base + 2 * VLAN_HLEN) | + FIELD_PREP(MTK_PPE_VLAN_MTU1_3TAG, + base + 3 * VLAN_HLEN); + ppe_w32(ppe, MTK_PPE_VLAN_MTU1, val); +} + void mtk_ppe_start(struct mtk_ppe *ppe) { u32 val; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h index 223f709e2704f8..ba85e39a155bf9 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe.h @@ -346,6 +346,7 @@ struct mtk_ppe { struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index); void mtk_ppe_deinit(struct mtk_eth *eth); +void mtk_ppe_update_mtu(struct mtk_ppe *ppe, int mtu); void mtk_ppe_start(struct mtk_ppe *ppe); int mtk_ppe_stop(struct mtk_ppe *ppe); int mtk_ppe_prepare_reset(struct mtk_ppe *ppe); diff --git a/drivers/net/ethernet/mellanox/mlx4/srq.c b/drivers/net/ethernet/mellanox/mlx4/srq.c index dd890f5d7b725c..8711689120f302 100644 --- a/drivers/net/ethernet/mellanox/mlx4/srq.c +++ b/drivers/net/ethernet/mellanox/mlx4/srq.c @@ -44,13 +44,14 @@ void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type) { struct mlx4_srq_table *srq_table = &mlx4_priv(dev)->srq_table; struct mlx4_srq *srq; + unsigned long flags; - rcu_read_lock(); + spin_lock_irqsave(&srq_table->lock, flags); srq = radix_tree_lookup(&srq_table->tree, srqn & (dev->caps.num_srqs - 1)); - rcu_read_unlock(); - if (srq) - refcount_inc(&srq->refcount); - else { + if (!srq || !refcount_inc_not_zero(&srq->refcount)) + srq = NULL; + spin_unlock_irqrestore(&srq_table->lock, flags); + if (!srq) { mlx4_warn(dev, "Async event for bogus SRQ %08x\n", srqn); return; } @@ -203,8 +204,8 @@ int mlx4_srq_alloc(struct mlx4_dev *dev, u32 pdn, u32 cqn, u16 xrcd, if (err) goto err_radix; - refcount_set(&srq->refcount, 1); init_completion(&srq->free); + refcount_set_release(&srq->refcount, 1); return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 05faad5083d9d5..145677ce96408f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */ +#include + #include "mlx5_core.h" #include "en.h" #include "ipsec.h" @@ -592,7 +594,6 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_wqe_aso_ctrl_seg *ctrl; struct mlx5e_hw_objs *res; struct mlx5_aso_wqe *wqe; - unsigned long expires; u8 ds_cnt; int ret; @@ -614,13 +615,8 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, mlx5e_ipsec_aso_copy(ctrl, data); mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl); - expires = jiffies + msecs_to_jiffies(10); - do { - ret = mlx5_aso_poll_cq(aso->aso, false); - if (ret) - /* We are in atomic context */ - udelay(10); - } while (ret && time_is_after_jiffies(expires)); + read_poll_timeout_atomic(mlx5_aso_poll_cq, ret, !ret, 10, + 10 * USEC_PER_MSEC, false, aso->aso, false); if (!ret) memcpy(sa_entry->ctx, aso->ctx, MLX5_ST_SZ_BYTES(ipsec_aso)); spin_unlock_bh(&aso->lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b6c12460b54a9e..0b8b44bbcb9ef9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -6756,6 +6756,14 @@ static int _mlx5e_probe(struct auxiliary_device *adev) goto err_resume; } + /* mlx5e_fix_features() returns early when the device is not present + * to avoid dereferencing cleared priv during profile changes. + * This also causes it to be a no-op during register_netdev(), where + * the device is not yet present. + * Trigger an additional features update that will actually work. + */ + mlx5e_update_features(netdev); + mlx5e_dcbnl_init_app(priv); mlx5_core_uplink_netdev_set(mdev, netdev); mlx5e_params_print_info(mdev, &priv->channels.params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 3f73d9b1115dd1..fab80c79ff071a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1907,7 +1907,7 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) err = mlx5_notifiers_init(dev); if (err) - goto err_hca_caps; + goto err_notifiers_init; /* The conjunction of sw_vhca_id with sw_owner_id will be a global * unique id per function which uses mlx5_core. @@ -1923,6 +1923,8 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) return 0; +err_notifiers_init: + mlx5_hca_caps_free(dev); err_hca_caps: mlx5_adev_cleanup(dev); err_adev_init: diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 3fa9d1910daa1e..8f331358c97256 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -139,7 +139,7 @@ void fbnic_up(struct fbnic_net *fbn) /* Enable Tx/Rx processing */ fbnic_napi_enable(fbn); - netif_tx_start_all_queues(fbn->netdev); + netif_tx_wake_all_queues(fbn->netdev); fbnic_service_task_start(fbn); diff --git a/drivers/net/ethernet/micrel/ks8851.h b/drivers/net/ethernet/micrel/ks8851.h index 31f75b4a67fd79..b795a3a6057118 100644 --- a/drivers/net/ethernet/micrel/ks8851.h +++ b/drivers/net/ethernet/micrel/ks8851.h @@ -408,10 +408,8 @@ struct ks8851_net { struct gpio_desc *gpio; struct mii_bus *mii_bus; - void (*lock)(struct ks8851_net *ks, - unsigned long *flags); - void (*unlock)(struct ks8851_net *ks, - unsigned long *flags); + void (*lock)(struct ks8851_net *ks); + void (*unlock)(struct ks8851_net *ks); unsigned int (*rdreg16)(struct ks8851_net *ks, unsigned int reg); void (*wrreg16)(struct ks8851_net *ks, diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 8048770958d60e..4afbb40bc0e4a6 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -28,25 +28,23 @@ /** * ks8851_lock - register access lock * @ks: The chip state - * @flags: Spinlock flags * * Claim chip register access lock */ -static void ks8851_lock(struct ks8851_net *ks, unsigned long *flags) +static void ks8851_lock(struct ks8851_net *ks) { - ks->lock(ks, flags); + ks->lock(ks); } /** * ks8851_unlock - register access unlock * @ks: The chip state - * @flags: Spinlock flags * * Release chip register access lock */ -static void ks8851_unlock(struct ks8851_net *ks, unsigned long *flags) +static void ks8851_unlock(struct ks8851_net *ks) { - ks->unlock(ks, flags); + ks->unlock(ks); } /** @@ -129,11 +127,10 @@ static void ks8851_set_powermode(struct ks8851_net *ks, unsigned pwrmode) static int ks8851_write_mac_addr(struct net_device *dev) { struct ks8851_net *ks = netdev_priv(dev); - unsigned long flags; u16 val; int i; - ks8851_lock(ks, &flags); + ks8851_lock(ks); /* * Wake up chip in case it was powered off when stopped; otherwise, @@ -149,7 +146,7 @@ static int ks8851_write_mac_addr(struct net_device *dev) if (!netif_running(dev)) ks8851_set_powermode(ks, PMECR_PM_SOFTDOWN); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); return 0; } @@ -163,12 +160,11 @@ static int ks8851_write_mac_addr(struct net_device *dev) static void ks8851_read_mac_addr(struct net_device *dev) { struct ks8851_net *ks = netdev_priv(dev); - unsigned long flags; u8 addr[ETH_ALEN]; u16 reg; int i; - ks8851_lock(ks, &flags); + ks8851_lock(ks); for (i = 0; i < ETH_ALEN; i += 2) { reg = ks8851_rdreg16(ks, KS_MAR(i)); @@ -177,7 +173,7 @@ static void ks8851_read_mac_addr(struct net_device *dev) } eth_hw_addr_set(dev, addr); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); } /** @@ -312,11 +308,10 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) { struct ks8851_net *ks = _ks; struct sk_buff_head rxq; - unsigned long flags; unsigned int status; struct sk_buff *skb; - ks8851_lock(ks, &flags); + ks8851_lock(ks); status = ks8851_rdreg16(ks, KS_ISR); ks8851_wrreg16(ks, KS_ISR, status); @@ -373,14 +368,17 @@ static irqreturn_t ks8851_irq(int irq, void *_ks) ks8851_wrreg16(ks, KS_RXCR1, rxc->rxcr1); } - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); if (status & IRQ_LCI) mii_check_link(&ks->mii); - if (status & IRQ_RXI) + if (status & IRQ_RXI) { + local_bh_disable(); while ((skb = __skb_dequeue(&rxq))) netif_rx(skb); + local_bh_enable(); + } return IRQ_HANDLED; } @@ -405,7 +403,6 @@ static void ks8851_flush_tx_work(struct ks8851_net *ks) static int ks8851_net_open(struct net_device *dev) { struct ks8851_net *ks = netdev_priv(dev); - unsigned long flags; int ret; ret = request_threaded_irq(dev->irq, NULL, ks8851_irq, @@ -418,7 +415,7 @@ static int ks8851_net_open(struct net_device *dev) /* lock the card, even if we may not actually be doing anything * else at the moment */ - ks8851_lock(ks, &flags); + ks8851_lock(ks); netif_dbg(ks, ifup, ks->netdev, "opening\n"); @@ -471,7 +468,7 @@ static int ks8851_net_open(struct net_device *dev) netif_dbg(ks, ifup, ks->netdev, "network device up\n"); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); mii_check_link(&ks->mii); return 0; } @@ -487,23 +484,22 @@ static int ks8851_net_open(struct net_device *dev) static int ks8851_net_stop(struct net_device *dev) { struct ks8851_net *ks = netdev_priv(dev); - unsigned long flags; netif_info(ks, ifdown, dev, "shutting down\n"); netif_stop_queue(dev); - ks8851_lock(ks, &flags); + ks8851_lock(ks); /* turn off the IRQs and ack any outstanding */ ks8851_wrreg16(ks, KS_IER, 0x0000); ks8851_wrreg16(ks, KS_ISR, 0xffff); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); /* stop any outstanding work */ ks8851_flush_tx_work(ks); flush_work(&ks->rxctrl_work); - ks8851_lock(ks, &flags); + ks8851_lock(ks); /* shutdown RX process */ ks8851_wrreg16(ks, KS_RXCR1, 0x0000); @@ -512,7 +508,7 @@ static int ks8851_net_stop(struct net_device *dev) /* set powermode to soft power down to save power */ ks8851_set_powermode(ks, PMECR_PM_SOFTDOWN); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); /* ensure any queued tx buffers are dumped */ while (!skb_queue_empty(&ks->txq)) { @@ -566,14 +562,13 @@ static netdev_tx_t ks8851_start_xmit(struct sk_buff *skb, static void ks8851_rxctrl_work(struct work_struct *work) { struct ks8851_net *ks = container_of(work, struct ks8851_net, rxctrl_work); - unsigned long flags; - ks8851_lock(ks, &flags); + ks8851_lock(ks); /* need to shutdown RXQ before modifying filter parameters */ ks8851_wrreg16(ks, KS_RXCR1, 0x00); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); } static void ks8851_set_rx_mode(struct net_device *dev) @@ -780,7 +775,6 @@ static int ks8851_set_eeprom(struct net_device *dev, { struct ks8851_net *ks = netdev_priv(dev); int offset = ee->offset; - unsigned long flags; int len = ee->len; u16 tmp; @@ -794,7 +788,7 @@ static int ks8851_set_eeprom(struct net_device *dev, if (!(ks->rc_ccr & CCR_EEPROM)) return -ENOENT; - ks8851_lock(ks, &flags); + ks8851_lock(ks); ks8851_eeprom_claim(ks); @@ -817,7 +811,7 @@ static int ks8851_set_eeprom(struct net_device *dev, eeprom_93cx6_wren(&ks->eeprom, false); ks8851_eeprom_release(ks); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); return 0; } @@ -827,7 +821,6 @@ static int ks8851_get_eeprom(struct net_device *dev, { struct ks8851_net *ks = netdev_priv(dev); int offset = ee->offset; - unsigned long flags; int len = ee->len; /* must be 2 byte aligned */ @@ -837,7 +830,7 @@ static int ks8851_get_eeprom(struct net_device *dev, if (!(ks->rc_ccr & CCR_EEPROM)) return -ENOENT; - ks8851_lock(ks, &flags); + ks8851_lock(ks); ks8851_eeprom_claim(ks); @@ -845,7 +838,7 @@ static int ks8851_get_eeprom(struct net_device *dev, eeprom_93cx6_multiread(&ks->eeprom, offset/2, (__le16 *)data, len/2); ks8851_eeprom_release(ks); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); return 0; } @@ -904,7 +897,6 @@ static int ks8851_phy_reg(int reg) static int ks8851_phy_read_common(struct net_device *dev, int phy_addr, int reg) { struct ks8851_net *ks = netdev_priv(dev); - unsigned long flags; int result; int ksreg; @@ -912,9 +904,9 @@ static int ks8851_phy_read_common(struct net_device *dev, int phy_addr, int reg) if (ksreg < 0) return ksreg; - ks8851_lock(ks, &flags); + ks8851_lock(ks); result = ks8851_rdreg16(ks, ksreg); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); return result; } @@ -949,14 +941,13 @@ static void ks8851_phy_write(struct net_device *dev, int phy, int reg, int value) { struct ks8851_net *ks = netdev_priv(dev); - unsigned long flags; int ksreg; ksreg = ks8851_phy_reg(reg); if (ksreg >= 0) { - ks8851_lock(ks, &flags); + ks8851_lock(ks); ks8851_wrreg16(ks, ksreg, value); - ks8851_unlock(ks, &flags); + ks8851_unlock(ks); } } diff --git a/drivers/net/ethernet/micrel/ks8851_par.c b/drivers/net/ethernet/micrel/ks8851_par.c index 78695be2570bfb..9f1c33f6ddec01 100644 --- a/drivers/net/ethernet/micrel/ks8851_par.c +++ b/drivers/net/ethernet/micrel/ks8851_par.c @@ -55,29 +55,27 @@ struct ks8851_net_par { /** * ks8851_lock_par - register access lock * @ks: The chip state - * @flags: Spinlock flags * * Claim chip register access lock */ -static void ks8851_lock_par(struct ks8851_net *ks, unsigned long *flags) +static void ks8851_lock_par(struct ks8851_net *ks) { struct ks8851_net_par *ksp = to_ks8851_par(ks); - spin_lock_irqsave(&ksp->lock, *flags); + spin_lock_bh(&ksp->lock); } /** * ks8851_unlock_par - register access unlock * @ks: The chip state - * @flags: Spinlock flags * * Release chip register access lock */ -static void ks8851_unlock_par(struct ks8851_net *ks, unsigned long *flags) +static void ks8851_unlock_par(struct ks8851_net *ks) { struct ks8851_net_par *ksp = to_ks8851_par(ks); - spin_unlock_irqrestore(&ksp->lock, *flags); + spin_unlock_bh(&ksp->lock); } /** @@ -233,7 +231,6 @@ static netdev_tx_t ks8851_start_xmit_par(struct sk_buff *skb, { struct ks8851_net *ks = netdev_priv(dev); netdev_tx_t ret = NETDEV_TX_OK; - unsigned long flags; unsigned int txqcr; u16 txmir; int err; @@ -241,7 +238,7 @@ static netdev_tx_t ks8851_start_xmit_par(struct sk_buff *skb, netif_dbg(ks, tx_queued, ks->netdev, "%s: skb %p, %d@%p\n", __func__, skb, skb->len, skb->data); - ks8851_lock_par(ks, &flags); + ks8851_lock_par(ks); txmir = ks8851_rdreg16_par(ks, KS_TXMIR) & 0x1fff; @@ -262,7 +259,7 @@ static netdev_tx_t ks8851_start_xmit_par(struct sk_buff *skb, ret = NETDEV_TX_BUSY; } - ks8851_unlock_par(ks, &flags); + ks8851_unlock_par(ks); return ret; } diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c index a161ae45743ab5..b9e68520278d08 100644 --- a/drivers/net/ethernet/micrel/ks8851_spi.c +++ b/drivers/net/ethernet/micrel/ks8851_spi.c @@ -71,11 +71,10 @@ struct ks8851_net_spi { /** * ks8851_lock_spi - register access lock * @ks: The chip state - * @flags: Spinlock flags * * Claim chip register access lock */ -static void ks8851_lock_spi(struct ks8851_net *ks, unsigned long *flags) +static void ks8851_lock_spi(struct ks8851_net *ks) { struct ks8851_net_spi *kss = to_ks8851_spi(ks); @@ -85,11 +84,10 @@ static void ks8851_lock_spi(struct ks8851_net *ks, unsigned long *flags) /** * ks8851_unlock_spi - register access unlock * @ks: The chip state - * @flags: Spinlock flags * * Release chip register access lock */ -static void ks8851_unlock_spi(struct ks8851_net *ks, unsigned long *flags) +static void ks8851_unlock_spi(struct ks8851_net *ks) { struct ks8851_net_spi *kss = to_ks8851_spi(ks); @@ -309,7 +307,6 @@ static void ks8851_tx_work(struct work_struct *work) struct ks8851_net_spi *kss; unsigned short tx_space; struct ks8851_net *ks; - unsigned long flags; struct sk_buff *txb; bool last; @@ -317,7 +314,7 @@ static void ks8851_tx_work(struct work_struct *work) ks = &kss->ks8851; last = skb_queue_empty(&ks->txq); - ks8851_lock_spi(ks, &flags); + ks8851_lock_spi(ks); while (!last) { txb = skb_dequeue(&ks->txq); @@ -343,7 +340,7 @@ static void ks8851_tx_work(struct work_struct *work) ks->tx_space = tx_space; spin_unlock_bh(&ks->statelock); - ks8851_unlock_spi(ks, &flags); + ks8851_unlock_spi(ks); } /** diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 786186c9a115f9..c2e855ff3ca9a8 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -2007,11 +2007,8 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) gc->dev = &pdev->dev; xa_init(&gc->irq_contexts); - if (gc->is_pf) - gc->mana_pci_debugfs = debugfs_create_dir("0", mana_debugfs_root); - else - gc->mana_pci_debugfs = debugfs_create_dir(pci_slot_name(pdev->slot), - mana_debugfs_root); + gc->mana_pci_debugfs = debugfs_create_dir(pci_name(pdev), + mana_debugfs_root); err = mana_gd_setup(pdev); if (err) diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 09a53c97754550..14d6f68eaa6958 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -2889,6 +2889,13 @@ static void mana_rss_table_init(struct mana_port_context *apc) ethtool_rxfh_indir_default(i, apc->num_queues); } +int mana_disable_vport_rx(struct mana_port_context *apc) +{ + return mana_cfg_vport_steering(apc, TRI_STATE_FALSE, false, false, + false); +} +EXPORT_SYMBOL_NS(mana_disable_vport_rx, "NET_MANA"); + int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, bool update_hash, bool update_tab) { @@ -3117,6 +3124,8 @@ static int mana_init_port(struct net_device *ndev) eth_hw_addr_set(ndev, apc->mac_addr); sprintf(vport, "vport%d", port_idx); apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); + debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, + &apc->speed); return 0; reset_apc: @@ -3273,10 +3282,12 @@ static int mana_dealloc_queues(struct net_device *ndev) */ apc->rss_state = TRI_STATE_FALSE; - err = mana_config_rss(apc, TRI_STATE_FALSE, false, false); + err = mana_disable_vport_rx(apc); if (err && mana_en_need_log(apc, err)) netdev_err(ndev, "Failed to disable vPort: %d\n", err); + mana_fence_rqs(apc); + /* Even in err case, still need to cleanup the vPort */ mana_destroy_vport(apc); @@ -3393,8 +3404,6 @@ static int mana_probe_port(struct mana_context *ac, int port_idx, netif_carrier_on(ndev); - debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, &apc->speed); - return 0; free_indir: @@ -3585,8 +3594,12 @@ int mana_probe(struct gdma_dev *gd, bool resuming) ac->gdma_dev = gd; gd->driver_data = ac; + + INIT_WORK(&ac->link_change_work, mana_link_state_handle); } + INIT_DELAYED_WORK(&ac->gf_stats_work, mana_gf_stats_work_handler); + err = mana_create_eq(ac); if (err) { dev_err(dev, "Failed to create EQs: %d\n", err); @@ -3602,8 +3615,6 @@ int mana_probe(struct gdma_dev *gd, bool resuming) if (!resuming) { ac->num_ports = num_ports; - - INIT_WORK(&ac->link_change_work, mana_link_state_handle); } else { if (ac->num_ports != num_ports) { dev_err(dev, "The number of vPorts changed: %d->%d\n", @@ -3632,10 +3643,9 @@ int mana_probe(struct gdma_dev *gd, bool resuming) if (!resuming) { for (i = 0; i < ac->num_ports; i++) { err = mana_probe_port(ac, i, &ac->ports[i]); - /* we log the port for which the probe failed and stop - * probes for subsequent ports. - * Note that we keep running ports, for which the probes - * were successful, unless add_adev fails too + /* Log the port for which the probe failed, stop probing + * subsequent ports, and skip add_adev. + * mana_remove() will clean up already-probed ports. */ if (err) { dev_err(dev, "Probe Failed for port %d\n", i); @@ -3649,10 +3659,9 @@ int mana_probe(struct gdma_dev *gd, bool resuming) enable_work(&apc->queue_reset_work); err = mana_attach(ac->ports[i]); rtnl_unlock(); - /* we log the port for which the attach failed and stop - * attach for subsequent ports - * Note that we keep running ports, for which the attach - * were successful, unless add_adev fails too + /* Log the port for which the attach failed, stop + * attaching subsequent ports, and skip add_adev. + * mana_remove() will clean up already-attached ports. */ if (err) { dev_err(dev, "Attach Failed for port %d\n", i); @@ -3661,9 +3670,9 @@ int mana_probe(struct gdma_dev *gd, bool resuming) } } - err = add_adev(gd, "eth"); + if (!err) + err = add_adev(gd, "eth"); - INIT_DELAYED_WORK(&ac->gf_stats_work, mana_gf_stats_work_handler); schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); out: @@ -3684,11 +3693,16 @@ void mana_remove(struct gdma_dev *gd, bool suspending) struct gdma_context *gc = gd->gdma_context; struct mana_context *ac = gd->driver_data; struct mana_port_context *apc; - struct device *dev = gc->dev; + struct device *dev; struct net_device *ndev; int err; int i; + if (!gc || !ac) + return; + + dev = gc->dev; + disable_work_sync(&ac->link_change_work); cancel_delayed_work_sync(&ac->gf_stats_work); @@ -3701,7 +3715,7 @@ void mana_remove(struct gdma_dev *gd, bool suspending) if (!ndev) { if (i == 0) dev_err(dev, "No net device to remove\n"); - goto out; + break; } apc = netdev_priv(ndev); @@ -3732,7 +3746,7 @@ void mana_remove(struct gdma_dev *gd, bool suspending) } mana_destroy_eq(ac); -out: + if (ac->per_port_queue_reset_wq) { destroy_workqueue(ac->per_port_queue_reset_wq); ac->per_port_queue_reset_wq = NULL; diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_target.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_target.c index 79470f198a62a6..9cf19446657c6c 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_target.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_target.c @@ -435,12 +435,17 @@ static int nfp_encode_basic_qdr(u64 addr, int dest_island, int cpp_tgt, /* Full Island ID and channel bits overlap? */ ret = nfp_decode_basic(addr, &v, cpp_tgt, mode, addr40, isld1, isld0); - if (ret) + if (ret) { + pr_warn("%s: decode dest_island failed: %d\n", __func__, ret); return ret; + } /* The current address won't go where expected? */ - if (dest_island != -1 && dest_island != v) + if (dest_island != -1 && dest_island != v) { + pr_warn("%s: dest_island mismatch: current (%d) != decoded (%d)\n", + __func__, dest_island, v); return -EINVAL; + } /* If dest_island was -1, we don't care where it goes. */ return 0; @@ -493,7 +498,7 @@ static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt, * the address but we can verify if the existing * contents will point to a valid island. */ - return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island, + return nfp_encode_basic_qdr(*addr, dest_island, cpp_tgt, mode, addr40, isld1, isld0); iid_lsb = addr40 ? 34 : 26; @@ -504,7 +509,7 @@ static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt, return 0; case 1: if (cpp_tgt == NFP_CPP_TARGET_QDR && !addr40) - return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island, + return nfp_encode_basic_qdr(*addr, dest_island, cpp_tgt, mode, addr40, isld1, isld0); idx_lsb = addr40 ? 39 : 31; @@ -530,7 +535,7 @@ static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt, * be set before hand and with them select an island. * So we need to confirm that it's at least plausible. */ - return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island, + return nfp_encode_basic_qdr(*addr, dest_island, cpp_tgt, mode, addr40, isld1, isld0); /* Make sure we compare against isldN values @@ -551,7 +556,7 @@ static int nfp_encode_basic(u64 *addr, int dest_island, int cpp_tgt, * iid<1> = addr<30> = channel<0> * channel<1> = addr<31> = Index */ - return nfp_encode_basic_qdr(*addr, cpp_tgt, dest_island, + return nfp_encode_basic_qdr(*addr, dest_island, cpp_tgt, mode, addr40, isld1, isld0); isld[0] &= ~3; diff --git a/drivers/net/ethernet/sfc/efx_devlink.c b/drivers/net/ethernet/sfc/efx_devlink.c index d842c60dfc100d..e5c6f81af48be9 100644 --- a/drivers/net/ethernet/sfc/efx_devlink.c +++ b/drivers/net/ethernet/sfc/efx_devlink.c @@ -531,7 +531,7 @@ static int efx_devlink_info_running_versions(struct efx_nic *efx, if (rc || outlength < MC_CMD_GET_VERSION_OUT_LEN) { netif_err(efx, drv, efx->net_dev, "mcdi MC_CMD_GET_VERSION failed\n"); - return rc; + return rc ?: -EIO; } /* Handle previous output */ diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c index 37f9417c7c0e04..fc04a23342cfc7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c @@ -47,7 +47,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, while (len != 0) { tx_q->tx_skbuff[entry] = NULL; - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); desc = tx_q->dma_tx + entry; if (len > bmax) { diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index d26e8a06302243..bd0fb5142d663d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -63,7 +63,7 @@ static inline bool dwmac_is_xmac(enum dwmac_core_type core_type) #define DMA_MIN_RX_SIZE 64 #define DMA_MAX_RX_SIZE 1024 #define DMA_DEFAULT_RX_SIZE 512 -#define STMMAC_GET_ENTRY(x, size) ((x + 1) & (size - 1)) +#define STMMAC_NEXT_ENTRY(x, size) ((x + 1) & (size - 1)) #undef FRAME_FILTER_DEBUG /* #define FRAME_FILTER_DEBUG */ diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c index 382d94a3b97209..78fc6aa5bbe954 100644 --- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c @@ -51,7 +51,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_RING_MODE, 0, false, skb->len); tx_q->tx_skbuff[entry] = NULL; - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); if (priv->extend_desc) desc = (struct dma_desc *)(tx_q->dma_etx + entry); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 13d3cac056bea5..81a6ab19a45bbc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -2744,7 +2744,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) xsk_tx_metadata_to_compl(meta, &tx_q->tx_skbuff_dma[entry].xsk_meta); - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); entry = tx_q->cur_tx; } u64_stats_update_begin(&txq_stats->napi_syncp); @@ -2915,7 +2915,7 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, stmmac_release_tx_desc(priv, p, priv->mode); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); } tx_q->dirty_tx = entry; @@ -4258,7 +4258,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, return false; stmmac_set_tx_owner(priv, p); - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); return true; } @@ -4286,7 +4286,7 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, while (tmp_len > 0) { dma_addr_t curr_addr; - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); @@ -4437,7 +4437,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_set_mss(priv, mss_desc, mss); tx_q->mss = mss; - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); } @@ -4541,7 +4541,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) * ndo_start_xmit will fill this descriptor the next time it's * called and stmmac_tx_clean may clean up to this descriptor. */ - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", @@ -4751,7 +4751,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) int len = skb_frag_size(frag); bool last_segment = (i == (nfrags - 1)); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[entry]); if (likely(priv->extend_desc)) @@ -4821,7 +4821,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) * ndo_start_xmit will fill this descriptor the next time it's * called and stmmac_tx_clean may clean up to this descriptor. */ - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); tx_q->cur_tx = entry; if (netif_msg_pktdata(priv)) { @@ -4990,7 +4990,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) dma_wmb(); stmmac_set_rx_owner(priv, p, use_rx_wd); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); } rx_q->dirty_rx = entry; rx_q->rx_tail_addr = rx_q->dma_rx_phy + @@ -5140,7 +5140,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); tx_q->cur_tx = entry; return STMMAC_XDP_TX; @@ -5374,7 +5374,7 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) dma_wmb(); stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); } if (rx_desc) { @@ -5469,9 +5469,12 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) break; /* Prefetch the next RX descriptor */ - rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, - priv->dma_conf.dma_rx_size); - next_entry = rx_q->cur_rx; + next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, + priv->dma_conf.dma_rx_size); + if (unlikely(next_entry == rx_q->dirty_rx)) + break; + + rx_q->cur_rx = next_entry; if (priv->extend_desc) np = (struct dma_desc *)(rx_q->dma_erx + next_entry); @@ -5609,7 +5612,6 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) dma_dir = page_pool_get_dma_dir(rx_q->page_pool); bufsz = DIV_ROUND_UP(priv->dma_conf.dma_buf_sz, PAGE_SIZE) * PAGE_SIZE; - limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); if (netif_msg_rx_status(priv)) { void *rx_head; @@ -5665,9 +5667,12 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) if (unlikely(status & dma_own)) break; - rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, - priv->dma_conf.dma_rx_size); - next_entry = rx_q->cur_rx; + next_entry = STMMAC_NEXT_ENTRY(rx_q->cur_rx, + priv->dma_conf.dma_rx_size); + if (unlikely(next_entry == rx_q->dirty_rx)) + break; + + rx_q->cur_rx = next_entry; if (priv->extend_desc) np = (struct dma_desc *)(rx_q->dma_erx + next_entry); diff --git a/drivers/net/ethernet/ti/Makefile b/drivers/net/ethernet/ti/Makefile index 6da50f4b7c2eae..f4276c9a776205 100644 --- a/drivers/net/ethernet/ti/Makefile +++ b/drivers/net/ethernet/ti/Makefile @@ -6,30 +6,30 @@ obj-$(CONFIG_TI_PRUETH) += icssm-prueth.o icssm-prueth-y := icssm/icssm_prueth.o icssm/icssm_prueth_switch.o icssm/icssm_switchdev.o -obj-$(CONFIG_TI_CPSW) += cpsw-common.o -obj-$(CONFIG_TI_DAVINCI_EMAC) += cpsw-common.o -obj-$(CONFIG_TI_CPSW_SWITCHDEV) += cpsw-common.o +ti-cpsw-common-y += cpsw-common.o davinci_cpdma.o +ti-cpsw-priv-y += cpsw_priv.o cpsw_ethtool.o +ti-cpsw-ale-y += cpsw_ale.o +ti-cpsw-sl-y += cpsw_sl.o obj-$(CONFIG_TLAN) += tlan.o -obj-$(CONFIG_TI_DAVINCI_EMAC) += ti_davinci_emac.o -ti_davinci_emac-y := davinci_emac.o davinci_cpdma.o +obj-$(CONFIG_TI_DAVINCI_EMAC) += davinci_emac.o ti-cpsw-common.o obj-$(CONFIG_TI_DAVINCI_MDIO) += davinci_mdio.o obj-$(CONFIG_TI_CPSW_PHY_SEL) += cpsw-phy-sel.o obj-$(CONFIG_TI_CPTS) += cpts.o -obj-$(CONFIG_TI_CPSW) += ti_cpsw.o -ti_cpsw-y := cpsw.o davinci_cpdma.o cpsw_ale.o cpsw_priv.o cpsw_sl.o cpsw_ethtool.o -obj-$(CONFIG_TI_CPSW_SWITCHDEV) += ti_cpsw_new.o -ti_cpsw_new-y := cpsw_switchdev.o cpsw_new.o davinci_cpdma.o cpsw_ale.o cpsw_sl.o cpsw_priv.o cpsw_ethtool.o +obj-$(CONFIG_TI_CPSW) += ti_cpsw.o ti-cpsw-common.o ti-cpsw-priv.o ti-cpsw-ale.o ti-cpsw-sl.o +ti_cpsw-y := cpsw.o +obj-$(CONFIG_TI_CPSW_SWITCHDEV) += ti_cpsw_new.o ti-cpsw-common.o ti-cpsw-priv.o ti-cpsw-ale.o ti-cpsw-sl.o +ti_cpsw_new-y := cpsw_switchdev.o cpsw_new.o -obj-$(CONFIG_TI_KEYSTONE_NETCP) += keystone_netcp.o -keystone_netcp-y := netcp_core.o cpsw_ale.o -obj-$(CONFIG_TI_KEYSTONE_NETCP_ETHSS) += keystone_netcp_ethss.o -keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o netcp_xgbepcsr.o cpsw_ale.o +obj-$(CONFIG_TI_KEYSTONE_NETCP) += keystone_netcp.o ti-cpsw-ale.o +keystone_netcp-y := netcp_core.o +obj-$(CONFIG_TI_KEYSTONE_NETCP_ETHSS) += keystone_netcp_ethss.o ti-cpsw-ale.o +keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o netcp_xgbepcsr.o obj-$(CONFIG_TI_K3_CPPI_DESC_POOL) += k3-cppi-desc-pool.o -obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o -ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o cpsw_ale.o +obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o ti-cpsw-sl.o ti-cpsw-ale.o +ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o am65-cpsw-ethtool.o ti-am65-cpsw-nuss-$(CONFIG_TI_AM65_CPSW_QOS) += am65-cpsw-qos.o ti-am65-cpsw-nuss-$(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV) += am65-cpsw-switchdev.o obj-$(CONFIG_TI_K3_AM65_CPTS) += am65-cpts.o diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index b0e18bdc2c8510..aa3531e844e871 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -706,7 +706,7 @@ static void cpsw_init_host_port(struct cpsw_priv *priv) struct cpsw_common *cpsw = priv->cpsw; /* soft reset the controller and initialize ale */ - soft_reset("cpsw", &cpsw->regs->soft_reset); + cpsw_soft_reset("cpsw", &cpsw->regs->soft_reset); cpsw_ale_start(cpsw->ale); /* switch to vlan aware mode */ diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index be7b6931922166..e202bba4948071 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -493,6 +493,7 @@ int cpsw_ale_flush_multicast(struct cpsw_ale *ale, int port_mask, int vid) } return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_flush_multicast); static inline void cpsw_ale_set_vlan_entry_type(u32 *ale_entry, int flags, u16 vid) @@ -530,6 +531,7 @@ int cpsw_ale_add_ucast(struct cpsw_ale *ale, const u8 *addr, int port, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_add_ucast); int cpsw_ale_del_ucast(struct cpsw_ale *ale, const u8 *addr, int port, int flags, u16 vid) @@ -545,6 +547,7 @@ int cpsw_ale_del_ucast(struct cpsw_ale *ale, const u8 *addr, int port, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_del_ucast); int cpsw_ale_add_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, int flags, u16 vid, int mcast_state) @@ -578,6 +581,7 @@ int cpsw_ale_add_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_add_mcast); int cpsw_ale_del_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, int flags, u16 vid) @@ -607,6 +611,7 @@ int cpsw_ale_del_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_del_mcast); /* ALE NetCP NU switch specific vlan functions */ static void cpsw_ale_set_vlan_mcast(struct cpsw_ale *ale, u32 *ale_entry, @@ -676,6 +681,7 @@ int cpsw_ale_add_vlan(struct cpsw_ale *ale, u16 vid, int port_mask, int untag, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_add_vlan); static void cpsw_ale_vlan_del_modify_int(struct cpsw_ale *ale, u32 *ale_entry, u16 vid, int port_mask) @@ -733,6 +739,7 @@ int cpsw_ale_vlan_del_modify(struct cpsw_ale *ale, u16 vid, int port_mask) return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_vlan_del_modify); int cpsw_ale_del_vlan(struct cpsw_ale *ale, u16 vid, int port_mask) { @@ -767,6 +774,7 @@ int cpsw_ale_del_vlan(struct cpsw_ale *ale, u16 vid, int port_mask) return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_del_vlan); int cpsw_ale_vlan_add_modify(struct cpsw_ale *ale, u16 vid, int port_mask, int untag_mask, int reg_mask, int unreg_mask) @@ -806,6 +814,7 @@ int cpsw_ale_vlan_add_modify(struct cpsw_ale *ale, u16 vid, int port_mask, return ret; } +EXPORT_SYMBOL_GPL(cpsw_ale_vlan_add_modify); void cpsw_ale_set_unreg_mcast(struct cpsw_ale *ale, int unreg_mcast_mask, bool add) @@ -833,6 +842,7 @@ void cpsw_ale_set_unreg_mcast(struct cpsw_ale *ale, int unreg_mcast_mask, cpsw_ale_write(ale, idx, ale_entry); } } +EXPORT_SYMBOL_GPL(cpsw_ale_set_unreg_mcast); static void cpsw_ale_vlan_set_unreg_mcast(struct cpsw_ale *ale, u32 *ale_entry, int allmulti) @@ -898,6 +908,7 @@ void cpsw_ale_set_allmulti(struct cpsw_ale *ale, int allmulti, int port) cpsw_ale_write(ale, idx, ale_entry); } } +EXPORT_SYMBOL_GPL(cpsw_ale_set_allmulti); struct ale_control_info { const char *name; @@ -1155,6 +1166,7 @@ int cpsw_ale_control_set(struct cpsw_ale *ale, int port, int control, return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_control_set); int cpsw_ale_control_get(struct cpsw_ale *ale, int port, int control) { @@ -1178,6 +1190,7 @@ int cpsw_ale_control_get(struct cpsw_ale *ale, int port, int control) tmp = readl_relaxed(ale->params.ale_regs + offset) >> shift; return tmp & BITMASK(info->bits); } +EXPORT_SYMBOL_GPL(cpsw_ale_control_get); int cpsw_ale_rx_ratelimit_mc(struct cpsw_ale *ale, int port, unsigned int ratelimit_pps) @@ -1200,6 +1213,7 @@ int cpsw_ale_rx_ratelimit_mc(struct cpsw_ale *ale, int port, unsigned int rateli port, val * ALE_RATE_LIMIT_MIN_PPS); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_rx_ratelimit_mc); int cpsw_ale_rx_ratelimit_bc(struct cpsw_ale *ale, int port, unsigned int ratelimit_pps) @@ -1222,6 +1236,7 @@ int cpsw_ale_rx_ratelimit_bc(struct cpsw_ale *ale, int port, unsigned int rateli port, val * ALE_RATE_LIMIT_MIN_PPS); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_rx_ratelimit_bc); static void cpsw_ale_timer(struct timer_list *t) { @@ -1311,6 +1326,7 @@ void cpsw_ale_start(struct cpsw_ale *ale) cpsw_ale_aging_start(ale); } +EXPORT_SYMBOL_GPL(cpsw_ale_start); void cpsw_ale_stop(struct cpsw_ale *ale) { @@ -1318,6 +1334,7 @@ void cpsw_ale_stop(struct cpsw_ale *ale) cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1); cpsw_ale_control_set(ale, 0, ALE_ENABLE, 0); } +EXPORT_SYMBOL_GPL(cpsw_ale_stop); static const struct reg_field ale_fields_cpsw[] = { /* CPSW_ALE_IDVER_REG */ @@ -1618,6 +1635,7 @@ struct cpsw_ale *cpsw_ale_create(struct cpsw_ale_params *params) cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1); return ale; } +EXPORT_SYMBOL_GPL(cpsw_ale_create); void cpsw_ale_dump(struct cpsw_ale *ale, u32 *data) { @@ -1628,6 +1646,7 @@ void cpsw_ale_dump(struct cpsw_ale *ale, u32 *data) data += ALE_ENTRY_WORDS; } } +EXPORT_SYMBOL_GPL(cpsw_ale_dump); void cpsw_ale_restore(struct cpsw_ale *ale, u32 *data) { @@ -1638,11 +1657,13 @@ void cpsw_ale_restore(struct cpsw_ale *ale, u32 *data) data += ALE_ENTRY_WORDS; } } +EXPORT_SYMBOL_GPL(cpsw_ale_restore); u32 cpsw_ale_get_num_entries(struct cpsw_ale *ale) { return ale ? ale->params.ale_entries : 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_get_num_entries); /* Reads the specified policer index into ALE POLICER registers */ static void cpsw_ale_policer_read_idx(struct cpsw_ale *ale, u32 idx) @@ -1745,3 +1766,7 @@ void cpsw_ale_classifier_setup_default(struct cpsw_ale *ale, int num_rx_ch) 1); } } +EXPORT_SYMBOL_GPL(cpsw_ale_classifier_setup_default); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TI N-Port Ethernet Switch Address Lookup Engine"); diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c index a43f75ee269edf..3f2682c461f9c7 100644 --- a/drivers/net/ethernet/ti/cpsw_ethtool.c +++ b/drivers/net/ethernet/ti/cpsw_ethtool.c @@ -144,6 +144,7 @@ u32 cpsw_get_msglevel(struct net_device *ndev) return priv->msg_enable; } +EXPORT_SYMBOL_GPL(cpsw_get_msglevel); void cpsw_set_msglevel(struct net_device *ndev, u32 value) { @@ -151,6 +152,7 @@ void cpsw_set_msglevel(struct net_device *ndev, u32 value) priv->msg_enable = value; } +EXPORT_SYMBOL_GPL(cpsw_set_msglevel); int cpsw_get_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, struct kernel_ethtool_coalesce *kernel_coal, @@ -161,6 +163,7 @@ int cpsw_get_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, coal->rx_coalesce_usecs = cpsw->coal_intvl; return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_coalesce); int cpsw_set_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, struct kernel_ethtool_coalesce *kernel_coal, @@ -220,6 +223,7 @@ int cpsw_set_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, return 0; } +EXPORT_SYMBOL_GPL(cpsw_set_coalesce); int cpsw_get_sset_count(struct net_device *ndev, int sset) { @@ -234,6 +238,7 @@ int cpsw_get_sset_count(struct net_device *ndev, int sset) return -EOPNOTSUPP; } } +EXPORT_SYMBOL_GPL(cpsw_get_sset_count); static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir) { @@ -271,6 +276,7 @@ void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data) break; } } +EXPORT_SYMBOL_GPL(cpsw_get_strings); void cpsw_get_ethtool_stats(struct net_device *ndev, struct ethtool_stats *stats, u64 *data) @@ -303,6 +309,7 @@ void cpsw_get_ethtool_stats(struct net_device *ndev, } } } +EXPORT_SYMBOL_GPL(cpsw_get_ethtool_stats); void cpsw_get_pauseparam(struct net_device *ndev, struct ethtool_pauseparam *pause) @@ -313,6 +320,7 @@ void cpsw_get_pauseparam(struct net_device *ndev, pause->rx_pause = priv->rx_pause ? true : false; pause->tx_pause = priv->tx_pause ? true : false; } +EXPORT_SYMBOL_GPL(cpsw_get_pauseparam); void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) { @@ -326,6 +334,7 @@ void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) if (cpsw->slaves[slave_no].phy) phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol); } +EXPORT_SYMBOL_GPL(cpsw_get_wol); int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) { @@ -338,6 +347,7 @@ int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) else return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_set_wol); int cpsw_get_regs_len(struct net_device *ndev) { @@ -346,6 +356,7 @@ int cpsw_get_regs_len(struct net_device *ndev) return cpsw_ale_get_num_entries(cpsw->ale) * ALE_ENTRY_WORDS * sizeof(u32); } +EXPORT_SYMBOL_GPL(cpsw_get_regs_len); void cpsw_get_regs(struct net_device *ndev, struct ethtool_regs *regs, void *p) { @@ -357,6 +368,7 @@ void cpsw_get_regs(struct net_device *ndev, struct ethtool_regs *regs, void *p) cpsw_ale_dump(cpsw->ale, reg); } +EXPORT_SYMBOL_GPL(cpsw_get_regs); int cpsw_ethtool_op_begin(struct net_device *ndev) { @@ -370,6 +382,7 @@ int cpsw_ethtool_op_begin(struct net_device *ndev) return ret; } +EXPORT_SYMBOL_GPL(cpsw_ethtool_op_begin); void cpsw_ethtool_op_complete(struct net_device *ndev) { @@ -377,6 +390,7 @@ void cpsw_ethtool_op_complete(struct net_device *ndev) pm_runtime_put(priv->cpsw->dev); } +EXPORT_SYMBOL_GPL(cpsw_ethtool_op_complete); void cpsw_get_channels(struct net_device *ndev, struct ethtool_channels *ch) { @@ -391,6 +405,7 @@ void cpsw_get_channels(struct net_device *ndev, struct ethtool_channels *ch) ch->tx_count = cpsw->tx_ch_num; ch->combined_count = 0; } +EXPORT_SYMBOL_GPL(cpsw_get_channels); int cpsw_get_link_ksettings(struct net_device *ndev, struct ethtool_link_ksettings *ecmd) @@ -405,6 +420,7 @@ int cpsw_get_link_ksettings(struct net_device *ndev, phy_ethtool_ksettings_get(cpsw->slaves[slave_no].phy, ecmd); return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_link_ksettings); int cpsw_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *ecmd) @@ -418,6 +434,7 @@ int cpsw_set_link_ksettings(struct net_device *ndev, return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy, ecmd); } +EXPORT_SYMBOL_GPL(cpsw_set_link_ksettings); int cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { @@ -430,6 +447,7 @@ int cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata) else return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_get_eee); int cpsw_nway_reset(struct net_device *ndev) { @@ -442,6 +460,7 @@ int cpsw_nway_reset(struct net_device *ndev) else return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_nway_reset); static void cpsw_suspend_data_pass(struct net_device *ndev) { @@ -639,6 +658,7 @@ int cpsw_set_channels_common(struct net_device *ndev, cpsw_fail(cpsw); return ret; } +EXPORT_SYMBOL_GPL(cpsw_set_channels_common); void cpsw_get_ringparam(struct net_device *ndev, struct ethtool_ringparam *ering, @@ -654,6 +674,7 @@ void cpsw_get_ringparam(struct net_device *ndev, ering->rx_max_pending = cpsw->descs_pool_size - CPSW_MAX_QUEUES; ering->rx_pending = cpdma_get_num_rx_descs(cpsw->dma); } +EXPORT_SYMBOL_GPL(cpsw_get_ringparam); int cpsw_set_ringparam(struct net_device *ndev, struct ethtool_ringparam *ering, @@ -700,6 +721,7 @@ int cpsw_set_ringparam(struct net_device *ndev, cpsw_fail(cpsw); return ret; } +EXPORT_SYMBOL_GPL(cpsw_set_ringparam); #if IS_ENABLED(CONFIG_TI_CPTS) int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *info) @@ -720,6 +742,7 @@ int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *inf (1 << HWTSTAMP_FILTER_PTP_V2_EVENT); return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_ts_info); #else int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *info) { @@ -729,4 +752,5 @@ int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *inf info->rx_filters = 0; return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_ts_info); #endif diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 7f42f58a4b031f..c5be359f3c663e 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -573,7 +573,7 @@ static void cpsw_init_host_port(struct cpsw_priv *priv) u32 control_reg; /* soft reset the controller and initialize ale */ - soft_reset("cpsw", &cpsw->regs->soft_reset); + cpsw_soft_reset("cpsw", &cpsw->regs->soft_reset); cpsw_ale_start(cpsw->ale); /* switch to vlan aware mode */ diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index bc4fdf17a99ec0..1f6f374551cb6e 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -32,6 +32,7 @@ #define CPTS_N_ETX_TS 4 int (*cpsw_slave_index)(struct cpsw_common *cpsw, struct cpsw_priv *priv); +EXPORT_SYMBOL_GPL(cpsw_slave_index); void cpsw_intr_enable(struct cpsw_common *cpsw) { @@ -40,6 +41,7 @@ void cpsw_intr_enable(struct cpsw_common *cpsw) cpdma_ctlr_int_ctrl(cpsw->dma, true); } +EXPORT_SYMBOL_GPL(cpsw_intr_enable); void cpsw_intr_disable(struct cpsw_common *cpsw) { @@ -48,6 +50,7 @@ void cpsw_intr_disable(struct cpsw_common *cpsw) cpdma_ctlr_int_ctrl(cpsw->dma, false); } +EXPORT_SYMBOL_GPL(cpsw_intr_disable); void cpsw_tx_handler(void *token, int len, int status) { @@ -82,6 +85,7 @@ void cpsw_tx_handler(void *token, int len, int status) ndev->stats.tx_packets++; ndev->stats.tx_bytes += len; } +EXPORT_SYMBOL_GPL(cpsw_tx_handler); irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id) { @@ -98,6 +102,7 @@ irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id) napi_schedule(&cpsw->napi_tx); return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(cpsw_tx_interrupt); irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id) { @@ -114,6 +119,7 @@ irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id) napi_schedule(&cpsw->napi_rx); return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(cpsw_rx_interrupt); irqreturn_t cpsw_misc_interrupt(int irq, void *dev_id) { @@ -126,6 +132,7 @@ irqreturn_t cpsw_misc_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(cpsw_misc_interrupt); int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget) { @@ -158,6 +165,7 @@ int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget) return num_tx; } +EXPORT_SYMBOL_GPL(cpsw_tx_mq_poll); int cpsw_tx_poll(struct napi_struct *napi_tx, int budget) { @@ -176,6 +184,7 @@ int cpsw_tx_poll(struct napi_struct *napi_tx, int budget) return num_tx; } +EXPORT_SYMBOL_GPL(cpsw_tx_poll); int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget) { @@ -208,6 +217,7 @@ int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget) return num_rx; } +EXPORT_SYMBOL_GPL(cpsw_rx_mq_poll); int cpsw_rx_poll(struct napi_struct *napi_rx, int budget) { @@ -226,6 +236,7 @@ int cpsw_rx_poll(struct napi_struct *napi_rx, int budget) return num_rx; } +EXPORT_SYMBOL_GPL(cpsw_rx_poll); void cpsw_rx_vlan_encap(struct sk_buff *skb) { @@ -268,14 +279,16 @@ void cpsw_rx_vlan_encap(struct sk_buff *skb) skb_pull(skb, VLAN_HLEN); } } +EXPORT_SYMBOL_GPL(cpsw_rx_vlan_encap); void cpsw_set_slave_mac(struct cpsw_slave *slave, struct cpsw_priv *priv) { slave_write(slave, mac_hi(priv->mac_addr), SA_HI); slave_write(slave, mac_lo(priv->mac_addr), SA_LO); } +EXPORT_SYMBOL_GPL(cpsw_set_slave_mac); -void soft_reset(const char *module, void __iomem *reg) +void cpsw_soft_reset(const char *module, void __iomem *reg) { unsigned long timeout = jiffies + HZ; @@ -286,6 +299,7 @@ void soft_reset(const char *module, void __iomem *reg) WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module); } +EXPORT_SYMBOL_GPL(cpsw_soft_reset); void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue) { @@ -305,6 +319,7 @@ void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue) netif_trans_update(ndev); netif_tx_wake_all_queues(ndev); } +EXPORT_SYMBOL_GPL(cpsw_ndo_tx_timeout); static int cpsw_get_common_speed(struct cpsw_common *cpsw) { @@ -343,6 +358,7 @@ int cpsw_need_resplit(struct cpsw_common *cpsw) return 1; } +EXPORT_SYMBOL_GPL(cpsw_need_resplit); void cpsw_split_res(struct cpsw_common *cpsw) { @@ -428,6 +444,7 @@ void cpsw_split_res(struct cpsw_common *cpsw) if (budget) cpsw->rxv[0].budget += budget; } +EXPORT_SYMBOL_GPL(cpsw_split_res); int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs, int ale_ageout, phys_addr_t desc_mem_phys, @@ -548,6 +565,7 @@ int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs, return ret; } +EXPORT_SYMBOL_GPL(cpsw_init_common); #if IS_ENABLED(CONFIG_TI_CPTS) @@ -678,6 +696,7 @@ int cpsw_hwtstamp_set(struct net_device *dev, return 0; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_set); int cpsw_hwtstamp_get(struct net_device *dev, struct kernel_hwtstamp_config *cfg) @@ -695,12 +714,14 @@ int cpsw_hwtstamp_get(struct net_device *dev, return 0; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_get); #else int cpsw_hwtstamp_get(struct net_device *dev, struct kernel_hwtstamp_config *cfg) { return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_set); int cpsw_hwtstamp_set(struct net_device *dev, struct kernel_hwtstamp_config *cfg, @@ -708,6 +729,7 @@ int cpsw_hwtstamp_set(struct net_device *dev, { return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_get); #endif /*CONFIG_TI_CPTS*/ int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate) @@ -758,6 +780,7 @@ int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate) cpsw_split_res(cpsw); return ret; } +EXPORT_SYMBOL_GPL(cpsw_ndo_set_tx_maxrate); static int cpsw_tc_to_fifo(int tc, int num_tc) { @@ -782,6 +805,7 @@ bool cpsw_shp_is_off(struct cpsw_priv *priv) return !val; } +EXPORT_SYMBOL_GPL(cpsw_shp_is_off); static void cpsw_fifo_shp_on(struct cpsw_priv *priv, int fifo, int on) { @@ -1043,6 +1067,7 @@ int cpsw_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type, return -EOPNOTSUPP; } } +EXPORT_SYMBOL_GPL(cpsw_ndo_setup_tc); void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) { @@ -1056,6 +1081,7 @@ void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) cpsw_set_fifo_rlimit(priv, fifo, bw); } } +EXPORT_SYMBOL_GPL(cpsw_cbs_resume); void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) { @@ -1078,6 +1104,7 @@ void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) slave_write(slave, tx_prio_map, tx_prio_rg); } +EXPORT_SYMBOL_GPL(cpsw_mqprio_resume); int cpsw_fill_rx_channels(struct cpsw_priv *priv) { @@ -1123,6 +1150,7 @@ int cpsw_fill_rx_channels(struct cpsw_priv *priv) return 0; } +EXPORT_SYMBOL_GPL(cpsw_fill_rx_channels); static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw, int size) @@ -1208,6 +1236,7 @@ void cpsw_destroy_xdp_rxqs(struct cpsw_common *cpsw) cpsw->page_pool[ch] = NULL; } } +EXPORT_SYMBOL_GPL(cpsw_destroy_xdp_rxqs); int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw) { @@ -1240,6 +1269,7 @@ int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw) return ret; } +EXPORT_SYMBOL_GPL(cpsw_create_xdp_rxqs); static int cpsw_xdp_prog_setup(struct cpsw_priv *priv, struct netdev_bpf *bpf) { @@ -1267,6 +1297,7 @@ int cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf) return -EINVAL; } } +EXPORT_SYMBOL_GPL(cpsw_ndo_bpf); int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, struct page *page, int port) @@ -1300,6 +1331,7 @@ int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, return ret; } +EXPORT_SYMBOL_GPL(cpsw_xdp_tx_frame); int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, struct page *page, int port, int *len) @@ -1362,6 +1394,7 @@ int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, page_pool_recycle_direct(cpsw->page_pool[ch], page); return ret; } +EXPORT_SYMBOL_GPL(cpsw_run_xdp); static int cpsw_qos_clsflower_add_policer(struct cpsw_priv *priv, struct netlink_ext_ack *extack, @@ -1564,3 +1597,7 @@ void cpsw_qos_clsflower_resume(struct cpsw_priv *priv) cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id, priv->ale_mc_ratelimit.rate_packet_ps); } +EXPORT_SYMBOL_GPL(cpsw_qos_clsflower_resume); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TI CPSW Ethernet Switch Driver"); diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h index acb6181c5c9e1b..fddd7a79f4b0fb 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.h +++ b/drivers/net/ethernet/ti/cpsw_priv.h @@ -458,7 +458,7 @@ int cpsw_tx_poll(struct napi_struct *napi_tx, int budget); int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget); int cpsw_rx_poll(struct napi_struct *napi_rx, int budget); void cpsw_rx_vlan_encap(struct sk_buff *skb); -void soft_reset(const char *module, void __iomem *reg); +void cpsw_soft_reset(const char *module, void __iomem *reg); void cpsw_set_slave_mac(struct cpsw_slave *slave, struct cpsw_priv *priv); void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue); int cpsw_need_resplit(struct cpsw_common *cpsw); diff --git a/drivers/net/ethernet/ti/cpsw_sl.c b/drivers/net/ethernet/ti/cpsw_sl.c index 0c7531cb0f3982..761719a348fa5f 100644 --- a/drivers/net/ethernet/ti/cpsw_sl.c +++ b/drivers/net/ethernet/ti/cpsw_sl.c @@ -200,6 +200,7 @@ u32 cpsw_sl_reg_read(struct cpsw_sl *sl, enum cpsw_sl_regs reg) dev_dbg(sl->dev, "cpsw_sl: reg: %04X r 0x%08X\n", sl->regs[reg], val); return val; } +EXPORT_SYMBOL_GPL(cpsw_sl_reg_read); void cpsw_sl_reg_write(struct cpsw_sl *sl, enum cpsw_sl_regs reg, u32 val) { @@ -212,6 +213,7 @@ void cpsw_sl_reg_write(struct cpsw_sl *sl, enum cpsw_sl_regs reg, u32 val) dev_dbg(sl->dev, "cpsw_sl: reg: %04X w 0x%08X\n", sl->regs[reg], val); writel(val, sl->sl_base + sl->regs[reg]); } +EXPORT_SYMBOL_GPL(cpsw_sl_reg_write); static const struct cpsw_sl_dev_id *cpsw_sl_match_id( const struct cpsw_sl_dev_id *id, @@ -252,6 +254,7 @@ struct cpsw_sl *cpsw_sl_get(const char *device_id, struct device *dev, return sl; } +EXPORT_SYMBOL_GPL(cpsw_sl_get); void cpsw_sl_reset(struct cpsw_sl *sl, unsigned long tmo) { @@ -270,6 +273,7 @@ void cpsw_sl_reset(struct cpsw_sl *sl, unsigned long tmo) if (cpsw_sl_reg_read(sl, CPSW_SL_SOFT_RESET) & CPSW_SL_SOFT_RESET_BIT) dev_err(sl->dev, "cpsw_sl failed to soft-reset.\n"); } +EXPORT_SYMBOL_GPL(cpsw_sl_reset); u32 cpsw_sl_ctl_set(struct cpsw_sl *sl, u32 ctl_funcs) { @@ -287,6 +291,7 @@ u32 cpsw_sl_ctl_set(struct cpsw_sl *sl, u32 ctl_funcs) return 0; } +EXPORT_SYMBOL_GPL(cpsw_sl_ctl_set); u32 cpsw_sl_ctl_clr(struct cpsw_sl *sl, u32 ctl_funcs) { @@ -304,11 +309,13 @@ u32 cpsw_sl_ctl_clr(struct cpsw_sl *sl, u32 ctl_funcs) return 0; } +EXPORT_SYMBOL_GPL(cpsw_sl_ctl_clr); void cpsw_sl_ctl_reset(struct cpsw_sl *sl) { cpsw_sl_reg_write(sl, CPSW_SL_MACCONTROL, 0); } +EXPORT_SYMBOL_GPL(cpsw_sl_ctl_reset); int cpsw_sl_wait_for_idle(struct cpsw_sl *sl, unsigned long tmo) { @@ -326,3 +333,7 @@ int cpsw_sl_wait_for_idle(struct cpsw_sl *sl, unsigned long tmo) return 0; } +EXPORT_SYMBOL_GPL(cpsw_sl_wait_for_idle); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TI Ethernet Switch media-access-controller (MAC) submodule"); diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c index d2eab5cd1e0c9f..41e89a19be5378 100644 --- a/drivers/net/ethernet/ti/davinci_cpdma.c +++ b/drivers/net/ethernet/ti/davinci_cpdma.c @@ -531,6 +531,7 @@ struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params) ctlr->num_chan = CPDMA_MAX_CHANNELS; return ctlr; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_create); int cpdma_ctlr_start(struct cpdma_ctlr *ctlr) { @@ -591,6 +592,7 @@ int cpdma_ctlr_start(struct cpdma_ctlr *ctlr) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_start); int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr) { @@ -623,6 +625,7 @@ int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_stop); int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr) { @@ -640,6 +643,7 @@ int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr) cpdma_desc_pool_destroy(ctlr); return ret; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_destroy); int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable) { @@ -660,21 +664,25 @@ int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_int_ctrl); void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value) { dma_reg_write(ctlr, CPDMA_MACEOIVECTOR, value); } +EXPORT_SYMBOL_GPL(cpdma_ctlr_eoi); u32 cpdma_ctrl_rxchs_state(struct cpdma_ctlr *ctlr) { return dma_reg_read(ctlr, CPDMA_RXINTSTATMASKED); } +EXPORT_SYMBOL_GPL(cpdma_ctrl_rxchs_state); u32 cpdma_ctrl_txchs_state(struct cpdma_ctlr *ctlr) { return dma_reg_read(ctlr, CPDMA_TXINTSTATMASKED); } +EXPORT_SYMBOL_GPL(cpdma_ctrl_txchs_state); static void cpdma_chan_set_descs(struct cpdma_ctlr *ctlr, int rx, int desc_num, @@ -802,6 +810,7 @@ int cpdma_chan_set_weight(struct cpdma_chan *ch, int weight) spin_unlock_irqrestore(&ctlr->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_set_weight); /* cpdma_chan_get_min_rate - get minimum allowed rate for channel * Should be called before cpdma_chan_set_rate. @@ -816,6 +825,7 @@ u32 cpdma_chan_get_min_rate(struct cpdma_ctlr *ctlr) return DIV_ROUND_UP(divident, divisor); } +EXPORT_SYMBOL_GPL(cpdma_chan_get_min_rate); /* cpdma_chan_set_rate - limits bandwidth for transmit channel. * The bandwidth * limited channels have to be in order beginning from lowest. @@ -860,6 +870,7 @@ int cpdma_chan_set_rate(struct cpdma_chan *ch, u32 rate) spin_unlock_irqrestore(&ctlr->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_set_rate); u32 cpdma_chan_get_rate(struct cpdma_chan *ch) { @@ -872,6 +883,7 @@ u32 cpdma_chan_get_rate(struct cpdma_chan *ch) return rate; } +EXPORT_SYMBOL_GPL(cpdma_chan_get_rate); struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num, cpdma_handler_fn handler, int rx_type) @@ -931,6 +943,7 @@ struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num, spin_unlock_irqrestore(&ctlr->lock, flags); return chan; } +EXPORT_SYMBOL_GPL(cpdma_chan_create); int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan) { @@ -943,6 +956,7 @@ int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan) return desc_num; } +EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num); int cpdma_chan_destroy(struct cpdma_chan *chan) { @@ -964,6 +978,7 @@ int cpdma_chan_destroy(struct cpdma_chan *chan) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_destroy); int cpdma_chan_get_stats(struct cpdma_chan *chan, struct cpdma_chan_stats *stats) @@ -976,6 +991,7 @@ int cpdma_chan_get_stats(struct cpdma_chan *chan, spin_unlock_irqrestore(&chan->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_get_stats); static void __cpdma_chan_submit(struct cpdma_chan *chan, struct cpdma_desc __iomem *desc) @@ -1100,6 +1116,7 @@ int cpdma_chan_idle_submit(struct cpdma_chan *chan, void *token, void *data, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_idle_submit); int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token, dma_addr_t data, int len, int directed) @@ -1125,6 +1142,7 @@ int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_idle_submit_mapped); int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data, int len, int directed) @@ -1150,6 +1168,7 @@ int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_submit); int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token, dma_addr_t data, int len, int directed) @@ -1175,6 +1194,7 @@ int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_submit_mapped); bool cpdma_check_free_tx_desc(struct cpdma_chan *chan) { @@ -1189,6 +1209,7 @@ bool cpdma_check_free_tx_desc(struct cpdma_chan *chan) spin_unlock_irqrestore(&chan->lock, flags); return free_tx_desc; } +EXPORT_SYMBOL_GPL(cpdma_check_free_tx_desc); static void __cpdma_chan_free(struct cpdma_chan *chan, struct cpdma_desc __iomem *desc, @@ -1289,6 +1310,7 @@ int cpdma_chan_process(struct cpdma_chan *chan, int quota) } return used; } +EXPORT_SYMBOL_GPL(cpdma_chan_process); int cpdma_chan_start(struct cpdma_chan *chan) { @@ -1308,6 +1330,7 @@ int cpdma_chan_start(struct cpdma_chan *chan) return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_start); int cpdma_chan_stop(struct cpdma_chan *chan) { @@ -1370,6 +1393,7 @@ int cpdma_chan_stop(struct cpdma_chan *chan) spin_unlock_irqrestore(&chan->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_stop); int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable) { @@ -1416,11 +1440,13 @@ int cpdma_get_num_rx_descs(struct cpdma_ctlr *ctlr) { return ctlr->num_rx_desc; } +EXPORT_SYMBOL_GPL(cpdma_get_num_rx_descs); int cpdma_get_num_tx_descs(struct cpdma_ctlr *ctlr) { return ctlr->num_tx_desc; } +EXPORT_SYMBOL_GPL(cpdma_get_num_tx_descs); int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc) { @@ -1442,3 +1468,4 @@ int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc) return ret; } +EXPORT_SYMBOL_GPL(cpdma_set_num_rx_descs); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index bee9e245e7927b..5315d5c92d0b21 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -2480,8 +2480,11 @@ int wx_sw_init(struct wx *wx) wx->oem_svid = pdev->subsystem_vendor; wx->oem_ssid = pdev->subsystem_device; wx->bus.device = PCI_SLOT(pdev->devfn); - wx->bus.func = FIELD_GET(WX_CFG_PORT_ST_LANID, - rd32(wx, WX_CFG_PORT_ST)); + if (pdev->is_virtfn) + wx->bus.func = PCI_FUNC(pdev->devfn); + else + wx->bus.func = FIELD_GET(WX_CFG_PORT_ST_LANID, + rd32(wx, WX_CFG_PORT_ST)); if (wx->oem_svid == PCI_VENDOR_ID_WANGXUN || pdev->is_virtfn) { diff --git a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c index ade2bfe563aaac..5478f2fdfce889 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c @@ -98,8 +98,8 @@ int wx_request_msix_irqs_vf(struct wx *wx) } } - err = request_threaded_irq(wx->msix_entry->vector, wx_msix_misc_vf, - NULL, IRQF_ONESHOT, netdev->name, wx); + err = request_irq(wx->msix_entry->vector, wx_msix_misc_vf, + 0, netdev->name, wx); if (err) { wx_err(wx, "request_irq for msix_other failed: %d\n", err); goto free_queue_irqs; diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index 0de051450a8237..54fa456fc66a74 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -867,7 +867,8 @@ static int txgbe_probe(struct pci_dev *pdev, "0x%08x", etrack_id); } - if (etrack_id < 0x20010) + if (wx->mac.type == wx_mac_sp && + ((etrack_id & 0xfffff) < 0x20010)) dev_warn(&pdev->dev, "Please upgrade the firmware to 0x20010 or above.\n"); err = txgbe_test_hostif(wx); diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c index 8ea7aa07ae4e9b..dc9f243146580b 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c @@ -657,7 +657,9 @@ void txgbe_remove_phy(struct txgbe *txgbe) return; case wx_mac_sp: if (txgbe->wx->media_type == wx_media_copper) { + rtnl_lock(); phylink_disconnect_phy(txgbe->wx->phylink); + rtnl_unlock(); phylink_destroy(txgbe->wx->phylink); return; } diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index e8949f55620905..fa0a3881cfb79c 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -2400,6 +2400,7 @@ static int gtp_genl_send_echo_req(struct sk_buff *skb, struct genl_info *info) return -ENODEV; } + local_bh_disable(); udp_tunnel_xmit_skb(rt, sk, skb_to_send, fl4.saddr, fl4.daddr, inet_dscp_to_dsfield(fl4.flowi4_dscp), @@ -2409,6 +2410,7 @@ static int gtp_genl_send_echo_req(struct sk_buff *skb, struct genl_info *info) !net_eq(sock_net(sk), dev_net(gtp->dev)), false, 0); + local_bh_enable(); return 0; } diff --git a/drivers/net/hamradio/6pack.c b/drivers/net/hamradio/6pack.c index 885992951e8a65..c8b2dc5c1becc0 100644 --- a/drivers/net/hamradio/6pack.c +++ b/drivers/net/hamradio/6pack.c @@ -391,7 +391,6 @@ static void sixpack_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { struct sixpack *sp; - size_t count1; if (!count) return; @@ -401,16 +400,16 @@ static void sixpack_receive_buf(struct tty_struct *tty, const u8 *cp, return; /* Read the characters out of the buffer */ - count1 = count; - while (count) { - count--; + while (count--) { if (fp && *fp++) { if (!test_and_set_bit(SIXPF_ERROR, &sp->flags)) sp->dev->stats.rx_errors++; + cp++; continue; } + sixpack_decode(sp, cp, 1); + cp++; } - sixpack_decode(sp, cp, count1); tty_unthrottle(tty); } diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index 4c3227e77898cf..624649484d6275 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -2044,6 +2044,7 @@ static int gsi_ring_setup(struct gsi *gsi) count = reg_decode(reg, NUM_EV_PER_EE, val); } else { reg = gsi_reg(gsi, HW_PARAM_4); + val = ioread32(gsi->virt + reg_offset(reg)); count = reg_decode(reg, EV_PER_EE, val); } if (!count) { diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index edead9c48d1fe4..216506eeef1fb3 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -361,7 +361,7 @@ static void ipa_qtime_config(struct ipa *ipa) { const struct reg *reg; u32 offset; - u32 val; + u32 val = 0; /* Timer clock divider must be disabled when we change the rate */ reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG); @@ -374,8 +374,8 @@ static void ipa_qtime_config(struct ipa *ipa) val |= reg_bit(reg, DPL_TIMESTAMP_SEL); } /* Configure tag and NAT Qtime timestamp resolution as well */ - val = reg_encode(reg, TAG_TIMESTAMP_LSB, TAG_TIMESTAMP_SHIFT); - val = reg_encode(reg, NAT_TIMESTAMP_LSB, NAT_TIMESTAMP_SHIFT); + val |= reg_encode(reg, TAG_TIMESTAMP_LSB, TAG_TIMESTAMP_SHIFT); + val |= reg_encode(reg, NAT_TIMESTAMP_LSB, NAT_TIMESTAMP_SHIFT); iowrite32(val, ipa->reg_virt + reg_offset(reg)); diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index f6cad0746a0222..6147ee8b1d78b1 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -2584,7 +2584,9 @@ static void macsec_inherit_tso_max(struct net_device *dev) netif_inherit_tso_max(dev, macsec->real_dev); } -static int macsec_update_offload(struct net_device *dev, enum macsec_offload offload) +static int macsec_update_offload(struct net_device *dev, + enum macsec_offload offload, + struct netlink_ext_ack *extack) { enum macsec_offload prev_offload; const struct macsec_ops *ops; @@ -2616,14 +2618,35 @@ static int macsec_update_offload(struct net_device *dev, enum macsec_offload off if (!ops) return -EOPNOTSUPP; - macsec->offload = offload; - ctx.secy = &macsec->secy; ret = offload == MACSEC_OFFLOAD_OFF ? macsec_offload(ops->mdo_del_secy, &ctx) : macsec_offload(ops->mdo_add_secy, &ctx); - if (ret) { - macsec->offload = prev_offload; + if (ret) return ret; + + /* Remove VLAN filters when disabling offload. */ + if (offload == MACSEC_OFFLOAD_OFF) { + vlan_drop_rx_ctag_filter_info(dev); + vlan_drop_rx_stag_filter_info(dev); + } + macsec->offload = offload; + /* Add VLAN filters when enabling offload. */ + if (prev_offload == MACSEC_OFFLOAD_OFF) { + ret = vlan_get_rx_ctag_filter_info(dev); + if (ret) { + NL_SET_ERR_MSG_FMT(extack, + "adding ctag VLAN filters failed, err %d", + ret); + goto rollback_offload; + } + ret = vlan_get_rx_stag_filter_info(dev); + if (ret) { + NL_SET_ERR_MSG_FMT(extack, + "adding stag VLAN filters failed, err %d", + ret); + vlan_drop_rx_ctag_filter_info(dev); + goto rollback_offload; + } } macsec_set_head_tail_room(dev); @@ -2633,6 +2656,12 @@ static int macsec_update_offload(struct net_device *dev, enum macsec_offload off netdev_update_features(dev); + return 0; + +rollback_offload: + macsec->offload = prev_offload; + macsec_offload(ops->mdo_del_secy, &ctx); + return ret; } @@ -2673,7 +2702,7 @@ static int macsec_upd_offload(struct sk_buff *skb, struct genl_info *info) offload = nla_get_u8(tb_offload[MACSEC_OFFLOAD_ATTR_TYPE]); if (macsec->offload != offload) - ret = macsec_update_offload(dev, offload); + ret = macsec_update_offload(dev, offload, info->extack); out: rtnl_unlock(); return ret; @@ -3486,7 +3515,8 @@ static netdev_tx_t macsec_start_xmit(struct sk_buff *skb, } #define MACSEC_FEATURES \ - (NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST) + (NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST | \ + NETIF_F_HW_VLAN_STAG_FILTER | NETIF_F_HW_VLAN_CTAG_FILTER) #define MACSEC_OFFLOAD_FEATURES \ (MACSEC_FEATURES | NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES | \ @@ -3707,6 +3737,29 @@ static int macsec_set_mac_address(struct net_device *dev, void *p) return err; } +static int macsec_vlan_rx_add_vid(struct net_device *dev, + __be16 proto, u16 vid) +{ + struct macsec_dev *macsec = netdev_priv(dev); + + if (!macsec_is_offloaded(macsec)) + return 0; + + return vlan_vid_add(macsec->real_dev, proto, vid); +} + +static int macsec_vlan_rx_kill_vid(struct net_device *dev, + __be16 proto, u16 vid) +{ + struct macsec_dev *macsec = netdev_priv(dev); + + if (!macsec_is_offloaded(macsec)) + return 0; + + vlan_vid_del(macsec->real_dev, proto, vid); + return 0; +} + static int macsec_change_mtu(struct net_device *dev, int new_mtu) { struct macsec_dev *macsec = macsec_priv(dev); @@ -3748,6 +3801,8 @@ static const struct net_device_ops macsec_netdev_ops = { .ndo_set_rx_mode = macsec_dev_set_rx_mode, .ndo_change_rx_flags = macsec_dev_change_rx_flags, .ndo_set_mac_address = macsec_set_mac_address, + .ndo_vlan_rx_add_vid = macsec_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = macsec_vlan_rx_kill_vid, .ndo_start_xmit = macsec_start_xmit, .ndo_get_stats64 = macsec_get_stats64, .ndo_get_iflink = macsec_get_iflink, @@ -3912,7 +3967,7 @@ static int macsec_changelink(struct net_device *dev, struct nlattr *tb[], offload = nla_get_u8(data[IFLA_MACSEC_OFFLOAD]); if (macsec->offload != offload) { macsec_offload_state_change = true; - ret = macsec_update_offload(dev, offload); + ret = macsec_update_offload(dev, offload, extack); if (ret) goto cleanup; } diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index a71f058eceef05..3073d67d0dffc0 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -352,6 +352,7 @@ static void macvlan_broadcast_enqueue(struct macvlan_port *port, const struct macvlan_dev *src, struct sk_buff *skb) { + u32 bc_queue_len_used = READ_ONCE(port->bc_queue_len_used); struct sk_buff *nskb; int err = -ENOMEM; @@ -362,7 +363,7 @@ static void macvlan_broadcast_enqueue(struct macvlan_port *port, MACVLAN_SKB_CB(nskb)->src = src; spin_lock(&port->bc_queue.lock); - if (skb_queue_len(&port->bc_queue) < port->bc_queue_len_used) { + if (skb_queue_len(&port->bc_queue) < bc_queue_len_used) { if (src) dev_hold(src->dev); __skb_queue_tail(&port->bc_queue, nskb); @@ -1681,6 +1682,7 @@ static size_t macvlan_get_size(const struct net_device *dev) + macvlan_get_size_mac(vlan) /* IFLA_MACVLAN_MACADDR */ + nla_total_size(4) /* IFLA_MACVLAN_BC_QUEUE_LEN */ + nla_total_size(4) /* IFLA_MACVLAN_BC_QUEUE_LEN_USED */ + + nla_total_size(4) /* IFLA_MACVLAN_BC_CUTOFF */ ); } @@ -1727,7 +1729,8 @@ static int macvlan_fill_info(struct sk_buff *skb, } if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN, vlan->bc_queue_len_req)) goto nla_put_failure; - if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN_USED, port->bc_queue_len_used)) + if (nla_put_u32(skb, IFLA_MACVLAN_BC_QUEUE_LEN_USED, + READ_ONCE(port->bc_queue_len_used))) goto nla_put_failure; if (port->bc_cutoff != 1 && nla_put_s32(skb, IFLA_MACVLAN_BC_CUTOFF, port->bc_cutoff)) @@ -1787,7 +1790,7 @@ static void update_port_bc_queue_len(struct macvlan_port *port) if (vlan->bc_queue_len_req > max_bc_queue_len_req) max_bc_queue_len_req = vlan->bc_queue_len_req; } - port->bc_queue_len_used = max_bc_queue_len_req; + WRITE_ONCE(port->bc_queue_len_used, max_bc_queue_len_req); } static int macvlan_device_event(struct notifier_block *unused, diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c index 15fe4d1163c1cd..ee2913758e54ed 100644 --- a/drivers/net/mctp/mctp-i2c.c +++ b/drivers/net/mctp/mctp-i2c.c @@ -496,8 +496,6 @@ static void mctp_i2c_xmit(struct mctp_i2c_dev *midev, struct sk_buff *skb) u8 *pecp; int rc; - fs = mctp_i2c_get_tx_flow_state(midev, skb); - hdr = (void *)skb_mac_header(skb); /* Sanity check that packet contents matches skb length, * and can't exceed MCTP_I2C_BUFSZ @@ -509,6 +507,8 @@ static void mctp_i2c_xmit(struct mctp_i2c_dev *midev, struct sk_buff *skb) return; } + fs = mctp_i2c_get_tx_flow_state(midev, skb); + if (skb_tailroom(skb) >= 1) { /* Linear case with space, we can just append the PEC */ skb_put(skb, 1); diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 3c9acd6e49e866..57dd6821a8aa90 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -497,6 +497,8 @@ static void trim_newline(char *s, size_t maxlen) size_t len; len = strnlen(s, maxlen); + if (!len) + return; if (s[len - 1] == '\n') s[len - 1] = '\0'; } @@ -750,7 +752,7 @@ static ssize_t enabled_store(struct config_item *item, unregister_netcons_consoles(); } - ret = strnlen(buf, count); + ret = count; /* Deferred cleanup */ netconsole_process_cleanups(); out_unlock: @@ -779,7 +781,7 @@ static ssize_t release_store(struct config_item *item, const char *buf, nt->release = release; - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -805,7 +807,7 @@ static ssize_t extended_store(struct config_item *item, const char *buf, goto out_unlock; nt->extended = extended; - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -815,6 +817,13 @@ static ssize_t dev_name_store(struct config_item *item, const char *buf, size_t count) { struct netconsole_target *nt = to_target(item); + size_t len = count; + + /* Account for a trailing newline appended by tools like echo */ + if (len && buf[len - 1] == '\n') + len--; + if (len >= IFNAMSIZ) + return -ENAMETOOLONG; dynamic_netconsole_mutex_lock(); if (nt->state == STATE_ENABLED) { @@ -828,7 +837,7 @@ static ssize_t dev_name_store(struct config_item *item, const char *buf, trim_newline(nt->np.dev_name, IFNAMSIZ); dynamic_netconsole_mutex_unlock(); - return strnlen(buf, count); + return count; } static ssize_t local_port_store(struct config_item *item, const char *buf, @@ -847,7 +856,7 @@ static ssize_t local_port_store(struct config_item *item, const char *buf, ret = kstrtou16(buf, 10, &nt->np.local_port); if (ret < 0) goto out_unlock; - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -869,7 +878,7 @@ static ssize_t remote_port_store(struct config_item *item, ret = kstrtou16(buf, 10, &nt->np.remote_port); if (ret < 0) goto out_unlock; - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -894,7 +903,7 @@ static ssize_t local_ip_store(struct config_item *item, const char *buf, goto out_unlock; nt->np.ipv6 = !!ipv6; - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -919,7 +928,7 @@ static ssize_t remote_ip_store(struct config_item *item, const char *buf, goto out_unlock; nt->np.ipv6 = !!ipv6; - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -955,7 +964,7 @@ static ssize_t remote_mac_store(struct config_item *item, const char *buf, goto out_unlock; memcpy(nt->np.remote_mac, remote_mac, ETH_ALEN); - ret = strnlen(buf, count); + ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); return ret; @@ -1070,26 +1079,30 @@ static ssize_t userdatum_value_store(struct config_item *item, const char *buf, size_t count) { struct userdatum *udm = to_userdatum(item); + char old_value[MAX_EXTRADATA_VALUE_LEN]; struct netconsole_target *nt; struct userdata *ud; ssize_t ret; - if (count > MAX_EXTRADATA_VALUE_LEN) + if (count >= MAX_EXTRADATA_VALUE_LEN) return -EMSGSIZE; mutex_lock(&netconsole_subsys.su_mutex); dynamic_netconsole_mutex_lock(); - - ret = strscpy(udm->value, buf, sizeof(udm->value)); - if (ret < 0) - goto out_unlock; + /* Snapshot for rollback if update_userdata() fails below */ + strscpy(old_value, udm->value, sizeof(old_value)); + /* count is bounded above, so strscpy() cannot truncate here */ + strscpy(udm->value, buf, sizeof(udm->value)); trim_newline(udm->value, sizeof(udm->value)); ud = to_userdata(item->ci_parent); nt = userdata_to_target(ud); ret = update_userdata(nt); - if (ret < 0) + if (ret < 0) { + /* Restore the previous value so it matches the live payload */ + strscpy(udm->value, old_value, sizeof(udm->value)); goto out_unlock; + } ret = count; out_unlock: dynamic_netconsole_mutex_unlock(); @@ -1131,7 +1144,7 @@ static ssize_t sysdata_msgid_enabled_store(struct config_item *item, disable_sysdata_feature(nt, SYSDATA_MSGID); unlock_ok: - ret = strnlen(buf, count); + ret = count; dynamic_netconsole_mutex_unlock(); mutex_unlock(&netconsole_subsys.su_mutex); return ret; @@ -1160,7 +1173,7 @@ static ssize_t sysdata_release_enabled_store(struct config_item *item, disable_sysdata_feature(nt, SYSDATA_RELEASE); unlock_ok: - ret = strnlen(buf, count); + ret = count; dynamic_netconsole_mutex_unlock(); mutex_unlock(&netconsole_subsys.su_mutex); return ret; @@ -1189,7 +1202,7 @@ static ssize_t sysdata_taskname_enabled_store(struct config_item *item, disable_sysdata_feature(nt, SYSDATA_TASKNAME); unlock_ok: - ret = strnlen(buf, count); + ret = count; dynamic_netconsole_mutex_unlock(); mutex_unlock(&netconsole_subsys.su_mutex); return ret; @@ -1223,7 +1236,7 @@ static ssize_t sysdata_cpu_nr_enabled_store(struct config_item *item, disable_sysdata_feature(nt, SYSDATA_CPU_NR); unlock_ok: - ret = strnlen(buf, count); + ret = count; dynamic_netconsole_mutex_unlock(); mutex_unlock(&netconsole_subsys.su_mutex); return ret; diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index e82de0fd315794..8f6d0a09e17619 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -829,7 +829,7 @@ static struct sk_buff *nsim_dev_trap_skb_build(void) skb->protocol = htons(ETH_P_IP); skb_set_network_header(skb, skb->len); - iph = skb_put(skb, sizeof(struct iphdr)); + iph = skb_put_zero(skb, sizeof(struct iphdr)); iph->protocol = IPPROTO_UDP; iph->saddr = in_aton("192.0.2.1"); iph->daddr = in_aton("198.51.100.1"); diff --git a/drivers/net/phy/dp83869.c b/drivers/net/phy/dp83869.c index 1f381d7b13ff3b..96a7d255f50fd9 100644 --- a/drivers/net/phy/dp83869.c +++ b/drivers/net/phy/dp83869.c @@ -31,6 +31,7 @@ #define DP83869_RGMIICTL 0x0032 #define DP83869_STRAP_STS1 0x006e #define DP83869_RGMIIDCTL 0x0086 +#define DP83869_ANA_PLL_PROG_PI 0x00c6 #define DP83869_RXFCFG 0x0134 #define DP83869_RXFPMD1 0x0136 #define DP83869_RXFPMD2 0x0137 @@ -826,12 +827,22 @@ static int dp83869_config_init(struct phy_device *phydev) dp83869_config_port_mirroring(phydev); /* Clock output selection if muxing property is set */ - if (dp83869->clk_output_sel != DP83869_CLK_O_SEL_REF_CLK) + if (dp83869->clk_output_sel != DP83869_CLK_O_SEL_REF_CLK) { + /* + * Table 7-121 in datasheet says we have to set register 0xc6 + * to value 0x10 before CLK_O_SEL can be modified. + */ + ret = phy_write_mmd(phydev, DP83869_DEVADDR, + DP83869_ANA_PLL_PROG_PI, 0x10); + if (ret) + return ret; + ret = phy_modify_mmd(phydev, DP83869_DEVADDR, DP83869_IO_MUX_CFG, DP83869_IO_MUX_CFG_CLK_O_SEL_MASK, dp83869->clk_output_sel << DP83869_IO_MUX_CFG_CLK_O_SEL_SHIFT); + } if (phy_interface_is_rgmii(phydev)) { ret = phy_write_mmd(phydev, DP83869_DEVADDR, DP83869_RGMIIDCTL, diff --git a/drivers/net/phy/mdio_bus_provider.c b/drivers/net/phy/mdio_bus_provider.c index 4b0637405740e7..fd691c5424ea19 100644 --- a/drivers/net/phy/mdio_bus_provider.c +++ b/drivers/net/phy/mdio_bus_provider.c @@ -294,8 +294,8 @@ int __mdiobus_register(struct mii_bus *bus, struct module *owner) return -EINVAL; if (bus->parent && bus->parent->of_node) - bus->parent->of_node->fwnode.flags |= - FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD; + fwnode_set_flag(&bus->parent->of_node->fwnode, + FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD); WARN(bus->state != MDIOBUS_ALLOCATED && bus->state != MDIOBUS_UNREGISTERED, diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c index 3bd415710bf3f8..f3696d9819d351 100644 --- a/drivers/net/phy/phy_device.c +++ b/drivers/net/phy/phy_device.c @@ -927,8 +927,8 @@ static int get_phy_c45_ids(struct mii_bus *bus, int addr, /* returning -ENODEV doesn't stop bus * scanning */ - return (phy_reg == -EIO || - phy_reg == -ENODEV) ? -ENODEV : -EIO; + return (ret == -EIO || + ret == -ENODEV) ? -ENODEV : -EIO; if (!ret) continue; diff --git a/drivers/net/phy/qcom/at803x.c b/drivers/net/phy/qcom/at803x.c index 2995b08bac9632..63726cf98cd423 100644 --- a/drivers/net/phy/qcom/at803x.c +++ b/drivers/net/phy/qcom/at803x.c @@ -524,7 +524,7 @@ static int at803x_config_init(struct phy_device *phydev) * behaviour but we still need to accommodate it. XNP is only needed * for 10Gbps support, so disable XNP. */ - return phy_modify(phydev, MII_ADVERTISE, MDIO_AN_CTRL1_XNP, 0); + return phy_modify(phydev, MII_ADVERTISE, ADVERTISE_XNP, 0); } static void at803x_link_change_notify(struct phy_device *phydev) diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index e9b41777be809a..192a5b94783e37 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -1057,6 +1057,9 @@ static int ppp_unattached_ioctl(struct net *net, struct ppp_file *pf, struct ppp_net *pn; int __user *p = (int __user *)arg; + if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) + return -EPERM; + switch (cmd) { case PPPIOCNEWUNIT: /* Create a new ppp unit */ @@ -2257,7 +2260,7 @@ ppp_do_recv(struct ppp *ppp, struct sk_buff *skb, struct channel *pch) */ static void __ppp_decompress_proto(struct sk_buff *skb) { - if (skb->data[0] & 0x01) + if (ppp_skb_is_compressed_proto(skb)) *(u8 *)skb_push(skb, 1) = 0x00; } diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c index 4275b393a45440..6992b3f6478198 100644 --- a/drivers/net/ppp/pppoe.c +++ b/drivers/net/ppp/pppoe.c @@ -424,7 +424,7 @@ static int pppoe_rcv(struct sk_buff *skb, struct net_device *dev, if (skb_mac_header_len(skb) < ETH_HLEN) goto drop; - if (!pskb_may_pull(skb, sizeof(struct pppoe_hdr))) + if (!pskb_may_pull(skb, PPPOE_SES_HLEN)) goto drop; ph = pppoe_hdr(skb); @@ -434,6 +434,12 @@ static int pppoe_rcv(struct sk_buff *skb, struct net_device *dev, if (skb->len < len) goto drop; + /* skb->data points to the PPP protocol header after skb_pull_rcsum. + * Drop PFC frames. + */ + if (ppp_skb_is_compressed_proto(skb)) + goto drop; + if (pskb_trim_rcsum(skb, len)) goto drop; diff --git a/drivers/net/slip/slhc.c b/drivers/net/slip/slhc.c index e3c785da3eef3d..1a9b27d5e256b9 100644 --- a/drivers/net/slip/slhc.c +++ b/drivers/net/slip/slhc.c @@ -80,9 +80,9 @@ #include static unsigned char *encode(unsigned char *cp, unsigned short n); -static long decode(unsigned char **cpp); +static long decode(unsigned char **cpp, const unsigned char *end); static unsigned char * put16(unsigned char *cp, unsigned short x); -static unsigned short pull16(unsigned char **cpp); +static long pull16(unsigned char **cpp, const unsigned char *end); /* Allocate compression data structure * slots must be in range 0 to 255 (zero meaning no compression) @@ -190,30 +190,34 @@ encode(unsigned char *cp, unsigned short n) return cp; } -/* Pull a 16-bit integer in host order from buffer in network byte order */ -static unsigned short -pull16(unsigned char **cpp) +/* Pull a 16-bit integer in host order from buffer in network byte order. + * Returns -1 if the buffer is exhausted, otherwise the 16-bit value. + */ +static long +pull16(unsigned char **cpp, const unsigned char *end) { - short rval; + long rval; + if (*cpp + 2 > end) + return -1; rval = *(*cpp)++; rval <<= 8; rval |= *(*cpp)++; return rval; } -/* Decode a number */ +/* Decode a number. Returns -1 if the buffer is exhausted. */ static long -decode(unsigned char **cpp) +decode(unsigned char **cpp, const unsigned char *end) { int x; + if (*cpp >= end) + return -1; x = *(*cpp)++; - if(x == 0){ - return pull16(cpp) & 0xffff; /* pull16 returns -1 on error */ - } else { - return x & 0xff; /* -1 if PULLCHAR returned error */ - } + if (x == 0) + return pull16(cpp, end); + return x & 0xff; } /* @@ -499,6 +503,7 @@ slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize) struct cstate *cs; int len, hdrlen; unsigned char *cp = icp; + const unsigned char *end = icp + isize; /* We've got a compressed packet; read the change byte */ comp->sls_i_compressed++; @@ -506,6 +511,8 @@ slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize) comp->sls_i_error++; return 0; } + if (!comp->rstate) + goto bad; changes = *cp++; if(changes & NEW_C){ /* Make sure the state index is in range, then grab the state. @@ -534,6 +541,8 @@ slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize) thp = &cs->cs_tcp; ip = &cs->cs_ip; + if (cp + 2 > end) + goto bad; thp->check = *(__sum16 *)cp; cp += 2; @@ -564,26 +573,26 @@ slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize) default: if(changes & NEW_U){ thp->urg = 1; - if((x = decode(&cp)) == -1) { + if((x = decode(&cp, end)) == -1) { goto bad; } thp->urg_ptr = htons(x); } else thp->urg = 0; if(changes & NEW_W){ - if((x = decode(&cp)) == -1) { + if((x = decode(&cp, end)) == -1) { goto bad; } thp->window = htons( ntohs(thp->window) + x); } if(changes & NEW_A){ - if((x = decode(&cp)) == -1) { + if((x = decode(&cp, end)) == -1) { goto bad; } thp->ack_seq = htonl( ntohl(thp->ack_seq) + x); } if(changes & NEW_S){ - if((x = decode(&cp)) == -1) { + if((x = decode(&cp, end)) == -1) { goto bad; } thp->seq = htonl( ntohl(thp->seq) + x); @@ -591,7 +600,7 @@ slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize) break; } if(changes & NEW_I){ - if((x = decode(&cp)) == -1) { + if((x = decode(&cp, end)) == -1) { goto bad; } ip->id = htons (ntohs (ip->id) + x); @@ -649,6 +658,10 @@ slhc_remember(struct slcompress *comp, unsigned char *icp, int isize) struct cstate *cs; unsigned int ihl; + if (!comp->rstate) { + comp->sls_i_error++; + return slhc_toss(comp); + } /* The packet is shorter than a legal IP header. * Also make sure isize is positive. */ diff --git a/drivers/net/usb/cdc-phonet.c b/drivers/net/usb/cdc-phonet.c index ad5121e9cf5d6f..165650ecef649f 100644 --- a/drivers/net/usb/cdc-phonet.c +++ b/drivers/net/usb/cdc-phonet.c @@ -157,11 +157,16 @@ static void rx_complete(struct urb *req) PAGE_SIZE); page = NULL; } - } else { + } else if (skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS) { skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 0, req->actual_length, PAGE_SIZE); page = NULL; + } else { + dev_kfree_skb_any(skb); + pnd->rx_skb = NULL; + skb = NULL; + dev->stats.rx_length_errors++; } if (req->actual_length < PAGE_SIZE) pnd->rx_skb = NULL; /* Last fragment */ diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index 0c83bbbea2e7c3..f69e7e1ab7788d 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -3890,7 +3890,7 @@ static void r8156_ups_en(struct r8152 *tp, bool enable) case RTL_VER_15: ocp_data = ocp_read_word(tp, MCU_TYPE_USB, USB_UPHY_XTAL); ocp_data &= ~OOBS_POLLING; - ocp_write_byte(tp, MCU_TYPE_USB, USB_UPHY_XTAL, ocp_data); + ocp_write_word(tp, MCU_TYPE_USB, USB_UPHY_XTAL, ocp_data); break; default: break; diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c index 4cda0643afb6eb..c880c95c41a5ec 100644 --- a/drivers/net/usb/rtl8150.c +++ b/drivers/net/usb/rtl8150.c @@ -683,6 +683,7 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb, struct net_device *netdev) { rtl8150_t *dev = netdev_priv(netdev); + unsigned int skb_len; int count, res; /* pad the frame and ensure terminating USB packet, datasheet 9.2.3 */ @@ -694,6 +695,8 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb, return NETDEV_TX_OK; } + skb_len = skb->len; + netif_stop_queue(netdev); dev->tx_skb = skb; usb_fill_bulk_urb(dev->tx_urb, dev->udev, usb_sndbulkpipe(dev->udev, 2), @@ -707,9 +710,16 @@ static netdev_tx_t rtl8150_start_xmit(struct sk_buff *skb, netdev->stats.tx_errors++; netif_start_queue(netdev); } + /* + * The URB was not submitted, so write_bulk_callback() will + * never run to free dev->tx_skb. Drop the skb here and + * clear tx_skb to avoid leaving a stale pointer. + */ + dev->tx_skb = NULL; + dev_kfree_skb_any(skb); } else { netdev->stats.tx_packets++; - netdev->stats.tx_bytes += skb->len; + netdev->stats.tx_bytes += skb_len; netif_trans_update(netdev); } diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index c0b9bc5574e232..67b913218144bd 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3748,6 +3748,12 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) queue_pairs); return -EINVAL; } + + /* Keep max_tx_vq in sync so that a later RSS command does not + * revert queue_pairs to a stale value. + */ + if (vi->has_rss) + vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs); succ: vi->curr_queue_pairs = queue_pairs; if (dev->flags & IFF_UP) { diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 8c009bcaa8e77d..91a97c3f46c4fa 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -1084,6 +1084,7 @@ static int do_vrf_add_slave(struct net_device *dev, struct net_device *port_dev, err: port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE; + synchronize_net(); return ret; } @@ -1103,10 +1104,16 @@ static int vrf_add_slave(struct net_device *dev, struct net_device *port_dev, } /* inverse of do_vrf_add_slave */ -static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev) +static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev, + bool needs_sync) { netdev_upper_dev_unlink(port_dev, dev); port_dev->priv_flags &= ~IFF_L3MDEV_SLAVE; + /* Make sure that concurrent RCU readers that identified the device + * as a VRF port see a VRF master or no master at all. + */ + if (needs_sync) + synchronize_net(); cycle_netdev(port_dev, NULL); @@ -1115,7 +1122,7 @@ static int do_vrf_del_slave(struct net_device *dev, struct net_device *port_dev) static int vrf_del_slave(struct net_device *dev, struct net_device *port_dev) { - return do_vrf_del_slave(dev, port_dev); + return do_vrf_del_slave(dev, port_dev, true); } static void vrf_dev_uninit(struct net_device *dev) @@ -1669,7 +1676,7 @@ static void vrf_dellink(struct net_device *dev, struct list_head *head) struct list_head *iter; netdev_for_each_lower_dev(dev, port_dev, iter) - vrf_del_slave(dev, port_dev); + do_vrf_del_slave(dev, port_dev, false); vrf_map_unregister_dev(dev); @@ -1801,7 +1808,7 @@ static int vrf_device_event(struct notifier_block *unused, goto out; vrf_dev = netdev_master_upper_dev_get(dev); - vrf_del_slave(vrf_dev, dev); + do_vrf_del_slave(vrf_dev, dev, false); } out: return NOTIFY_DONE; diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index 46a71ec36af870..67b07ee2d66003 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -411,12 +411,11 @@ static struct rtnl_link_ops link_ops __read_mostly = { .newlink = wg_newlink, }; -static void wg_netns_pre_exit(struct net *net) +static void __net_exit wg_netns_exit_rtnl(struct net *net, struct list_head *dev_kill_list) { struct wg_device *wg; struct wg_peer *peer; - rtnl_lock(); list_for_each_entry(wg, &device_list, device_list) { if (rcu_access_pointer(wg->creating_net) == net) { pr_debug("%s: Creating namespace exiting\n", wg->dev->name); @@ -429,11 +428,10 @@ static void wg_netns_pre_exit(struct net *net) mutex_unlock(&wg->device_update_lock); } } - rtnl_unlock(); } -static struct pernet_operations pernet_ops = { - .pre_exit = wg_netns_pre_exit +static struct pernet_operations pernet_ops __read_mostly = { + .exit_rtnl = wg_netns_exit_rtnl }; int __init wg_device_init(void) diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.c b/drivers/net/wireless/ath/ath10k/wmi-tlv.c index ec8e91707f84a1..01f2d1fa9d7d92 100644 --- a/drivers/net/wireless/ath/ath10k/wmi-tlv.c +++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.c @@ -3,7 +3,7 @@ * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. - * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include "core.h" #include "debug.h" @@ -14,6 +14,7 @@ #include "wmi-tlv.h" #include "p2p.h" #include "testmode.h" +#include "txrx.h" #include /***************/ @@ -224,8 +225,9 @@ static int ath10k_wmi_tlv_parse_peer_stats_info(struct ath10k *ar, u16 tag, u16 const void *ptr, void *data) { const struct wmi_tlv_peer_stats_info *stat = ptr; - struct ieee80211_sta *sta; + u32 vdev_id = *(u32 *)data; struct ath10k_sta *arsta; + struct ath10k_peer *peer; if (tag != WMI_TLV_TAG_STRUCT_PEER_STATS_INFO) return -EPROTO; @@ -241,20 +243,20 @@ static int ath10k_wmi_tlv_parse_peer_stats_info(struct ath10k *ar, u16 tag, u16 __le32_to_cpu(stat->last_tx_rate_code), __le32_to_cpu(stat->last_tx_bitrate_kbps)); - rcu_read_lock(); - sta = ieee80211_find_sta_by_ifaddr(ar->hw, stat->peer_macaddr.addr, NULL); - if (!sta) { - rcu_read_unlock(); - ath10k_warn(ar, "not found station for peer stats\n"); + guard(spinlock_bh)(&ar->data_lock); + + peer = ath10k_peer_find(ar, vdev_id, stat->peer_macaddr.addr); + if (!peer || !peer->sta) { + ath10k_warn(ar, "not found %s with vdev id %u mac addr %pM for peer stats\n", + peer ? "sta" : "peer", vdev_id, stat->peer_macaddr.addr); return -EINVAL; } - arsta = (struct ath10k_sta *)sta->drv_priv; + arsta = (struct ath10k_sta *)peer->sta->drv_priv; arsta->rx_rate_code = __le32_to_cpu(stat->last_rx_rate_code); arsta->rx_bitrate_kbps = __le32_to_cpu(stat->last_rx_bitrate_kbps); arsta->tx_rate_code = __le32_to_cpu(stat->last_tx_rate_code); arsta->tx_bitrate_kbps = __le32_to_cpu(stat->last_tx_bitrate_kbps); - rcu_read_unlock(); return 0; } @@ -266,6 +268,7 @@ static int ath10k_wmi_tlv_op_pull_peer_stats_info(struct ath10k *ar, const struct wmi_tlv_peer_stats_info_ev *ev; const void *data; u32 num_peer_stats; + u32 vdev_id; int ret; tb = ath10k_wmi_tlv_parse_alloc(ar, skb->data, skb->len, GFP_ATOMIC); @@ -284,15 +287,16 @@ static int ath10k_wmi_tlv_op_pull_peer_stats_info(struct ath10k *ar, } num_peer_stats = __le32_to_cpu(ev->num_peers); + vdev_id = __le32_to_cpu(ev->vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv peer stats info update peer vdev id %d peers %i more data %d\n", - __le32_to_cpu(ev->vdev_id), + vdev_id, num_peer_stats, __le32_to_cpu(ev->more_data)); ret = ath10k_wmi_tlv_iter(ar, data, ath10k_wmi_tlv_len(data), - ath10k_wmi_tlv_parse_peer_stats_info, NULL); + ath10k_wmi_tlv_parse_peer_stats_info, &vdev_id); if (ret) ath10k_warn(ar, "failed to parse stats info tlv: %d\n", ret); diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index e4ee2ba1f669e9..c06ee110a90f08 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -1557,12 +1557,15 @@ static int ath11k_mac_setup_bcn_tmpl_ema(struct ath11k_vif *arvif, if (!beacons || !beacons->cnt) { ath11k_warn(arvif->ar->ab, "failed to get ema beacon templates from mac80211\n"); - return -EPERM; + ret = -EPERM; + goto free; } if (tx_arvif == arvif) { - if (ath11k_mac_set_vif_params(tx_arvif, beacons->bcn[0].skb)) - return -EINVAL; + if (ath11k_mac_set_vif_params(tx_arvif, beacons->bcn[0].skb)) { + ret = -EINVAL; + goto free; + } } else { arvif->wpaie_present = tx_arvif->wpaie_present; } @@ -1589,11 +1592,11 @@ static int ath11k_mac_setup_bcn_tmpl_ema(struct ath11k_vif *arvif, } } - ieee80211_beacon_free_ema_list(beacons); - if (tx_arvif != arvif && !nontx_vif_params_set) - return -EINVAL; /* Profile not found in the beacons */ + ret = -EINVAL; /* Profile not found in the beacons */ +free: + ieee80211_beacon_free_ema_list(beacons); return ret; } @@ -1622,19 +1625,22 @@ static int ath11k_mac_setup_bcn_tmpl_mbssid(struct ath11k_vif *arvif, } if (tx_arvif == arvif) { - if (ath11k_mac_set_vif_params(tx_arvif, bcn)) - return -EINVAL; + if (ath11k_mac_set_vif_params(tx_arvif, bcn)) { + ret = -EINVAL; + goto free; + } } else if (!ath11k_mac_set_nontx_vif_params(tx_arvif, arvif, bcn)) { - return -EINVAL; + ret = -EINVAL; + goto free; } ret = ath11k_wmi_bcn_tmpl(ar, arvif->vdev_id, &offs, bcn, 0); - kfree_skb(bcn); - if (ret) ath11k_warn(ab, "failed to submit beacon template command: %d\n", ret); +free: + kfree_skb(bcn); return ret; } diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.c b/drivers/net/wireless/ath/ath12k/dp_htt.c index e71bb71a6020e0..9c19d9707abfb5 100644 --- a/drivers/net/wireless/ath/ath12k/dp_htt.c +++ b/drivers/net/wireless/ath/ath12k/dp_htt.c @@ -205,16 +205,9 @@ ath12k_update_per_peer_tx_stats(struct ath12k_pdev_dp *dp_pdev, if (!(usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_RATE))) return; - if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) { + if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) is_ampdu = HTT_USR_CMPLTN_IS_AMPDU(usr_stats->cmpltn_cmn.flags); - tx_retry_failed = - __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) - - __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success); - tx_retry_count = - HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) + - HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags); - } if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_ACK_BA_STATUS)) { @@ -223,10 +216,19 @@ ath12k_update_per_peer_tx_stats(struct ath12k_pdev_dp *dp_pdev, HTT_PPDU_STATS_ACK_BA_INFO_NUM_MSDU_M); tid = le32_get_bits(usr_stats->ack_ba.info, HTT_PPDU_STATS_ACK_BA_INFO_TID_NUM); - } - if (common->fes_duration_us) - tx_duration = le32_to_cpu(common->fes_duration_us); + if (usr_stats->tlv_flags & BIT(HTT_PPDU_STATS_TAG_USR_COMPLTN_COMMON)) { + tx_retry_failed = + __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_tried) - + __le16_to_cpu(usr_stats->cmpltn_cmn.mpdu_success); + tx_retry_count = + HTT_USR_CMPLTN_LONG_RETRY(usr_stats->cmpltn_cmn.flags) + + HTT_USR_CMPLTN_SHORT_RETRY(usr_stats->cmpltn_cmn.flags); + } + + if (common->fes_duration_us) + tx_duration = le32_to_cpu(common->fes_duration_us); + } user_rate = &usr_stats->rate; flags = HTT_USR_RATE_PREAMBLE(user_rate->rate_flags); diff --git a/drivers/net/wireless/ath/ath12k/hal.h b/drivers/net/wireless/ath/ath12k/hal.h index 43e3880f825798..bf4f7dbae86693 100644 --- a/drivers/net/wireless/ath/ath12k/hal.h +++ b/drivers/net/wireless/ath/ath12k/hal.h @@ -268,21 +268,28 @@ enum hal_rx_reception_type { }; enum hal_rx_legacy_rate { - HAL_RX_LEGACY_RATE_1_MBPS, - HAL_RX_LEGACY_RATE_2_MBPS, - HAL_RX_LEGACY_RATE_5_5_MBPS, - HAL_RX_LEGACY_RATE_6_MBPS, - HAL_RX_LEGACY_RATE_9_MBPS, - HAL_RX_LEGACY_RATE_11_MBPS, - HAL_RX_LEGACY_RATE_12_MBPS, - HAL_RX_LEGACY_RATE_18_MBPS, - HAL_RX_LEGACY_RATE_24_MBPS, - HAL_RX_LEGACY_RATE_36_MBPS, - HAL_RX_LEGACY_RATE_48_MBPS, - HAL_RX_LEGACY_RATE_54_MBPS, + HAL_RX_LEGACY_RATE_LP_1_MBPS, + HAL_RX_LEGACY_RATE_LP_2_MBPS, + HAL_RX_LEGACY_RATE_LP_5_5_MBPS, + HAL_RX_LEGACY_RATE_LP_11_MBPS, + HAL_RX_LEGACY_RATE_SP_2_MBPS, + HAL_RX_LEGACY_RATE_SP_5_5_MBPS, + HAL_RX_LEGACY_RATE_SP_11_MBPS, HAL_RX_LEGACY_RATE_INVALID, }; +enum hal_rx_legacy_rates_ofdm { + HAL_RX_LEGACY_RATE_OFDM_48_MBPS, + HAL_RX_LEGACY_RATE_OFDM_24_MBPS, + HAL_RX_LEGACY_RATE_OFDM_12_MBPS, + HAL_RX_LEGACY_RATE_OFDM_6_MBPS, + HAL_RX_LEGACY_RATE_OFDM_54_MBPS, + HAL_RX_LEGACY_RATE_OFDM_36_MBPS, + HAL_RX_LEGACY_RATE_OFDM_18_MBPS, + HAL_RX_LEGACY_RATE_OFDM_9_MBPS, + HAL_RX_LEGACY_RATE_OFDM_INVALID, +}; + enum hal_ring_type { HAL_REO_DST, HAL_REO_EXCEPTION, diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index b253d1e3f40524..fa36e984c74b2a 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -164,30 +164,31 @@ static const struct ieee80211_channel ath12k_6ghz_channels[] = { CHAN6G(233, 7115, 0), }; +#define ATH12K_MAC_RATE_A_M(bps, code) \ + { .bitrate = (bps), .hw_value = (code),\ + .flags = IEEE80211_RATE_MANDATORY_A } + +#define ATH12K_MAC_RATE_B(bps, code, code_short) \ + { .bitrate = (bps), .hw_value = (code), .hw_value_short = (code_short),\ + .flags = IEEE80211_RATE_SHORT_PREAMBLE } + static struct ieee80211_rate ath12k_legacy_rates[] = { { .bitrate = 10, .hw_value = ATH12K_HW_RATE_CCK_LP_1M }, - { .bitrate = 20, - .hw_value = ATH12K_HW_RATE_CCK_LP_2M, - .hw_value_short = ATH12K_HW_RATE_CCK_SP_2M, - .flags = IEEE80211_RATE_SHORT_PREAMBLE }, - { .bitrate = 55, - .hw_value = ATH12K_HW_RATE_CCK_LP_5_5M, - .hw_value_short = ATH12K_HW_RATE_CCK_SP_5_5M, - .flags = IEEE80211_RATE_SHORT_PREAMBLE }, - { .bitrate = 110, - .hw_value = ATH12K_HW_RATE_CCK_LP_11M, - .hw_value_short = ATH12K_HW_RATE_CCK_SP_11M, - .flags = IEEE80211_RATE_SHORT_PREAMBLE }, - - { .bitrate = 60, .hw_value = ATH12K_HW_RATE_OFDM_6M }, - { .bitrate = 90, .hw_value = ATH12K_HW_RATE_OFDM_9M }, - { .bitrate = 120, .hw_value = ATH12K_HW_RATE_OFDM_12M }, - { .bitrate = 180, .hw_value = ATH12K_HW_RATE_OFDM_18M }, - { .bitrate = 240, .hw_value = ATH12K_HW_RATE_OFDM_24M }, - { .bitrate = 360, .hw_value = ATH12K_HW_RATE_OFDM_36M }, - { .bitrate = 480, .hw_value = ATH12K_HW_RATE_OFDM_48M }, - { .bitrate = 540, .hw_value = ATH12K_HW_RATE_OFDM_54M }, + ATH12K_MAC_RATE_B(20, ATH12K_HW_RATE_CCK_LP_2M, + ATH12K_HW_RATE_CCK_SP_2M), + ATH12K_MAC_RATE_B(55, ATH12K_HW_RATE_CCK_LP_5_5M, + ATH12K_HW_RATE_CCK_SP_5_5M), + ATH12K_MAC_RATE_B(110, ATH12K_HW_RATE_CCK_LP_11M, + ATH12K_HW_RATE_CCK_SP_11M), + ATH12K_MAC_RATE_A_M(60, ATH12K_HW_RATE_OFDM_6M), + ATH12K_MAC_RATE_A_M(90, ATH12K_HW_RATE_OFDM_9M), + ATH12K_MAC_RATE_A_M(120, ATH12K_HW_RATE_OFDM_12M), + ATH12K_MAC_RATE_A_M(180, ATH12K_HW_RATE_OFDM_18M), + ATH12K_MAC_RATE_A_M(240, ATH12K_HW_RATE_OFDM_24M), + ATH12K_MAC_RATE_A_M(360, ATH12K_HW_RATE_OFDM_36M), + ATH12K_MAC_RATE_A_M(480, ATH12K_HW_RATE_OFDM_48M), + ATH12K_MAC_RATE_A_M(540, ATH12K_HW_RATE_OFDM_54M), }; static const int @@ -732,11 +733,17 @@ u8 ath12k_mac_hw_rate_to_idx(const struct ieee80211_supported_band *sband, if (ath12k_mac_bitrate_is_cck(rate->bitrate) != cck) continue; - if (rate->hw_value == hw_rate) + /* To handle 802.11a PPDU type */ + if ((!cck) && (rate->hw_value == hw_rate) && + (rate->flags & IEEE80211_RATE_MANDATORY_A)) return i; + /* To handle 802.11b short PPDU type */ else if (rate->flags & IEEE80211_RATE_SHORT_PREAMBLE && rate->hw_value_short == hw_rate) return i; + /* To handle 802.11b long PPDU type */ + else if (rate->hw_value == hw_rate) + return i; } return 0; diff --git a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c index c9cea597a92eb6..77f5d23be78d61 100644 --- a/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c +++ b/drivers/net/wireless/ath/ath12k/wifi7/dp_mon.c @@ -405,6 +405,42 @@ ath12k_wifi7_dp_mon_hal_rx_parse_user_info(const struct hal_receive_user_info *r } } +static __always_inline u8 +ath12k_wifi7_hal_mon_map_legacy_rate_to_hw_rate(u8 rate) +{ + u8 ath12k_rate; + + /* Map hal_rx_legacy_rate to ath12k_hw_rate_cck */ + switch (rate) { + case HAL_RX_LEGACY_RATE_LP_1_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_LP_1M; + break; + case HAL_RX_LEGACY_RATE_LP_2_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_LP_2M; + break; + case HAL_RX_LEGACY_RATE_LP_5_5_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_LP_5_5M; + break; + case HAL_RX_LEGACY_RATE_LP_11_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_LP_11M; + break; + case HAL_RX_LEGACY_RATE_SP_2_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_SP_2M; + break; + case HAL_RX_LEGACY_RATE_SP_5_5_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_SP_5_5M; + break; + case HAL_RX_LEGACY_RATE_SP_11_MBPS: + ath12k_rate = ATH12K_HW_RATE_CCK_SP_11M; + break; + default: + ath12k_rate = rate; + break; + } + + return ath12k_rate; +} + static void ath12k_wifi7_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb, struct hal_rx_mon_ppdu_info *ppdu_info) @@ -415,25 +451,32 @@ ath12k_wifi7_dp_mon_parse_l_sig_b(const struct hal_rx_lsig_b_info *lsigb, rate = u32_get_bits(info0, HAL_RX_LSIG_B_INFO_INFO0_RATE); switch (rate) { case 1: - rate = HAL_RX_LEGACY_RATE_1_MBPS; + rate = HAL_RX_LEGACY_RATE_LP_1_MBPS; break; case 2: - case 5: - rate = HAL_RX_LEGACY_RATE_2_MBPS; + rate = HAL_RX_LEGACY_RATE_LP_2_MBPS; break; case 3: - case 6: - rate = HAL_RX_LEGACY_RATE_5_5_MBPS; + rate = HAL_RX_LEGACY_RATE_LP_5_5_MBPS; break; case 4: + rate = HAL_RX_LEGACY_RATE_LP_11_MBPS; + break; + case 5: + rate = HAL_RX_LEGACY_RATE_SP_2_MBPS; + break; + case 6: + rate = HAL_RX_LEGACY_RATE_SP_5_5_MBPS; + break; case 7: - rate = HAL_RX_LEGACY_RATE_11_MBPS; + rate = HAL_RX_LEGACY_RATE_SP_11_MBPS; break; default: rate = HAL_RX_LEGACY_RATE_INVALID; + break; } - ppdu_info->rate = rate; + ppdu_info->rate = ath12k_wifi7_hal_mon_map_legacy_rate_to_hw_rate(rate); ppdu_info->cck_flag = 1; } @@ -447,31 +490,32 @@ ath12k_wifi7_dp_mon_parse_l_sig_a(const struct hal_rx_lsig_a_info *lsiga, rate = u32_get_bits(info0, HAL_RX_LSIG_A_INFO_INFO0_RATE); switch (rate) { case 8: - rate = HAL_RX_LEGACY_RATE_48_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_48_MBPS; break; case 9: - rate = HAL_RX_LEGACY_RATE_24_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_24_MBPS; break; case 10: - rate = HAL_RX_LEGACY_RATE_12_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_12_MBPS; break; case 11: - rate = HAL_RX_LEGACY_RATE_6_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_6_MBPS; break; case 12: - rate = HAL_RX_LEGACY_RATE_54_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_54_MBPS; break; case 13: - rate = HAL_RX_LEGACY_RATE_36_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_36_MBPS; break; case 14: - rate = HAL_RX_LEGACY_RATE_18_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_18_MBPS; break; case 15: - rate = HAL_RX_LEGACY_RATE_9_MBPS; + rate = HAL_RX_LEGACY_RATE_OFDM_9_MBPS; break; default: - rate = HAL_RX_LEGACY_RATE_INVALID; + rate = HAL_RX_LEGACY_RATE_OFDM_INVALID; + break; } ppdu_info->rate = rate; diff --git a/drivers/net/wireless/ath/ath5k/base.c b/drivers/net/wireless/ath/ath5k/base.c index 05c9c07591fcb1..6ca31d4ea437bd 100644 --- a/drivers/net/wireless/ath/ath5k/base.c +++ b/drivers/net/wireless/ath/ath5k/base.c @@ -1738,7 +1738,8 @@ ath5k_tx_frame_completed(struct ath5k_hw *ah, struct sk_buff *skb, } info->status.rates[ts->ts_final_idx].count = ts->ts_final_retry; - info->status.rates[ts->ts_final_idx + 1].idx = -1; + if (ts->ts_final_idx + 1 < IEEE80211_TX_MAX_RATES) + info->status.rates[ts->ts_final_idx + 1].idx = -1; if (unlikely(ts->ts_status)) { ah->stats.ack_fail++; diff --git a/drivers/net/wireless/broadcom/b43/xmit.c b/drivers/net/wireless/broadcom/b43/xmit.c index 7651b1bdb59266..f0b082596637ff 100644 --- a/drivers/net/wireless/broadcom/b43/xmit.c +++ b/drivers/net/wireless/broadcom/b43/xmit.c @@ -702,7 +702,8 @@ void b43_rx(struct b43_wldev *dev, struct sk_buff *skb, const void *_rxhdr) * key index, but the ucode passed it slightly different. */ keyidx = b43_kidx_to_raw(dev, keyidx); - B43_WARN_ON(keyidx >= ARRAY_SIZE(dev->key)); + if (B43_WARN_ON(keyidx >= ARRAY_SIZE(dev->key))) + goto drop; if (dev->key[keyidx].algorithm != B43_SEC_ALGO_NONE) { wlhdr_len = ieee80211_hdrlen(fctl); diff --git a/drivers/net/wireless/broadcom/b43legacy/xmit.c b/drivers/net/wireless/broadcom/b43legacy/xmit.c index efd63f4ce74f2b..ee199d4eaf039a 100644 --- a/drivers/net/wireless/broadcom/b43legacy/xmit.c +++ b/drivers/net/wireless/broadcom/b43legacy/xmit.c @@ -476,7 +476,8 @@ void b43legacy_rx(struct b43legacy_wldev *dev, * key index, but the ucode passed it slightly different. */ keyidx = b43legacy_kidx_to_raw(dev, keyidx); - B43legacy_WARN_ON(keyidx >= dev->max_nr_keys); + if (B43legacy_WARN_ON(keyidx >= dev->max_nr_keys)) + goto drop; if (dev->key[keyidx].algorithm != B43legacy_SEC_ALGO_NONE) { /* Remove PROTECTED flag to mark it as decrypted. */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile index e5ca0f51182271..6fd805023500be 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/Makefile @@ -25,7 +25,11 @@ brcmfmac-objs += \ btcoex.o \ vendor.o \ pno.o \ - xtlv.o + join_param.o \ + scan_param.o \ + xtlv.o \ + interface_create.o + brcmfmac-$(CONFIG_BRCMFMAC_PROTO_BCDC) += \ bcdc.o \ fwsignal.o diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index fe31051a9e11b1..5efd7f6d757a4c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -107,6 +107,7 @@ struct brcmf_bus_ops { void (*debugfs_create)(struct device *dev); int (*reset)(struct device *dev); void (*remove)(struct device *dev); + void (*d2h_mb_rx)(struct device *dev, u32 data); }; @@ -286,6 +287,15 @@ static inline void brcmf_bus_remove(struct brcmf_bus *bus) bus->ops->remove(bus->dev); } +static inline +void brcmf_bus_d2h_mb_rx(struct brcmf_bus *bus, u32 data) +{ + if (!bus->ops->d2h_mb_rx) + return; + + return bus->ops->d2h_mb_rx(bus->dev, data); +} + /* * interface functions from common layer */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index cea02b33b79857..7b76385f183b0a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -32,7 +32,11 @@ #include "vendor.h" #include "bus.h" #include "common.h" +#include "feature.h" #include "fwvid.h" +#include "xtlv.h" +#include "ratespec.h" +#include "interface_create.h" #define BRCMF_SCAN_IE_LEN_MAX 2048 @@ -64,6 +68,8 @@ #define RSN_CAP_MFPR_MASK BIT(6) #define RSN_CAP_MFPC_MASK BIT(7) #define RSN_PMKID_COUNT_LEN 2 +#define DPP_AKM_SUITE_TYPE 2 +#define WLAN_AKM_SUITE_DPP SUITE(WLAN_OUI_WFA, DPP_AKM_SUITE_TYPE) #define VNDR_IE_CMD_LEN 4 /* length of the set command * string :"add", "del" (+ NUL) @@ -76,10 +82,6 @@ #define DOT11_BCN_PRB_FIXED_LEN 12 /* beacon/probe fixed length */ -#define BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS 320 -#define BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS 400 -#define BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS 20 - #define BRCMF_SCAN_CHANNEL_TIME 40 #define BRCMF_SCAN_UNASSOC_TIME 40 #define BRCMF_SCAN_PASSIVE_TIME 120 @@ -98,9 +100,6 @@ #define PKT_TOKEN_IDX 15 #define IDLE_TOKEN_IDX 12 -#define BRCMF_ASSOC_PARAMS_FIXED_SIZE \ - (sizeof(struct brcmf_assoc_params_le) - sizeof(u16)) - #define BRCMF_MAX_CHANSPEC_LIST \ (BRCMF_DCMD_MEDLEN / sizeof(__le32) - 1) @@ -124,6 +123,13 @@ struct cca_msrmnt_query { u32 time_req; }; +/* algo bit vector */ +#define KEY_ALGO_MASK(_algo) (1 << (_algo)) + +/* start enum value for BSS properties */ +#define WL_WSEC_INFO_BSS_BASE 0x0100 +#define WL_WSEC_INFO_BSS_ALGOS (WL_WSEC_INFO_BSS_BASE + 6) + static bool check_vif_up(struct brcmf_cfg80211_vif *vif) { if (!test_bit(BRCMF_VIF_STATUS_READY, &vif->sme_state)) { @@ -178,6 +184,15 @@ static struct ieee80211_rate __wl_rates[] = { .max_power = 30, \ } +#define CHAN6G(_channel) { \ + .band = NL80211_BAND_6GHZ, \ + .center_freq = ((_channel == 2) ? 5935 : 5950 + (5 * (_channel))), \ + .hw_value = (_channel), \ + .max_antenna_gain = 0, \ + .max_power = 30, \ +} + + static struct ieee80211_channel __wl_2ghz_channels[] = { CHAN2G(1, 2412), CHAN2G(2, 2417), CHAN2G(3, 2422), CHAN2G(4, 2427), CHAN2G(5, 2432), CHAN2G(6, 2437), CHAN2G(7, 2442), CHAN2G(8, 2447), @@ -194,6 +209,23 @@ static struct ieee80211_channel __wl_5ghz_channels[] = { CHAN5G(153), CHAN5G(157), CHAN5G(161), CHAN5G(165) }; +static struct ieee80211_channel __wl_6ghz_channels[] = { + CHAN6G(1), CHAN6G(2), CHAN6G(5), CHAN6G(9), CHAN6G(13), + CHAN6G(17), CHAN6G(21), CHAN6G(25), CHAN6G(29), CHAN6G(33), + CHAN6G(37), CHAN6G(41), CHAN6G(45), CHAN6G(49), CHAN6G(53), + CHAN6G(57), CHAN6G(61), CHAN6G(65), CHAN6G(69), CHAN6G(73), + CHAN6G(77), CHAN6G(81), CHAN6G(85), CHAN6G(89), CHAN6G(93), + CHAN6G(97), CHAN6G(101), CHAN6G(105), CHAN6G(109), CHAN6G(113), + CHAN6G(117), CHAN6G(121), CHAN6G(125), CHAN6G(129), CHAN6G(133), + CHAN6G(137), CHAN6G(141), CHAN6G(145), CHAN6G(149), CHAN6G(153), + CHAN6G(157), CHAN6G(161), CHAN6G(165), CHAN6G(169), CHAN6G(173), + CHAN6G(177), CHAN6G(181), CHAN6G(185), CHAN6G(189), CHAN6G(193), + CHAN6G(197), CHAN6G(201), CHAN6G(205), CHAN6G(209), CHAN6G(213), + CHAN6G(217), CHAN6G(221), CHAN6G(225), CHAN6G(229), CHAN6G(233), +}; + +struct ieee80211_sband_iftype_data sdata[NUM_NL80211_BANDS]; + /* Band templates duplicated per wiphy. The channel info * above is added to the band during setup. */ @@ -209,6 +241,12 @@ static const struct ieee80211_supported_band __wl_band_5ghz = { .n_bitrates = wl_a_rates_size, }; +static const struct ieee80211_supported_band __wl_band_6ghz = { + .band = NL80211_BAND_6GHZ, + .bitrates = wl_a_rates, + .n_bitrates = wl_a_rates_size, +}; + /* This is to override regulatory domains defined in cfg80211 module (reg.c) * By default world regulatory domain defined in reg.c puts the flags * NL80211_RRF_NO_IR for 5GHz channels (for * 36..48 and 149..165). @@ -217,35 +255,43 @@ static const struct ieee80211_supported_band __wl_band_5ghz = { * domain are to be done here. */ static const struct ieee80211_regdomain brcmf_regdom = { - .n_reg_rules = 4, + .n_reg_rules = 5, .alpha2 = "99", .reg_rules = { /* IEEE 802.11b/g, channels 1..11 */ - REG_RULE(2412-10, 2472+10, 40, 6, 20, 0), + REG_RULE(2412 - 10, 2472 + 10, 40, 6, 20, 0), /* If any */ /* IEEE 802.11 channel 14 - Only JP enables * this and for 802.11b only */ - REG_RULE(2484-10, 2484+10, 20, 6, 20, 0), + REG_RULE(2484 - 10, 2484 + 10, 20, 6, 20, 0), /* IEEE 802.11a, channel 36..64 */ - REG_RULE(5150-10, 5350+10, 160, 6, 20, 0), + REG_RULE(5150 - 10, 5350 + 10, 160, 6, 20, 0), /* IEEE 802.11a, channel 100..165 */ - REG_RULE(5470-10, 5850+10, 160, 6, 20, 0), } + REG_RULE(5470 - 10, 5850 + 10, 160, 6, 20, 0), + /* IEEE 802.11ax, 6E */ + REG_RULE(5935 - 10, 7115 + 10, 160, 6, 20, 0), } }; /* Note: brcmf_cipher_suites is an array of int defining which cipher suites * are supported. A pointer to this array and the number of entries is passed * on to upper layers. AES_CMAC defines whether or not the driver supports MFP. - * So the cipher suite AES_CMAC has to be the last one in the array, and when - * device does not support MFP then the number of suites will be decreased by 1 + * MFP support includes a few other suites, so if MFP is not supported, + * then the number of suites will be decreased by 4 */ static const u32 brcmf_cipher_suites[] = { WLAN_CIPHER_SUITE_WEP40, WLAN_CIPHER_SUITE_WEP104, WLAN_CIPHER_SUITE_TKIP, WLAN_CIPHER_SUITE_CCMP, - /* Keep as last entry: */ - WLAN_CIPHER_SUITE_AES_CMAC + WLAN_CIPHER_SUITE_CCMP_256, + WLAN_CIPHER_SUITE_GCMP, + WLAN_CIPHER_SUITE_GCMP_256, + /* Keep as last 4 entries: */ + WLAN_CIPHER_SUITE_AES_CMAC, + WLAN_CIPHER_SUITE_BIP_CMAC_256, + WLAN_CIPHER_SUITE_BIP_GMAC_128, + WLAN_CIPHER_SUITE_BIP_GMAC_256 }; /* Vendor specific ie. id = 221, oui and type defines exact ie */ @@ -267,48 +313,6 @@ struct parsed_vndr_ies { struct parsed_vndr_ie_info ie_info[VNDR_IE_PARSE_LIMIT]; }; -#define WL_INTERFACE_CREATE_VER_1 1 -#define WL_INTERFACE_CREATE_VER_2 2 -#define WL_INTERFACE_CREATE_VER_3 3 -#define WL_INTERFACE_CREATE_VER_MAX WL_INTERFACE_CREATE_VER_3 - -#define WL_INTERFACE_MAC_DONT_USE 0x0 -#define WL_INTERFACE_MAC_USE 0x2 - -#define WL_INTERFACE_CREATE_STA 0x0 -#define WL_INTERFACE_CREATE_AP 0x1 - -struct wl_interface_create_v1 { - u16 ver; /* structure version */ - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u32 wlc_index; /* optional for wlc index */ -}; - -struct wl_interface_create_v2 { - u16 ver; /* structure version */ - u8 pad1[2]; - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u8 iftype; /* type of interface created */ - u8 pad2; - u32 wlc_index; /* optional for wlc index */ -}; - -struct wl_interface_create_v3 { - u16 ver; /* structure version */ - u16 len; /* length of structure + data */ - u16 fixed_len; /* length of structure */ - u8 iftype; /* type of interface created */ - u8 wlc_index; /* optional for wlc index */ - u32 flags; /* flags for operation */ - u8 mac_addr[ETH_ALEN]; /* MAC address */ - u8 bssid[ETH_ALEN]; /* optional for BSSID */ - u8 if_index; /* interface index request */ - u8 pad[3]; - u8 data[]; /* Optional for specific data */ -}; - static u8 nl80211_band_to_fwil(enum nl80211_band band) { switch (band) { @@ -316,6 +320,8 @@ static u8 nl80211_band_to_fwil(enum nl80211_band band) return WLC_BAND_2G; case NL80211_BAND_5GHZ: return WLC_BAND_5G; + case NL80211_BAND_6GHZ: + return WLC_BAND_6G; default: WARN_ON(1); break; @@ -323,8 +329,25 @@ static u8 nl80211_band_to_fwil(enum nl80211_band band) return 0; } -static u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, - struct cfg80211_chan_def *ch) +static int nl80211_band_to_chanspec_band(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return BRCMU_CHAN_BAND_2G; + case NL80211_BAND_5GHZ: + return BRCMU_CHAN_BAND_5G; + case NL80211_BAND_6GHZ: + return BRCMU_CHAN_BAND_6G; + case NL80211_BAND_60GHZ: + default: + WARN_ON_ONCE(1); + // Choose a safe default + return BRCMU_CHAN_BAND_2G; + } +} + +u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, + struct cfg80211_chan_def *ch) { struct brcmu_chan ch_inf; s32 primary_offset; @@ -382,17 +405,7 @@ static u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, default: WARN_ON_ONCE(1); } - switch (ch->chan->band) { - case NL80211_BAND_2GHZ: - ch_inf.band = BRCMU_CHAN_BAND_2G; - break; - case NL80211_BAND_5GHZ: - ch_inf.band = BRCMU_CHAN_BAND_5G; - break; - case NL80211_BAND_60GHZ: - default: - WARN_ON_ONCE(1); - } + ch_inf.band = nl80211_band_to_chanspec_band(ch->chan->band); d11inf->encchspec(&ch_inf); brcmf_dbg(TRACE, "chanspec: 0x%x\n", ch_inf.chspec); @@ -404,6 +417,7 @@ u16 channel_to_chanspec(struct brcmu_d11inf *d11inf, { struct brcmu_chan ch_inf; + ch_inf.band = nl80211_band_to_chanspec_band(ch->band); ch_inf.chnum = ieee80211_frequency_to_channel(ch->center_freq); ch_inf.bw = BRCMU_CHAN_BW_20; d11inf->encchspec(&ch_inf); @@ -581,231 +595,6 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) ADDR_INDIRECT); } -static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) -{ - int bsscfgidx; - - for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) { - /* bsscfgidx 1 is reserved for legacy P2P */ - if (bsscfgidx == 1) - continue; - if (!drvr->iflist[bsscfgidx]) - return bsscfgidx; - } - - return -ENOMEM; -} - -static void brcmf_set_vif_sta_macaddr(struct brcmf_if *ifp, u8 *mac_addr) -{ - u8 mac_idx = ifp->drvr->sta_mac_idx; - - /* set difference MAC address with locally administered bit */ - memcpy(mac_addr, ifp->mac_addr, ETH_ALEN); - mac_addr[0] |= 0x02; - mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; - mac_idx++; - mac_idx = mac_idx % 2; - ifp->drvr->sta_mac_idx = mac_idx; -} - -static int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) -{ - struct wl_interface_create_v1 iface_v1; - struct wl_interface_create_v2 iface_v2; - struct wl_interface_create_v3 iface_v3; - u32 iface_create_ver; - int err; - - /* interface_create version 1 */ - memset(&iface_v1, 0, sizeof(iface_v1)); - iface_v1.ver = WL_INTERFACE_CREATE_VER_1; - iface_v1.flags = WL_INTERFACE_CREATE_STA | - WL_INTERFACE_MAC_USE; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v1.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v1.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v1, - sizeof(iface_v1)); - if (err) { - brcmf_info("failed to create interface(v1), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v1)\n"); - return 0; - } - - /* interface_create version 2 */ - memset(&iface_v2, 0, sizeof(iface_v2)); - iface_v2.ver = WL_INTERFACE_CREATE_VER_2; - iface_v2.flags = WL_INTERFACE_MAC_USE; - iface_v2.iftype = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v2.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v2.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v2, - sizeof(iface_v2)); - if (err) { - brcmf_info("failed to create interface(v2), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v2)\n"); - return 0; - } - - /* interface_create version 3+ */ - /* get supported version from firmware side */ - iface_create_ver = 0; - err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", - &iface_create_ver); - if (err) { - brcmf_err("fail to get supported version, err=%d\n", err); - return -EOPNOTSUPP; - } - - switch (iface_create_ver) { - case WL_INTERFACE_CREATE_VER_3: - memset(&iface_v3, 0, sizeof(iface_v3)); - iface_v3.ver = WL_INTERFACE_CREATE_VER_3; - iface_v3.flags = WL_INTERFACE_MAC_USE; - iface_v3.iftype = WL_INTERFACE_CREATE_STA; - if (!is_zero_ether_addr(macaddr)) - memcpy(iface_v3.mac_addr, macaddr, ETH_ALEN); - else - brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v3, - sizeof(iface_v3)); - - if (!err) - brcmf_dbg(INFO, "interface created(v3)\n"); - break; - default: - brcmf_err("not support interface create(v%d)\n", - iface_create_ver); - err = -EOPNOTSUPP; - break; - } - - if (err) { - brcmf_info("station interface creation failed (%d)\n", - err); - return -EIO; - } - - return 0; -} - -static int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) -{ - struct wl_interface_create_v1 iface_v1; - struct wl_interface_create_v2 iface_v2; - struct wl_interface_create_v3 iface_v3; - u32 iface_create_ver; - struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_mbss_ssid_le mbss_ssid_le; - int bsscfgidx; - int err; - - /* interface_create version 1 */ - memset(&iface_v1, 0, sizeof(iface_v1)); - iface_v1.ver = WL_INTERFACE_CREATE_VER_1; - iface_v1.flags = WL_INTERFACE_CREATE_AP | - WL_INTERFACE_MAC_USE; - - brcmf_set_vif_sta_macaddr(ifp, iface_v1.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v1, - sizeof(iface_v1)); - if (err) { - brcmf_info("failed to create interface(v1), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v1)\n"); - return 0; - } - - /* interface_create version 2 */ - memset(&iface_v2, 0, sizeof(iface_v2)); - iface_v2.ver = WL_INTERFACE_CREATE_VER_2; - iface_v2.flags = WL_INTERFACE_MAC_USE; - iface_v2.iftype = WL_INTERFACE_CREATE_AP; - - brcmf_set_vif_sta_macaddr(ifp, iface_v2.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v2, - sizeof(iface_v2)); - if (err) { - brcmf_info("failed to create interface(v2), err=%d\n", - err); - } else { - brcmf_dbg(INFO, "interface created(v2)\n"); - return 0; - } - - /* interface_create version 3+ */ - /* get supported version from firmware side */ - iface_create_ver = 0; - err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", - &iface_create_ver); - if (err) { - brcmf_err("fail to get supported version, err=%d\n", err); - return -EOPNOTSUPP; - } - - switch (iface_create_ver) { - case WL_INTERFACE_CREATE_VER_3: - memset(&iface_v3, 0, sizeof(iface_v3)); - iface_v3.ver = WL_INTERFACE_CREATE_VER_3; - iface_v3.flags = WL_INTERFACE_MAC_USE; - iface_v3.iftype = WL_INTERFACE_CREATE_AP; - brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - - err = brcmf_fil_iovar_data_get(ifp, "interface_create", - &iface_v3, - sizeof(iface_v3)); - - if (!err) - brcmf_dbg(INFO, "interface created(v3)\n"); - break; - default: - brcmf_err("not support interface create(v%d)\n", - iface_create_ver); - err = -EOPNOTSUPP; - break; - } - - if (err) { - brcmf_info("Does not support interface_create (%d)\n", - err); - memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le)); - bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr); - if (bsscfgidx < 0) - return bsscfgidx; - - mbss_ssid_le.bsscfgidx = cpu_to_le32(bsscfgidx); - mbss_ssid_le.SSID_len = cpu_to_le32(5); - sprintf(mbss_ssid_le.SSID, "ssid%d", bsscfgidx); - - err = brcmf_fil_bsscfg_data_set(ifp, "bsscfg:ssid", &mbss_ssid_le, - sizeof(mbss_ssid_le)); - - if (err < 0) - bphy_err(drvr, "setting ssid failed %d\n", err); - } - - return err; -} - /** * brcmf_apsta_add_vif() - create a new AP or STA virtual interface * @@ -1058,134 +847,11 @@ bool brcmf_is_apmode_operating(struct wiphy *wiphy) return ret; } -static void brcmf_scan_params_v2_to_v1(struct brcmf_scan_params_v2_le *params_v2_le, - struct brcmf_scan_params_le *params_le) -{ - size_t params_size; - u32 ch; - int n_channels, n_ssids; - - memcpy(¶ms_le->ssid_le, ¶ms_v2_le->ssid_le, - sizeof(params_le->ssid_le)); - memcpy(¶ms_le->bssid, ¶ms_v2_le->bssid, - sizeof(params_le->bssid)); - - params_le->bss_type = params_v2_le->bss_type; - params_le->scan_type = le32_to_cpu(params_v2_le->scan_type); - params_le->nprobes = params_v2_le->nprobes; - params_le->active_time = params_v2_le->active_time; - params_le->passive_time = params_v2_le->passive_time; - params_le->home_time = params_v2_le->home_time; - params_le->channel_num = params_v2_le->channel_num; - - ch = le32_to_cpu(params_v2_le->channel_num); - n_channels = ch & BRCMF_SCAN_PARAMS_COUNT_MASK; - n_ssids = ch >> BRCMF_SCAN_PARAMS_NSSID_SHIFT; - - params_size = sizeof(u16) * n_channels; - if (n_ssids > 0) { - params_size = roundup(params_size, sizeof(u32)); - params_size += sizeof(struct brcmf_ssid_le) * n_ssids; - } - - memcpy(¶ms_le->channel_list[0], - ¶ms_v2_le->channel_list[0], params_size); -} - -static void brcmf_escan_prep(struct brcmf_cfg80211_info *cfg, - struct brcmf_scan_params_v2_le *params_le, - struct cfg80211_scan_request *request) -{ - u32 n_ssids; - u32 n_channels; - s32 i; - s32 offset; - u16 chanspec; - char *ptr; - int length; - struct brcmf_ssid_le ssid_le; - - eth_broadcast_addr(params_le->bssid); - - length = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; - - params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); - params_le->bss_type = DOT11_BSSTYPE_ANY; - params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_ACTIVE); - params_le->channel_num = 0; - params_le->nprobes = cpu_to_le32(-1); - params_le->active_time = cpu_to_le32(-1); - params_le->passive_time = cpu_to_le32(-1); - params_le->home_time = cpu_to_le32(-1); - memset(¶ms_le->ssid_le, 0, sizeof(params_le->ssid_le)); - - /* Scan abort */ - if (!request) { - length += sizeof(u16); - params_le->channel_num = cpu_to_le32(1); - params_le->channel_list[0] = cpu_to_le16(-1); - params_le->length = cpu_to_le16(length); - return; - } - - n_ssids = request->n_ssids; - n_channels = request->n_channels; - - /* Copy channel array if applicable */ - brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", - n_channels); - if (n_channels > 0) { - length += roundup(sizeof(u16) * n_channels, sizeof(u32)); - for (i = 0; i < n_channels; i++) { - chanspec = channel_to_chanspec(&cfg->d11inf, - request->channels[i]); - brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n", - request->channels[i]->hw_value, chanspec); - params_le->channel_list[i] = cpu_to_le16(chanspec); - } - } else { - brcmf_dbg(SCAN, "Scanning all channels\n"); - } - - /* Copy ssid array if applicable */ - brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); - if (n_ssids > 0) { - offset = offsetof(struct brcmf_scan_params_v2_le, channel_list) + - n_channels * sizeof(u16); - offset = roundup(offset, sizeof(u32)); - length += sizeof(ssid_le) * n_ssids; - ptr = (char *)params_le + offset; - for (i = 0; i < n_ssids; i++) { - memset(&ssid_le, 0, sizeof(ssid_le)); - ssid_le.SSID_len = - cpu_to_le32(request->ssids[i].ssid_len); - memcpy(ssid_le.SSID, request->ssids[i].ssid, - request->ssids[i].ssid_len); - if (!ssid_le.SSID_len) - brcmf_dbg(SCAN, "%d: Broadcast scan\n", i); - else - brcmf_dbg(SCAN, "%d: scan for %.32s size=%d\n", - i, ssid_le.SSID, ssid_le.SSID_len); - memcpy(ptr, &ssid_le, sizeof(ssid_le)); - ptr += sizeof(ssid_le); - } - } else { - brcmf_dbg(SCAN, "Performing passive scan\n"); - params_le->scan_type = cpu_to_le32(BRCMF_SCANTYPE_PASSIVE); - } - params_le->length = cpu_to_le16(length); - /* Adding mask to channel numbers */ - params_le->channel_num = - cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | - (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); -} - s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bool aborted, bool fw_abort) { struct brcmf_pub *drvr = cfg->pub; - struct brcmf_scan_params_v2_le params_v2_le; struct cfg80211_scan_request *scan_request; u64 reqid; u32 bucket; @@ -1201,25 +867,16 @@ s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, timer_delete_sync(&cfg->escan_timeout); if (fw_abort) { + u32 len; + void *data = drvr->scan_param_handler.get_struct_for_request(cfg, &len, NULL); + if (!data){ + bphy_err(drvr, "Scan abort failed to prepare abort struct\n"); + return 0; + } /* Do a scan abort to stop the driver's scan engine */ brcmf_dbg(SCAN, "ABORT scan in firmware\n"); - - brcmf_escan_prep(cfg, ¶ms_v2_le, NULL); - - /* E-Scan (or anyother type) can be aborted by SCAN */ - if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_v2_le, - sizeof(params_v2_le)); - } else { - struct brcmf_scan_params_le params_le; - - brcmf_scan_params_v2_to_v1(¶ms_v2_le, ¶ms_le); - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, - ¶ms_le, - sizeof(params_le)); - } - + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SCAN, data, len); + kfree(data); if (err) bphy_err(drvr, "Scan abort failed\n"); } @@ -1443,19 +1100,24 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, struct cfg80211_scan_request *request) { struct brcmf_pub *drvr = cfg->pub; - s32 params_size = BRCMF_SCAN_PARAMS_V2_FIXED_SIZE + - offsetof(struct brcmf_escan_params_le, params_v2_le); + u32 struct_size = 0; + void *prepped_params = NULL; + u32 params_size = 0; struct brcmf_escan_params_le *params; s32 err = 0; brcmf_dbg(SCAN, "E-SCAN START\n"); - if (request != NULL) { - /* Allocate space for populating ssids in struct */ - params_size += sizeof(u32) * ((request->n_channels + 1) / 2); - - /* Allocate space for populating ssids in struct */ - params_size += sizeof(struct brcmf_ssid_le) * request->n_ssids; + prepped_params = drvr->scan_param_handler.get_struct_for_request(cfg, &struct_size, request); + if (!prepped_params) { + err = -EINVAL; + goto exit; + } + params_size = struct_size + + offsetof(struct brcmf_escan_params_le, params_v4_le); + if (!params_size) { + err = -EINVAL; + goto exit; } params = kzalloc(params_size, GFP_KERNEL); @@ -1463,27 +1125,14 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, err = -ENOMEM; goto exit; } - BUG_ON(params_size + sizeof("escan") >= BRCMF_DCMD_MEDLEN); - brcmf_escan_prep(cfg, ¶ms->params_v2_le, request); - - params->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION_V2); - - if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SCAN_V2)) { - struct brcmf_escan_params_le *params_v1; - - params_size -= BRCMF_SCAN_PARAMS_V2_FIXED_SIZE; - params_size += BRCMF_SCAN_PARAMS_FIXED_SIZE; - params_v1 = kzalloc(params_size, GFP_KERNEL); - if (!params_v1) { - err = -ENOMEM; - goto exit_params; - } - params_v1->version = cpu_to_le32(BRCMF_ESCAN_REQ_VERSION); - brcmf_scan_params_v2_to_v1(¶ms->params_v2_le, ¶ms_v1->params_le); - kfree(params); - params = params_v1; - } + /* Copy into the largest part */ + unsafe_memcpy( + ¶ms->params_v4_le, prepped_params, struct_size, + /* A composite flex-array that is at least as large as the memcpy due to the allocation above */); + /* We can now free the original prepped parameters */ + kfree(prepped_params); + params->version = cpu_to_le32(drvr->scan_param_handler.version); params->action = cpu_to_le16(WL_ESCAN_ACTION_START); params->sync_id = cpu_to_le16(0x1234); @@ -1495,7 +1144,6 @@ brcmf_run_escan(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp, bphy_err(drvr, "error (%d)\n", err); } -exit_params: kfree(params); exit: return err; @@ -1779,21 +1427,19 @@ static void brcmf_link_down(struct brcmf_cfg80211_vif *vif, u16 reason, brcmf_dbg(TRACE, "Exit\n"); } -static s32 -brcmf_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_ibss_params *params) +static s32 brcmf_cfg80211_join_ibss(struct wiphy *wiphy, + struct net_device *ndev, + struct cfg80211_ibss_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; struct brcmf_pub *drvr = cfg->pub; - struct brcmf_join_params join_params; - size_t join_params_size = 0; - s32 err = 0; + void *join_params; + u32 join_params_size = 0; s32 wsec = 0; s32 bcnprd; - u16 chanspec; - u32 ssid_len; + s32 err = 0; brcmf_dbg(TRACE, "Enter\n"); if (!check_vif_up(ifp->vif)) @@ -1867,58 +1513,39 @@ brcmf_cfg80211_join_ibss(struct wiphy *wiphy, struct net_device *ndev, goto done; } - /* Configure required join parameter */ - memset(&join_params, 0, sizeof(struct brcmf_join_params)); - - /* SSID */ - ssid_len = min_t(u32, params->ssid_len, IEEE80211_MAX_SSID_LEN); - memcpy(join_params.ssid_le.SSID, params->ssid, ssid_len); - join_params.ssid_le.SSID_len = cpu_to_le32(ssid_len); - join_params_size = sizeof(join_params.ssid_le); - - /* BSSID */ if (params->bssid) { - memcpy(join_params.params_le.bssid, params->bssid, ETH_ALEN); - join_params_size += BRCMF_ASSOC_PARAMS_FIXED_SIZE; memcpy(profile->bssid, params->bssid, ETH_ALEN); } else { - eth_broadcast_addr(join_params.params_le.bssid); eth_zero_addr(profile->bssid); } - /* Channel */ + cfg->ibss_starter = false; + cfg->channel = 0; if (params->chandef.chan) { - u32 target_channel; + u16 chanspec; + cfg->channel = ieee80211_frequency_to_channel( + params->chandef.chan->center_freq); + /* adding chanspec */ + chanspec = chandef_to_chanspec(&cfg->d11inf, ¶ms->chandef); - cfg->channel = - ieee80211_frequency_to_channel( - params->chandef.chan->center_freq); - if (params->channel_fixed) { - /* adding chanspec */ - chanspec = chandef_to_chanspec(&cfg->d11inf, - ¶ms->chandef); - join_params.params_le.chanspec_list[0] = - cpu_to_le16(chanspec); - join_params.params_le.chanspec_num = cpu_to_le32(1); - join_params_size += sizeof(join_params.params_le); - } - - /* set channel for starter */ - target_channel = cfg->channel; - err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_CHANNEL, - target_channel); + /* set chanspec */ + err = brcmf_fil_iovar_int_set(ifp, "chanspec", chanspec); if (err) { - bphy_err(drvr, "WLC_SET_CHANNEL failed (%d)\n", err); + bphy_err(drvr, "Setting chanspec failed (%d)\n", err); goto done; } - } else - cfg->channel = 0; - - cfg->ibss_starter = false; - + } - err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, join_params_size); + join_params = drvr->join_param_handler.get_struct_for_ibss( + cfg, &join_params_size, params); + if (!join_params) { + bphy_err(drvr, "Converting join params failed\n"); + goto done; + } + err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, join_params, + join_params_size); + /* Free params no matter what */ + kfree(join_params); if (err) { bphy_err(drvr, "WLC_SET_SSID failed (%d)\n", err); goto done; @@ -1969,6 +1596,10 @@ static s32 brcmf_set_wpa_version(struct net_device *ndev, if (drvr->bus_if->fwvid == BRCMF_FWVENDOR_CYW && sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_SAE) val = WPA3_AUTH_SAE_PSK; + else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_SAE) + val = WPA3_AUTH_SAE_PSK; + else if (sme->crypto.akm_suites[0] == WLAN_AKM_SUITE_OWE) + val = WPA3_AUTH_OWE; else val = WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED; } else if (sme->crypto.wpa_versions & NL80211_WPA_VERSION_3) { @@ -2026,6 +1657,48 @@ static s32 brcmf_set_auth_type(struct net_device *ndev, return err; } +static s32 brcmf_set_wsec_info_algos(struct brcmf_if *ifp, u32 algos, u32 mask) +{ + struct brcmf_pub *drvr = ifp->drvr; + s32 err = 0; + struct brcmf_wsec_info *wsec_info; + struct brcmf_xtlv *wsec_info_tlv; + u16 tlv_data_len; + u8 tlv_data[8]; + u32 param_len; + u8 *buf; + + brcmf_dbg(TRACE, "Enter\n"); + + buf = kzalloc(sizeof(struct brcmf_wsec_info) + sizeof(tlv_data), + GFP_KERNEL); + if (!buf) { + bphy_err(drvr, "unable to allocate.\n"); + return -ENOMEM; + } + wsec_info = (struct brcmf_wsec_info *)buf; + wsec_info->version = BRCMF_WSEC_INFO_VER; + wsec_info_tlv = + (struct brcmf_xtlv *)(buf + + offsetof(struct brcmf_wsec_info, tlvs)); + wsec_info->num_tlvs++; + tlv_data_len = sizeof(tlv_data); + memcpy(tlv_data, &algos, sizeof(algos)); + memcpy(tlv_data + sizeof(algos), &mask, sizeof(mask)); + brcmf_xtlv_pack_header(wsec_info_tlv, WL_WSEC_INFO_BSS_ALGOS, + tlv_data_len, tlv_data, 0); + + param_len = offsetof(struct brcmf_wsec_info, tlvs) + + offsetof(struct brcmf_wsec_info_tlv, data) + tlv_data_len; + + err = brcmf_fil_bsscfg_data_set(ifp, "wsec_info", buf, param_len); + if (err) + brcmf_err("set wsec_info_error:%d\n", err); + + kfree(buf); + return err; +} + static s32 brcmf_set_wsec_mode(struct net_device *ndev, struct cfg80211_connect_params *sme) @@ -2038,6 +1711,8 @@ brcmf_set_wsec_mode(struct net_device *ndev, s32 gval = 0; s32 wsec; s32 err = 0; + u32 algos = 0; + u32 mask = 0; if (sme->crypto.n_ciphers_pairwise) { switch (sme->crypto.ciphers_pairwise[0]) { @@ -2054,6 +1729,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, case WLAN_CIPHER_SUITE_AES_CMAC: pval = AES_ENABLED; break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("This chip does not support GCMP\n"); + return -EOPNOTSUPP; + } + pval = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; default: bphy_err(drvr, "invalid cipher pairwise (%d)\n", sme->crypto.ciphers_pairwise[0]); @@ -2075,6 +1759,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, case WLAN_CIPHER_SUITE_AES_CMAC: gval = AES_ENABLED; break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("This chip does not support GCMP\n"); + return -EOPNOTSUPP; + } + gval = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; default: bphy_err(drvr, "invalid cipher group (%d)\n", sme->crypto.cipher_group); @@ -2083,6 +1776,7 @@ brcmf_set_wsec_mode(struct net_device *ndev, } brcmf_dbg(CONN, "pval (%d) gval (%d)\n", pval, gval); + brcmf_dbg(CONN, "algos (0x%x) mask (0x%x)\n", algos, mask); /* In case of privacy, but no security and WPS then simulate */ /* setting AES. WPS-2.0 allows no security */ if (brcmf_find_wpsie(sme->ie, sme->ie_len) && !pval && !gval && @@ -2095,6 +1789,15 @@ brcmf_set_wsec_mode(struct net_device *ndev, bphy_err(drvr, "error (%d)\n", err); return err; } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_dbg(CONN, "set_wsec_info algos (0x%x) mask (0x%x)\n", + algos, mask); + err = brcmf_set_wsec_info_algos(ifp, algos, mask); + if (err) { + brcmf_err("set wsec_info error (%d)\n", err); + return err; + } + } sec = &profile->sec; sec->cipher_pairwise = sme->crypto.ciphers_pairwise[0]; @@ -2118,9 +1821,13 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) u16 rsn_cap; u32 mfp; u16 count; + s32 okc_enable; + u16 pmkid_count; + const u8 *group_mgmt_cs = NULL; profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; profile->is_ft = false; + profile->is_okc = false; if (!sme->crypto.n_akm_suites) return 0; @@ -2137,13 +1844,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) val = WPA_AUTH_UNSPECIFIED; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_PSK: val = WPA_AUTH_PSK; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) { @@ -2152,11 +1861,15 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) val = WPA2_AUTH_UNSPECIFIED; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_8021X_SHA256: val = WPA2_AUTH_1X_SHA256; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_PSK_SHA256: val = WPA2_AUTH_PSK_SHA256; @@ -2169,14 +1882,35 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->is_ft = true; if (sme->want_1x) profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; case WLAN_AKM_SUITE_FT_PSK: val = WPA2_AUTH_PSK | WPA2_AUTH_FT; profile->is_ft = true; + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; + break; + case WLAN_AKM_SUITE_DPP: + val = WFA_AUTH_DPP; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_NONE; + break; + case WLAN_AKM_SUITE_OWE: + val = WPA3_AUTH_OWE; + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; + break; + case WLAN_AKM_SUITE_8021X_SUITE_B_192: + val = WPA3_AUTH_1X_SUITE_B_SHA384; + if (sme->want_1x) + profile->use_fwsup = BRCMF_PROFILE_FWSUP_1X; + else + profile->use_fwsup = BRCMF_PROFILE_FWSUP_ROAM; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } } else if (val & WPA3_AUTH_SAE_PSK) { @@ -2189,17 +1923,38 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) profile->is_ft = true; break; default: - bphy_err(drvr, "invalid akm suite (%d)\n", - sme->crypto.akm_suites[0]); + bphy_err(drvr, "invalid cipher group (%d)\n", + sme->crypto.cipher_group); return -EINVAL; } if (sme->crypto.sae_pwd) { profile->use_fwsup = BRCMF_PROFILE_FWSUP_SAE; } } - - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) + if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X) || + (profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM)) { brcmf_dbg(INFO, "using 1X offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } else if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE && + (val == WPA3_AUTH_SAE_PSK)) { + brcmf_dbg(INFO, "not using SAE offload\n"); + err = brcmf_fil_bsscfg_int_get(netdev_priv(ndev), "okc_enable", + &okc_enable); + if (err) { + bphy_err(drvr, "get okc_enable failed (%d)\n", err); + } else { + brcmf_dbg(INFO, "get okc_enable (%d)\n", okc_enable); + profile->is_okc = okc_enable; + } + } + if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) brcmf_dbg(INFO, "using SAE offload\n"); @@ -2235,14 +1990,47 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) mfp = BRCMF_MFP_REQUIRED; else if (rsn_cap & RSN_CAP_MFPC_MASK) mfp = BRCMF_MFP_CAPABLE; + /* In case of dpp, very low tput is observed if MFPC is set in + * firmmare. Firmware needs to ensure that MFPC is not set when + * MFPR was requested from fmac. However since this change being + * specific to DPP, fmac needs to set wpa_auth prior to mfp, so + * that firmware can use this info to prevent MFPC being set in + * case of dpp. + */ + if (val == WFA_AUTH_DPP) { + brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); + err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", + val); + if (err) { + bphy_err(drvr, "could not set wpa_auth (%d)\n", err); + return err; + } + } + brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "mfp", mfp); + offset += RSN_CAP_LEN; + if (mfp && (ie_len - offset >= RSN_PMKID_COUNT_LEN)) { + pmkid_count = ie[offset] + (ie[offset + 1] << 8); + offset += RSN_PMKID_COUNT_LEN + (pmkid_count * WLAN_PMKID_LEN); + if (ie_len - offset >= WPA_IE_MIN_OUI_LEN) { + group_mgmt_cs = &ie[offset]; + if (memcmp(group_mgmt_cs, RSN_OUI, TLV_OUI_LEN) == 0) { + brcmf_fil_bsscfg_data_set(ifp, "bip", + (void *)group_mgmt_cs, + WPA_IE_MIN_OUI_LEN); + } + } + } skip_mfp_config: - brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); - err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", val); - if (err) { - bphy_err(drvr, "could not set wpa_auth (%d)\n", err); - return err; + if (val != WFA_AUTH_DPP) { + brcmf_dbg(CONN, "setting wpa_auth to 0x%0x\n", val); + err = brcmf_fil_bsscfg_int_set(netdev_priv(ndev), "wpa_auth", + val); + if (err) { + bphy_err(drvr, "could not set wpa_auth (%d)\n", err); + return err; + } } return err; @@ -2375,52 +2163,51 @@ static void brcmf_set_join_pref(struct brcmf_if *ifp, static s32 brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, - struct cfg80211_connect_params *sme) + struct cfg80211_connect_params *params) { struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_if *ifp = netdev_priv(ndev); struct brcmf_cfg80211_profile *profile = &ifp->vif->profile; - struct ieee80211_channel *chan = sme->channel; + struct ieee80211_channel *chan = params->channel; struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_join_params join_params; - size_t join_params_size; + void *join_params; + u32 join_params_size; + void *fallback_join_params; + u32 fallback_join_params_size; const struct brcmf_tlv *rsn_ie; const struct brcmf_vs_tlv *wpa_ie; const void *ie; u32 ie_len; - struct brcmf_ext_join_params_le *ext_join_params; - u16 chanspec; s32 err = 0; - u32 ssid_len; brcmf_dbg(TRACE, "Enter\n"); if (!check_vif_up(ifp->vif)) return -EIO; - if (!sme->ssid) { + if (!params->ssid) { bphy_err(drvr, "Invalid ssid\n"); return -EOPNOTSUPP; } - if (sme->channel_hint) - chan = sme->channel_hint; + if (params->channel_hint) + chan = params->channel_hint; - if (sme->bssid_hint) - sme->bssid = sme->bssid_hint; + if (params->bssid_hint) + params->bssid = params->bssid_hint; if (ifp->vif == cfg->p2p.bss_idx[P2PAPI_BSSCFG_PRIMARY].vif) { /* A normal (non P2P) connection request setup. */ ie = NULL; ie_len = 0; /* find the WPA_IE */ - wpa_ie = brcmf_find_wpaie((u8 *)sme->ie, sme->ie_len); + wpa_ie = brcmf_find_wpaie((u8 *)params->ie, params->ie_len); if (wpa_ie) { ie = wpa_ie; ie_len = wpa_ie->len + TLV_HDR_LEN; } else { /* find the RSN_IE */ - rsn_ie = brcmf_parse_tlvs((const u8 *)sme->ie, - sme->ie_len, + rsn_ie = brcmf_parse_tlvs((const u8 *)params->ie, + params->ie_len, WLAN_EID_RSN); if (rsn_ie) { ie = rsn_ie; @@ -2431,7 +2218,7 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, } err = brcmf_vif_set_mgmt_ie(ifp->vif, BRCMF_VNDR_IE_ASSOCREQ_FLAG, - sme->ie, sme->ie_len); + params->ie, params->ie_len); if (err) bphy_err(drvr, "Set Assoc REQ IE Failed\n"); else @@ -2442,166 +2229,129 @@ brcmf_cfg80211_connect(struct wiphy *wiphy, struct net_device *ndev, if (chan) { cfg->channel = ieee80211_frequency_to_channel(chan->center_freq); - chanspec = channel_to_chanspec(&cfg->d11inf, chan); - brcmf_dbg(CONN, "channel=%d, center_req=%d, chanspec=0x%04x\n", - cfg->channel, chan->center_freq, chanspec); + brcmf_dbg(CONN, "channel=%d, center_req=%d\n", + cfg->channel, chan->center_freq); } else { cfg->channel = 0; - chanspec = 0; } - brcmf_dbg(INFO, "ie (%p), ie_len (%zd)\n", sme->ie, sme->ie_len); + brcmf_dbg(INFO, "ie (%p), ie_len (%zd)\n", params->ie, params->ie_len); - err = brcmf_set_wpa_version(ndev, sme); + err = brcmf_set_wpa_version(ndev, params); if (err) { bphy_err(drvr, "wl_set_wpa_version failed (%d)\n", err); goto done; } - sme->auth_type = brcmf_war_auth_type(ifp, sme->auth_type); - err = brcmf_set_auth_type(ndev, sme); + params->auth_type = brcmf_war_auth_type(ifp, params->auth_type); + err = brcmf_set_auth_type(ndev, params); if (err) { bphy_err(drvr, "wl_set_auth_type failed (%d)\n", err); goto done; } - err = brcmf_set_wsec_mode(ndev, sme); + err = brcmf_set_wsec_mode(ndev, params); if (err) { bphy_err(drvr, "wl_set_set_cipher failed (%d)\n", err); goto done; } - err = brcmf_set_key_mgmt(ndev, sme); + err = brcmf_set_key_mgmt(ndev, params); if (err) { bphy_err(drvr, "wl_set_key_mgmt failed (%d)\n", err); goto done; } - err = brcmf_set_sharedkey(ndev, sme); - if (err) { - bphy_err(drvr, "brcmf_set_sharedkey failed (%d)\n", err); - goto done; - } - - if (sme->crypto.psk && - profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) { - if (WARN_ON(profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE)) { - err = -EINVAL; - goto done; - } - brcmf_dbg(INFO, "using PSK offload\n"); - profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; - } - - if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) { - /* enable firmware supplicant for this interface */ - err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 1); - if (err < 0) { - bphy_err(drvr, "failed to enable fw supplicant\n"); - goto done; - } - } - - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); - else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { - /* clean up user-space RSNE */ - err = brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0); - if (err) { - bphy_err(drvr, "failed to clean up user-space RSNE\n"); - goto done; - } - err = brcmf_fwvid_set_sae_password(ifp, &sme->crypto); - if (!err && sme->crypto.psk) - err = brcmf_set_pmk(ifp, sme->crypto.psk, - BRCMF_WSEC_MAX_PSK_LEN); - } - if (err) - goto done; - - /* Join with specific BSSID and cached SSID - * If SSID is zero join based on BSSID only - */ - join_params_size = offsetof(struct brcmf_ext_join_params_le, assoc_le) + - offsetof(struct brcmf_assoc_params_le, chanspec_list); - if (cfg->channel) - join_params_size += sizeof(u16); - ext_join_params = kzalloc_obj(*ext_join_params); - if (ext_join_params == NULL) { - err = -ENOMEM; + err = brcmf_set_sharedkey(ndev, params); + if (err) { + bphy_err(drvr, "brcmf_set_sharedkey failed (%d)\n", err); goto done; } - ssid_len = min_t(u32, sme->ssid_len, IEEE80211_MAX_SSID_LEN); - ext_join_params->ssid_le.SSID_len = cpu_to_le32(ssid_len); - memcpy(&ext_join_params->ssid_le.SSID, sme->ssid, ssid_len); - if (ssid_len < IEEE80211_MAX_SSID_LEN) - brcmf_dbg(CONN, "SSID \"%s\", len (%d)\n", - ext_join_params->ssid_le.SSID, ssid_len); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_FWSUP)) { + if (params->crypto.psk) { + if ((profile->use_fwsup != BRCMF_PROFILE_FWSUP_SAE) && + (profile->use_fwsup != BRCMF_PROFILE_FWSUP_PSK)) { + if (WARN_ON(profile->use_fwsup != + BRCMF_PROFILE_FWSUP_NONE)) { + err = -EINVAL; + goto done; + } + brcmf_dbg(INFO, "using PSK offload\n"); + profile->use_fwsup = BRCMF_PROFILE_FWSUP_PSK; + } + } - /* Set up join scan parameters */ - ext_join_params->scan_le.scan_type = -1; - ext_join_params->scan_le.home_time = cpu_to_le32(-1); + if (profile->use_fwsup != BRCMF_PROFILE_FWSUP_NONE) { + /* enable firmware supplicant for this interface */ + err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 1); + if (err < 0) { + bphy_err(drvr, + "failed to enable fw supplicant\n"); + goto done; + } + } else { + err = brcmf_fil_iovar_int_set(ifp, "sup_wpa", 0); + } - if (sme->bssid) - memcpy(&ext_join_params->assoc_le.bssid, sme->bssid, ETH_ALEN); - else - eth_broadcast_addr(ext_join_params->assoc_le.bssid); + if ((profile->use_fwsup == BRCMF_PROFILE_FWSUP_PSK) && + params->crypto.psk) + err = brcmf_set_pmk(ifp, params->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); + else if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_SAE) { + /* clean up user-space RSNE */ + if (brcmf_fil_iovar_data_set(ifp, "wpaie", NULL, 0)) { + bphy_err( + drvr, + "failed to clean up user-space RSNE\n"); + goto done; + } + err = brcmf_fwvid_set_sae_password(ifp, ¶ms->crypto); + if (!err && params->crypto.psk) + err = brcmf_set_pmk(ifp, params->crypto.psk, + BRCMF_WSEC_MAX_PSK_LEN); + } + if (err) + goto done; + } + brcmf_set_join_pref(ifp, ¶ms->bss_select); + if (params->ssid_len < IEEE80211_MAX_SSID_LEN) + brcmf_dbg(CONN, "SSID \"%s\", len (%zu)\n", params->ssid, + params->ssid_len); + join_params = drvr->join_param_handler.get_struct_for_connect( + cfg, &join_params_size, params); - if (cfg->channel) { - ext_join_params->assoc_le.chanspec_num = cpu_to_le32(1); + if (join_params) { + err = brcmf_fil_bsscfg_data_set(ifp, "join", join_params, + join_params_size); - ext_join_params->assoc_le.chanspec_list[0] = - cpu_to_le16(chanspec); - /* Increase dwell time to receive probe response or detect - * beacon from target AP at a noisy air only during connect - * command. - */ - ext_join_params->scan_le.active_time = - cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS); - ext_join_params->scan_le.passive_time = - cpu_to_le32(BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS); - /* To sync with presence period of VSDB GO send probe request - * more frequently. Probe request will be stopped when it gets - * probe response from target AP/GO. - */ - ext_join_params->scan_le.nprobes = - cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS / - BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS); - } else { - ext_join_params->scan_le.active_time = cpu_to_le32(-1); - ext_join_params->scan_le.passive_time = cpu_to_le32(-1); - ext_join_params->scan_le.nprobes = cpu_to_le32(-1); + /* We only free the join parameters if we were successful. + * Otherwise they are used to extract the fallback, below */ + if (!err) { + kfree(join_params); + /* This is it. join command worked, we are done */ + goto done; + } + /* For versions >= 1, this should have worked, so report the error */ + if (drvr->join_param_handler.version >= 1) { + bphy_err(drvr, "Failed to use join iovar to join: %d\n", + err); + } } - brcmf_set_join_pref(ifp, &sme->bss_select); - - err = brcmf_fil_bsscfg_data_set(ifp, "join", ext_join_params, - join_params_size); - kfree(ext_join_params); - if (!err) - /* This is it. join command worked, we are done */ + /* Fallback to using WLC_SET_SSID approach, which just uses join_params parts of the structure */ + fallback_join_params = drvr->join_param_handler.get_join_from_ext_join( + join_params, &fallback_join_params_size); + if (!fallback_join_params) { + bphy_err(drvr, "Unable to generate fallback join params\n"); + kfree(join_params); goto done; - - /* join command failed, fallback to set ssid */ - memset(&join_params, 0, sizeof(join_params)); - join_params_size = sizeof(join_params.ssid_le); - - memcpy(&join_params.ssid_le.SSID, sme->ssid, ssid_len); - join_params.ssid_le.SSID_len = cpu_to_le32(ssid_len); - - if (sme->bssid) - memcpy(join_params.params_le.bssid, sme->bssid, ETH_ALEN); - else - eth_broadcast_addr(join_params.params_le.bssid); - - if (cfg->channel) { - join_params.params_le.chanspec_list[0] = cpu_to_le16(chanspec); - join_params.params_le.chanspec_num = cpu_to_le32(1); - join_params_size += sizeof(join_params.params_le); } err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_SSID, - &join_params, join_params_size); + fallback_join_params, + fallback_join_params_size); + + kfree(join_params); + kfree(fallback_join_params); if (err) bphy_err(drvr, "BRCMF_C_SET_SSID failed (%d)\n", err); @@ -2807,6 +2557,8 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, s32 val; s32 wsec; s32 err; + u32 algos = 0; + u32 mask = 0; u8 keybuf[8]; bool ext_key; @@ -2890,6 +2642,30 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, val = AES_ENABLED; brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_CCMP\n"); break; + case WLAN_CIPHER_SUITE_GCMP_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("the low layer not support GCMP\n"); + err = -EOPNOTSUPP; + goto done; + } + key->algo = CRYPTO_ALGO_AES_GCM256; + val = AES_ENABLED; + brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_GCMP_256\n"); + algos = KEY_ALGO_MASK(CRYPTO_ALGO_AES_GCM256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + break; + case WLAN_CIPHER_SUITE_BIP_GMAC_256: + if (!brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_err("the low layer not support GCMP\n"); + err = -EOPNOTSUPP; + goto done; + } + key->algo = CRYPTO_ALGO_BIP_GMAC256; + val = AES_ENABLED; + algos = KEY_ALGO_MASK(CRYPTO_ALGO_BIP_GMAC256); + mask = algos | KEY_ALGO_MASK(CRYPTO_ALGO_AES_CCM); + brcmf_dbg(CONN, "WLAN_CIPHER_SUITE_BIP_GMAC_256\n"); + break; default: bphy_err(drvr, "Invalid cipher (0x%x)\n", params->cipher); err = -EINVAL; @@ -2911,6 +2687,17 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev, bphy_err(drvr, "set wsec error (%d)\n", err); goto done; } + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GCMP)) { + brcmf_dbg(CONN, + "set_wsdec_info algos (0x%x) mask (0x%x)\n", + algos, mask); + err = brcmf_set_wsec_info_algos(ifp, algos, mask); + if (err) { + brcmf_err("set wsec_info error (%d)\n", err); + return err; + } + } + done: brcmf_dbg(TRACE, "Exit\n"); @@ -3131,6 +2918,70 @@ brcmf_cfg80211_get_station_ibss(struct brcmf_if *ifp, return 0; } +static void brcmf_convert_ratespec_to_rateinfo(u32 ratespec, + struct rate_info *rateinfo) +{ + /* First extract the bandwidth info */ + switch (ratespec & BRCMF_RSPEC_BW_MASK) { + case BRCMF_RSPEC_BW_20MHZ: + rateinfo->bw = RATE_INFO_BW_20; + break; + case BRCMF_RSPEC_BW_40MHZ: + rateinfo->bw = RATE_INFO_BW_40; + break; + case BRCMF_RSPEC_BW_80MHZ: + rateinfo->bw = RATE_INFO_BW_80; + break; + case BRCMF_RSPEC_BW_160MHZ: + rateinfo->bw = RATE_INFO_BW_160; + break; + case BRCMF_RSPEC_BW_320MHZ: + rateinfo->bw = RATE_INFO_BW_320; + break; + default: + /* Fill in nothing */ + break; + } + if (BRCMF_RSPEC_ISHT(ratespec)) { + rateinfo->flags |= RATE_INFO_FLAGS_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_HT_MCS_MASK; + } else if (BRCMF_RSPEC_ISVHT(ratespec)) { + rateinfo->flags |= RATE_INFO_FLAGS_VHT_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_VHT_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_VHT_NSS_MASK) >> + BRCMF_RSPEC_VHT_NSS_SHIFT; + } else if (BRCMF_RSPEC_ISHE(ratespec)) { + u32 ltf_gi = BRCMF_RSPEC_HE_LTF_GI(ratespec); + + rateinfo->flags |= RATE_INFO_FLAGS_HE_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_HE_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_HE_NSS_MASK) >> + BRCMF_RSPEC_HE_NSS_SHIFT; + rateinfo->he_dcm = BRCMF_RSPEC_HE_DCM(ratespec); + if (HE_IS_GI_0_8us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_0_8; + } else if (HE_IS_GI_1_6us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_1_6; + } else if (HE_IS_GI_3_2us(ltf_gi)) { + rateinfo->he_gi = NL80211_RATE_INFO_HE_GI_3_2; + } + } else if (BRCMF_RSPEC_ISEHT(ratespec)) { + u32 ltf_gi = BRCMF_RSPEC_EHT_LTF_GI(ratespec); + + rateinfo->flags |= RATE_INFO_FLAGS_EHT_MCS; + rateinfo->mcs = ratespec & BRCMF_RSPEC_EHT_MCS_MASK; + rateinfo->nss = (ratespec & BRCMF_RSPEC_EHT_NSS_MASK) >> + BRCMF_RSPEC_EHT_NSS_SHIFT; + if (EHT_IS_GI_0_8us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_0_8; + } else if (EHT_IS_GI_1_6us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_1_6; + } else if (EHT_IS_GI_3_2us(ltf_gi)) { + rateinfo->eht_gi = NL80211_RATE_INFO_EHT_GI_3_2; + } + } +} + static s32 brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, const u8 *mac, struct station_info *sinfo) @@ -3148,6 +2999,8 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, s32 count_rssi = 0; int rssi; u32 i; + u16 struct_ver; + u16 info_len; brcmf_dbg(TRACE, "Enter, MAC %pM\n", mac); if (!check_vif_up(ifp->vif)) @@ -3171,7 +3024,9 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, goto done; } } - brcmf_dbg(TRACE, "version %d\n", le16_to_cpu(sta_info_le.ver)); + info_len = le16_to_cpu(sta_info_le.len); + struct_ver = le16_to_cpu(sta_info_le.ver); + brcmf_dbg(TRACE, "version %d\n", struct_ver); sinfo->filled = BIT_ULL(NL80211_STA_INFO_INACTIVE_TIME); sinfo->inactive_time = le32_to_cpu(sta_info_le.idle) * 1000; sta_flags = le32_to_cpu(sta_info_le.flags); @@ -3205,12 +3060,13 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, sinfo->rxrate.legacy = le32_to_cpu(sta_info_le.rx_rate) / 100; } - if (le16_to_cpu(sta_info_le.ver) >= 4) { + if (struct_ver >= 4) { sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES); sinfo->tx_bytes = le64_to_cpu(sta_info_le.tx_tot_bytes); sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES); sinfo->rx_bytes = le64_to_cpu(sta_info_le.rx_tot_bytes); } + for (i = 0; i < BRCMF_ANT_MAX; i++) { if (sta_info_le.rssi[i] == 0 || sta_info_le.rx_lastpkt_rssi[i] == 0) @@ -3249,6 +3105,25 @@ brcmf_cfg80211_get_station(struct wiphy *wiphy, struct net_device *ndev, } } } + /* Some version 7 structs have ratespecs from the last packet. */ + if (struct_ver >= 7) { + if (info_len >= sizeof(sta_info_le)) { + brcmf_convert_ratespec_to_rateinfo( + le32_to_cpu(sta_info_le.v7.tx_rspec), + &sinfo->txrate); + brcmf_convert_ratespec_to_rateinfo( + le32_to_cpu(sta_info_le.v7.rx_rspec), + &sinfo->rxrate); + } else { + /* We didn't get the fields we were expecting, fallback to nrate */ + u32 nrate = 0; + err = brcmf_fil_iovar_int_get(ifp, "nrate", &nrate); + if (!err) { + brcmf_convert_ratespec_to_rateinfo( + nrate, &sinfo->txrate); + } + } + } done: brcmf_dbg(TRACE, "Exit\n"); return err; @@ -3349,6 +3224,7 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, struct cfg80211_bss *bss; enum nl80211_band band; struct brcmu_chan ch; + u16 chanspec; u16 channel; u32 freq; u16 notify_capability; @@ -3362,20 +3238,41 @@ static s32 brcmf_inform_single_bss(struct brcmf_cfg80211_info *cfg, return -EINVAL; } + chanspec = le16_to_cpu(bi->chanspec); if (!bi->ctl_ch) { - ch.chspec = le16_to_cpu(bi->chanspec); + ch.chspec = chanspec; cfg->d11inf.decchspec(&ch); bi->ctl_ch = ch.control_ch_num; } channel = bi->ctl_ch; - if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; - else + if (CHSPEC_IS6G(chanspec)) + band = NL80211_BAND_6GHZ; + else if (CHSPEC_IS5G(chanspec)) band = NL80211_BAND_5GHZ; + else + band = NL80211_BAND_2GHZ; freq = ieee80211_channel_to_frequency(channel, band); + if (!freq) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, channel, band, bi->chanspec); + + /* We ignore this BSS ID rather than try to continue on. + * Otherwise we will cause an OOPs because our frequency is 0. + * The main case this occurs is some new frequency band + * we have not seen before, and if we return an error, + * we will cause the scan to fail. It seems better to + * report the error, skip this BSS, and move on. + */ + return 0; + } bss_data.chan = ieee80211_get_channel(wiphy, freq); + if (!bss_data.chan) { + brcmf_err("Could not convert frequency into channel for channel %d, band %d, chanspec was %04x\n", + channel, band, bi->chanspec); + return 0; + } bss_data.boottime_ns = ktime_to_ns(ktime_get_boottime()); notify_capability = le16_to_cpu(bi->capability); @@ -3424,8 +3321,9 @@ static s32 brcmf_inform_bss(struct brcmf_cfg80211_info *cfg) bss_list = (struct brcmf_scan_results *)cfg->escan_info.escan_buf; if (bss_list->count != 0 && - bss_list->version != BRCMF_BSS_INFO_VERSION) { - bphy_err(drvr, "Version %d != WL_BSS_INFO_VERSION\n", + (bss_list->version < BRCMF_BSS_INFO_MIN_VERSION || + bss_list->version > BRCMF_BSS_INFO_MAX_VERSION)) { + bphy_err(drvr, "BSS info version %d unsupported\n", bss_list->version); return -EOPNOTSUPP; } @@ -3463,7 +3361,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, buf = kzalloc(WL_BSS_INFO_MAX, GFP_KERNEL); if (buf == NULL) { err = -ENOMEM; - goto CleanUp; + goto cleanup; } *(__le32 *)buf = cpu_to_le32(WL_BSS_INFO_MAX); @@ -3472,7 +3370,7 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, buf, WL_BSS_INFO_MAX); if (err) { bphy_err(drvr, "WLC_GET_BSS_INFO failed: %d\n", err); - goto CleanUp; + goto cleanup; } bi = (struct brcmf_bss_info_le *)(buf + 4); @@ -3482,10 +3380,18 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, if (ch.band == BRCMU_CHAN_BAND_2G) band = wiphy->bands[NL80211_BAND_2GHZ]; - else + else if (ch.band == BRCMU_CHAN_BAND_5G) band = wiphy->bands[NL80211_BAND_5GHZ]; + else + band = wiphy->bands[NL80211_BAND_6GHZ]; freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, bi->chanspec); + goto cleanup; + } + cfg->channel = freq; notify_channel = ieee80211_get_channel(wiphy, freq); @@ -3508,12 +3414,12 @@ static s32 brcmf_inform_ibss(struct brcmf_cfg80211_info *cfg, if (!bss) { err = -ENOMEM; - goto CleanUp; + goto cleanup; } cfg80211_put_bss(wiphy, bss); -CleanUp: +cleanup: kfree(buf); @@ -3765,17 +3671,11 @@ brcmf_alloc_internal_escan_request(struct wiphy *wiphy, u32 n_netinfo) { } static int brcmf_internal_escan_add_info(struct cfg80211_scan_request *req, - u8 *ssid, u8 ssid_len, u8 channel) + u8 *ssid, u8 ssid_len, u8 channel, enum nl80211_band band) { struct ieee80211_channel *chan; - enum nl80211_band band; int freq, i; - if (channel <= CH_MAX_2G_CHANNEL) - band = NL80211_BAND_2GHZ; - else - band = NL80211_BAND_5GHZ; - freq = ieee80211_channel_to_frequency(channel, band); if (!freq) return -EINVAL; @@ -3831,53 +3731,30 @@ static int brcmf_start_internal_escan(struct brcmf_if *ifp, u32 fwmap, return 0; } -static struct brcmf_pno_net_info_le * -brcmf_get_netinfo_array(struct brcmf_pno_scanresults_le *pfn_v1) -{ - struct brcmf_pno_scanresults_v2_le *pfn_v2; - struct brcmf_pno_net_info_le *netinfo; - - switch (pfn_v1->version) { - default: - WARN_ON(1); - fallthrough; - case cpu_to_le32(1): - netinfo = (struct brcmf_pno_net_info_le *)(pfn_v1 + 1); - break; - case cpu_to_le32(2): - pfn_v2 = (struct brcmf_pno_scanresults_v2_le *)pfn_v1; - netinfo = (struct brcmf_pno_net_info_le *)(pfn_v2 + 1); - break; - } - - return netinfo; -} - /* PFN result doesn't have all the info which are required by the supplicant * (For e.g IEs) Do a target Escan so that sched scan results are reported * via wl_inform_single_bss in the required format. Escan does require the * scan request in the form of cfg80211_scan_request. For timebeing, create * cfg80211_scan_request one out of the received PNO event. */ -static s32 -brcmf_notify_sched_scan_results(struct brcmf_if *ifp, - const struct brcmf_event_msg *e, void *data) +static s32 brcmf_notify_sched_scan_results(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, + void *data) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_cfg80211_info *cfg = drvr->config; - struct brcmf_pno_net_info_le *netinfo, *netinfo_start; struct cfg80211_scan_request *request = NULL; struct wiphy *wiphy = cfg_to_wiphy(cfg); int i, err = 0; - struct brcmf_pno_scanresults_le *pfn_result; u32 bucket_map; u32 result_count; u32 status; - u32 datalen; + u32 min_data_len; brcmf_dbg(SCAN, "Enter\n"); + min_data_len = drvr->pno_handler.get_min_data_len(); - if (e->datalen < (sizeof(*pfn_result) + sizeof(*netinfo))) { + if (e->datalen < min_data_len) { brcmf_dbg(SCAN, "Event data too small. Ignore\n"); return 0; } @@ -3887,9 +3764,8 @@ brcmf_notify_sched_scan_results(struct brcmf_if *ifp, return 0; } - pfn_result = (struct brcmf_pno_scanresults_le *)data; - result_count = le32_to_cpu(pfn_result->count); - status = le32_to_cpu(pfn_result->status); + result_count = drvr->pno_handler.get_result_count(data); + status = drvr->pno_handler.get_result_status(data); /* PFN event is limited to fit 512 bytes so we may get * multiple NET_FOUND events. For now place a warning here. @@ -3900,38 +3776,33 @@ brcmf_notify_sched_scan_results(struct brcmf_if *ifp, bphy_err(drvr, "FALSE PNO Event. (pfn_count == 0)\n"); goto out_err; } - - netinfo_start = brcmf_get_netinfo_array(pfn_result); - datalen = e->datalen - ((void *)netinfo_start - (void *)pfn_result); - if (datalen < result_count * sizeof(*netinfo)) { - bphy_err(drvr, "insufficient event data\n"); + err = drvr->pno_handler.validate_pfn_results(data, e->datalen); + if (err) { + bphy_err(drvr, "Invalid escan results (%d)", err); goto out_err; } - - request = brcmf_alloc_internal_escan_request(wiphy, - result_count); + request = brcmf_alloc_internal_escan_request(wiphy, result_count); if (!request) { err = -ENOMEM; goto out_err; } - bucket_map = 0; for (i = 0; i < result_count; i++) { - netinfo = &netinfo_start[i]; - - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; - brcmf_dbg(SCAN, "SSID:%.32s Channel:%d\n", - netinfo->SSID, netinfo->channel); - bucket_map |= brcmf_pno_get_bucket_map(cfg->pno, netinfo); - err = brcmf_internal_escan_add_info(request, - netinfo->SSID, - netinfo->SSID_len, - netinfo->channel); + u8 channel; + enum nl80211_band band; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + + drvr->pno_handler.get_result_info(data, i, &ssid, &ssid_len, + &channel, &band); + brcmf_dbg(SCAN, "SSID:%.32s Channel:%d Band:%d\n", ssid, + channel, band); + bucket_map |= drvr->pno_handler.get_bucket_map(data, i, cfg->pno); + err = brcmf_internal_escan_add_info(request, ssid, ssid_len, + channel, band); if (err) goto out_err; } - if (!bucket_map) goto free_req; @@ -4034,48 +3905,50 @@ static s32 brcmf_config_wowl_pattern(struct brcmf_if *ifp, u8 cmd[4], return ret; } -static s32 -brcmf_wowl_nd_results(struct brcmf_if *ifp, const struct brcmf_event_msg *e, - void *data) +static s32 brcmf_wowl_nd_results(struct brcmf_if *ifp, + const struct brcmf_event_msg *e, void *data) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_cfg80211_info *cfg = drvr->config; - struct brcmf_pno_scanresults_le *pfn_result; - struct brcmf_pno_net_info_le *netinfo; + u32 min_data_len; + u8 channel; + enum nl80211_band band; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + u8 ssid_len; + u32 result_count; brcmf_dbg(SCAN, "Enter\n"); - if (e->datalen < (sizeof(*pfn_result) + sizeof(*netinfo))) { + min_data_len = drvr->pno_handler.get_min_data_len(); + + if (e->datalen < min_data_len) { brcmf_dbg(SCAN, "Event data too small. Ignore\n"); return 0; } - pfn_result = (struct brcmf_pno_scanresults_le *)data; if (e->event_code == BRCMF_E_PFN_NET_LOST) { brcmf_dbg(SCAN, "PFN NET LOST event. Ignore\n"); return 0; } - if (le32_to_cpu(pfn_result->count) < 1) { + result_count = drvr->pno_handler.get_result_count(data); + if (result_count < 1) { bphy_err(drvr, "Invalid result count, expected 1 (%d)\n", - le32_to_cpu(pfn_result->count)); + result_count); return -EINVAL; } - netinfo = brcmf_get_netinfo_array(pfn_result); - if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) - netinfo->SSID_len = IEEE80211_MAX_SSID_LEN; - memcpy(cfg->wowl.nd->ssid.ssid, netinfo->SSID, netinfo->SSID_len); - cfg->wowl.nd->ssid.ssid_len = netinfo->SSID_len; + drvr->pno_handler.get_result_info(data, 0, &ssid, &ssid_len, &channel, + &band); + memcpy(cfg->wowl.nd->ssid.ssid, ssid, ssid_len); + cfg->wowl.nd->ssid.ssid_len = ssid_len; cfg->wowl.nd->n_channels = 1; cfg->wowl.nd->channels[0] = - ieee80211_channel_to_frequency(netinfo->channel, - netinfo->channel <= CH_MAX_2G_CHANNEL ? - NL80211_BAND_2GHZ : NL80211_BAND_5GHZ); + ieee80211_channel_to_frequency(channel, band); + cfg->wowl.nd_info->n_matches = 1; cfg->wowl.nd_info->matches[0] = cfg->wowl.nd; - /* Inform (the resume task) that the net detect information was recvd */ cfg->wowl.nd_data_completed = true; wake_up(&cfg->wowl.nd_data_wait); @@ -5149,6 +5022,25 @@ brcmf_cfg80211_start_ap(struct wiphy *wiphy, struct net_device *ndev, settings->inactivity_timeout); dev_role = ifp->vif->wdev.iftype; mbss = ifp->vif->mbss; + /* Bring firmware into correct state for AP mode*/ + if (dev_role == NL80211_IFTYPE_AP) { + brcmf_dbg(TRACE, "set AP mode\n"); + err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_AP, 1); + if (err < 0) { + bphy_err(drvr, "setting AP mode failed %d\n", + err); + goto exit; + } + + bss_enable.bsscfgidx = cpu_to_le32(ifp->bsscfgidx); + bss_enable.enable = cpu_to_le32(WLC_AP_IOV_OP_MANUAL_AP_BSSCFG_CREATE); + err = brcmf_fil_iovar_data_set(ifp, "bss", &bss_enable, + sizeof(bss_enable)); + if (err < 0) { + bphy_err(drvr, "AP role set error, %d\n", err); + goto exit; + } + } /* store current 11d setting */ if (brcmf_fil_cmd_int_get(ifp, BRCMF_C_GET_REGULATORY, @@ -5742,6 +5634,9 @@ static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, case BRCMU_CHAN_BAND_5G: band = NL80211_BAND_5GHZ; break; + case BRCMU_CHAN_BAND_6G: + band = NL80211_BAND_6GHZ; + break; } switch (ch.bw) { @@ -5763,9 +5658,19 @@ static int brcmf_cfg80211_get_channel(struct wiphy *wiphy, } freq = ieee80211_channel_to_frequency(ch.control_ch_num, band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, chanspec); + return -EINVAL; + } chandef->chan = ieee80211_get_channel(wiphy, freq); chandef->width = width; chandef->center_freq1 = ieee80211_channel_to_frequency(ch.chnum, band); + if (chandef->center_freq1 == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.chnum, ch.band, chanspec); + return -EINVAL; + } chandef->center_freq2 = 0; return 0; @@ -5930,17 +5835,29 @@ static int brcmf_cfg80211_set_pmk(struct wiphy *wiphy, struct net_device *dev, const struct cfg80211_pmk_conf *conf) { struct brcmf_if *ifp; - + struct brcmf_pub *drvr; + int ret; brcmf_dbg(TRACE, "enter\n"); /* expect using firmware supplicant for 1X */ ifp = netdev_priv(dev); - if (WARN_ON(ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X)) + drvr = ifp->drvr; + if (WARN_ON((ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_1X) && + (ifp->vif->profile.use_fwsup != BRCMF_PROFILE_FWSUP_ROAM) && + (ifp->vif->profile.is_ft != true) && + (ifp->vif->profile.is_okc != true))) return -EINVAL; if (conf->pmk_len > BRCMF_WSEC_MAX_PSK_LEN) return -ERANGE; + if (ifp->vif->profile.is_okc) { + ret = brcmf_fil_iovar_data_set(ifp, "okc_info_pmk", conf->pmk, + conf->pmk_len); + if (ret < 0) + bphy_err(drvr, "okc_info_pmk iovar failed: ret=%d\n", + ret); + } return brcmf_set_pmk(ifp, conf->pmk, conf->pmk_len); } @@ -6399,6 +6316,46 @@ static s32 brcmf_get_assoc_ies(struct brcmf_cfg80211_info *cfg, return err; } +static bool brcmf_has_pmkid(const u8 *parse, u32 len) +{ + const struct brcmf_tlv *rsn_ie; + const u8 *ie; + u32 ie_len; + u32 offset; + u16 count; + + rsn_ie = brcmf_parse_tlvs(parse, len, WLAN_EID_RSN); + if (!rsn_ie) + goto done; + ie = (const u8 *)rsn_ie; + ie_len = rsn_ie->len + TLV_HDR_LEN; + /* Skip group data cipher suite */ + offset = TLV_HDR_LEN + WPA_IE_VERSION_LEN + WPA_IE_MIN_OUI_LEN; + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip pairwise cipher suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + WPA_IE_SUITE_COUNT_LEN >= ie_len) + goto done; + /* Skip auth key management suite(s) */ + count = ie[offset] + (ie[offset + 1] << 8); + offset += WPA_IE_SUITE_COUNT_LEN + (count * WPA_IE_MIN_OUI_LEN); + if (offset + RSN_CAP_LEN >= ie_len) + goto done; + /* Skip rsn capabilities */ + offset += RSN_CAP_LEN; + if (offset + RSN_PMKID_COUNT_LEN > ie_len) + goto done; + /* Extract PMKID count */ + count = ie[offset] + (ie[offset + 1] << 8); + if (count) + return true; + +done: + return false; +} + static s32 brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, struct net_device *ndev, @@ -6443,10 +6400,17 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, if (ch.band == BRCMU_CHAN_BAND_2G) band = wiphy->bands[NL80211_BAND_2GHZ]; - else + else if (ch.band == BRCMU_CHAN_BAND_5G) band = wiphy->bands[NL80211_BAND_5GHZ]; + else + band = wiphy->bands[NL80211_BAND_6GHZ]; freq = ieee80211_channel_to_frequency(ch.control_ch_num, band->band); + if (freq == 0) { + brcmf_err("Invalid frequency %d returned for channel %d, band %d. chanspec was %04x\n", + freq, ch.control_ch_num, ch.band, bi->chanspec); + goto done; + } notify_channel = ieee80211_get_channel(wiphy, freq); done: @@ -6462,11 +6426,16 @@ brcmf_bss_roaming_done(struct brcmf_cfg80211_info *cfg, cfg80211_roamed(ndev, &roam_info, GFP_KERNEL); brcmf_dbg(CONN, "Report roaming result\n"); - if (profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X && profile->is_ft) { - cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, GFP_KERNEL); + if (((profile->use_fwsup == BRCMF_PROFILE_FWSUP_1X || + profile->use_fwsup == BRCMF_PROFILE_FWSUP_ROAM) && + (brcmf_has_pmkid(roam_info.req_ie, roam_info.req_ie_len) || + profile->is_ft || profile->is_okc))) { + cfg80211_port_authorized(ndev, profile->bssid, NULL, 0, + GFP_KERNEL); brcmf_dbg(CONN, "Report port authorized\n"); } + clear_bit(BRCMF_VIF_STATUS_CONNECTING, &ifp->vif->sme_state); set_bit(BRCMF_VIF_STATUS_CONNECTED, &ifp->vif->sme_state); brcmf_dbg(TRACE, "Exit\n"); return err; @@ -6925,8 +6894,6 @@ static s32 brcmf_dongle_roam(struct brcmf_if *ifp) if (err) bphy_err(drvr, "WLC_SET_ROAM_TRIGGER error (%d)\n", err); - roam_delta[0] = cpu_to_le32(WL_ROAM_DELTA); - roam_delta[1] = cpu_to_le32(BRCM_BAND_ALL); err = brcmf_fil_cmd_data_set(ifp, BRCMF_C_SET_ROAM_DELTA, (void *)roam_delta, sizeof(roam_delta)); if (err) @@ -7019,15 +6986,34 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, goto fail_pbuf; } + /* Changing regulatory domain may change power limits upwards. + * To ensure that we correctly set the new band info, copy the original + * info first. + */ band = wiphy->bands[NL80211_BAND_2GHZ]; - if (band) + if (band) { + memcpy(band->channels, &__wl_2ghz_channels, + sizeof(__wl_2ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_2ghz_channels); for (i = 0; i < band->n_channels; i++) band->channels[i].flags = IEEE80211_CHAN_DISABLED; + } band = wiphy->bands[NL80211_BAND_5GHZ]; - if (band) + if (band) { + memcpy(band->channels, &__wl_5ghz_channels, + sizeof(__wl_5ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_5ghz_channels); for (i = 0; i < band->n_channels; i++) band->channels[i].flags = IEEE80211_CHAN_DISABLED; - + } + band = wiphy->bands[NL80211_BAND_6GHZ]; + if (band) { + memcpy(band->channels, &__wl_6ghz_channels, + sizeof(__wl_6ghz_channels)); + band->n_channels = ARRAY_SIZE(__wl_6ghz_channels); + for (i = 0; i < band->n_channels; i++) + band->channels[i].flags = IEEE80211_CHAN_DISABLED; + } total = le32_to_cpu(list->count); if (total > BRCMF_MAX_CHANSPEC_LIST) { bphy_err(drvr, "Invalid count of channel Spec. (%u)\n", @@ -7044,6 +7030,8 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, band = wiphy->bands[NL80211_BAND_2GHZ]; } else if (ch.band == BRCMU_CHAN_BAND_5G) { band = wiphy->bands[NL80211_BAND_5GHZ]; + } else if (ch.band == BRCMU_CHAN_BAND_6G) { + band = wiphy->bands[NL80211_BAND_6GHZ]; } else { bphy_err(drvr, "Invalid channel Spec. 0x%x.\n", ch.chspec); @@ -7065,6 +7053,7 @@ static int brcmf_construct_chaninfo(struct brcmf_cfg80211_info *cfg, break; } } + if (!channel) { /* It seems firmware supports some channel we never * considered. Something new in IEEE standard? @@ -7137,17 +7126,25 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) struct brcmu_chan ch; u32 num_chan; int i, j; + s32 updown; /* verify support for bw_cap command */ - val = WLC_BAND_5G; + val = WLC_BAND_2G; err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &val); - + brcmf_dbg(INFO, "Check bw_cap support:%d\n", err); if (!err) { + /* Setting the bw_cap is DOWN restricted. */ + updown = 0; + brcmf_fil_cmd_data_set(ifp, BRCMF_C_DOWN, &updown, sizeof(s32)); /* only set 2G bandwidth using bw_cap command */ band_bwcap.band = cpu_to_le32(WLC_BAND_2G); band_bwcap.bw_cap = cpu_to_le32(WLC_BW_CAP_40MHZ); err = brcmf_fil_iovar_data_set(ifp, "bw_cap", &band_bwcap, sizeof(band_bwcap)); + brcmf_dbg(INFO, "set bw_cap support:%d\n", err); + brcmf_c_set_joinpref_default(ifp); + updown = 1; + brcmf_fil_cmd_data_set(ifp, BRCMF_C_UP, &updown, sizeof(s32)); } else { brcmf_dbg(INFO, "fallback to mimo_bw_cap\n"); val = WLC_N_BW_40ALL; @@ -7209,7 +7206,7 @@ static int brcmf_enable_bw40_2g(struct brcmf_cfg80211_info *cfg) return err; } -static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) +static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[4], bool has_6g) { struct brcmf_pub *drvr = ifp->drvr; u32 band, mimo_bwcap; @@ -7217,17 +7214,29 @@ static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) band = WLC_BAND_2G; err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); - if (!err) { - bw_cap[NL80211_BAND_2GHZ] = band; - band = WLC_BAND_5G; - err = brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); - if (!err) { - bw_cap[NL80211_BAND_5GHZ] = band; - return; - } - WARN_ON(1); + if (err) + goto fallback; + bw_cap[NL80211_BAND_2GHZ] = band; + band = WLC_BAND_5G; + err |= brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); + if (err) + goto fallback; + bw_cap[NL80211_BAND_5GHZ] = band; + if (!has_6g) return; - } + band = WLC_BAND_6G; + err |= brcmf_fil_iovar_int_query(ifp, "bw_cap", &band); + /* Prior to the introduction of 6g, this function only + * did fallback in the case of 2g and 5g -failing. + * As mimo_bwcap does not have 6g bwcap info anyway, + * we keep that behavior. + */ + if (err) + return; + bw_cap[NL80211_BAND_6GHZ] = band; + return; +fallback: + brcmf_dbg(INFO, "fallback to mimo_bw_cap info\n"); err = brcmf_fil_iovar_int_get(ifp, "mimo_bw_cap", &mimo_bwcap); if (err) @@ -7251,8 +7260,11 @@ static void brcmf_get_bwcap(struct brcmf_if *ifp, u32 bw_cap[]) } static void brcmf_update_ht_cap(struct ieee80211_supported_band *band, - u32 bw_cap[2], u32 nchain) + u32 bw_cap[4], u32 nrxchain) { + /* Not supported in 6G band */ + if (band->band == NL80211_BAND_6GHZ) + return; band->ht_cap.ht_supported = true; if (bw_cap[band->band] & WLC_BW_40MHZ_BIT) { band->ht_cap.cap |= IEEE80211_HT_CAP_SGI_40; @@ -7262,32 +7274,49 @@ static void brcmf_update_ht_cap(struct ieee80211_supported_band *band, band->ht_cap.cap |= IEEE80211_HT_CAP_DSSSCCK40; band->ht_cap.ampdu_factor = IEEE80211_HT_MAX_AMPDU_64K; band->ht_cap.ampdu_density = IEEE80211_HT_MPDU_DENSITY_16; - memset(band->ht_cap.mcs.rx_mask, 0xff, nchain); + memset(band->ht_cap.mcs.rx_mask, 0xff, nrxchain); band->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED; } -static __le16 brcmf_get_mcs_map(u32 nchain, enum ieee80211_vht_mcs_support supp) +static __le16 brcmf_get_mcs_map(u32 nstreams, + enum ieee80211_vht_mcs_support supp) { u16 mcs_map; int i; - for (i = 0, mcs_map = 0xFFFF; i < nchain; i++) + for (i = 0, mcs_map = 0xFFFF; i < nstreams; i++) mcs_map = (mcs_map << 2) | supp; return cpu_to_le16(mcs_map); } static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, - u32 bw_cap[2], u32 nchain, u32 txstreams, - u32 txbf_bfe_cap, u32 txbf_bfr_cap) + u32 bw_cap[4], u32 txstreams, u32 rxstreams, + u32 txbf_bfe_cap, u32 txbf_bfr_cap, + u32 ldpc_cap, u32 stbc_rx, u32 stbc_tx) { __le16 mcs_map; - /* not allowed in 2.4G band */ - if (band->band == NL80211_BAND_2GHZ) + /* not allowed in 2.4G or 6G band */ + if (band->band == NL80211_BAND_2GHZ || band->band == NL80211_BAND_6GHZ) return; band->vht_cap.vht_supported = true; + band->vht_cap.vht_mcs.tx_highest = cpu_to_le16(433 * txstreams); + band->vht_cap.vht_mcs.rx_highest = cpu_to_le16(433 * rxstreams); + + band->vht_cap.cap |= IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN | + IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN; + + if (ldpc_cap) + band->vht_cap.cap |= IEEE80211_VHT_CAP_RXLDPC; + if (stbc_tx) + band->vht_cap.cap |= IEEE80211_VHT_CAP_TXSTBC; + + if (stbc_rx) + band->vht_cap.cap |= + (stbc_rx << IEEE80211_VHT_CAP_RXSTBC_SHIFT); + /* 80MHz is mandatory */ band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_80; if (bw_cap[band->band] & WLC_BW_160MHZ_BIT) { @@ -7295,8 +7324,10 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, band->vht_cap.cap |= IEEE80211_VHT_CAP_SHORT_GI_160; } /* all support 256-QAM */ - mcs_map = brcmf_get_mcs_map(nchain, IEEE80211_VHT_MCS_SUPPORT_0_9); + mcs_map = brcmf_get_mcs_map(rxstreams, IEEE80211_VHT_MCS_SUPPORT_0_9); band->vht_cap.vht_mcs.rx_mcs_map = mcs_map; + mcs_map = brcmf_get_mcs_map(txstreams, IEEE80211_VHT_MCS_SUPPORT_0_9); + band->vht_cap.vht_mcs.tx_mcs_map = mcs_map; /* Beamforming support information */ @@ -7312,11 +7343,129 @@ static void brcmf_update_vht_cap(struct ieee80211_supported_band *band, if ((txbf_bfe_cap || txbf_bfr_cap) && (txstreams > 1)) { band->vht_cap.cap |= (2 << IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT); - band->vht_cap.cap |= ((txstreams - 1) << - IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT); + band->vht_cap.cap |= + ((txstreams - 1) + << IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT); band->vht_cap.cap |= IEEE80211_VHT_CAP_VHT_LINK_ADAPTATION_VHT_MRQ_MFB; } + /* AMPDU length limit, support max 1MB (2 ^ (13 + 7)) */ + band->vht_cap.cap |= + (7 << IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_SHIFT); +} + +static void brcmf_update_he_cap(struct ieee80211_supported_band *band, + struct ieee80211_sband_iftype_data *data) +{ + int idx = 1; + struct ieee80211_sta_he_cap *he_cap = &data->he_cap; + struct ieee80211_he_cap_elem *he_cap_elem = &he_cap->he_cap_elem; + struct ieee80211_he_mcs_nss_supp *he_mcs = &he_cap->he_mcs_nss_supp; + struct ieee80211_he_6ghz_capa *he_6ghz_capa = &data->he_6ghz_capa; + + if (!data) { + brcmf_err("failed to allocate sdata\n"); + return; + } + + data->types_mask = BIT(NL80211_IFTYPE_STATION); + he_cap->has_he = true; + + /* HE MAC Capabilities Information */ + he_cap_elem->mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE | + IEEE80211_HE_MAC_CAP0_TWT_REQ | + IEEE80211_HE_MAC_CAP0_TWT_RES; + + he_cap_elem->mac_cap_info[1] = + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_8US | + IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US; + + he_cap_elem->mac_cap_info[2] = IEEE80211_HE_MAC_CAP2_BSR | + IEEE80211_HE_MAC_CAP2_BCAST_TWT; + + he_cap_elem->mac_cap_info[3] = + IEEE80211_HE_MAC_CAP3_OMI_CONTROL | + IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1 | + IEEE80211_HE_MAC_CAP3_FLEX_TWT_SCHED; + + he_cap_elem->mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU; + + /* HE PHY Capabilities Information */ + he_cap_elem->phy_cap_info[0] = + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G | + IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G; + ; + + he_cap_elem->phy_cap_info[1] = + IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD; + + he_cap_elem->phy_cap_info[2] = + IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US | + IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO | + IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO; + + he_cap_elem->phy_cap_info[3] = + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_TX_QPSK | + IEEE80211_HE_PHY_CAP3_DCM_MAX_TX_NSS_2 | + IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM | + IEEE80211_HE_PHY_CAP3_SU_BEAMFORMER; + + he_cap_elem->phy_cap_info[4] = + IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_MASK | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4 | + IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_8; + + he_cap_elem->phy_cap_info[5] = + IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK | + IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK | + IEEE80211_HE_PHY_CAP5_BEAMFORMEE_NUM_SND_DIM_UNDER_80MHZ_2; + + he_cap_elem->phy_cap_info[6] = + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_42_SU | + IEEE80211_HE_PHY_CAP6_CODEBOOK_SIZE_75_MU | + IEEE80211_HE_PHY_CAP6_TRIG_SU_BEAMFORMING_FB | + IEEE80211_HE_PHY_CAP6_TRIG_MU_BEAMFORMING_PARTIAL_BW_FB | + IEEE80211_HE_PHY_CAP6_TRIG_CQI_FB | + IEEE80211_HE_PHY_CAP6_PARTIAL_BW_EXT_RANGE | + IEEE80211_HE_PHY_CAP6_PPE_THRESHOLD_PRESENT; + + he_cap_elem->phy_cap_info[7] = + IEEE80211_HE_PHY_CAP7_HE_SU_MU_PPDU_4XLTF_AND_08_US_GI | + IEEE80211_HE_PHY_CAP7_MAX_NC_1; + + he_cap_elem->phy_cap_info[8] = + IEEE80211_HE_PHY_CAP8_HE_ER_SU_PPDU_4XLTF_AND_08_US_GI | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_40MHZ_HE_PPDU_IN_2G | + IEEE80211_HE_PHY_CAP8_20MHZ_IN_160MHZ_HE_PPDU | + IEEE80211_HE_PHY_CAP8_80MHZ_IN_160MHZ_HE_PPDU; + + he_cap_elem->phy_cap_info[9] = + IEEE80211_HE_PHY_CAP9_TX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_1024_QAM_LESS_THAN_242_TONE_RU | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_COMP_SIGB | + IEEE80211_HE_PHY_CAP9_RX_FULL_BW_SU_USING_MU_WITH_NON_COMP_SIGB; + + /* HE Supported MCS and NSS Set */ + he_mcs->rx_mcs_80 = cpu_to_le16(0xfffa); + he_mcs->tx_mcs_80 = cpu_to_le16(0xfffa); + he_mcs->rx_mcs_160 = cpu_to_le16(0xfffa); + he_mcs->tx_mcs_160 = cpu_to_le16(0xfffa); + /* HE 6 GHz band capabilities */ + if (band->band == NL80211_BAND_6GHZ) { + u16 capa = 0; + + capa = FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START, + IEEE80211_HT_MPDU_DENSITY_8) | + FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP, + IEEE80211_VHT_MAX_AMPDU_1024K) | + FIELD_PREP(IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN, + IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454); + he_6ghz_capa->capa = cpu_to_le16(capa); + } + band->n_iftype_data = idx; + band->iftype_data = data; } static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) @@ -7326,26 +7475,49 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) struct wiphy *wiphy = cfg_to_wiphy(cfg); u32 nmode; u32 vhtmode = 0; - u32 bw_cap[2] = { WLC_BW_20MHZ_BIT, WLC_BW_20MHZ_BIT }; + /* 2GHZ, 5GHZ, 60GHZ, 6GHZ */ + u32 bw_cap[4] = { 0, 0, 0, 0 }; u32 rxchain; - u32 nchain; + u32 txchain; + u32 nrxchain; + u32 ntxchain; int err; s32 i; struct ieee80211_supported_band *band; u32 txstreams = 0; + u32 rxstreams = 0; u32 txbf_bfe_cap = 0; u32 txbf_bfr_cap = 0; + u8 he_enable; + struct brcmf_he_defcap he_cap; + u32 ldpc_cap = 0; + u32 stbc_rx = 0; + u32 stbc_tx = 0; (void)brcmf_fil_iovar_int_get(ifp, "vhtmode", &vhtmode); + (void)brcmf_fil_iovar_int_get(ifp, "ldpc_cap", &ldpc_cap); + (void)brcmf_fil_iovar_int_get(ifp, "stbc_rx", &stbc_rx); + (void)brcmf_fil_iovar_int_get(ifp, "stbc_tx", &stbc_tx); + err = brcmf_fil_xtlv_int8_get(ifp, "he", BRCMF_HE_CMD_ENABLE, + &he_enable); + if (!err && he_enable) { + brcmf_fil_xtlv_data_get(ifp, "he", BRCMF_HE_CMD_DEFCAP, &he_cap, + sizeof(he_cap)); + brcmf_dbg_hex_dump(BRCMF_INFO_ON(), he_cap.mac_cap, 6, + "default HE mac cap\n"); + brcmf_dbg_hex_dump(BRCMF_INFO_ON(), he_cap.phy_cap, 11, + "default HE phy cap\n"); + } err = brcmf_fil_iovar_int_get(ifp, "nmode", &nmode); if (err) { bphy_err(drvr, "nmode error (%d)\n", err); - } else { - brcmf_get_bwcap(ifp, bw_cap); } - brcmf_dbg(INFO, "nmode=%d, vhtmode=%d, bw_cap=(%d, %d)\n", + brcmf_get_bwcap(ifp, bw_cap, he_enable != 0); + brcmf_dbg(INFO, + "nmode=%d, vhtmode=%d, bw_cap=(%d, %d, %d), he_enable=%d\n", nmode, vhtmode, bw_cap[NL80211_BAND_2GHZ], - bw_cap[NL80211_BAND_5GHZ]); + bw_cap[NL80211_BAND_5GHZ], bw_cap[NL80211_BAND_6GHZ], + he_enable); err = brcmf_fil_iovar_int_get(ifp, "rxchain", &rxchain); if (err) { @@ -7355,12 +7527,31 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) else bphy_err(drvr, "rxchain error (%d)\n", err); - nchain = 1; + nrxchain = 1; + rxchain = 1; } else { - for (nchain = 0; rxchain; nchain++) + for (nrxchain = 0; rxchain; nrxchain++) rxchain = rxchain & (rxchain - 1); } - brcmf_dbg(INFO, "nchain=%d\n", nchain); + brcmf_dbg(INFO, "nrxchain=%d\n", nrxchain); + err = brcmf_fil_iovar_int_get(ifp, "txchain", &txchain); + if (err) { + /* rxchain unsupported by firmware of older chips */ + if (err == -EBADE) + bphy_info_once(drvr, "rxchain unsupported\n"); + else + bphy_err(drvr, "rxchain error (%d)\n", err); + + ntxchain = 1; + txchain = 1; + } else { + for (ntxchain = 0; txchain; ntxchain++) + txchain = txchain & (txchain - 1); + } + brcmf_dbg(INFO, "ntxchain=%d\n", ntxchain); + + wiphy->available_antennas_rx = nrxchain; + wiphy->available_antennas_tx = ntxchain; err = brcmf_construct_chaninfo(cfg, bw_cap); if (err) { @@ -7369,6 +7560,7 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) } if (vhtmode) { + (void)brcmf_fil_iovar_int_get(ifp, "rxstreams", &rxstreams); (void)brcmf_fil_iovar_int_get(ifp, "txstreams", &txstreams); (void)brcmf_fil_iovar_int_get(ifp, "txbf_bfe_cap", &txbf_bfe_cap); @@ -7382,10 +7574,13 @@ static int brcmf_setup_wiphybands(struct brcmf_cfg80211_info *cfg) continue; if (nmode) - brcmf_update_ht_cap(band, bw_cap, nchain); + brcmf_update_ht_cap(band, bw_cap, nrxchain); if (vhtmode) - brcmf_update_vht_cap(band, bw_cap, nchain, txstreams, - txbf_bfe_cap, txbf_bfr_cap); + brcmf_update_vht_cap(band, bw_cap, txstreams, rxstreams, + txbf_bfe_cap, txbf_bfr_cap, + ldpc_cap, stbc_rx, stbc_tx); + if (he_enable) + brcmf_update_he_cap(band, &sdata[band->band]); } return 0; @@ -7639,7 +7834,7 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) struct ieee80211_supported_band *band; u16 max_interfaces = 0; bool gscan; - __le32 bandlist[3]; + __le32 bandlist[16]; u32 n_bands; int err, i; @@ -7707,6 +7902,13 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) } if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_SAE_EXT)) wiphy->features |= NL80211_FEATURE_SAE; + + /* High accuracy and low power scans are always supported. */ + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_HIGH_ACCURACY_SCAN); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_LOW_POWER_SCAN); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_LOW_SPAN_SCAN); + wiphy->features |= NL80211_FEATURE_LOW_PRIORITY_SCAN; + wiphy->mgmt_stypes = brcmf_txrx_stypes; wiphy->max_remain_on_channel_duration = 5000; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_PNO)) { @@ -7762,12 +7964,27 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) band->n_channels = ARRAY_SIZE(__wl_5ghz_channels); wiphy->bands[NL80211_BAND_5GHZ] = band; } - } + if (bandlist[i] == cpu_to_le32(WLC_BAND_6G)) { + band = kmemdup(&__wl_band_6ghz, sizeof(__wl_band_6ghz), + GFP_KERNEL); + if (!band) + return -ENOMEM; + + band->channels = kmemdup(&__wl_6ghz_channels, + sizeof(__wl_6ghz_channels), + GFP_KERNEL); + if (!band->channels) { + kfree(band); + return -ENOMEM; + } + band->n_channels = ARRAY_SIZE(__wl_6ghz_channels); + wiphy->bands[NL80211_BAND_6GHZ] = band; + } + } if (wiphy->bands[NL80211_BAND_5GHZ] && brcmf_feat_is_enabled(ifp, BRCMF_FEAT_DOT11H)) - wiphy_ext_feature_set(wiphy, - NL80211_EXT_FEATURE_DFS_OFFLOAD); + wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_DFS_OFFLOAD); wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); @@ -8266,9 +8483,17 @@ static void brcmf_cfg80211_reg_notifier(struct wiphy *wiphy, } err = brcmf_translate_country_code(ifp->drvr, req->alpha2, &ccreq); - if (err) - return; - + if (err) { + /* Because we ignore the default country code above, + * we will start out in our custom reg domain, but the chip + * may already be set to the right country. + * As such, we force the bands to be re-set the first + * time we try to set a country for real. + */ + if (err != -EAGAIN || !cfg->force_band_setup) + return; + } + cfg->force_band_setup = false; err = brcmf_fil_iovar_data_set(ifp, "country", &ccreq, sizeof(ccreq)); if (err) { bphy_err(drvr, "Firmware rejected country setting\n"); @@ -8297,6 +8522,10 @@ static void brcmf_free_wiphy(struct wiphy *wiphy) kfree(wiphy->bands[NL80211_BAND_5GHZ]->channels); kfree(wiphy->bands[NL80211_BAND_5GHZ]); } + if (wiphy->bands[NL80211_BAND_6GHZ]) { + kfree(wiphy->bands[NL80211_BAND_6GHZ]->channels); + kfree(wiphy->bands[NL80211_BAND_6GHZ]); + } #if IS_ENABLED(CONFIG_PM) if (wiphy->wowlan != &brcmf_wowlan_support) kfree(wiphy->wowlan); @@ -8331,6 +8560,7 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->pub = drvr; init_vif_event(&cfg->vif_event); INIT_LIST_HEAD(&cfg->vif_list); + cfg->force_band_setup = true; vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); if (IS_ERR(vif)) @@ -8388,18 +8618,21 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_DUMP_OBSS)) ops->dump_survey = brcmf_cfg80211_dump_survey; - err = wiphy_register(wiphy); - if (err < 0) { - bphy_err(drvr, "Could not register wiphy device (%d)\n", err); - goto priv_out; - } - + /* We have to configure the bands before we register the wiphy device + * because it requires that band capabilities be correct. + */ err = brcmf_setup_wiphybands(cfg); if (err) { bphy_err(drvr, "Setting wiphy bands failed (%d)\n", err); goto wiphy_unreg_out; } + err = wiphy_register(wiphy); + if (err < 0) { + bphy_err(drvr, "Could not register wiphy device (%d)\n", err); + goto priv_out; + } + /* If cfg80211 didn't disable 40MHz HT CAP in wiphy_register(), * setup 40MHz in 2GHz band and enable OBSS scanning. */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 6ceb301429054c..dec5ff1190f94e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -8,6 +8,7 @@ /* for brcmu_d11inf */ #include +#include #include "core.h" #include "fwil_types.h" @@ -127,7 +128,8 @@ enum brcmf_profile_fwsup { BRCMF_PROFILE_FWSUP_NONE, BRCMF_PROFILE_FWSUP_PSK, BRCMF_PROFILE_FWSUP_1X, - BRCMF_PROFILE_FWSUP_SAE + BRCMF_PROFILE_FWSUP_SAE, + BRCMF_PROFILE_FWSUP_ROAM }; /** @@ -172,6 +174,7 @@ struct brcmf_cfg80211_profile { enum brcmf_profile_fwsup use_fwsup; u16 use_fwauth; bool is_ft; + bool is_okc; }; /** @@ -354,6 +357,7 @@ struct brcmf_cfg80211_wowl { * @dongle_up: indicate whether dongle up or not. * @roam_on: on/off switch for dongle self-roaming. * @scan_tried: indicates if first scan attempted. + * @force_band_setup: indicates if we should force band setup * @dcmd_buf: dcmd buffer. * @extra_buf: mainly to grab assoc information. * @debugfsdir: debugfs folder for this device. @@ -384,6 +388,7 @@ struct brcmf_cfg80211_info { bool pwr_save; bool dongle_up; bool scan_tried; + bool force_band_setup; u8 *dcmd_buf; u8 *extra_buf; struct dentry *debugfsdir; @@ -413,6 +418,22 @@ struct brcmf_tlv { u8 data[]; }; +static inline enum nl80211_band fwil_band_to_nl80211(u16 band) +{ + switch (band) { + case WLC_BAND_2G: + return NL80211_BAND_2GHZ; + case WLC_BAND_5G: + return NL80211_BAND_5GHZ; + case WLC_BAND_6G: + return NL80211_BAND_6GHZ; + default: + WARN_ON(1); + break; + } + return 0; +} + static inline struct wiphy *cfg_to_wiphy(struct brcmf_cfg80211_info *cfg) { return cfg->wiphy; @@ -480,6 +501,8 @@ s32 brcmf_vif_set_mgmt_ie(struct brcmf_cfg80211_vif *vif, s32 pktflag, s32 brcmf_vif_clear_mgmt_ies(struct brcmf_cfg80211_vif *vif); u16 channel_to_chanspec(struct brcmu_d11inf *d11inf, struct ieee80211_channel *ch); +u16 chandef_to_chanspec(struct brcmu_d11inf *d11inf, + struct cfg80211_chan_def *ch); bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, unsigned long state); void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c index a790f1693b82eb..7eed9248a22bcd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c @@ -162,6 +162,15 @@ struct sbconfig { #define SRCI_SRBSZ_SHIFT 0 #define SR_BSZ_BASE 14 +#define SYSMEM_SRCI_ROMNB_MASK 0x3e0 +#define SYSMEM_SRCI_ROMNB_SHIFT 5 +#define SYSMEM_SRCI_SRNB_MASK 0x1f +#define SYSMEM_SRCI_SRNB_SHIFT 0 +#define SYSMEM_SRCI_NEW_ROMNB_MASK 0xff000000 +#define SYSMEM_SRCI_NEW_ROMNB_SHIFT 24 +#define SYSMEM_SRCI_NEW_SRNB_MASK 0xff0000 +#define SYSMEM_SRCI_NEW_SRNB_SHIFT 16 + struct sbsocramregs { u32 coreinfo; u32 bwalloc; @@ -436,25 +445,11 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, { struct brcmf_chip_priv *ci; int count; - struct brcmf_core *d11core2 = NULL; - struct brcmf_core_priv *d11priv2 = NULL; ci = core->chip; - /* special handle two D11 cores reset */ - if (core->pub.id == BCMA_CORE_80211) { - d11core2 = brcmf_chip_get_d11core(&ci->pub, 1); - if (d11core2) { - brcmf_dbg(INFO, "found two d11 cores, reset both\n"); - d11priv2 = container_of(d11core2, - struct brcmf_core_priv, pub); - } - } - /* must disable first to work for arbitrary current core state */ brcmf_chip_ai_coredisable(core, prereset, reset); - if (d11priv2) - brcmf_chip_ai_coredisable(d11priv2, prereset, reset); count = 0; while (ci->ops->read32(ci->ctx, core->wrapbase + BCMA_RESET_CTL) & @@ -466,30 +461,9 @@ static void brcmf_chip_ai_resetcore(struct brcmf_core_priv *core, u32 prereset, usleep_range(40, 60); } - if (d11priv2) { - count = 0; - while (ci->ops->read32(ci->ctx, - d11priv2->wrapbase + BCMA_RESET_CTL) & - BCMA_RESET_CTL_RESET) { - ci->ops->write32(ci->ctx, - d11priv2->wrapbase + BCMA_RESET_CTL, - 0); - count++; - if (count > 50) - break; - usleep_range(40, 60); - } - } - ci->ops->write32(ci->ctx, core->wrapbase + BCMA_IOCTL, postreset | BCMA_IOCTL_CLK); ci->ops->read32(ci->ctx, core->wrapbase + BCMA_IOCTL); - - if (d11priv2) { - ci->ops->write32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL, - postreset | BCMA_IOCTL_CLK); - ci->ops->read32(ci->ctx, d11priv2->wrapbase + BCMA_IOCTL); - } } char *brcmf_chip_name(u32 id, u32 rev, char *buf, uint len) @@ -659,6 +633,7 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) u32 memsize = 0; u32 coreinfo; u32 idx; + u32 nrb; u32 nb; u32 banksize; @@ -666,10 +641,16 @@ static u32 brcmf_chip_sysmem_ramsize(struct brcmf_core_priv *sysmem) brcmf_chip_resetcore(&sysmem->pub, 0, 0, 0); coreinfo = brcmf_chip_core_read32(sysmem, SYSMEMREGOFFS(coreinfo)); - nb = (coreinfo & SRCI_SRNB_MASK) >> SRCI_SRNB_SHIFT; + if (sysmem->pub.rev >= 12) { + nrb = (coreinfo & SYSMEM_SRCI_NEW_ROMNB_MASK) >> SYSMEM_SRCI_NEW_ROMNB_SHIFT; + nb = (coreinfo & SYSMEM_SRCI_NEW_SRNB_MASK) >> SYSMEM_SRCI_NEW_SRNB_SHIFT; + } else { + nrb = (coreinfo & SYSMEM_SRCI_ROMNB_MASK) >> SYSMEM_SRCI_ROMNB_SHIFT; + nb = (coreinfo & SYSMEM_SRCI_SRNB_MASK) >> SYSMEM_SRCI_SRNB_SHIFT; + } for (idx = 0; idx < nb; idx++) { - brcmf_chip_socram_banksize(sysmem, idx, &banksize); + brcmf_chip_socram_banksize(sysmem, idx + nrb, &banksize); memsize += banksize; } @@ -731,6 +712,7 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci) case BRCM_CC_4366_CHIP_ID: case BRCM_CC_43664_CHIP_ID: case BRCM_CC_43666_CHIP_ID: + case BRCM_CC_4388_CHIP_ID: return 0x200000; case BRCM_CC_4355_CHIP_ID: case BRCM_CC_4359_CHIP_ID: @@ -1007,18 +989,33 @@ static int brcmf_chip_recognition(struct brcmf_chip_priv *ci) core = brcmf_chip_add_core(ci, BCMA_CORE_CHIPCOMMON, SI_ENUM_BASE_DEFAULT, 0); + if (IS_ERR(core)) + return PTR_ERR(core); + brcmf_chip_sb_corerev(ci, core); core = brcmf_chip_add_core(ci, BCMA_CORE_SDIO_DEV, BCM4329_CORE_BUS_BASE, 0); + if (IS_ERR(core)) + return PTR_ERR(core); + brcmf_chip_sb_corerev(ci, core); core = brcmf_chip_add_core(ci, BCMA_CORE_INTERNAL_MEM, BCM4329_CORE_SOCRAM_BASE, 0); + if (IS_ERR(core)) + return PTR_ERR(core); + brcmf_chip_sb_corerev(ci, core); core = brcmf_chip_add_core(ci, BCMA_CORE_ARM_CM3, BCM4329_CORE_ARM_BASE, 0); + if (IS_ERR(core)) + return PTR_ERR(core); + brcmf_chip_sb_corerev(ci, core); core = brcmf_chip_add_core(ci, BCMA_CORE_80211, 0x18001000, 0); + if (IS_ERR(core)) + return PTR_ERR(core); + brcmf_chip_sb_corerev(ci, core); } else if (socitype == SOCI_AI) { ci->iscoreup = brcmf_chip_ai_iscoreup; @@ -1338,14 +1335,15 @@ static inline void brcmf_chip_ca7_set_passive(struct brcmf_chip_priv *chip) { struct brcmf_core *core; + int i; brcmf_chip_disable_arm(chip, BCMA_CORE_ARM_CA7); - core = brcmf_chip_get_core(&chip->pub, BCMA_CORE_80211); - brcmf_chip_resetcore(core, D11_BCMA_IOCTL_PHYRESET | - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN, - D11_BCMA_IOCTL_PHYCLOCKEN); + /* Disable the cores only and let the firmware enable them. */ + for (i = 0; (core = brcmf_chip_get_d11core(&chip->pub, i)); i++) + brcmf_chip_coredisable(core, D11_BCMA_IOCTL_PHYRESET | + D11_BCMA_IOCTL_PHYCLOCKEN, + D11_BCMA_IOCTL_PHYCLOCKEN); } static bool brcmf_chip_ca7_set_active(struct brcmf_chip_priv *chip, u32 rstvec) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index b6027d83e6ab57..e3a9ca5526c459 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -13,6 +13,7 @@ #include "core.h" #include "bus.h" #include "debug.h" +#include "fweh.h" #include "fwil.h" #include "fwil_types.h" #include "tracepoint.h" @@ -265,7 +266,6 @@ static int brcmf_c_process_cal_blob(struct brcmf_if *ifp) int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) { struct brcmf_pub *drvr = ifp->drvr; - struct brcmf_fweh_info *fweh = drvr->fweh; u8 buf[BRCMF_DCMD_SMLEN]; struct brcmf_bus *bus; struct brcmf_rev_info_le revinfo; @@ -411,27 +411,6 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp) brcmf_c_set_joinpref_default(ifp); - /* Setup event_msgs, enable E_IF */ - err = brcmf_fil_iovar_data_get(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); - if (err) { - bphy_err(drvr, "Get event_msgs error (%d)\n", err); - goto done; - } - /* - * BRCMF_E_IF can safely be used to set the appropriate bit - * in the event_mask as the firmware event code is guaranteed - * to match the value of BRCMF_E_IF because it is old cruft - * that all vendors have. - */ - setbit(fweh->event_mask, BRCMF_E_IF); - err = brcmf_fil_iovar_data_set(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); - if (err) { - bphy_err(drvr, "Set event_msgs error (%d)\n", err); - goto done; - } - /* Setup default scan channel time */ err = brcmf_fil_cmd_int_set(ifp, BRCMF_C_SET_SCAN_CHANNEL_TIME, BRCMF_DEFAULT_SCAN_CHANNEL_TIME); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index ec170647800dad..0d0ae2870a37e8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1227,7 +1227,14 @@ static int brcmf_bus_started(struct brcmf_pub *drvr, struct cfg80211_ops *ops) if (ret < 0) goto fail; - brcmf_feat_attach(drvr); + ret = brcmf_feat_attach(drvr); + if (ret) + goto fail; + + /* Setup event_msgs, enable E_IF */ + ret = brcmf_fweh_init_events(ifp); + if (ret) + goto fail; ret = brcmf_proto_init_done(drvr); if (ret < 0) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 399b6810e394de..4b52a3aa855de8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -97,6 +97,68 @@ struct brcmf_rev_info { u32 nvramrev; }; +struct brcmf_pno_info; +enum nl80211_band; +/** + * struct pno_struct_handler + */ +struct pno_struct_handler { + u8 version; + int (*pno_config)(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn); + u32 (*get_min_data_len)(void); + u32 (*get_result_count)(void *data); + u32 (*get_result_status)(void *data); + int (*validate_pfn_results)(void *data, u32 event_datalen); + u32 (*get_bucket_map)(void *data, int idx, struct brcmf_pno_info *pi); + int (*get_result_info)(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], u8 *ssid_len, + u8 *channel, enum nl80211_band *band); +}; + +struct cfg80211_scan_request; +struct scan_param_struct_handler { + u8 version; + void *(*get_struct_for_request)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request); +}; + +struct cfg80211_ibss_params; +struct cfg80211_connect_params; + +/** + * struct join_param_struct_handler - Handler for different join parameter versions + * + * There are a number of different, incompatible structures and interface versions for join/extended join parameters + * We abstract away the actual structures used, so that code does not have to worry about filling in structs properly. + * + * This interface deliberately takes and returns opaque structures. + * + * @version - Interface version the firmware supports/uses + * @get_struct_for_ibss - Return a join parameter structure for a set of IBSS parameters. + * This structure can be used to join the passed BSS. + * @get_struct_for_connect - Return an extended join parameter structure for a set of connect + * parameters. This structure can be used to join the SSID specified in the parameters. + * @get_join_from_ext_join - When an extended join does not work, we fall back to a regular join. + * This function produces a join parameter struture from an extended join one. + */ +struct join_param_struct_handler { + u8 version; + /* This returns a join_param type struct */ + void *(*get_struct_for_ibss)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params); + /* This returns an ext_join_param type struct */ + void *(*get_struct_for_connect)(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params); + /* This returns the join param portion of an ext_join_param type struct. + * The memory returned is separately allocated from the passed-in struct. + */ + void *(*get_join_from_ext_join)(void *ext_join_param, u32 *struct_size); +}; + /* Common structure for module and instance linkage */ struct brcmf_pub { /* Linkage ponters */ @@ -145,6 +207,10 @@ struct brcmf_pub { u8 sta_mac_idx; const struct brcmf_fwvid_ops *vops; void *vdata; + u16 cnt_ver; + struct pno_struct_handler pno_handler; + struct scan_param_struct_handler scan_param_handler; + struct join_param_struct_handler join_param_handler; }; /* forward declarations */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 9bb5f709d41a27..432d93ae8fb854 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -85,6 +85,7 @@ do { \ #define BRCMF_FIL_ON() (brcmf_msg_level & BRCMF_FIL_VAL) #define BRCMF_FWCON_ON() (brcmf_msg_level & BRCMF_FWCON_VAL) #define BRCMF_SCAN_ON() (brcmf_msg_level & BRCMF_SCAN_VAL) +#define BRCMF_INFO_ON() (brcmf_msg_level & BRCMF_INFO_VAL) #else /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ @@ -104,6 +105,7 @@ do { \ #define BRCMF_FIL_ON() 0 #define BRCMF_FWCON_ON() 0 #define BRCMF_SCAN_ON() 0 +#define BRCMF_INFO_ON() 0 #endif /* defined(DEBUG) || defined(CONFIG_BRCM_TRACING) */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c index 488364ef8ff2a1..d823ced048454a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c @@ -16,9 +16,24 @@ #include "fwvid.h" #include "feature.h" #include "common.h" +#include "pno.h" +#include "scan_param.h" +#include "join_param.h" #define BRCMF_FW_UNSUPPORTED 23 +/* MIN branch version supporting join iovar versioning */ +#define MIN_JOINEXT_V1_FW_MAJOR 17u +/* Branch/es supporting join iovar versioning prior to + * MIN_JOINEXT_V1_FW_MAJOR + */ +#define MIN_JOINEXT_V1_BR2_FW_MAJOR 16 +#define MIN_JOINEXT_V1_BR2_FW_MINOR 1 + +#define MIN_JOINEXT_V1_BR1_FW_MAJOR 14 +#define MIN_JOINEXT_V1_BR1_FW_MINOR_2 2 +#define MIN_JOINEXT_V1_BR1_FW_MINOR_4 4 + /* * expand feature list to array of feature strings. */ @@ -45,6 +60,7 @@ static const struct brcmf_feat_fwcap brcmf_fwcap_map[] = { { BRCMF_FEAT_SAE, "sae " }, { BRCMF_FEAT_FWAUTH, "idauth" }, { BRCMF_FEAT_SAE_EXT, "sae_ext" }, + { BRCMF_FEAT_GCMP, "gcmp"} }; #ifdef DEBUG @@ -136,7 +152,7 @@ struct brcmf_feat_wlcfeat { static const struct brcmf_feat_wlcfeat brcmf_feat_wlcfeat_map[] = { { 12, 0, BIT(BRCMF_FEAT_PMKID_V2) }, - { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) }, + { 13, 0, BIT(BRCMF_FEAT_PMKID_V3) } }; static void brcmf_feat_wlc_version_overrides(struct brcmf_pub *drv) @@ -286,9 +302,12 @@ static int brcmf_feat_fwcap_debugfs_read(struct seq_file *seq, void *data) return 0; } -void brcmf_feat_attach(struct brcmf_pub *drvr) +int brcmf_feat_attach(struct brcmf_pub *drvr) { struct brcmf_if *ifp = brcmf_get_ifp(drvr, 0); + struct brcmf_join_version_le join_ver; + struct brcmf_scan_version_le scan_ver; + struct brcmf_pno_param_v3_le pno_params; struct brcmf_pno_macaddr_le pfn_mac; struct brcmf_gscan_config gscan_cfg; u32 wowl_cap; @@ -331,6 +350,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_TDLS, "tdls_enable"); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_MFP, "mfp"); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_DUMP_OBSS, "dump_obss"); + brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_EVENT_MSGS_EXT, "event_msgs_ext"); pfn_mac.version = BRCMF_PFN_MACADDR_CFG_VER; err = brcmf_fil_iovar_data_get(ifp, "pfn_macaddr", &pfn_mac, @@ -339,13 +359,71 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) ifp->drvr->feat_flags |= BIT(BRCMF_FEAT_SCAN_RANDOM_MAC); brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_FWSUP, "sup_wpa"); - brcmf_feat_iovar_int_get(ifp, BRCMF_FEAT_SCAN_V2, "scan_ver"); + + err = brcmf_fil_iovar_data_get(ifp, "join_ver", &join_ver, sizeof(join_ver)); + if (!err) { + u16 ver = le16_to_cpu(join_ver.join_ver_major); + err = brcmf_join_param_setup_for_version(drvr, ver); + } else { + /* Default to version 0, unless it is one of the firmware branches + * that doesn't have a join_ver iovar but are still version 1 */ + u8 version = 0; + struct brcmf_wlc_version_le ver; + err = brcmf_fil_iovar_data_get(ifp, "wlc_ver", &ver, + sizeof(ver)); + if (!err) { + u16 major = le16_to_cpu(ver.wlc_ver_major); + u16 minor = le16_to_cpu(ver.wlc_ver_minor); + if (((major == MIN_JOINEXT_V1_BR1_FW_MAJOR) && + ((minor == MIN_JOINEXT_V1_BR1_FW_MINOR_2) || + (minor == MIN_JOINEXT_V1_BR1_FW_MINOR_4))) || + ((major == MIN_JOINEXT_V1_BR2_FW_MAJOR) && + (minor >= MIN_JOINEXT_V1_BR2_FW_MINOR)) || + (major >= MIN_JOINEXT_V1_FW_MAJOR)) { + version = 1; + } + } + err = brcmf_join_param_setup_for_version(drvr, version); + } + if (err) { + bphy_err(drvr, "Error setting up join structure handler: %d\n", + err); + return err; + } + err = brcmf_fil_iovar_data_get(ifp, "scan_ver", &scan_ver, + sizeof(scan_ver)); + if (!err) { + u16 ver = le16_to_cpu(scan_ver.scan_ver_major); + err = brcmf_scan_param_setup_for_version(drvr, ver); + } else { + /* Default to version 1. */ + err = brcmf_scan_param_setup_for_version(drvr, 1); + } + if (err) { + bphy_err(drvr, "Error setting up scan structure handler: %d\n", + err); + return err; + } + /* See what version of PFN scan is supported*/ + err = brcmf_fil_iovar_data_get(ifp, "pno_set", &pno_params, + sizeof(pno_params)); + if (!err) { + err = brcmf_pno_setup_for_version( + drvr, le16_to_cpu(pno_params.version)); + } else { + /* Default to version 2, supported by all chips we support. */ + err = brcmf_pno_setup_for_version(drvr, 2); + } + if (err) { + bphy_err(drvr, "Error setting up escan structure handler: %d\n", + err); + return err; + } brcmf_feat_wlc_version_overrides(drvr); brcmf_feat_firmware_overrides(drvr); brcmf_fwvid_feat_attach(ifp); - if (drvr->settings->feature_disable) { brcmf_dbg(INFO, "Features: 0x%02x, disable: 0x%02x\n", ifp->drvr->feat_flags, @@ -365,6 +443,7 @@ void brcmf_feat_attach(struct brcmf_pub *drvr) /* no quirks */ break; } + return 0; } void brcmf_feat_debugfs_create(struct brcmf_pub *drvr) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h index 31f8695ca41765..be271ca0fca588 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.h @@ -30,8 +30,12 @@ * SAE: simultaneous authentication of equals * FWAUTH: Firmware authenticator * DUMP_OBSS: Firmware has capable to dump obss info to support ACS - * SCAN_V2: Version 2 scan params * SAE_EXT: SAE authentication handled by user-space supplicant + * PMKID_V2: Version 2 PMKID + * PMKID_V3: Version 3 PMKID + * EVENT_MSGS_EXT: Event messages extension + * JOIN_V1: Version 1 join struct + * GCMP: GCMP Cipher suite support */ #define BRCMF_FEAT_LIST \ BRCMF_FEAT_DEF(MBSS) \ @@ -56,10 +60,11 @@ BRCMF_FEAT_DEF(SAE) \ BRCMF_FEAT_DEF(FWAUTH) \ BRCMF_FEAT_DEF(DUMP_OBSS) \ - BRCMF_FEAT_DEF(SCAN_V2) \ BRCMF_FEAT_DEF(PMKID_V2) \ BRCMF_FEAT_DEF(PMKID_V3) \ - BRCMF_FEAT_DEF(SAE_EXT) + BRCMF_FEAT_DEF(SAE_EXT) \ + BRCMF_FEAT_DEF(EVENT_MSGS_EXT) \ + BRCMF_FEAT_DEF(GCMP) /* * Quirks: @@ -97,8 +102,10 @@ enum brcmf_feat_quirk { * brcmf_feat_attach() - determine features and quirks. * * @drvr: driver instance. + * + * Return: 0 in case of success, error code otherwise. */ -void brcmf_feat_attach(struct brcmf_pub *drvr); +int brcmf_feat_attach(struct brcmf_pub *drvr); /** * brcmf_feat_debugfs_create() - create debugfs entries. diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index 1cff4ba7694303..eec4b10e1f6741 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -11,8 +11,10 @@ #include "core.h" #include "debug.h" #include "tracepoint.h" +#include "feature.h" #include "fweh.h" #include "fwil.h" +#include "fwil_types.h" #include "proto.h" #include "bus.h" #include "fwvid.h" @@ -430,6 +432,67 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, drvr->fweh->evt_handler[evt_handler_idx] = NULL; } +/** + * brcmf_fweh_init_events() - initialize event handling. + * + * @ifp: primary interface object. + */ +int brcmf_fweh_init_events(struct brcmf_if *ifp) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_eventmsgs_ext_le *eventmsgs; + size_t size = sizeof(*eventmsgs) + drvr->fweh->event_mask_len; + int err; + + eventmsgs = kzalloc(size, GFP_KERNEL); + if(!eventmsgs) + return -ENOMEM; + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_NONE; + eventmsgs->len = drvr->fweh->event_mask_len; + eventmsgs->maxgetsize = drvr->fweh->event_mask_len; + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_get(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_get(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + + if (err) { + bphy_err(drvr, "Get event_msgs error (%d)\n", err); + kfree(eventmsgs); + return err; + } + + brcmf_dbg(EVENT, "Event mask len: driver=%d fw=%d\n", + drvr->fweh->event_mask_len, eventmsgs->len); + + /* want to handle IF event as well */ + brcmf_dbg(EVENT, "enable event IF\n"); + setbit(eventmsgs->mask, BRCMF_E_IF); + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_SET_MASK; + eventmsgs->len = drvr->fweh->event_mask_len; + + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_set(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + + if (err) + bphy_err(drvr, "Set event_msgs error (%d)\n", err); + + kfree(eventmsgs); + return err; +} + /** * brcmf_fweh_activate_events() - enables firmware events registered. * @@ -437,32 +500,47 @@ void brcmf_fweh_unregister(struct brcmf_pub *drvr, */ int brcmf_fweh_activate_events(struct brcmf_if *ifp) { - struct brcmf_fweh_info *fweh = ifp->drvr->fweh; - enum brcmf_fweh_event_code code; + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_eventmsgs_ext_le *eventmsgs; + size_t size = sizeof(*eventmsgs) + drvr->fweh->event_mask_len; int i, err; - memset(fweh->event_mask, 0, fweh->event_mask_len); - for (i = 0; i < fweh->num_event_codes; i++) { - if (fweh->evt_handler[i]) { - brcmf_fweh_map_fwevt_code(fweh, i, &code); + eventmsgs = kzalloc(size, GFP_KERNEL); + if(!eventmsgs) + return -ENOMEM; + + for (i = 0; i < drvr->fweh->num_event_codes; i++) { + if (drvr->fweh->evt_handler[i]) { brcmf_dbg(EVENT, "enable event %s\n", - brcmf_fweh_event_name(code)); - setbit(fweh->event_mask, i); + brcmf_fweh_event_name(i)); + setbit(eventmsgs->mask, i); } } /* want to handle IF event as well */ brcmf_dbg(EVENT, "enable event IF\n"); - setbit(fweh->event_mask, BRCMF_E_IF); + setbit(eventmsgs->mask, BRCMF_E_IF); + + eventmsgs->version = EVENTMSGS_VER; + eventmsgs->command = EVENTMSGS_SET_MASK; + eventmsgs->len = drvr->fweh->event_mask_len; /* allow per-vendor method to activate firmware events */ if (!brcmf_fwvid_activate_events(ifp)) return 0; - err = brcmf_fil_iovar_data_set(ifp, "event_msgs", fweh->event_mask, - fweh->event_mask_len); + if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_EVENT_MSGS_EXT)) + err = brcmf_fil_iovar_data_set(ifp, "event_msgs_ext", + eventmsgs, size); + else + err = brcmf_fil_iovar_data_set(ifp, "event_msgs", + drvr->fweh->event_mask, + drvr->fweh->event_mask_len); + if (err) - bphy_err(fweh->drvr, "Set event_msgs error (%d)\n", err); + bphy_err(drvr, "Set event_msgs error (%d)\n", err); + + kfree(eventmsgs); return err; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h index e327dd58d29c95..53c4b58e6323cc 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.h @@ -356,6 +356,7 @@ int brcmf_fweh_register(struct brcmf_pub *drvr, enum brcmf_fweh_event_code code, void *data)); void brcmf_fweh_unregister(struct brcmf_pub *drvr, enum brcmf_fweh_event_code code); +int brcmf_fweh_init_events(struct brcmf_if *ifp); int brcmf_fweh_activate_events(struct brcmf_if *ifp); void brcmf_fweh_process_event(struct brcmf_pub *drvr, struct brcmf_event *event_packet, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index e74a23e11830c1..7b8f809cdc412d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -18,7 +18,8 @@ #define BRCMF_ARP_OL_HOST_AUTO_REPLY 0x00000004 #define BRCMF_ARP_OL_PEER_AUTO_REPLY 0x00000008 -#define BRCMF_BSS_INFO_VERSION 109 /* curr ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MIN_VERSION 109 /* min ver of brcmf_bss_info_le struct */ +#define BRCMF_BSS_INFO_MAX_VERSION 112 /* max ver of brcmf_bss_info_le struct */ #define BRCMF_BSS_RSSI_ON_CHANNEL 0x0004 #define BRCMF_STA_BRCM 0x00000001 /* Running a Broadcom driver */ @@ -46,12 +47,10 @@ #define BRCMF_STA_DWDS_CAP 0x01000000 /* DWDS CAP */ #define BRCMF_STA_DWDS 0x02000000 /* DWDS active */ -/* size of brcmf_scan_params not including variable length array */ -#define BRCMF_SCAN_PARAMS_FIXED_SIZE 64 -#define BRCMF_SCAN_PARAMS_V2_FIXED_SIZE 72 - /* version of brcmf_scan_params structure */ #define BRCMF_SCAN_PARAMS_VERSION_V2 2 +#define BRCMF_SCAN_PARAMS_VERSION_V3 3 +#define BRCMF_SCAN_PARAMS_VERSION_V4 4 /* masks for channel and ssid count */ #define BRCMF_SCAN_PARAMS_COUNT_MASK 0x0000ffff @@ -62,16 +61,26 @@ #define BRCMF_SCANTYPE_ACTIVE 0 #define BRCMF_SCANTYPE_PASSIVE 1 +/* Additional scanning flags */ +#define BRCMF_SCANFLAGS_LOW_PRIO 0x2 +#define BRCMF_SCANFLAGS_LOW_POWER 0x1000 +#define BRCMF_SCANFLAGS_HIGH_ACCURACY 0x2000 +#define BRCMF_SCANFLAGS_LOW_SPAN 0x4000 + +/* scan ssid_type flags */ +#define BRCMF_SCANSSID_INC_RNR 0x02 /* Include RNR channels*/ + #define BRCMF_WSEC_MAX_PSK_LEN 32 #define BRCMF_WSEC_PASSPHRASE BIT(0) -#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 128 +#define BRCMF_WSEC_MAX_SAE_PASSWORD_LEN 256 /* primary (ie tx) key */ #define BRCMF_PRIMARY_KEY (1 << 1) #define DOT11_BSSTYPE_ANY 2 #define BRCMF_ESCAN_REQ_VERSION 1 #define BRCMF_ESCAN_REQ_VERSION_V2 2 +#define BRCMF_ESCAN_REQ_VERSION_V3 3 #define BRCMF_MAXRATES_IN_SET 16 /* max # of rates in rateset */ @@ -320,29 +329,57 @@ struct brcmf_bss_info_le { __le16 beacon_period; /* units are Kusec */ __le16 capability; /* Capability information */ u8 SSID_len; - u8 SSID[32]; + u8 SSID[IEEE80211_MAX_SSID_LEN]; + u8 bcnflags; /* additional flags w.r.t. beacon */ struct { __le32 count; /* # rates in this set */ u8 rates[16]; /* rates in 500kbps units w/hi bit set if basic */ } rateset; /* supported rates */ __le16 chanspec; /* chanspec for bss */ __le16 atim_window; /* units are Kusec */ - u8 dtim_period; /* DTIM period */ + u8 dtim_period; /* DTIM period */ + u8 accessnet; /* from beacon interwork IE (if bcnflags) */ __le16 RSSI; /* receive signal strength (in dBm) */ s8 phy_noise; /* noise (in dBm) */ u8 n_cap; /* BSS is 802.11N Capable */ + u8 he_cap; /* BSS is he capable */ + u8 load; /* BSS Load from QBSS load IE if available */ /* 802.11N BSS Capabilities (based on HT_CAP_*): */ __le32 nbss_cap; u8 ctl_ch; /* 802.11N BSS control channel number */ - __le32 reserved32[1]; /* Reserved for expansion of BSS properties */ + u8 reserved1[3]; /* Reserved for expansion of BSS properties */ + __le16 vht_rxmcsmap; /* VHT rx mcs map (802.11ac IE, VHT_CAP_MCS_MAP_*) */ + __le16 vht_txmcsmap; /* VHT tx mcs map (802.11ac IE, VHT_CAP_MCS_MAP_*) */ u8 flags; /* flags */ - u8 reserved[3]; /* Reserved for expansion of BSS properties */ + u8 vht_cap; /* BSS is vht capable */ + u8 reserved2[2]; /* Reserved for expansion of BSS properties */ u8 basic_mcs[BRCMF_MCSSET_LEN]; /* 802.11N BSS required MCS set */ __le16 ie_offset; /* offset at which IEs start, from beginning */ + u8 reserved3[2]; /* Reserved for expansion of BSS properties */ __le32 ie_length; /* byte length of Information Elements */ __le16 SNR; /* average SNR of during frame reception */ + __le16 vht_mcsmap; /**< STA's Associated vhtmcsmap */ + __le16 vht_mcsmap_prop; /**< STA's Associated prop vhtmcsmap */ + __le16 vht_txmcsmap_prop; /**< prop VHT tx mcs prop */ + __le32 he_mcsmap; /**< STA's Associated hemcsmap */ + __le32 he_rxmcsmap; /**< HE rx mcs map (802.11ax IE, HE_CAP_MCS_MAP_*) */ + __le32 he_txmcsmap; /**< HE tx mcs map (802.11ax IE, HE_CAP_MCS_MAP_*) */ + __le32 timestamp[2]; /* Beacon Timestamp for FAKEAP req */ + /* V112 fields follow */ + u8 eht_cap; /* BSS is EHT capable */ + u8 reserved4[3]; /* Reserved for expansion of BSS properties */ + /* by the spec. it is maximum 16 streams hence all mcs code for all nss may not fit + * in a 32 bit mcs nss map but since this field only reflects the common mcs nss map + * between that of the peer and our device so it's probably ok to make it 32 bit and + * allow only a limited number of nss e.g. upto 8 of them in the map given the fact + * that our device probably won't exceed 4 streams anyway... + */ + __le32 eht_mcsmap; /* STA's associated EHT mcs code map */ + /* FIXME: change the following mcs code map to uint32 if all mcs+nss can fit in */ + u8 eht_rxmcsmap[6]; /* EHT rx mcs code map */ + u8 eht_txmcsmap[6]; /* EHT tx mcs code map */ /* Add new fields here */ /* variable length Information Elements */ }; @@ -366,23 +403,23 @@ struct brcmf_ssid8_le { }; struct brcmf_scan_params_le { - struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ - u8 bssid[ETH_ALEN]; /* default: bcast */ - s8 bss_type; /* default: any, + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT */ - u8 scan_type; /* flags, 0 use default */ - __le32 nprobes; /* -1 use default, number of probes per channel */ - __le32 active_time; /* -1 use default, dwell time per channel for + u8 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for * active scanning */ - __le32 passive_time; /* -1 use default, dwell time per channel + __le32 passive_time; /* -1 use default, dwell time per channel * for passive scanning */ __le32 home_time; /* -1 use default, dwell time for the * home channel between channel scans */ - __le32 channel_num; /* count of channels and ssids that follow + __le32 channel_num; /* count of channels and ssids that follow * * low half is count of channels in * channel_list, 0 means default (use all @@ -398,56 +435,125 @@ struct brcmf_scan_params_le { * fixed parameter portion is assumed, otherwise * ssid in the fixed portion is ignored */ - union { - __le16 padding; /* Reserve space for at least 1 entry for abort - * which uses an on stack brcmf_scan_params_le - */ - DECLARE_FLEX_ARRAY(__le16, channel_list); /* chanspecs */ - }; + __le16 channel_list[]; /* chanspecs */ }; struct brcmf_scan_params_v2_le { - __le16 version; /* structure version */ - __le16 length; /* structure length */ - struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ - u8 bssid[ETH_ALEN]; /* default: bcast */ - s8 bss_type; /* default: any, - * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT - */ - u8 pad; - __le32 scan_type; /* flags, 0 use default */ - __le32 nprobes; /* -1 use default, number of probes per channel */ - __le32 active_time; /* -1 use default, dwell time per channel for - * active scanning - */ - __le32 passive_time; /* -1 use default, dwell time per channel - * for passive scanning - */ - __le32 home_time; /* -1 use default, dwell time for the - * home channel between channel scans - */ - __le32 channel_num; /* count of channels and ssids that follow - * - * low half is count of channels in - * channel_list, 0 means default (use all - * available channels) - * - * high half is entries in struct brcmf_ssid - * array that follows channel_list, aligned for - * s32 (4 bytes) meaning an odd channel count - * implies a 2-byte pad between end of - * channel_list and first ssid - * - * if ssid count is zero, single ssid in the - * fixed parameter portion is assumed, otherwise - * ssid in the fixed portion is ignored - */ - union { - __le16 padding; /* Reserve space for at least 1 entry for abort - * which uses an on stack brcmf_scan_params_v2_le - */ - DECLARE_FLEX_ARRAY(__le16, channel_list); /* chanspecs */ - }; + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 PAD; + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ +}; + +struct brcmf_scan_params_v3_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 ssid_type; /* short vs regular SSID */ + __le32 scan_type; /* flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ +}; + +struct brcmf_scan_params_v4_le { + __le16 version; /* structure version */ + __le16 length; /* structure length */ + struct brcmf_ssid_le ssid_le; /* default: {0, ""} */ + u8 bssid[ETH_ALEN]; /* default: bcast */ + s8 bss_type; /* default: any, + * DOT11_BSSTYPE_ANY/INFRASTRUCTURE/INDEPENDENT + */ + u8 ssid_type; /* short vs regular SSID */ + __le32 scan_type; /* flags, 0 use default */ + __le32 scan_type_ext; /* ext flags, 0 use default */ + __le32 nprobes; /* -1 use default, number of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the + * home channel between channel scans + */ + __le32 channel_num; /* count of channels and ssids that follow + * + * low half is count of channels in + * channel_list, 0 means default (use all + * available channels) + * + * high half is entries in struct brcmf_ssid + * array that follows channel_list, aligned for + * s32 (4 bytes) meaning an odd channel count + * implies a 2-byte pad between end of + * channel_list and first ssid + * + * if ssid count is zero, single ssid in the + * fixed parameter portion is assumed, otherwise + * ssid in the fixed portion is ignored + */ + __le16 channel_list[]; /* chanspecs */ }; struct brcmf_scan_results { @@ -464,6 +570,8 @@ struct brcmf_escan_params_le { union { struct brcmf_scan_params_le params_le; struct brcmf_scan_params_v2_le params_v2_le; + struct brcmf_scan_params_v3_le params_v3_le; + struct brcmf_scan_params_v4_le params_v4_le; }; }; @@ -482,11 +590,67 @@ struct brcmf_escan_result_le { struct brcmf_assoc_params_le { /* 00:00:00:00:00:00: broadcast scan */ u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; + /* 0: all available channels, otherwise count of chanspecs in + * chanspec_list */ + __le32 chanspec_num; + /* list of chanspecs */ + __le16 chanspec_list[]; +}; + +struct brcmf_assoc_params_v1_le { + __le16 version; + __le16 flags; + /* 00:00:00:00:00:00: broadcast scan */ + u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; /* 0: all available channels, otherwise count of chanspecs in * chanspec_list */ __le32 chanspec_num; /* list of chanspecs */ - __le16 chanspec_list[1]; + __le16 chanspec_list[]; +}; + +/* ML assoc and scan params */ +struct brcmf_ml_assoc_scan_params_v1_le { + /* whether to follow strictly ordered assoc ? */ + u8 ml_assoc_mode; + /* to identify whether ml scan needs to be triggered */ + u8 ml_scan_mode; + u8 pad[2]; +}; + +struct brcmf_assoc_params_v2_le { + __le16 version; + __le16 flags; + /* 00:00:00:00:00:00: broadcast scan */ + u8 bssid[ETH_ALEN]; + /* 0: use chanspec_num, and the single bssid, + * otherwise count of chanspecs in chanspec_list + * AND paired bssids following chanspec_list + * also, chanspec_num has to be set to zero + * for bssid list to be used + */ + __le16 bssid_cnt; + /* Multilink association and scan params */ + struct brcmf_ml_assoc_scan_params_v1_le ml_assoc_scan_params; + /* 0: all available channels, otherwise count of chanspecs in + * chanspec_list */ + __le32 chanspec_num; + /* list of chanspecs */ + __le16 chanspec_list[]; }; /** @@ -511,9 +675,19 @@ struct brcmf_join_params { struct brcmf_assoc_params_le params_le; }; +struct brcmf_join_params_v1 { + struct brcmf_ssid_le ssid_le; + struct brcmf_assoc_params_v1_le params_le; +}; +struct brcmf_join_params_v2 { + struct brcmf_ssid_le ssid_le; + struct brcmf_assoc_params_v2_le params_le; +}; + /* scan params for extended join */ struct brcmf_join_scan_params_le { u8 scan_type; /* 0 use default, active or passive scan */ + u8 PAD[3]; __le32 nprobes; /* -1 use default, nr of probes per channel */ __le32 active_time; /* -1 use default, dwell time per channel for * active scanning @@ -526,6 +700,23 @@ struct brcmf_join_scan_params_le { */ }; +/* scan params for extended join */ +struct brcmf_join_scan_params_v1_le { + u8 scan_type; /* 0 use default, active or passive scan */ + u8 ml_scan_mode; /* 0 scan ML channels in RNR, 1 scan only provided channels */ + u8 PAD[2]; + __le32 nprobes; /* -1 use default, nr of probes per channel */ + __le32 active_time; /* -1 use default, dwell time per channel for + * active scanning + */ + __le32 passive_time; /* -1 use default, dwell time per channel + * for passive scanning + */ + __le32 home_time; /* -1 use default, dwell time for the home + * channel between channel scans + */ +}; + /* extended join params */ struct brcmf_ext_join_params_le { struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ @@ -533,6 +724,24 @@ struct brcmf_ext_join_params_le { struct brcmf_assoc_params_le assoc_le; }; +/* extended join params */ +struct brcmf_ext_join_params_v1_le { + __le16 version; + u16 pad; + struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ + struct brcmf_join_scan_params_le scan_le; + struct brcmf_assoc_params_v1_le assoc_le; +}; + +/* extended join params v2 */ +struct brcmf_ext_join_params_v2_le { + __le16 version; + u16 pad; + struct brcmf_ssid_le ssid_le; /* {0, ""}: wildcard scan */ + struct brcmf_join_scan_params_v1_le scan_le; + struct brcmf_assoc_params_v2_le assoc_le; +}; + struct brcmf_wsec_key { u32 index; /* key index */ u32 len; /* key length */ @@ -580,11 +789,15 @@ struct brcmf_wsec_key_le { * @key_len: number of octets in key material. * @flags: key handling qualifiers. * @key: PMK key material. + * @opt_len: optional field length + * @opt_tlvs: optional fields in TLV format */ struct brcmf_wsec_pmk_le { __le16 key_len; __le16 flags; u8 key[BRCMF_WSEC_MAX_SAE_PASSWORD_LEN]; + __le16 opt_len; + u8 opt_tlvs[]; }; /** @@ -611,13 +824,17 @@ struct brcmf_channel_info_le { __le32 scan_channel; }; +#define BRCMF_MAX_ASSOC_OUI_NUM 6 +#define BRCMF_ASSOC_OUI_LEN 3 struct brcmf_sta_info_le { __le16 ver; /* version of this struct */ __le16 len; /* length in bytes of this structure */ __le16 cap; /* sta's advertised capabilities */ + u16 PAD; __le32 flags; /* flags defined below */ __le32 idle; /* time since data pkt rx'd from sta */ u8 ea[ETH_ALEN]; /* Station address */ + u16 PAD2; __le32 count; /* # rates in this set */ u8 rates[BRCMF_MAXRATES_IN_SET]; /* rates in 500kbps units */ /* w/hi bit set if basic */ @@ -649,6 +866,7 @@ struct brcmf_sta_info_le { __le16 aid; /* association ID */ __le16 ht_capabilities; /* advertised ht caps */ __le16 vht_flags; /* converted vht flags */ + u16 PAD3; __le32 tx_pkts_retry_cnt; /* # of frames where a retry was * exhausted. */ @@ -701,6 +919,13 @@ struct brcmf_sta_info_le { __le32 tx_rspec; /* Rate of last successful tx frame */ __le32 rx_rspec; /* Rate of last successful rx frame */ __le32 wnm_cap; /* wnm capabilities */ + __le16 he_flags; /* converted he flags */ + u16 PAD; + struct { + u8 count; + u8 oui[BRCMF_MAX_ASSOC_OUI_NUM][BRCMF_ASSOC_OUI_LEN]; + } vendor_oui; + u8 link_bw; } v7; }; }; @@ -833,6 +1058,30 @@ struct brcmf_wlc_version_le { __le16 wlc_ver_minor; }; +/** + * struct brcmf_join_version_le - join interface version + */ +struct brcmf_join_version_le { + __le16 version; /**< version of the structure */ + __le16 length; /**< length of the entire structure */ + + /* join interface version numbers */ + __le16 join_ver_major; /**< join interface major version number */ + u8 pad[2]; +}; +#define BRCMF_JOIN_VERSION_VERSION 1 + +/** + * struct brcmf_scan_version_le - scan interface version + */ +struct brcmf_scan_version_le { + __le16 version; + __le16 length; + __le16 scan_ver_major; +}; + +#define BRCMF_SCAN_VERSION_VERSION 1 + /** * struct brcmf_assoclist_le - request assoc list. * @@ -1009,6 +1258,46 @@ struct brcmf_pno_param_le { __le32 slow_freq; }; +/** + * struct brcmf_pno_param_le - PNO scan configuration parameters + * + * @version: PNO parameters version. + * @length: Length of PNO structure + * @scan_freq: scan frequency. + * @lost_network_timeout: #sec. to declare discovered network as lost. + * @flags: Bit field to control features of PFN such as sort criteria auto + * enable switch and background scan. + * @rssi_margin: Margin to avoid jitter for choosing a PFN based on RSSI sort + * criteria. + * @bestn: number of best networks in each scan. + * @mscan: number of scans recorded. + * @repeat: minimum number of scan intervals before scan frequency changes + * in adaptive scan. + * @exp: exponent of 2 for maximum scan interval. + * @slow_freq: slow scan period. + * @min_bound: min bound for scan time randomization + * @max_bound: max bound for scan time randomization + * @pfn_lp_scan_disable: unused + * @pfn_lp_scan_cnt: allow interleaving lp scan with hp scan + */ +struct brcmf_pno_param_v3_le { + __le16 version; + __le16 length; + __le32 scan_freq; + __le32 lost_network_timeout; + __le16 flags; + __le16 rssi_margin; + u8 bestn; + u8 mscan; + u8 repeat; + u8 exp; + __le32 slow_freq; + u8 min_bound; + u8 max_bound; + u8 pfn_lp_scan_disable; + u8 pfn_lp_scan_cnt; +}; + /** * struct brcmf_pno_config_le - PNO channel configuration. * @@ -1062,6 +1351,28 @@ struct brcmf_pno_net_info_le { __le16 timestamp; }; +/** + * struct brcmf_pno_net_info_v3_le - information per found network. + * + * @bssid: BSS network identifier. + * @chanspec: channel spec. + * @SSID_len: length of ssid. + * @SSID: ssid characters. + * @flags: flags + * @RSSI: receive signal strength (in dBm). + * @timestamp: age in seconds. + */ +struct brcmf_pno_net_info_v3_le { + u8 bssid[6]; + u16 chanspec; + u8 SSID_len; + u8 padding; + u16 flags; + u8 SSID[32]; + __le16 RSSI; + __le16 timestamp; +}; + /** * struct brcmf_pno_scanresults_le - result returned in PNO NET FOUND event. * @@ -1082,6 +1393,14 @@ struct brcmf_pno_scanresults_v2_le { __le32 scan_ch_bucket; }; +/* V2 and V3 structs are the same */ +struct brcmf_pno_scanresults_v3_le { + __le32 version; + __le32 status; + __le32 count; + __le32 scan_ch_bucket; +}; + /** * struct brcmf_pno_macaddr_le - to configure PNO macaddr randomization. * @@ -1236,4 +1555,141 @@ struct brcmf_mkeep_alive_pkt_le { u8 data[]; } __packed; +enum event_msgs_ext_command { + EVENTMSGS_NONE = 0, + EVENTMSGS_SET_BIT = 1, + EVENTMSGS_RESET_BIT = 2, + EVENTMSGS_SET_MASK = 3 +}; + +#define EVENTMSGS_VER 1 + +/** + * struct brcmf_eventmsgs_ext_le - new event message mask commands + * + * @version: EVENTMSGS_VER + * @command: one of enum event_msgs_ext_command + * @len: for set, the mask size from the application to the firmware. + * for get, the actual firmware mask size. + * @maxgetsize: for get, the max size that the application can read from + * the firmware. + */ +struct brcmf_eventmsgs_ext_le { + u8 version; + u8 command; + u8 len; + u8 maxgetsize; + u8 mask[]; +}; + +/* version of the brcmf_wl_wsec_info structure */ +#define BRCMF_WSEC_INFO_VER 1 + +/* tlv used to return wl_wsec_info properties */ +struct brcmf_wsec_info_tlv { + u16 type; + u16 len; /* data length */ + u8 data[1]; /* data follows */ +}; + +/* input/output data type for wsec_info iovar */ +struct brcmf_wsec_info { + u8 version; /* structure version */ + u8 pad[2]; + u8 num_tlvs; + struct brcmf_wsec_info_tlv tlvs[1]; /* tlv data follows */ +}; + +/* HE top level command IDs */ +enum { + BRCMF_HE_CMD_ENABLE = 0, + BRCMF_HE_CMD_FEATURES = 1, + BRCMF_HE_CMD_SR = 2, + BRCMF_HE_CMD_TESTBED = 3, + BRCMF_HE_CMD_BSR_SUPPORT = 4, + BRCMF_HE_CMD_BSSCOLOR = 5, + BRCMF_HE_CMD_PARTIAL_BSSCOLOR = 6, + BRCMF_HE_CMD_CAP = 7, + BRCMF_HE_CMD_OMI = 8, + BRCMF_HE_CMD_RANGE_EXT = 9, + BRCMF_HE_CMD_RTSDURTHRESH = 10, + BRCMF_HE_CMD_PEDURATION = 11, + BRCMF_HE_CMD_MUEDCA = 12, + BRCMF_HE_CMD_DYNFRAG = 13, + BRCMF_HE_CMD_PPET = 14, + BRCMF_HE_CMD_HTC = 15, + BRCMF_HE_CMD_AXMODE = 16, + BRCMF_HE_CMD_FRAGTX = 17, + BRCMF_HE_CMD_DEFCAP = 18, +}; + +#define BRCMF_HE_VER_1 1 + +struct brcmf_he_bsscolor { + u8 color; /* 1..63, on get returns currently in use color */ + u8 disabled; /* 0/1, 0 means disabled is false, so coloring is enabled */ + u8 switch_count; /* 0, immediate programming, 1 .. 255 beacon count down */ + u8 PAD; +}; + +struct brcmf_he_omi { + u8 peer[ETH_ALEN]; /* leave it all 0s' for non-AP */ + u8 rx_nss; /* 0..7 */ + u8 channel_width; /* 0:20, 1:40, 2:80, 3:160 */ + u8 ul_mu_disable; /* 0|1 */ + u8 tx_nsts; /* 0..7 */ + u8 er_su_disable; /* 0|1 */ + u8 dl_mumimo_resound; /* 0|1 */ + u8 ul_mu_data_disable; /* 0|1 */ + u8 tx_override; /* 0, only used for testbed AP */ + u8 PAD[2]; +}; + +struct brcmf_he_edca_v1 { + u8 aci_aifsn; + u8 ecw_min_max; + u8 muedca_timer; + u8 PAD; +}; + +#define BRCMF_AC_COUNT 4 +struct brcmf_he_muedca_v1 { + /* structure control */ + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + struct brcmf_he_edca_v1 ac_param_ap[BRCMF_AC_COUNT]; + struct brcmf_he_edca_v1 ac_param_sta[BRCMF_AC_COUNT]; +}; + +#define BRCMF_HE_SR_VER_1 1 + +#define SRC_PSR_DIS 0x01 +#define SRC_NON_SRG_OBSS_PD_SR_DIS 0x02 +#define SRC_NON_SRG_OFFSET_PRESENT 0x04 +#define SRC_SRG_INFORMATION_PRESENT 0x08 +#define SRC_HESIGA_SPATIAL_REUSE_VALUE15_ALLOWED 0x10 + +#define HE_SR_SRG_INFO_LEN 18 + +struct brcmf_he_sr_v1 { + /* structure control */ + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + u8 enabled; + u8 src; /* SR control, see above defines. */ + u8 non_srg_offset; /* Non-SRG Offset */ + u8 srg[HE_SR_SRG_INFO_LEN]; /* SRG Information */ +}; + +#define BRCMF_HE_DEFCAP_VER_1 1 + +struct brcmf_he_defcap { + __le16 version; /* structure version */ + __le16 length; /* data length (starting after this field) */ + u8 bsscfg_type; + u8 bsscfg_subtype; + u8 mac_cap[6]; + u8 phy_cap[11]; +}; + #endif /* FWIL_TYPES_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c new file mode 100644 index 00000000000000..1f40ff8d632c25 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +/* This file handles firmware-side interface creation */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include "cfg80211.h" +#include "debug.h" +#include "fwil.h" +#include "proto.h" +#include "bus.h" +#include "common.h" +#include "interface_create.h" + +#define BRCMF_INTERFACE_CREATE_VER_1 1 +#define BRCMF_INTERFACE_CREATE_VER_2 2 +#define BRCMF_INTERFACE_CREATE_VER_3 3 +#define BRCMF_INTERFACE_CREATE_VER_MAX BRCMF_INTERFACE_CREATE_VER_3 + +/* These sets of flags specify whether to use various fields in the interface create structures */ + +/* This is only used with version 0 or 1 */ +#define BRCMF_INTERFACE_CREATE_STA (0 << 0) +#define BRCMF_INTERFACE_CREATE_AP (1 << 0) + +#define BRCMF_INTERFACE_MAC_DONT_USE (0 << 1) +#define BRCMF_INTERFACE_MAC_USE (1 << 1) + +#define BRCMF_INTERFACE_WLC_INDEX_DONT_USE (0 << 2) +#define BRCMF_INTERFACE_WLC_INDEX_USE (1 << 2) + +#define BRCMF_INTERFACE_IF_INDEX_DONT_USE (0 << 3) +#define BRCMF_INTERFACE_IF_INDEX_USE (1 << 3) + +#define BRCMF_INTERFACE_BSSID_DONT_USE (0 << 4) +#define BRCMF_INTERFACE_BSSID_USE (1 << 4) + +/* + * From revision >= 2 Bit 0 of flags field will not be used for STA or AP interface creation. + * "iftype" field shall be used for identifying the interface type. + */ +enum brcmf_interface_type { + BRCMF_INTERFACE_TYPE_STA = 0, + BRCMF_INTERFACE_TYPE_AP = 1, + /* The missing number here is deliberate */ + BRCMF_INTERFACE_TYPE_NAN = 3, + BRCMF_INTERFACE_TYPE_P2P_GO = 4, + BRCMF_INTERFACE_TYPE_P2P_GC = 5, + BRCMF_INTERFACE_TYPE_P2P_DISC = 6, + BRCMF_INTERFACE_TYPE_IBSS = 7, + BRCMF_INTERFACE_TYPE_MESH = 8 +}; + + +/* All sources treat these structures as being host endian. + * However, firmware treats it as little endian, so we do as well */ + +struct brcmf_interface_create_v1 { + __le16 ver; /* structure version */ + u8 pad1[2]; + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 pad2[2]; + __le32 wlc_index; /* optional for wlc index */ +}; + +struct brcmf_interface_create_v2 { + __le16 ver; /* structure version */ + u8 pad1[2]; + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 iftype; /* type of interface created */ + u8 pad2; + u32 wlc_index; /* optional for wlc index */ +}; + +struct brcmf_interface_create_v3 { + __le16 ver; /* structure version */ + __le16 len; /* length of structure + data */ + __le16 fixed_len; /* length of structure */ + u8 iftype; /* type of interface created */ + u8 wlc_index; /* optional for wlc index */ + __le32 flags; /* flags for operation */ + u8 mac_addr[ETH_ALEN]; /* MAC address */ + u8 bssid[ETH_ALEN]; /* optional for BSSID */ + u8 if_index; /* interface index request */ + u8 pad[3]; + u8 data[]; /* Optional for specific data */ +}; + +static int brcmf_get_first_free_bsscfgidx(struct brcmf_pub *drvr) +{ + int bsscfgidx; + + for (bsscfgidx = 0; bsscfgidx < BRCMF_MAX_IFS; bsscfgidx++) { + /* bsscfgidx 1 is reserved for legacy P2P */ + if (bsscfgidx == 1) + continue; + if (!drvr->iflist[bsscfgidx]) + return bsscfgidx; + } + + return -ENOMEM; +} + +static void brcmf_set_vif_sta_macaddr(struct brcmf_if *ifp, u8 *mac_addr) +{ + u8 mac_idx = ifp->drvr->sta_mac_idx; + + /* set difference MAC address with locally administered bit */ + memcpy(mac_addr, ifp->mac_addr, ETH_ALEN); + mac_addr[0] |= 0x02; + mac_addr[3] ^= mac_idx ? 0xC0 : 0xA0; + mac_idx++; + mac_idx = mac_idx % 2; + ifp->drvr->sta_mac_idx = mac_idx; +} + +static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, + enum brcmf_interface_type if_type, + u8 *macaddr) +{ + switch (version) { + case BRCMF_INTERFACE_CREATE_VER_1: { + struct brcmf_interface_create_v1 iface_v1 = {}; + u32 flags = if_type; + + iface_v1.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_1); + if (macaddr) { + flags |= BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v1.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v1.mac_addr); + } + iface_v1.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v1, sizeof(iface_v1)); + } + case BRCMF_INTERFACE_CREATE_VER_2: { + struct brcmf_interface_create_v2 iface_v2 = {}; + u32 flags = 0; + + iface_v2.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_2); + iface_v2.iftype = if_type; + if (macaddr) { + flags = BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v2.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v2.mac_addr); + } + iface_v2.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v2, sizeof(iface_v2)); + } + case BRCMF_INTERFACE_CREATE_VER_3: { + struct brcmf_interface_create_v3 iface_v3 = {}; + u32 flags = 0; + + iface_v3.ver = cpu_to_le16(BRCMF_INTERFACE_CREATE_VER_3); + iface_v3.iftype = if_type; + iface_v3.len = cpu_to_le16(sizeof(iface_v3)); + iface_v3.fixed_len = cpu_to_le16(sizeof(iface_v3)); + if (macaddr) { + flags = BRCMF_INTERFACE_MAC_USE; + if (!is_zero_ether_addr(macaddr)) + memcpy(iface_v3.mac_addr, macaddr, ETH_ALEN); + else + brcmf_set_vif_sta_macaddr(ifp, + iface_v3.mac_addr); + } + iface_v3.flags = cpu_to_le32(flags); + return brcmf_fil_iovar_data_get(ifp, "interface_create", + &iface_v3, sizeof(iface_v3)); + } + default: + bphy_err(ifp->drvr, "Unknown interface create version:%d\n", + version); + return -EINVAL; + } +} +static int brcmf_cfg80211_request_if(struct brcmf_if *ifp, + enum brcmf_interface_type if_type, + u8 *macaddr) +{ + s32 err; + u32 iface_create_ver; + + /* Query the creation version, see if the firmware knows */ + iface_create_ver = 0; + err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", + &iface_create_ver); + if (!err) { + err = brcmf_cfg80211_request_if_internal(ifp, iface_create_ver, + if_type, macaddr); + if (!err) { + brcmf_info("interface created (version %d)\n", + iface_create_ver); + } else { + bphy_err(ifp->drvr, + "failed to create interface (version %d):%d\n", + iface_create_ver, err); + } + return err; + } + /* Either version one or version two */ + err = brcmf_cfg80211_request_if_internal( + ifp, if_type, BRCMF_INTERFACE_CREATE_VER_2, macaddr); + if (!err) { + brcmf_info("interface created (version 2)\n"); + return 0; + } + err = brcmf_cfg80211_request_if_internal( + ifp, if_type, BRCMF_INTERFACE_CREATE_VER_1, macaddr); + if (!err) { + brcmf_info("interface created (version 1)\n"); + return 0; + } + bphy_err(ifp->drvr, + "interface creation failed, tried query, v2, v1: %d\n", err); + return -EINVAL; +} + +int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) +{ + return brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_STA, + macaddr); +} + +int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) +{ + int err; + + err = brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_AP, NULL); + if (err) { + struct brcmf_mbss_ssid_le mbss_ssid_le; + int bsscfgidx; + + brcmf_info("Does not support interface_create (%d)\n", err); + memset(&mbss_ssid_le, 0, sizeof(mbss_ssid_le)); + bsscfgidx = brcmf_get_first_free_bsscfgidx(ifp->drvr); + if (bsscfgidx < 0) + return bsscfgidx; + + mbss_ssid_le.bsscfgidx = cpu_to_le32(bsscfgidx); + mbss_ssid_le.SSID_len = cpu_to_le32(5); + sprintf(mbss_ssid_le.SSID, "ssid%d", bsscfgidx); + + err = brcmf_fil_bsscfg_data_set(ifp, "bsscfg:ssid", + &mbss_ssid_le, + sizeof(mbss_ssid_le)); + + if (err < 0) + bphy_err(ifp->drvr, "setting ssid failed %d\n", err); + } + return err; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h new file mode 100644 index 00000000000000..669fa1508b67f6 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_INTERFACE_CREATE_H_ +#define _BRCMF_INTERFACE_CREATE_H_ +#include "core.h" + +int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr); +int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp); + +#endif /* _BRCMF_INTERFACE_CREATE_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c new file mode 100644 index 00000000000000..4f026571c7e7eb --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ +#include +#include + +#include "core.h" +#include "debug.h" +#include "fwil_types.h" +#include "cfg80211.h" +#include "join_param.h" + +/* These defaults are the same as found in the DHD drivers, and represent + * reasonable defaults for various scan dwell and probe times. */ +#define BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS 320 +#define BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS 400 +#define BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS 20 + +/* Most of the actual structure fields we fill in are the same for various versions + * However, due to various incompatible changes and variants, the fields are not always + * in the same place. + * This makes for code duplication, so we try to commonize setting fields where it makes sense. + */ + +static void brcmf_joinscan_set_ssid(struct brcmf_ssid_le *ssid_le, + const u8 *ssid, u32 ssid_len) +{ + ssid_len = min_t(u32, ssid_len, IEEE80211_MAX_SSID_LEN); + ssid_le->SSID_len = cpu_to_le32(ssid_len); + memcpy(ssid_le->SSID, ssid, ssid_len); +} + +static void brcmf_joinscan_set_bssid(u8 out_bssid[6], const u8 *in_bssid) +{ + if (in_bssid) { + memcpy(out_bssid, in_bssid, ETH_ALEN); + } else { + eth_broadcast_addr(out_bssid); + } +} + +/* Create a single channel chanspec list from a wireless stack channel */ +static void brcmf_joinscan_set_single_chanspec_from_channel( + struct brcmf_cfg80211_info *cfg, struct ieee80211_channel *chan, + __le32 *chanspec_count, __le16 (*chanspec_list)[]) +{ + u16 chanspec = channel_to_chanspec(&cfg->d11inf, chan); + *chanspec_count = cpu_to_le32(1); + (*chanspec_list)[0] = cpu_to_le16(chanspec); +} + +/* Create a single channel chanspec list from a wireless stack chandef */ +static void brcmf_joinscan_set_single_chanspec_from_chandef( + struct brcmf_cfg80211_info *cfg, struct cfg80211_chan_def *chandef, + __le32 *chanspec_count, __le16 (*chanspec_list)[]) +{ + u16 chanspec = chandef_to_chanspec(&cfg->d11inf, chandef); + *chanspec_count = cpu_to_le32(1); + (*chanspec_list)[0] = cpu_to_le16(chanspec); +} + +static void *brcmf_get_struct_for_ibss_v0(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params) +{ + struct brcmf_join_params *join_params; + + u32 join_params_size = struct_size(join_params, params_le.chanspec_list, + params->chandef.chan != NULL); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + bphy_err(cfg, "Unable to allocate memory for join params\n"); + return NULL; + } + brcmf_joinscan_set_ssid(&join_params->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_bssid(join_params->params_le.bssid, params->bssid); + /* Channel */ + if (cfg->channel) { + brcmf_joinscan_set_single_chanspec_from_chandef( + cfg, ¶ms->chandef, + &join_params->params_le.chanspec_num, + &join_params->params_le.chanspec_list); + } + return join_params; +} + +static void * +brcmf_get_prepped_struct_for_ibss_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_ibss_params *params) +{ + struct brcmf_join_params_v1 *join_params; + u32 join_params_size = struct_size(join_params, params_le.chanspec_list, + params->chandef.chan != NULL); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + bphy_err(cfg, "Unable to allocate memory for join params\n"); + return NULL; + } + join_params->params_le.version = cpu_to_le16(1); + brcmf_joinscan_set_ssid(&join_params->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_bssid(join_params->params_le.bssid, params->bssid); + /* Channel */ + if (cfg->channel) { + brcmf_joinscan_set_single_chanspec_from_chandef( + cfg, ¶ms->chandef, + &join_params->params_le.chanspec_num, + &join_params->params_le.chanspec_list); + } + return join_params; +} + +static void +brcmf_joinscan_set_common_v0v1_params(struct brcmf_join_scan_params_le *scan_le, + bool have_channel) +{ + /* Set up join scan parameters */ + scan_le->scan_type = 0; + scan_le->home_time = cpu_to_le32(-1); + + if (have_channel) { + /* Increase dwell time to receive probe response or detect + * beacon from target AP at a noisy air only during connect + * command. + */ + scan_le->active_time = + cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS); + scan_le->passive_time = + cpu_to_le32(BRCMF_SCAN_JOIN_PASSIVE_DWELL_TIME_MS); + /* To sync with presence period of VSDB GO send probe request + * more frequently. Probe request will be stopped when it gets + * probe response from target AP/GO. + */ + scan_le->nprobes = + cpu_to_le32(BRCMF_SCAN_JOIN_ACTIVE_DWELL_TIME_MS / + BRCMF_SCAN_JOIN_PROBE_INTERVAL_MS); + } else { + scan_le->active_time = cpu_to_le32(-1); + scan_le->passive_time = cpu_to_le32(-1); + scan_le->nprobes = cpu_to_le32(-1); + } +} +static void * +brcmf_get_struct_for_connect_v0(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params) +{ + struct brcmf_ext_join_params_le *ext_v0; + u32 join_params_size = + struct_size(ext_v0, assoc_le.chanspec_list, cfg->channel != 0); + + *struct_size = join_params_size; + ext_v0 = kzalloc(join_params_size, GFP_KERNEL); + if (!ext_v0) { + bphy_err( + cfg, + "Could not allocate memory for extended join parameters\n"); + return NULL; + } + brcmf_joinscan_set_ssid(&ext_v0->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_common_v0v1_params(&ext_v0->scan_le, + cfg->channel != 0); + brcmf_joinscan_set_bssid(ext_v0->assoc_le.bssid, params->bssid); + if (cfg->channel) { + struct ieee80211_channel *chan = params->channel_hint ? + params->channel_hint : + params->channel; + brcmf_joinscan_set_single_chanspec_from_channel( + cfg, chan, &ext_v0->assoc_le.chanspec_num, + &ext_v0->assoc_le.chanspec_list); + } + return ext_v0; +} + +static void * +brcmf_get_struct_for_connect_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_connect_params *params) +{ + struct brcmf_ext_join_params_v1_le *ext_v1; + u32 join_params_size = + struct_size(ext_v1, assoc_le.chanspec_list, cfg->channel != 0); + + *struct_size = join_params_size; + ext_v1 = kzalloc(join_params_size, GFP_KERNEL); + if (!ext_v1) { + bphy_err( + cfg, + "Could not allocate memory for extended join parameters\n"); + return NULL; + } + ext_v1->version = cpu_to_le16(1); + ext_v1->assoc_le.version = cpu_to_le16(1); + brcmf_joinscan_set_ssid(&ext_v1->ssid_le, params->ssid, + params->ssid_len); + brcmf_joinscan_set_common_v0v1_params(&ext_v1->scan_le, + cfg->channel != 0); + brcmf_joinscan_set_bssid(ext_v1->assoc_le.bssid, params->bssid); + if (cfg->channel) { + struct ieee80211_channel *chan = params->channel_hint ? + params->channel_hint : + params->channel; + brcmf_joinscan_set_single_chanspec_from_channel( + cfg, chan, &ext_v1->assoc_le.chanspec_num, + &ext_v1->assoc_le.chanspec_list); + } + return ext_v1; +} + +static void *brcmf_get_join_from_ext_join_v0(void *ext_join, u32 *struct_size) +{ + struct brcmf_ext_join_params_le *ext_join_v0 = + (struct brcmf_ext_join_params_le *)ext_join; + u32 chanspec_num = le32_to_cpu(ext_join_v0->assoc_le.chanspec_num); + struct brcmf_join_params *join_params; + u32 join_params_size = + struct_size(join_params, params_le.chanspec_list, chanspec_num); + u32 assoc_size = struct_size_t(struct brcmf_assoc_params_le, + chanspec_list, chanspec_num); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + return NULL; + } + memcpy(&join_params->ssid_le, &ext_join_v0->ssid_le, + sizeof(ext_join_v0->ssid_le)); + memcpy(&join_params->params_le, &ext_join_v0->assoc_le, assoc_size); + + return join_params; +} + +static void *brcmf_get_join_from_ext_join_v1(void *ext_join, u32 *struct_size) +{ + struct brcmf_ext_join_params_v1_le *ext_join_v1 = + (struct brcmf_ext_join_params_v1_le *)ext_join; + u32 chanspec_num = le32_to_cpu(ext_join_v1->assoc_le.chanspec_num); + struct brcmf_join_params_v1 *join_params; + u32 join_params_size = + struct_size(join_params, params_le.chanspec_list, chanspec_num); + u32 assoc_size = struct_size_t(struct brcmf_assoc_params_le, + chanspec_list, chanspec_num); + + *struct_size = join_params_size; + join_params = kzalloc(join_params_size, GFP_KERNEL); + if (!join_params) { + return NULL; + } + memcpy(&join_params->ssid_le, &ext_join_v1->ssid_le, + sizeof(ext_join_v1->ssid_le)); + memcpy(&join_params->params_le, &ext_join_v1->assoc_le, assoc_size); + + return join_params; +} + +int brcmf_join_param_setup_for_version(struct brcmf_pub *drvr, u8 version) +{ + drvr->join_param_handler.version = version; + switch (version) { + case 0: + drvr->join_param_handler.get_struct_for_ibss = + brcmf_get_struct_for_ibss_v0; + drvr->join_param_handler.get_struct_for_connect = + brcmf_get_struct_for_connect_v0; + drvr->join_param_handler.get_join_from_ext_join = + brcmf_get_join_from_ext_join_v0; + break; + case 1: + drvr->join_param_handler.get_struct_for_ibss = + brcmf_get_prepped_struct_for_ibss_v1; + drvr->join_param_handler.get_struct_for_connect = + brcmf_get_struct_for_connect_v1; + drvr->join_param_handler.get_join_from_ext_join = + brcmf_get_join_from_ext_join_v1; + break; + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h new file mode 100644 index 00000000000000..f549fe2a740823 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/join_param.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_JOIN_PARAM_H +#define _BRCMF_JOIN_PARAM_H + +struct brcmf_pub; + +/** + * brcmf_join_param_setup_for_version() - Setup the driver to handle join structures + * + * There are a number of different structures and interface versions for join/extended join parameters + * This sets up the driver to handle a particular interface version. + * + * @drvr Driver structure to setup + * @ver Interface version + * Return: %0 if okay, error code otherwise + */ +int brcmf_join_param_setup_for_version(struct brcmf_pub *drvr, u8 ver); +#endif /* _BRCMF_JOIN_PARAM_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index ba1ce1552e0f43..eb3985a85ca250 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -47,9 +47,35 @@ #define MSGBUF_TYPE_RX_CMPLT 0x12 #define MSGBUF_TYPE_LPBK_DMAXFER 0x13 #define MSGBUF_TYPE_LPBK_DMAXFER_CMPLT 0x14 +#define MSGBUF_TYPE_FLOW_RING_RESUME 0x15 +#define MSGBUF_TYPE_FLOW_RING_RESUME_CMPLT 0x16 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND 0x17 +#define MSGBUF_TYPE_FLOW_RING_SUSPEND_CMPLT 0x18 +#define MSGBUF_TYPE_INFO_BUF_POST 0x19 +#define MSGBUF_TYPE_INFO_BUF_CMPLT 0x1A +#define MSGBUF_TYPE_H2D_RING_CREATE 0x1B +#define MSGBUF_TYPE_D2H_RING_CREATE 0x1C +#define MSGBUF_TYPE_H2D_RING_CREATE_CMPLT 0x1D +#define MSGBUF_TYPE_D2H_RING_CREATE_CMPLT 0x1E +#define MSGBUF_TYPE_H2D_RING_CONFIG 0x1F +#define MSGBUF_TYPE_D2H_RING_CONFIG 0x20 +#define MSGBUF_TYPE_H2D_RING_CONFIG_CMPLT 0x21 +#define MSGBUF_TYPE_D2H_RING_CONFIG_CMPLT 0x22 +#define MSGBUF_TYPE_H2D_MAILBOX_DATA 0x23 +#define MSGBUF_TYPE_D2H_MAILBOX_DATA 0x24 +#define MSGBUF_TYPE_TIMSTAMP_BUFPOST 0x25 +#define MSGBUF_TYPE_HOSTTIMSTAMP 0x26 +#define MSGBUF_TYPE_HOSTTIMSTAMP_CMPLT 0x27 +#define MSGBUF_TYPE_FIRMWARE_TIMESTAMP 0x28 +#define MSGBUF_TYPE_SNAPSHOT_UPLOAD 0x29 +#define MSGBUF_TYPE_SNAPSHOT_CMPLT 0x2A +#define MSGBUF_TYPE_H2D_RING_DELETE 0x2B +#define MSGBUF_TYPE_D2H_RING_DELETE 0x2C +#define MSGBUF_TYPE_H2D_RING_DELETE_CMPLT 0x2D +#define MSGBUF_TYPE_D2H_RING_DELETE_CMPLT 0x2E #define NR_TX_PKTIDS 2048 -#define NR_RX_PKTIDS 1024 +#define NR_RX_PKTIDS 2048 #define BRCMF_IOCTL_REQ_PKTID 0xFFFE @@ -218,6 +244,19 @@ struct msgbuf_flowring_flush_resp { __le32 rsvd0[3]; }; +struct msgbuf_h2d_mailbox_data { + struct msgbuf_common_hdr msg; + __le32 data; + __le32 rsvd0[7]; +}; + +struct msgbuf_d2h_mailbox_data { + struct msgbuf_common_hdr msg; + struct msgbuf_completion_hdr compl_hdr; + __le32 data; + __le32 rsvd0[2]; +}; + struct brcmf_msgbuf_work_item { struct list_head queue; u32 flowid; @@ -1285,6 +1324,16 @@ brcmf_msgbuf_process_flow_ring_delete_response(struct brcmf_msgbuf *msgbuf, } +static void brcmf_msgbuf_process_d2h_mailbox_data(struct brcmf_msgbuf *msgbuf, + void *buf) +{ + struct msgbuf_d2h_mailbox_data *d2h_mb_data = buf; + struct brcmf_pub *drvr = msgbuf->drvr; + + brcmf_bus_d2h_mb_rx(drvr->bus_if, le32_to_cpu(d2h_mb_data->data)); +} + + static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_pub *drvr = msgbuf->drvr; @@ -1327,6 +1376,10 @@ static void brcmf_msgbuf_process_msgtype(struct brcmf_msgbuf *msgbuf, void *buf) brcmf_dbg(MSGBUF, "MSGBUF_TYPE_RX_CMPLT\n"); brcmf_msgbuf_process_rx_complete(msgbuf, buf); break; + case MSGBUF_TYPE_D2H_MAILBOX_DATA: + brcmf_dbg(MSGBUF, "MSGBUF_TYPE_D2H_MAILBOX_DATA\n"); + brcmf_msgbuf_process_d2h_mailbox_data(msgbuf, buf); + break; default: bphy_err(drvr, "Unsupported msgtype %d\n", msg->msgtype); break; @@ -1465,6 +1518,38 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid) } } + +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data) +{ + struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd; + struct brcmf_commonring *commonring; + struct msgbuf_h2d_mailbox_data *request; + void *ret_ptr; + int err; + + commonring = msgbuf->commonrings[BRCMF_H2D_MSGRING_CONTROL_SUBMIT]; + brcmf_commonring_lock(commonring); + ret_ptr = brcmf_commonring_reserve_for_write(commonring); + if (!ret_ptr) { + bphy_err(drvr, "Failed to reserve space in commonring\n"); + brcmf_commonring_unlock(commonring); + return -ENOMEM; + } + + request = (struct msgbuf_h2d_mailbox_data *)ret_ptr; + request->msg.msgtype = MSGBUF_TYPE_H2D_MAILBOX_DATA; + request->msg.ifidx = -1; + request->msg.flags = 0; + request->msg.request_id = 0; + request->data = data; + + err = brcmf_commonring_write_complete(commonring); + brcmf_commonring_unlock(commonring); + + return err; +} + + #ifdef DEBUG static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h index 6a849f4a94dd7f..0ed48cf13d93cf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h @@ -8,10 +8,10 @@ #ifdef CONFIG_BRCMFMAC_PROTO_MSGBUF #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_MAX_ITEM 64 -#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 1024 +#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM 2048 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_MAX_ITEM 64 #define BRCMF_D2H_MSGRING_TX_COMPLETE_MAX_ITEM 1024 -#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 1024 +#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM 2048 #define BRCMF_H2D_TXFLOWRING_MAX_ITEM 512 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE 40 @@ -32,6 +32,7 @@ int brcmf_proto_msgbuf_rx_trigger(struct device *dev); void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid); int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr); void brcmf_proto_msgbuf_detach(struct brcmf_pub *drvr); +int brcmf_msgbuf_h2d_mb_write(struct brcmf_pub *drvr, u32 data); #else static inline int brcmf_proto_msgbuf_attach(struct brcmf_pub *drvr) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c index e1752a513c733d..06d2933162b26f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c @@ -1784,8 +1784,8 @@ bool brcmf_p2p_send_action_frame(struct brcmf_if *ifp, /* do not configure anything. it will be */ /* sent with a default configuration */ } else { - bphy_err(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", - category, action); + bphy_info_once(drvr, "Unknown Frame: category 0x%x, action 0x%x\n", + category, action); return false; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 45b342ea06373b..596041e6f354ae 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -71,6 +71,8 @@ BRCMF_FW_CLM_DEF(4377B3, "brcmfmac4377b3-pcie"); BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie"); BRCMF_FW_CLM_DEF(4378B3, "brcmfmac4378b3-pcie"); BRCMF_FW_CLM_DEF(4387C2, "brcmfmac4387c2-pcie"); +BRCMF_FW_CLM_DEF(4388B0, "brcmfmac4388b0-pcie"); +BRCMF_FW_CLM_DEF(4388C0, "brcmfmac4388c0-pcie"); BRCMF_FW_CLM_DEF(54591, "brcmfmac54591-pcie"); /* firmware config files */ @@ -112,6 +114,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0x0000000F, 4378B1), /* revision ID 3 */ BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFE0, 4378B3), /* revision ID 5 */ BRCMF_FW_ENTRY(BRCM_CC_4387_CHIP_ID, 0xFFFFFFFF, 4387C2), /* revision ID 7 */ + BRCMF_FW_ENTRY(BRCM_CC_4388_CHIP_ID, 0x0000000F, 4388B0), + BRCMF_FW_ENTRY(BRCM_CC_4388_CHIP_ID, 0xFFFFFFF0, 4388C0), /* revision ID 4 */ }; #define BRCMF_PCIE_FW_UP_TIMEOUT 5000 /* msec */ @@ -219,11 +223,64 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_SHARED_VERSION_MASK 0x00FF #define BRCMF_PCIE_SHARED_DMA_INDEX 0x10000 #define BRCMF_PCIE_SHARED_DMA_2B_IDX 0x100000 +#define BRCMF_PCIE_SHARED_USE_MAILBOX 0x2000000 +#define BRCMF_PCIE_SHARED_TIMESTAMP_DB0 0x8000000 #define BRCMF_PCIE_SHARED_HOSTRDY_DB1 0x10000000 +#define BRCMF_PCIE_SHARED_NO_OOB_DW 0x20000000 +#define BRCMF_PCIE_SHARED_INBAND_DS 0x40000000 +#define BRCMF_PCIE_SHARED_DAR 0x80000000 + +#define BRCMF_PCIE_SHARED2_EXTENDED_TRAP_DATA 0x1 +#define BRCMF_PCIE_SHARED2_TXSTATUS_METADATA 0x2 +#define BRCMF_PCIE_SHARED2_BT_LOGGING 0x4 +#define BRCMF_PCIE_SHARED2_SNAPSHOT_UPLOAD 0x8 +#define BRCMF_PCIE_SHARED2_SUBMIT_COUNT_WAR 0x10 +#define BRCMF_PCIE_SHARED2_FAST_DELETE_RING 0x20 +#define BRCMF_PCIE_SHARED2_EVTBUF_MAX_MASK 0xC0 +#define BRCMF_PCIE_SHARED2_PKT_TX_STATUS 0x100 +#define BRCMF_PCIE_SHARED2_FW_SMALL_MEMDUMP 0x200 +#define BRCMF_PCIE_SHARED2_FW_HC_ON_TRAP 0x400 +#define BRCMF_PCIE_SHARED2_HSCB 0x800 +#define BRCMF_PCIE_SHARED2_EDL_RING 0x1000 +#define BRCMF_PCIE_SHARED2_DEBUG_BUF_DEST 0x2000 +#define BRCMF_PCIE_SHARED2_PCIE_ENUM_RESET_FLR 0x4000 +#define BRCMF_PCIE_SHARED2_PKT_TIMESTAMP 0x8000 +#define BRCMF_PCIE_SHARED2_HP2P 0x10000 +#define BRCMF_PCIE_SHARED2_HWA 0x20000 +#define BRCMF_PCIE_SHARED2_TRAP_ON_HOST_DB7 0x40000 +#define BRCMF_PCIE_SHARED2_DURATION_SCALE 0x100000 +#define BRCMF_PCIE_SHARED2_D2H_D11_TX_STATUS 0x40000000 +#define BRCMF_PCIE_SHARED2_H2D_D11_TX_STATUS 0x80000000 #define BRCMF_PCIE_FLAGS_HTOD_SPLIT 0x4000 #define BRCMF_PCIE_FLAGS_DTOH_SPLIT 0x8000 +#define BRCMF_HOSTCAP_PCIEAPI_VERSION_MASK 0x000000FF +#define BRCMF_HOSTCAP_H2D_VALID_PHASE 0x00000100 +#define BRCMF_HOSTCAP_H2D_ENABLE_TRAP_ON_BADPHASE 0x00000200 +#define BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY 0x400 +#define BRCMF_HOSTCAP_DB0_TIMESTAMP 0x800 +#define BRCMF_HOSTCAP_DS_NO_OOB_DW 0x1000 +#define BRCMF_HOSTCAP_DS_INBAND_DW 0x2000 +#define BRCMF_HOSTCAP_H2D_IDMA 0x4000 +#define BRCMF_HOSTCAP_H2D_IFRM 0x8000 +#define BRCMF_HOSTCAP_H2D_DAR 0x10000 +#define BRCMF_HOSTCAP_EXTENDED_TRAP_DATA 0x20000 +#define BRCMF_HOSTCAP_TXSTATUS_METADATA 0x40000 +#define BRCMF_HOSTCAP_BT_LOGGING 0x80000 +#define BRCMF_HOSTCAP_SNAPSHOT_UPLOAD 0x100000 +#define BRCMF_HOSTCAP_FAST_DELETE_RING 0x200000 +#define BRCMF_HOSTCAP_PKT_TXSTATUS 0x400000 +#define BRCMF_HOSTCAP_UR_FW_NO_TRAP 0x800000 +#define BRCMF_HOSTCAP_HSCB 0x2000000 +#define BRCMF_HOSTCAP_EXT_TRAP_DBGBUF 0x4000000 +#define BRCMF_HOSTCAP_EDL_RING 0x10000000 +#define BRCMF_HOSTCAP_PKT_TIMESTAMP 0x20000000 +#define BRCMF_HOSTCAP_PKT_HP2P 0x40000000 +#define BRCMF_HOSTCAP_HWA 0x80000000 +#define BRCMF_HOSTCAP2_DURATION_SCALE_MASK 0x3F + +#define BRCMF_SHARED_FLAGS_OFFSET 0 #define BRCMF_SHARED_MAX_RXBUFPOST_OFFSET 34 #define BRCMF_SHARED_RING_BASE_OFFSET 52 #define BRCMF_SHARED_RX_DATAOFFSET_OFFSET 36 @@ -235,6 +292,11 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_SHARED_DMA_SCRATCH_ADDR_OFFSET 56 #define BRCMF_SHARED_DMA_RINGUPD_LEN_OFFSET 64 #define BRCMF_SHARED_DMA_RINGUPD_ADDR_OFFSET 68 +#define BRCMF_SHARED_FLAGS2_OFFSET 80 +#define BRCMF_SHARED_HOST_CAP_OFFSET 84 +#define BRCMF_SHARED_FLAGS3_OFFSET 108 +#define BRCMF_SHARED_HOST_CAP2_OFFSET 112 +#define BRCMF_SHARED_HOST_CAP3_OFFSET 116 #define BRCMF_RING_H2D_RING_COUNT_OFFSET 0 #define BRCMF_RING_D2H_RING_COUNT_OFFSET 1 @@ -280,6 +342,7 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = { #define BRCMF_PCIE_CFGREG_PML1_SUB_CTRL1 0x248 #define BRCMF_PCIE_CFGREG_REG_BAR2_CONFIG 0x4E0 #define BRCMF_PCIE_CFGREG_REG_BAR3_CONFIG 0x4F4 +#define BRCMF_PCIE_CFGREG_TLCNTRL_5 0x814 #define BRCMF_PCIE_LINK_STATUS_CTRL_ASPM_ENAB 3 /* Magic number at a magic location to find RAM size */ @@ -299,6 +362,8 @@ struct brcmf_pcie_console { struct brcmf_pcie_shared_info { u32 tcm_base_address; u32 flags; + u32 flags2; + u32 flags3; struct brcmf_pcie_ringbuf *commonrings[BRCMF_NROF_COMMON_MSGRINGS]; struct brcmf_pcie_ringbuf *flowrings; u16 max_rxbufpost; @@ -315,6 +380,7 @@ struct brcmf_pcie_shared_info { void *ringupd; dma_addr_t ringupd_dmahandle; u8 version; + bool mb_via_ctl; }; #define BRCMF_OTP_MAX_PARAM_LEN 16 @@ -331,6 +397,7 @@ struct brcmf_pciedev_info { bool in_irq; struct pci_dev *pdev; char fw_name[BRCMF_FW_NAME_LEN]; + char sig_name[BRCMF_FW_NAME_LEN]; char nvram_name[BRCMF_FW_NAME_LEN]; char clm_name[BRCMF_FW_NAME_LEN]; char txcap_name[BRCMF_FW_NAME_LEN]; @@ -339,14 +406,16 @@ struct brcmf_pciedev_info { const struct brcmf_pcie_reginfo *reginfo; void __iomem *regs; void __iomem *tcm; - u32 ram_base; - u32 ram_size; + u32 fw_size; + bool skip_reset_vector; struct brcmf_chip *ci; u32 coreid; struct brcmf_pcie_shared_info shared; wait_queue_head_t mbdata_resp_wait; bool mbdata_completed; bool irq_allocated; + bool irq_ready; + bool have_msi; bool wowl_enabled; u8 dma_idx_sz; void *idxbuf; @@ -433,8 +502,6 @@ struct brcmf_pcie_reginfo { u32 intmask; u32 mailboxint; u32 mailboxmask; - u32 h2d_mailbox_0; - u32 h2d_mailbox_1; u32 int_d2h_db; u32 int_fn0; }; @@ -443,8 +510,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_default = { .intmask = BRCMF_PCIE_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_MB_INT_D2H_DB, .int_fn0 = BRCMF_PCIE_MB_INT_FN0, }; @@ -453,8 +518,6 @@ static const struct brcmf_pcie_reginfo brcmf_reginfo_64 = { .intmask = BRCMF_PCIE_64_PCIE2REG_INTMASK, .mailboxint = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT, .mailboxmask = BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK, - .h2d_mailbox_0 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, - .h2d_mailbox_1 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, .int_d2h_db = BRCMF_PCIE_64_MB_INT_D2H_DB, .int_fn0 = 0, }; @@ -493,6 +556,19 @@ brcmf_pcie_write_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset, iowrite32(value, address); } +static u32 +brcmf_pcie_read_pcie32(struct brcmf_pciedev_info *devinfo, u32 reg_offset) +{ + return brcmf_pcie_read_reg32(devinfo, 0x2000 + reg_offset); +} + + +static void +brcmf_pcie_write_pcie32(struct brcmf_pciedev_info *devinfo, u32 reg_offset, + u32 value) +{ + brcmf_pcie_write_reg32(devinfo, 0x2000 + reg_offset, value); +} static u8 brcmf_pcie_read_tcm8(struct brcmf_pciedev_info *devinfo, u32 mem_offset) @@ -686,8 +762,30 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) /* Watchdog reset */ brcmf_pcie_select_core(devinfo, BCMA_CORE_CHIPCOMMON); - WRITECC32(devinfo, watchdog, 4); - msleep(100); + core = brcmf_chip_get_chipcommon(devinfo->ci); + + if (core->rev >= 65) { + u32 mask = CC_WD_SSRESET_PCIE_F0_EN; + + core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); + if (core->rev < 66) + mask |= CC_WD_SSRESET_PCIE_ALL_FN_EN; + + val = READCC32(devinfo, watchdog); + val &= ~CC_WD_ENABLE_MASK; + val |= mask; + WRITECC32(devinfo, watchdog, val); + val &= ~CC_WD_COUNTER_MASK; + val |= 4; + WRITECC32(devinfo, watchdog, val); + msleep(10); + val = READCC32(devinfo, intstatus); + val |= mask; + WRITECC32(devinfo, intstatus, val); + } else { + WRITECC32(devinfo, watchdog, 4); + msleep(100); + } /* Restore ASPM */ brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); @@ -697,14 +795,14 @@ static void brcmf_pcie_reset_device(struct brcmf_pciedev_info *devinfo) core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2); if (core->rev <= 13) { for (i = 0; i < ARRAY_SIZE(cfg_offset); i++) { - brcmf_pcie_write_reg32(devinfo, + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, cfg_offset[i]); - val = brcmf_pcie_read_reg32(devinfo, + val = brcmf_pcie_read_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); brcmf_dbg(PCIE, "config offset 0x%04x, value 0x%04x\n", cfg_offset[i], val); - brcmf_pcie_write_reg32(devinfo, + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, val); } @@ -718,9 +816,9 @@ static void brcmf_pcie_attach(struct brcmf_pciedev_info *devinfo) /* BAR1 window may not be sized properly */ brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, 0x4e0); - config = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); - brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, config); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, 0x4e0); + config = brcmf_pcie_read_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, config); device_wakeup_enable(&devinfo->pdev->dev); } @@ -739,6 +837,21 @@ static int brcmf_pcie_enter_download_state(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_ARMCR4REG_BANKPDA, 0); } + + /* Ensure all IRQs are masked so the firmware doesn't get + * a hostready notification too early. + */ + + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, 0); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, + 0xffffffff); + + pci_write_config_dword(devinfo->pdev, BRCMF_PCIE_REG_INTMASK, 0); + + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGADDR, + BRCMF_PCIE_CFGREG_TLCNTRL_5); + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_CONFIGDATA, + 0xffffffff); return 0; } @@ -769,6 +882,19 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) u32 i; shared = &devinfo->shared; + + if (shared->mb_via_ctl) { + struct pci_dev *pdev = devinfo->pdev; + struct brcmf_bus *bus = dev_get_drvdata(&pdev->dev); + int ret; + + ret = brcmf_msgbuf_h2d_mb_write(bus->drvr, htod_mb_data); + if (ret < 0) + brcmf_err(bus, "Failed to send H2D mailbox data (%d)\n", + ret); + return ret; + } + addr = shared->htod_mb_data_addr; cur_htod_mb_data = brcmf_pcie_read_tcm32(devinfo, addr); @@ -796,8 +922,29 @@ brcmf_pcie_send_mb_data(struct brcmf_pciedev_info *devinfo, u32 htod_mb_data) return 0; } +static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo, u32 data) +{ + brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", data); + if (data & BRCMF_D2H_DEV_DS_ENTER_REQ) { + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); + brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); + brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); + } + if (data & BRCMF_D2H_DEV_DS_EXIT_NOTE) + brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); + if (data & BRCMF_D2H_DEV_D3_ACK) { + brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); + devinfo->mbdata_completed = true; + wake_up(&devinfo->mbdata_resp_wait); + } + if (data & BRCMF_D2H_DEV_FWHALT) { + brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); + brcmf_fw_crashed(&devinfo->pdev->dev); + } +} -static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) + +static void brcmf_pcie_poll_mb_data(struct brcmf_pciedev_info *devinfo) { struct brcmf_pcie_shared_info *shared; u32 addr; @@ -812,23 +959,16 @@ static void brcmf_pcie_handle_mb_data(struct brcmf_pciedev_info *devinfo) brcmf_pcie_write_tcm32(devinfo, addr, 0); - brcmf_dbg(PCIE, "D2H_MB_DATA: 0x%04x\n", dtoh_mb_data); - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_ENTER_REQ) { - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP REQ\n"); - brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_DS_ACK); - brcmf_dbg(PCIE, "D2H_MB_DATA: sent DEEP SLEEP ACK\n"); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_DS_EXIT_NOTE) - brcmf_dbg(PCIE, "D2H_MB_DATA: DEEP SLEEP EXIT\n"); - if (dtoh_mb_data & BRCMF_D2H_DEV_D3_ACK) { - brcmf_dbg(PCIE, "D2H_MB_DATA: D3 ACK\n"); - devinfo->mbdata_completed = true; - wake_up(&devinfo->mbdata_resp_wait); - } - if (dtoh_mb_data & BRCMF_D2H_DEV_FWHALT) { - brcmf_dbg(PCIE, "D2H_MB_DATA: FW HALT\n"); - brcmf_fw_crashed(&devinfo->pdev->dev); - } + brcmf_pcie_handle_mb_data(devinfo, dtoh_mb_data); +} + + +static void brcmf_pcie_d2h_mb_rx(struct device *dev, u32 data) +{ + struct brcmf_bus *bus_if = dev_get_drvdata(dev); + struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie; + + brcmf_pcie_handle_mb_data(buspub->devinfo, data); } @@ -907,33 +1047,45 @@ static void brcmf_pcie_bus_console_read(struct brcmf_pciedev_info *devinfo, static void brcmf_pcie_intr_disable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, 0); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, 0); + + devinfo->irq_ready = false; } static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxmask, devinfo->reginfo->int_d2h_db | devinfo->reginfo->int_fn0); + + devinfo->irq_ready = true; } static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo) { - if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) - brcmf_pcie_write_reg32(devinfo, - devinfo->reginfo->h2d_mailbox_1, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) { + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1, 1); + else + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1); + } } static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg) { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)arg; - if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint)) { + if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint)) { brcmf_pcie_intr_disable(devinfo); brcmf_dbg(PCIE, "Enter\n"); return IRQ_WAKE_THREAD; } + + /* mailboxint is cleared by the firmware in MSI mode */ + if (devinfo->have_msi) + return IRQ_WAKE_THREAD; + return IRQ_NONE; } @@ -944,19 +1096,19 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg) u32 status; devinfo->in_irq = true; - status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); + status = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint); brcmf_dbg(PCIE, "Enter %x\n", status); if (status) { - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, status); if (status & devinfo->reginfo->int_fn0) - brcmf_pcie_handle_mb_data(devinfo); - if (status & devinfo->reginfo->int_d2h_db) { - if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) - brcmf_proto_msgbuf_rx_trigger( - &devinfo->pdev->dev); - } + brcmf_pcie_poll_mb_data(devinfo); } + if (devinfo->have_msi || status & devinfo->reginfo->int_d2h_db) { + if (devinfo->state == BRCMFMAC_PCIE_STATE_UP && devinfo->irq_ready) + brcmf_proto_msgbuf_rx_trigger(&devinfo->pdev->dev); + } + brcmf_pcie_bus_console_read(devinfo, false); if (devinfo->state == BRCMFMAC_PCIE_STATE_UP) brcmf_pcie_intr_enable(devinfo); @@ -974,7 +1126,10 @@ static int brcmf_pcie_request_irq(struct brcmf_pciedev_info *devinfo) brcmf_dbg(PCIE, "Enter\n"); - pci_enable_msi(pdev); + devinfo->have_msi = pci_enable_msi(pdev) >= 0; + if (devinfo->have_msi) + brcmf_dbg(PCIE, "MSI enabled\n"); + if (request_threaded_irq(pdev->irq, brcmf_pcie_quick_check_isr, brcmf_pcie_isr_thread, IRQF_SHARED, "brcmf_pcie_intr", devinfo)) { @@ -1010,8 +1165,8 @@ static void brcmf_pcie_release_irq(struct brcmf_pciedev_info *devinfo) if (devinfo->in_irq) brcmf_err(bus, "Still in IRQ (processing) !!!\n"); - status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint); - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status); + status = brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->mailboxint); + brcmf_pcie_write_pcie32(devinfo, devinfo->reginfo->mailboxint, status); devinfo->irq_allocated = false; } @@ -1063,7 +1218,10 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx) brcmf_dbg(PCIE, "RING !\n"); /* Any arbitrary value will do, lets use 1 */ - brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->h2d_mailbox_0, 1); + if (devinfo->shared.flags & BRCMF_PCIE_SHARED_DAR) + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0, 1); + else + brcmf_pcie_write_pcie32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1); return 0; } @@ -1589,6 +1747,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = { .get_blob = brcmf_pcie_get_blob, .reset = brcmf_pcie_reset, .debugfs_create = brcmf_pcie_debugfs_create, + .d2h_mb_rx = brcmf_pcie_d2h_mb_rx, }; @@ -1620,12 +1779,16 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); struct brcmf_pcie_shared_info *shared; + u32 host_cap; + u32 host_cap2; u32 addr; shared = &devinfo->shared; shared->tcm_base_address = sharedram_addr; - shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr); + shared->flags = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS_OFFSET); + shared->version = (u8)(shared->flags & BRCMF_PCIE_SHARED_VERSION_MASK); brcmf_dbg(PCIE, "PCIe protocol version %d\n", shared->version); if ((shared->version > BRCMF_PCIE_MAX_SHARED_VERSION) || @@ -1666,29 +1829,223 @@ brcmf_pcie_init_share_ram_info(struct brcmf_pciedev_info *devinfo, brcmf_pcie_bus_console_init(devinfo); brcmf_pcie_bus_console_read(devinfo, false); + /* Features added in revision 6 follow */ + if (shared->version < 6) + return 0; + + shared->flags2 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS2_OFFSET); + shared->flags3 = brcmf_pcie_read_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_FLAGS3_OFFSET); + + /* Check which mailbox mechanism to use */ + if (!(shared->flags & BRCMF_PCIE_SHARED_USE_MAILBOX)) + shared->mb_via_ctl = true; + + /* Update host support flags */ + host_cap = shared->version; + host_cap2 = 0; + + if (shared->flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1) + host_cap |= BRCMF_HOSTCAP_H2D_ENABLE_HOSTRDY; + + if (shared->flags & BRCMF_PCIE_SHARED_DAR) + host_cap |= BRCMF_HOSTCAP_H2D_DAR; + + /* Disable DS: this is not currently properly supported */ + host_cap |= BRCMF_HOSTCAP_DS_NO_OOB_DW; + + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP_OFFSET, host_cap); + brcmf_pcie_write_tcm32(devinfo, sharedram_addr + + BRCMF_SHARED_HOST_CAP2_OFFSET, host_cap2); + return 0; } -struct brcmf_random_seed_footer { +struct brcmf_rtlv_footer { __le32 length; __le32 magic; }; +/** struct brcmf_fw_memmap_region - start/end of memory regions for chip + */ +struct brcmf_fw_memmap_region { + u32 start; + u32 end; +}; + +/** struct brcmf_fw_memmap + * + * @reset_vec - Reset vector - read only + * @int_vec - copied from ram, jumps here on success + * @rom - bootloader at rom start + * @mmap - struct/memory map written by host + * @vstatus - verification status + * @fw - firmware + * @sig - firwmare signature + * @heap - region for heap allocations + * @stack - region for stack allocations + * @prng - PRNG data, may be 0 length + * @nvram - NVRAM data + */ +struct brcmf_fw_memmap { + struct brcmf_fw_memmap_region reset_vec; + struct brcmf_fw_memmap_region int_vec; + struct brcmf_fw_memmap_region rom; + struct brcmf_fw_memmap_region mmap; + struct brcmf_fw_memmap_region vstatus; + struct brcmf_fw_memmap_region fw; + struct brcmf_fw_memmap_region sig; + struct brcmf_fw_memmap_region heap; + struct brcmf_fw_memmap_region stack; + struct brcmf_fw_memmap_region prng; + struct brcmf_fw_memmap_region nvram; +}; + +#define BRCMF_BL_HEAP_START_GAP 0x1000 +#define BRCMF_BL_HEAP_SIZE 0x10000 #define BRCMF_RANDOM_SEED_MAGIC 0xfeedc0de #define BRCMF_RANDOM_SEED_LENGTH 0x100 +#define BRCMF_FW_SIG_MAGIC 0xfeedfe51 +#define BRCMF_NVRAM_SIG_MAGIC 0xfeedfe52 +#define BRCMF_MEMMAP_MAGIC 0xfeedfe53 +#define BRCMF_VSTATUS_MAGIC 0xfeedfe54 +#define BRCMF_VSTATUS_SIZE 0x28 +#define BRCMF_END_MAGIC 0xfeed0e2d -static noinline_for_stack void -brcmf_pcie_provide_random_bytes(struct brcmf_pciedev_info *devinfo, u32 address) +static int brcmf_alloc_rtlv(struct brcmf_pciedev_info *devinfo, u32 *address, u32 type, u32 length) { + struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); + u32 fw_top = devinfo->ci->rambase + devinfo->fw_size; + u32 ram_start = ALIGN(fw_top + BRCMF_BL_HEAP_START_GAP, 4); + u32 ram_end = ram_start + BRCMF_BL_HEAP_SIZE; + u32 start_addr; + struct brcmf_rtlv_footer footer = { + .magic = type, + }; + + length = ALIGN(length, 4); + start_addr = *address - length - sizeof(struct brcmf_rtlv_footer); + + if (length > 0xffff || start_addr > *address || start_addr < ram_end) { + brcmf_err(bus, "failed to allocate 0x%x bytes for rTLV type 0x%x\n", + length, type); + return -ENOMEM; + } + + /* Random seed does not use the length check code */ + if (type == BRCMF_RANDOM_SEED_MAGIC) + footer.length = length; + else + footer.length = length | ((length ^ 0xffff) << 16); + + memcpy_toio(devinfo->tcm + *address - sizeof(struct brcmf_rtlv_footer), + &footer, sizeof(struct brcmf_rtlv_footer)); + + *address = start_addr; + + return 0; +} + +static noinline_for_stack int +brcmf_pcie_add_random_seed(struct brcmf_pciedev_info *devinfo, u32 *address) +{ + int err; u8 randbuf[BRCMF_RANDOM_SEED_LENGTH]; + err = brcmf_alloc_rtlv(devinfo, address, + BRCMF_RANDOM_SEED_MAGIC, BRCMF_RANDOM_SEED_LENGTH); + if (err) + return err; + + /* Some Apple chips/firmwares expect a buffer of random + * data to be present before NVRAM + */ + brcmf_dbg(PCIE, "Download random seed\n"); + get_random_bytes(randbuf, BRCMF_RANDOM_SEED_LENGTH); - memcpy_toio(devinfo->tcm + address, randbuf, BRCMF_RANDOM_SEED_LENGTH); + memcpy_toio(devinfo->tcm + *address, randbuf, BRCMF_RANDOM_SEED_LENGTH); + + return 0; +} + +static int brcmf_pcie_add_signature(struct brcmf_pciedev_info *devinfo, + u32 *address, const struct firmware *fwsig) +{ + int err; + struct brcmf_fw_memmap memmap; + + brcmf_dbg(PCIE, "Download firmware signature\n"); + + memset(&memmap, 0, sizeof(memmap)); + + memmap.sig.end = *address; + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_FW_SIG_MAGIC, fwsig->size); + if (err) + return err; + memmap.sig.start = *address; + + memmap.vstatus.end = *address; + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_VSTATUS_MAGIC, BRCMF_VSTATUS_SIZE); + if (err) + return err; + memmap.vstatus.start = *address; + + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_MEMMAP_MAGIC, sizeof(memmap)); + if (err) + return err; + + memmap.fw.start = devinfo->ci->rambase; + memmap.fw.end = memmap.fw.start + devinfo->fw_size; + memmap.heap.start = ALIGN(memmap.fw.end + BRCMF_BL_HEAP_START_GAP, 4); + memmap.heap.end = memmap.heap.start + BRCMF_BL_HEAP_SIZE; + + if (memmap.heap.end > *address) + return -ENOMEM; + + memcpy_toio(devinfo->tcm + memmap.sig.start, fwsig->data, fwsig->size); + memset_io(devinfo->tcm + memmap.vstatus.start, 0, BRCMF_VSTATUS_SIZE); + memcpy_toio(devinfo->tcm + *address, &memmap, sizeof(memmap)); + + err = brcmf_alloc_rtlv(devinfo, address, BRCMF_END_MAGIC, 0); + if (err) + return err; + + devinfo->skip_reset_vector = true; + + return 0; +} + +static int brcmf_pcie_populate_footers(struct brcmf_pciedev_info *devinfo, + u32 *address, const struct firmware *fwsig) +{ + int err; + + /* We only do this for Apple firmwares. If any other + * production firmwares are found to need this, the condition + * needs to be adjusted. + */ + if (!devinfo->fwseed) + return 0; + + err = brcmf_pcie_add_random_seed(devinfo, address); + if (err) + return err; + + if (fwsig) { + err = brcmf_pcie_add_signature(devinfo, address, fwsig); + if (err) + return err; + } + + return 0; } static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, - const struct firmware *fw, void *nvram, - u32 nvram_len) + const struct firmware *fw, + const struct firmware *fwsig, + void *nvram, u32 nvram_len) { struct brcmf_bus *bus = dev_get_drvdata(&devinfo->pdev->dev); u32 sharedram_addr; @@ -1708,6 +2065,7 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, (void *)fw->data, fw->size); resetintr = get_unaligned_le32(fw->data); + devinfo->fw_size = fw->size; release_firmware(fw); /* reset last 4 bytes of RAM address. to be used for shared @@ -1715,37 +2073,31 @@ static int brcmf_pcie_download_fw_nvram(struct brcmf_pciedev_info *devinfo, */ brcmf_pcie_write_ram32(devinfo, devinfo->ci->ramsize - 4, 0); + address = devinfo->ci->rambase + devinfo->ci->ramsize; + if (nvram) { brcmf_dbg(PCIE, "Download NVRAM %s\n", devinfo->nvram_name); - address = devinfo->ci->rambase + devinfo->ci->ramsize - - nvram_len; + address -= nvram_len; memcpy_toio(devinfo->tcm + address, nvram, nvram_len); brcmf_fw_nvram_free(nvram); - if (devinfo->fwseed) { - size_t rand_len = BRCMF_RANDOM_SEED_LENGTH; - struct brcmf_random_seed_footer footer = { - .length = cpu_to_le32(rand_len), - .magic = cpu_to_le32(BRCMF_RANDOM_SEED_MAGIC), - }; - - /* Some chips/firmwares expect a buffer of random - * data to be present before NVRAM - */ - brcmf_dbg(PCIE, "Download random seed\n"); - - address -= sizeof(footer); - memcpy_toio(devinfo->tcm + address, &footer, - sizeof(footer)); - - address -= rand_len; - brcmf_pcie_provide_random_bytes(devinfo, address); - } + err = brcmf_pcie_populate_footers(devinfo, &address, fwsig); + if (err) + brcmf_err(bus, "failed to populate firmware footers err=%d\n", err); } else { brcmf_dbg(PCIE, "No matching NVRAM file found %s\n", devinfo->nvram_name); } + release_firmware(fwsig); + + /* Clear free TCM. This isn't really necessary, but it + * makes debugging memory dumps a lot easier since we + * don't get a bunch of junk filling up the free space. + */ + memset_io(devinfo->tcm + devinfo->ci->rambase + devinfo->fw_size, + 0, address - devinfo->fw_size - devinfo->ci->rambase); + sharedram_addr_written = brcmf_pcie_read_ram32(devinfo, devinfo->ci->ramsize - 4); @@ -1889,9 +2241,9 @@ static int brcmf_pcie_buscore_reset(void *ctx, struct brcmf_chip *chip) else reg = BRCMF_PCIE_PCIE2REG_MAILBOXINT; - val = brcmf_pcie_read_reg32(devinfo, reg); + val = brcmf_pcie_read_pcie32(devinfo, reg); if (val != 0xffffffff) - brcmf_pcie_write_reg32(devinfo, reg, val); + brcmf_pcie_write_pcie32(devinfo, reg, val); return 0; } @@ -1902,7 +2254,8 @@ static void brcmf_pcie_buscore_activate(void *ctx, struct brcmf_chip *chip, { struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)ctx; - brcmf_pcie_write_tcm32(devinfo, 0, rstvec); + if (!devinfo->skip_reset_vector) + brcmf_pcie_write_tcm32(devinfo, 0, rstvec); } @@ -2073,6 +2426,11 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) base = 0x113c; words = 0x170; break; + case BRCM_CC_4388_CHIP_ID: + coreid = BCMA_CORE_GCI; + base = 0x115c; + words = 0x150; + break; default: /* OTP not supported on this chip */ return 0; @@ -2131,11 +2489,12 @@ static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo) #define BRCMF_PCIE_FW_NVRAM 1 #define BRCMF_PCIE_FW_CLM 2 #define BRCMF_PCIE_FW_TXCAP 3 +#define BRCMF_PCIE_FW_SIG 4 static void brcmf_pcie_setup(struct device *dev, int ret, struct brcmf_fw_request *fwreq) { - const struct firmware *fw; + const struct firmware *fw, *fwsig; void *nvram; struct brcmf_bus *bus; struct brcmf_pciedev *pcie_bus_dev; @@ -2154,6 +2513,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, brcmf_pcie_attach(devinfo); fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary; + fwsig = fwreq->items[BRCMF_PCIE_FW_SIG].binary; nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data; nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len; devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary; @@ -2164,6 +2524,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret, if (ret) { brcmf_err(bus, "Failed to get RAM info\n"); release_firmware(fw); + release_firmware(fwsig); brcmf_fw_nvram_free(nvram); goto fail; } @@ -2175,7 +2536,15 @@ static void brcmf_pcie_setup(struct device *dev, int ret, */ brcmf_pcie_adjust_ramsize(devinfo, (u8 *)fw->data, fw->size); - ret = brcmf_pcie_download_fw_nvram(devinfo, fw, nvram, nvram_len); + /* Newer firmwares will signal firmware boot via MSI, so make sure we + * initialize that upfront. + */ + brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); + ret = brcmf_pcie_request_irq(devinfo); + if (ret) + goto fail; + + ret = brcmf_pcie_download_fw_nvram(devinfo, fw, fwsig, nvram, nvram_len); if (ret) goto fail; @@ -2190,9 +2559,6 @@ static void brcmf_pcie_setup(struct device *dev, int ret, goto fail; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); - ret = brcmf_pcie_request_irq(devinfo); - if (ret) - goto fail; /* hook the commonrings in the bus structure. */ for (i = 0; i < BRCMF_NROF_COMMON_MSGRINGS; i++) @@ -2239,6 +2605,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) { ".txt", devinfo->nvram_name }, { ".clm_blob", devinfo->clm_name }, { ".txcap_blob", devinfo->txcap_name }, + { ".sig", devinfo->sig_name }, }; fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev, @@ -2249,6 +2616,8 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo) return NULL; fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_SIG].type = BRCMF_FW_TYPE_BINARY; + fwreq->items[BRCMF_PCIE_FW_SIG].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM; fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL; fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY; @@ -2667,12 +3036,13 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus); /* Check if device is still up and running, if so we are ready */ - if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) { + if (brcmf_pcie_read_pcie32(devinfo, devinfo->reginfo->intmask) != 0) { brcmf_dbg(PCIE, "Try to wakeup device....\n"); + /* Set the device up, so we can write the MB data message in ring mode */ + devinfo->state = BRCMFMAC_PCIE_STATE_UP; if (brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_D0_INFORM)) goto cleanup; brcmf_dbg(PCIE, "Hot resume, continue....\n"); - devinfo->state = BRCMFMAC_PCIE_STATE_UP; brcmf_pcie_select_core(devinfo, BCMA_CORE_PCIE2); brcmf_bus_change_state(bus, BRCMF_BUS_UP); brcmf_pcie_intr_enable(devinfo); @@ -2682,6 +3052,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev) } cleanup: + devinfo->state = BRCMFMAC_PCIE_STATE_DOWN; brcmf_chip_detach(devinfo->ci); devinfo->ci = NULL; pdev = devinfo->pdev; @@ -2749,6 +3120,7 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = { BRCMF_PCIE_DEVICE(BRCM_PCIE_4377_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_4387_DEVICE_ID, WCC_SEED), + BRCMF_PCIE_DEVICE(BRCM_PCIE_4388_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(BRCM_PCIE_43752_DEVICE_ID, WCC_SEED), BRCMF_PCIE_DEVICE(CY_PCIE_54591_DEVICE_ID, CYW), { /* end: all zeroes */ } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c index d9fc94076791dc..42323da8754862 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c @@ -12,8 +12,10 @@ #include "fwil_types.h" #include "cfg80211.h" #include "pno.h" +#include "feature.h" -#define BRCMF_PNO_VERSION 2 +#define BRCMF_PNO_VERSION_2 2 +#define BRCMF_PNO_VERSION_3 3 #define BRCMF_PNO_REPEAT 4 #define BRCMF_PNO_FREQ_EXPO_MAX 3 #define BRCMF_PNO_IMMEDIATE_SCAN_BIT 3 @@ -99,8 +101,62 @@ static int brcmf_pno_channel_config(struct brcmf_if *ifp, return brcmf_fil_iovar_data_set(ifp, "pfn_cfg", cfg, sizeof(*cfg)); } -static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, - u32 mscan, u32 bestn) +static int brcmf_pno_config_v3(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) +{ + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_pno_param_v3_le pfn_param; + u16 flags; + u32 pfnmem; + s32 err; + + memset(&pfn_param, 0, sizeof(pfn_param)); + pfn_param.version = cpu_to_le16(BRCMF_PNO_VERSION_3); + pfn_param.length = cpu_to_le16(sizeof(struct brcmf_pno_param_v3_le)); + + /* set extra pno params */ + flags = BIT(BRCMF_PNO_IMMEDIATE_SCAN_BIT) | + BIT(BRCMF_PNO_ENABLE_ADAPTSCAN_BIT); + pfn_param.repeat = BRCMF_PNO_REPEAT; + pfn_param.exp = BRCMF_PNO_FREQ_EXPO_MAX; + + /* set up pno scan fr */ + pfn_param.scan_freq = cpu_to_le32(scan_freq); + + if (mscan) { + pfnmem = bestn; + + /* set bestn in firmware */ + err = brcmf_fil_iovar_int_set(ifp, "pfnmem", pfnmem); + if (err < 0) { + bphy_err(drvr, "failed to set pfnmem\n"); + goto exit; + } + /* get max mscan which the firmware supports */ + err = brcmf_fil_iovar_int_get(ifp, "pfnmem", &pfnmem); + if (err < 0) { + bphy_err(drvr, "failed to get pfnmem\n"); + goto exit; + } + mscan = min_t(u32, mscan, pfnmem); + pfn_param.mscan = mscan; + pfn_param.bestn = bestn; + flags |= BIT(BRCMF_PNO_ENABLE_BD_SCAN_BIT); + brcmf_dbg(INFO, "mscan=%d, bestn=%d\n", mscan, bestn); + } + + pfn_param.flags = cpu_to_le16(flags); + err = brcmf_fil_iovar_data_set(ifp, "pfn_set", &pfn_param, + sizeof(pfn_param)); + if (err) + bphy_err(drvr, "pfn_set failed, err=%d\n", err); + +exit: + return err; +} + +static int brcmf_pno_config_v2(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) { struct brcmf_pub *drvr = ifp->drvr; struct brcmf_pno_param_le pfn_param; @@ -109,7 +165,7 @@ static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, s32 err; memset(&pfn_param, 0, sizeof(pfn_param)); - pfn_param.version = cpu_to_le32(BRCMF_PNO_VERSION); + pfn_param.version = cpu_to_le32(BRCMF_PNO_VERSION_2); /* set extra pno params */ flags = BIT(BRCMF_PNO_IMMEDIATE_SCAN_BIT) | @@ -152,6 +208,12 @@ static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, return err; } +static int brcmf_pno_config(struct brcmf_if *ifp, u32 scan_freq, u32 mscan, + u32 bestn) +{ + return ifp->drvr->pno_handler.pno_config(ifp, scan_freq, mscan, bestn); +} + static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi) { struct brcmf_pub *drvr = ifp->drvr; @@ -275,7 +337,7 @@ static int brcmf_pno_get_bucket_channels(struct cfg80211_sched_scan_request *r, { u32 n_chan = le32_to_cpu(pno_cfg->channel_num); u16 chan; - int i, err = 0; + int i, err; for (i = 0; i < r->n_channels; i++) { if (n_chan >= BRCMF_NUMCHANNELS) { @@ -562,9 +624,82 @@ u64 brcmf_pno_find_reqid_by_bucket(struct brcmf_pno_info *pi, u32 bucket) return reqid; } -u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, - struct brcmf_pno_net_info_le *ni) + +static struct brcmf_pno_net_info_le * +brcmf_get_netinfo_array(void *pfn_v1_data) +{ + struct brcmf_pno_scanresults_le *pfn_v1 = + (struct brcmf_pno_scanresults_le *)pfn_v1_data; + struct brcmf_pno_scanresults_v2_le *pfn_v2; + struct brcmf_pno_net_info_le *netinfo = NULL; + + switch (pfn_v1->version) { + default: + WARN_ON(1); + fallthrough; + case cpu_to_le32(1): + netinfo = (struct brcmf_pno_net_info_le *)(pfn_v1 + 1); + break; + case cpu_to_le32(2): + pfn_v2 = (struct brcmf_pno_scanresults_v2_le *)pfn_v1; + netinfo = (struct brcmf_pno_net_info_le *)(pfn_v2 + 1); + break; + case cpu_to_le32(3): + brcmf_err("Need to use brcmf_get_netinfo_v3_array\n"); + break; + } + + return netinfo; +} + +static struct brcmf_pno_net_info_v3_le * +brcmf_get_netinfo_v3_array(void*pfn_v3_data) +{ + struct brcmf_pno_scanresults_v3_le *pfn_v3 = + (struct brcmf_pno_scanresults_v3_le *)pfn_v3_data; + return (struct brcmf_pno_net_info_v3_le *) (pfn_v3 + 1); +} + +static u32 brcmf_pno_get_bucket_map(void *data, int idx, struct brcmf_pno_info *pi) +{ + + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(data); + struct brcmf_pno_net_info_le *ni = &netinfo_start[idx]; + struct cfg80211_sched_scan_request *req; + struct cfg80211_match_set *ms; + u32 bucket_map = 0; + int i, j; + + mutex_lock(&pi->req_lock); + for (i = 0; i < pi->n_reqs; i++) { + req = pi->reqs[i]; + + if (!req->n_match_sets) + continue; + for (j = 0; j < req->n_match_sets; j++) { + ms = &req->match_sets[j]; + if (ms->ssid.ssid_len == ni->SSID_len && + !memcmp(ms->ssid.ssid, ni->SSID, ni->SSID_len)) { + bucket_map |= BIT(i); + break; + } + if (is_valid_ether_addr(ms->bssid) && + !memcmp(ms->bssid, ni->bssid, ETH_ALEN)) { + bucket_map |= BIT(i); + break; + } + } + } + mutex_unlock(&pi->req_lock); + return bucket_map; +} + +static u32 brcmf_pno_get_bucket_map_v3(void *data, int idx, struct brcmf_pno_info *pi) { + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(data); + struct brcmf_pno_net_info_v3_le *ni = &netinfo_v3_start[idx]; struct cfg80211_sched_scan_request *req; struct cfg80211_match_set *ms; u32 bucket_map = 0; @@ -593,3 +728,148 @@ u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, mutex_unlock(&pi->req_lock); return bucket_map; } + +static u32 brcmf_pno_min_data_len(void) +{ + return sizeof(struct brcmf_pno_scanresults_le) + + sizeof(struct brcmf_pno_net_info_le); +} +static u32 brcmf_pno_min_data_len_v3(void) +{ + return sizeof(struct brcmf_pno_scanresults_v3_le) + + sizeof(struct brcmf_pno_net_info_v3_le); +} + +static int brcmf_pno_validate_pfn_results_v3(void *data, u32 eventlen) +{ + struct brcmf_pno_scanresults_v3_le *scanresult = + (struct brcmf_pno_scanresults_v3_le *)data; + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(scanresult); + u32 datalen; + + if (!netinfo_v3_start) { + brcmf_err("did not get netinfo_v3 data\n"); + return -EINVAL; + } + datalen = eventlen - ((void *)netinfo_v3_start - (void *)data); + if (datalen < le32_to_cpu(scanresult->count) * sizeof(struct brcmf_pno_net_info_v3_le)) { + brcmf_err("insufficient event data\n"); + return -EINVAL; + } + return 0; +} + +static int brcmf_pno_validate_pfn_results(void *data, u32 eventlen) +{ + struct brcmf_pno_scanresults_le *scanresult = + (struct brcmf_pno_scanresults_le *)data; + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(scanresult); + u32 datalen; + + if (!netinfo_start) { + brcmf_err("did not get netinfo data\n"); + return -EINVAL; + } + datalen = eventlen - ((void *)netinfo_start - (void *)data); + if (datalen < le32_to_cpu(scanresult->count) * sizeof(struct brcmf_pno_net_info_le)) { + brcmf_err("insufficient event data\n"); + return -EINVAL; + } + return 0; +} + +static int brcmf_pno_get_result_info(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], + u8 *ssid_len, u8 *channel, + enum nl80211_band *band) +{ + struct brcmf_pno_scanresults_le *scanresult = + (struct brcmf_pno_scanresults_le *)data; + struct brcmf_pno_net_info_le *netinfo_start = + brcmf_get_netinfo_array(scanresult); + struct brcmf_pno_net_info_le *netinfo = &netinfo_start[result_idx]; + + *channel = netinfo->channel; + *band = netinfo->channel <= CH_MAX_2G_CHANNEL ? NL80211_BAND_2GHZ : + NL80211_BAND_5GHZ; + *ssid_len = netinfo->SSID_len; + if (netinfo->SSID_len > IEEE80211_MAX_SSID_LEN) + *ssid_len = IEEE80211_MAX_SSID_LEN; + memcpy(ssid, netinfo->SSID, *ssid_len); + + return 0; +} + +static int brcmf_pno_get_result_info_v3(void *data, int result_idx, + u8 (*ssid)[IEEE80211_MAX_SSID_LEN], + u8 *ssid_len, u8 *channel, + enum nl80211_band *band) +{ + struct brcmf_pno_scanresults_v3_le *scanresult = + (struct brcmf_pno_scanresults_v3_le *)data; + struct brcmf_pno_net_info_v3_le *netinfo_v3_start = + brcmf_get_netinfo_v3_array(scanresult); + struct brcmf_pno_net_info_v3_le *netinfo_v3 = + &netinfo_v3_start[result_idx]; + + *channel = CHSPEC_CHANNEL(netinfo_v3->chanspec); + *band = fwil_band_to_nl80211(CHSPEC_BAND(netinfo_v3->chanspec)); + *ssid_len = netinfo_v3->SSID_len; + if (netinfo_v3->SSID_len > IEEE80211_MAX_SSID_LEN) + *ssid_len = IEEE80211_MAX_SSID_LEN; + memcpy(ssid, netinfo_v3->SSID, *ssid_len); + + return 0; +} + +/* The count and status fields are in the same place for v1/2/3 */ +static u32 brcmf_pno_get_result_count_v123(void *data) +{ + struct brcmf_pno_scanresults_le *results = + (struct brcmf_pno_scanresults_le *)data; + return le32_to_cpu(results->count); +} +static u32 brcmf_pno_get_result_status_v123(void *data) +{ + struct brcmf_pno_scanresults_le *results = + (struct brcmf_pno_scanresults_le *)data; + return le32_to_cpu(results->status); +} + +int brcmf_pno_setup_for_version(struct brcmf_pub *drvr, u8 vers) +{ + /* The first supported version by this driver was version 2. + * The v2 functions handle version one structures if handed to them, + * but the config was always set to interface version 2. */ + switch (vers) { + case BRCMF_PNO_VERSION_2: { + drvr->pno_handler.version = BRCMF_PNO_VERSION_2; + drvr->pno_handler.pno_config = brcmf_pno_config_v2; + drvr->pno_handler.get_result_count = brcmf_pno_get_result_count_v123; + drvr->pno_handler.get_result_status = brcmf_pno_get_result_status_v123; + drvr->pno_handler.get_bucket_map = brcmf_pno_get_bucket_map; + drvr->pno_handler.get_min_data_len = brcmf_pno_min_data_len; + drvr->pno_handler.get_result_info = brcmf_pno_get_result_info; + drvr->pno_handler.validate_pfn_results = + brcmf_pno_validate_pfn_results; + break; + } + case BRCMF_PNO_VERSION_3: { + drvr->pno_handler.version = BRCMF_PNO_VERSION_3; + drvr->pno_handler.pno_config = brcmf_pno_config_v3; + drvr->pno_handler.get_result_count = brcmf_pno_get_result_count_v123; + drvr->pno_handler.get_result_status = brcmf_pno_get_result_status_v123; + drvr->pno_handler.get_bucket_map = brcmf_pno_get_bucket_map_v3; + drvr->pno_handler.get_min_data_len = brcmf_pno_min_data_len_v3; + drvr->pno_handler.get_result_info = brcmf_pno_get_result_info_v3; + drvr->pno_handler.validate_pfn_results = + brcmf_pno_validate_pfn_results_v3; + break; + } + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h index 25d406019ac340..0163c762f5385a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.h @@ -61,12 +61,12 @@ void brcmf_pno_detach(struct brcmf_cfg80211_info *cfg); u64 brcmf_pno_find_reqid_by_bucket(struct brcmf_pno_info *pi, u32 bucket); /** - * brcmf_pno_get_bucket_map - determine bucket map for given netinfo. + * brcmf_pno_setup_for_version - setup our PNO handler for whatever version structures + * are supported by the chip * - * @pi: pno instance used. - * @netinfo: netinfo to compare with bucket configuration. + * @cfg: CFG to fill in. + * @vers: Version to use */ -u32 brcmf_pno_get_bucket_map(struct brcmf_pno_info *pi, - struct brcmf_pno_net_info_le *netinfo); +int brcmf_pno_setup_for_version(struct brcmf_pub *drvr, u8 vers); #endif /* _BRCMF_PNO_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h new file mode 100644 index 00000000000000..37e722daab14d4 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/ratespec.h @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef BRCMFMAC_RATESPEC_H +#define BRCMFMAC_RATESPEC_H +/* Rate spec. definitions */ +/* for BRCMF_RSPEC_ENCODING field >= BRCMF_RSPEC_ENCODING_HE, backward compatible */ + +/**< Legacy rate or MCS or MCS + NSS */ +#define BRCMF_RSPEC_RATE_MASK 0x000000FFu +/**< Tx chain expansion beyond Nsts */ +#define BRCMF_RSPEC_TXEXP_MASK 0x00000300u +#define BRCMF_RSPEC_TXEXP_SHIFT 8u +/* EHT GI indices */ +#define BRCMF_RSPEC_EHT_GI_MASK 0x00000C00u +#define BRCMF_RSPEC_EHT_GI_SHIFT 10u +/* HE GI indices */ +#define BRCMF_RSPEC_HE_GI_MASK 0x00000C00u +#define BRCMF_RSPEC_HE_GI_SHIFT 10u +/**< Range extension mask */ +#define BRCMF_RSPEC_ER_MASK 0x0000C000u +#define BRCMF_RSPEC_ER_SHIFT 14u +/**< Range extension tone config */ +#define BRCMF_RSPEC_ER_TONE_MASK 0x00004000u +#define BRCMF_RSPEC_ER_TONE_SHIFT 14u +/**< Range extension enable */ +#define BRCMF_RSPEC_ER_ENAB_MASK 0x00008000u +#define BRCMF_RSPEC_ER_ENAB_SHIFT 15u +/**< Bandwidth */ +#define BRCMF_RSPEC_BW_MASK 0x00070000u +#define BRCMF_RSPEC_BW_SHIFT 16u +/**< Dual Carrier Modulation */ +#define BRCMF_RSPEC_DCM 0x00080000u +#define BRCMF_RSPEC_DCM_SHIFT 19u +/**< STBC expansion, Nsts = 2 * Nss */ +#define BRCMF_RSPEC_STBC 0x00100000u +#define BRCMF_RSPEC_TXBF 0x00200000u +#define BRCMF_RSPEC_LDPC 0x00400000u +/* HT/VHT SGI indication */ +#define BRCMF_RSPEC_SGI 0x00800000u +/**< DSSS short preable - Encoding 0 */ +#define BRCMF_RSPEC_SHORT_PREAMBLE 0x00800000u +/**< Encoding of RSPEC_RATE field */ +#define BRCMF_RSPEC_ENCODING_MASK 0x07000000u +#define BRCMF_RSPEC_ENCODING_SHIFT 24u +#define BRCMF_RSPEC_OVERRIDE_RATE 0x40000000u /**< override rate only */ +#define BRCMF_RSPEC_OVERRIDE_MODE 0x80000000u /**< override both rate & mode */ + +/* ======== RSPEC_EHT_GI|RSPEC_SGI fields for EHT ======== */ +/* 11be Draft 0.4 Table 36-35:Common field for non-OFDMA transmission. + * Table 36-32 Common field for OFDMA transmission + */ +#define BRCMF_RSPEC_EHT_LTF_GI(rspec) \ + (((rspec) & BRCMF_RSPEC_EHT_GI_MASK) >> BRCMF_RSPEC_EHT_GI_SHIFT) +#define BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us (0x0u) +#define BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us (0x1u) +#define BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us (0x2u) +#define BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us (0x3u) +#define WL_EHT_GI_TO_RSPEC(gi) \ + ((u32)(((gi) << BRCMF_RSPEC_EHT_GI_SHIFT) & \ + BRCMF_RSPEC_EHT_GI_MASK)) +#define WL_EHT_GI_TO_RSPEC_SET(rspec, gi) \ + ((rspec & (~BRCMF_RSPEC_EHT_GI_MASK)) | WL_EHT_GI_TO_RSPEC(gi)) + +/* Macros for EHT LTF and GI */ +#define EHT_IS_2X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us)) +#define EHT_IS_4X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us)) + +#define EHT_IS_GI_0_8us(gi) \ + (((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_0_8us)) +#define EHT_IS_GI_1_6us(gi) ((gi) == BRCMF_RSPEC_EHT_2x_LTF_GI_1_6us) +#define EHT_IS_GI_3_2us(gi) ((gi) == BRCMF_RSPEC_EHT_4x_LTF_GI_3_2us) + +/* ======== RSPEC_HE_GI|RSPEC_SGI fields for HE ======== */ + +/* GI for HE */ +#define BRCMF_RSPEC_HE_LTF_GI(rspec) \ + (((rspec) & BRCMF_RSPEC_HE_GI_MASK) >> BRCMF_RSPEC_HE_GI_SHIFT) +#define BRCMF_RSPEC_HE_1x_LTF_GI_0_8us (0x0u) +#define BRCMF_RSPEC_HE_2x_LTF_GI_0_8us (0x1u) +#define BRCMF_RSPEC_HE_2x_LTF_GI_1_6us (0x2u) +#define BRCMF_RSPEC_HE_4x_LTF_GI_3_2us (0x3u) +#define BRCMF_RSPEC_ISHEGI(rspec) \ + (RSPEC_HE_LTF_GI(rspec) > BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) +#define HE_GI_TO_RSPEC(gi) \ + (((u32)(gi) << BRCMF_RSPEC_HE_GI_SHIFT) & BRCMF_RSPEC_HE_GI_MASK) +#define HE_GI_TO_RSPEC_SET(rspec, gi) \ + ((rspec & (~BRCMF_RSPEC_HE_GI_MASK)) | HE_GI_TO_RSPEC(gi)) + +/* Macros for HE LTF and GI */ +#define HE_IS_1X_LTF(gi) ((gi) == BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) +#define HE_IS_2X_LTF(gi) \ + (((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_1_6us)) +#define HE_IS_4X_LTF(gi) ((gi) == BRCMF_RSPEC_HE_4x_LTF_GI_3_2us) + +#define HE_IS_GI_0_8us(gi) \ + (((gi) == BRCMF_RSPEC_HE_1x_LTF_GI_0_8us) || \ + ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_0_8us)) +#define HE_IS_GI_1_6us(gi) ((gi) == BRCMF_RSPEC_HE_2x_LTF_GI_1_6us) +#define HE_IS_GI_3_2us(gi) ((gi) == BRCMF_RSPEC_HE_4x_LTF_GI_3_2us) + +/* RSPEC Macros for extracting and using HE-ER and DCM */ +#define BRCMF_RSPEC_HE_DCM(rspec) \ + (((rspec) & BRCMF_RSPEC_DCM) >> BRCMF_RSPEC_DCM_SHIFT) +#define BRCMF_RSPEC_HE_ER(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_MASK) >> BRCMF_RSPEC_ER_SHIFT) +#define BRCMF_RSPEC_HE_ER_ENAB(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_ENAB_MASK) >> BRCMF_RSPEC_ER_ENAB_SHIFT) +#define BRCMF_RSPEC_HE_ER_TONE(rspec) \ + (((rspec) & BRCMF_RSPEC_ER_TONE_MASK) >> BRCMF_RSPEC_ER_TONE_SHIFT) +/* ======== RSPEC_RATE field ======== */ + +/* Encoding 0 - legacy rate */ +/* DSSS, CCK, and OFDM rates in [500kbps] units */ +#define BRCMF_RSPEC_LEGACY_RATE_MASK 0x0000007F +#define WLC_RATE_1M 2 +#define WLC_RATE_2M 4 +#define WLC_RATE_5M5 11 +#define WLC_RATE_11M 22 +#define WLC_RATE_6M 12 +#define WLC_RATE_9M 18 +#define WLC_RATE_12M 24 +#define WLC_RATE_18M 36 +#define WLC_RATE_24M 48 +#define WLC_RATE_36M 72 +#define WLC_RATE_48M 96 +#define WLC_RATE_54M 108 + +/* Encoding 1 - HT MCS */ +/**< HT MCS value mask in rspec */ +#define BRCMF_RSPEC_HT_MCS_MASK 0x0000007F + +/* Encoding >= 2 */ +/* NSS & MCS values mask in rspec */ +#define BRCMF_RSPEC_NSS_MCS_MASK 0x000000FF +/* mimo MCS value mask in rspec */ +#define BRCMF_RSPEC_MCS_MASK 0x0000000F +/* mimo NSS value mask in rspec */ +#define BRCMF_RSPEC_NSS_MASK 0x000000F0 +/* mimo NSS value shift in rspec */ +#define BRCMF_RSPEC_NSS_SHIFT 4 + +/* Encoding 2 - VHT MCS + NSS */ +/**< VHT MCS value mask in rspec */ +#define BRCMF_RSPEC_VHT_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< VHT Nss value mask in rspec */ +#define BRCMF_RSPEC_VHT_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< VHT Nss value shift in rspec */ +#define BRCMF_RSPEC_VHT_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +/* Encoding 3 - HE MCS + NSS */ +/**< HE MCS value mask in rspec */ +#define BRCMF_RSPEC_HE_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< HE Nss value mask in rspec */ +#define BRCMF_RSPEC_HE_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< HE Nss value shift in rpsec */ +#define BRCMF_RSPEC_HE_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +#define BRCMF_RSPEC_HE_NSS_UNSPECIFIED 0xf + +/* Encoding 4 - EHT MCS + NSS */ +/**< EHT MCS value mask in rspec */ +#define BRCMF_RSPEC_EHT_MCS_MASK BRCMF_RSPEC_MCS_MASK +/**< EHT Nss value mask in rspec */ +#define BRCMF_RSPEC_EHT_NSS_MASK BRCMF_RSPEC_NSS_MASK +/**< EHT Nss value shift in rpsec */ +#define BRCMF_RSPEC_EHT_NSS_SHIFT BRCMF_RSPEC_NSS_SHIFT + +/* ======== RSPEC_BW field ======== */ + +#define BRCMF_RSPEC_BW_UNSPECIFIED 0u +#define BRCMF_RSPEC_BW_20MHZ 0x00010000u +#define BRCMF_RSPEC_BW_40MHZ 0x00020000u +#define BRCMF_RSPEC_BW_80MHZ 0x00030000u +#define BRCMF_RSPEC_BW_160MHZ 0x00040000u +#define BRCMF_RSPEC_BW_320MHZ 0x00060000u + +/* ======== RSPEC_ENCODING field ======== */ + +/* NOTE: Assuming the rate field is always NSS+MCS starting from VHT encoding! + * Modify/fix RSPEC_ISNSSMCS() macro if above condition changes any time. + */ +/**< Legacy rate is stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_RATE 0x00000000u +/**< HT MCS is stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_HT 0x01000000u +/**< VHT MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_VHT 0x02000000u +/**< HE MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_HE 0x03000000u +/**< EHT MCS and NSS are stored in RSPEC_RATE */ +#define BRCMF_RSPEC_ENCODE_EHT 0x04000000u + +/** + * =============================== + * Handy macros to parse rate spec + * =============================== + */ +#define BRCMF_RSPEC_BW(rspec) ((rspec) & BRCMF_RSPEC_BW_MASK) +#define BRCMF_RSPEC_IS20MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_20MHZ) +#define BRCMF_RSPEC_IS40MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_40MHZ) +#define BRCMF_RSPEC_IS80MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_80MHZ) +#define BRCMF_RSPEC_IS160MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_160MHZ) +#if defined(WL_BW320MHZ) +#define BRCMF_RSPEC_IS320MHZ(rspec) (RSPEC_BW(rspec) == BRCMF_RSPEC_BW_320MHZ) +#else +#define BRCMF_RSPEC_IS320MHZ(rspec) (FALSE) +#endif /* WL_BW320MHZ */ + +#define BRCMF_RSPEC_BW_GE(rspec, rspec_bw) (RSPEC_BW(rspec) >= rspec_bw) +#define BRCMF_RSPEC_BW_LE(rspec, rspec_bw) (RSPEC_BW(rspec) <= rspec_bw) +#define BRCMF_RSPEC_BW_GT(rspec, rspec_bw) (!RSPEC_BW_LE(rspec, rspec_bw)) +#define BRCMF_RSPEC_BW_LT(rspec, rspec_bw) (!RSPEC_BW_GE(rspec, rspec_bw)) + +#define BRCMF_RSPEC_ISSGI(rspec) (((rspec) & BRCMF_RSPEC_SGI) != 0) +#define BRCMF_RSPEC_ISLDPC(rspec) (((rspec) & BRCMF_RSPEC_LDPC) != 0) +#define BRCMF_RSPEC_ISSTBC(rspec) (((rspec) & BRCMF_RSPEC_STBC) != 0) +#define BRCMF_RSPEC_ISTXBF(rspec) (((rspec) & BRCMF_RSPEC_TXBF) != 0) + +#define BRCMF_RSPEC_TXEXP(rspec) \ + (((rspec) & BRCMF_RSPEC_TXEXP_MASK) >> BRCMF_RSPEC_TXEXP_SHIFT) + +#define BRCMF_RSPEC_ENCODE(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >> BRCMF_RSPEC_ENCODING_SHIFT) +#define BRCMF_RSPEC_ISLEGACY(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_RATE) + +#define BRCMF_RSPEC_ISHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_HT) +#define BRCMF_RSPEC_ISVHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_VHT) +#define BRCMF_RSPEC_ISHE(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_HE) +#define BRCMF_RSPEC_ISEHT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) == BRCMF_RSPEC_ENCODE_EHT) + +/* fast check if rate field is NSS+MCS format (starting from VHT ratespec) */ +#define BRCMF_RSPEC_ISVHTEXT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >= BRCMF_RSPEC_ENCODE_VHT) +/* fast check if rate field is NSS+MCS format (starting from HE ratespec) */ +#define BRCMF_RSPEC_ISHEEXT(rspec) \ + (((rspec) & BRCMF_RSPEC_ENCODING_MASK) >= BRCMF_RSPEC_ENCODE_HE) + +#endif /* BRCMFMAC_RATESPEC_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c new file mode 100644 index 00000000000000..4f634509d25256 --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ +#include +#include + +#include "core.h" +#include "debug.h" +#include "fwil_types.h" +#include "cfg80211.h" +#include "scan_param.h" + +static void brcmf_scan_param_set_defaults(u8 (*bssid)[ETH_ALEN], s8 *bss_type, __le32 *channel_num, + __le32 *nprobes, __le32 *active_time, + __le32 *passive_time, + __le32 *home_time) +{ + eth_broadcast_addr(*bssid); + *bss_type = DOT11_BSSTYPE_ANY; + *channel_num = 0; + *nprobes = cpu_to_le32(-1); + *active_time = cpu_to_le32(-1); + *passive_time = cpu_to_le32(-1); + *home_time = cpu_to_le32(-1); +} + +static void brcmf_scan_param_copy_chanspecs( + struct brcmf_cfg80211_info *cfg, __le16 (*dest_channels)[], + struct ieee80211_channel **in_channels, u32 n_channels) +{ + int i; + for (i = 0; i < n_channels; i++) { + u32 chanspec = + channel_to_chanspec(&cfg->d11inf, in_channels[i]); + brcmf_dbg(SCAN, "Chan : %d, Channel spec: %x\n", + in_channels[i]->hw_value, chanspec); + (*dest_channels)[i] = cpu_to_le16(chanspec); + } +} + +static void brcmf_scan_param_copy_ssids(char *dest_ssids, + struct cfg80211_ssid *in_ssids, + u32 n_ssids) +{ + int i; + for (i = 0; i < n_ssids; i++) { + struct brcmf_ssid_le ssid_le; + memset(&ssid_le, 0, sizeof(ssid_le)); + ssid_le.SSID_len = cpu_to_le32(in_ssids[i].ssid_len); + memcpy(ssid_le.SSID, in_ssids[i].ssid, in_ssids[i].ssid_len); + if (!ssid_le.SSID_len) + brcmf_dbg(SCAN, "%d: Broadcast scan\n", i); + else + brcmf_dbg(SCAN, "%d: scan for %.32s size=%d\n", i, + ssid_le.SSID, ssid_le.SSID_len); + memcpy(dest_ssids, &ssid_le, sizeof(ssid_le)); + dest_ssids += sizeof(ssid_le); + } +} + +/* The scan parameter structures have an array of SSID's that appears at the end in some cases. + * In these cases, the chan list is really the lower half of a pair, the upper half is a ssid number, + * and then after all of that there is an array of SSIDs */ +static u32 +brcmf_scan_param_tail_size(const struct cfg80211_scan_request *request, + u32 params_size) +{ + if (request != NULL) { + /* Allocate space for populating ssid upper half in struct */ + params_size += sizeof(u32) * ((request->n_channels + 1) / 2); + /* Allocate space for populating ssids in struct */ + params_size += sizeof(struct brcmf_ssid_le) * request->n_ssids; + } else { + params_size += sizeof(u16); + } + return params_size; +} + +static u32 brcmf_nl80211_scan_flags_to_scan_flags(u32 nl80211_flags) +{ + u32 scan_flags = 0; + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_SPAN) { + scan_flags |= BRCMF_SCANFLAGS_LOW_SPAN; + brcmf_dbg(SCAN, "requested low span scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_HIGH_ACCURACY) { + scan_flags |= BRCMF_SCANFLAGS_HIGH_ACCURACY; + brcmf_dbg(SCAN, "requested high accuracy scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_POWER) { + scan_flags |= BRCMF_SCANFLAGS_LOW_POWER; + brcmf_dbg(SCAN, "requested low power scan\n"); + } + if (nl80211_flags & NL80211_SCAN_FLAG_LOW_PRIORITY) { + scan_flags |= BRCMF_SCANFLAGS_LOW_PRIO; + brcmf_dbg(SCAN, "requested low priority scan\n"); + } + return scan_flags; +} + +static void * +brcmf_scan_param_get_prepped_struct_v1(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_le); + u32 length; + struct brcmf_scan_params_le *params_le = NULL; + u8 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type =scan_type; + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v2(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v2_le); + u32 length; + struct brcmf_scan_params_v2_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v2_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V2); + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v2_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v3(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v3_le); + u32 length; + struct brcmf_scan_params_v3_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v3_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V3); + params_le->ssid_type = 0; + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v3_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); + + /* Include RNR results if requested */ + if (request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + params_le->ssid_type |= BRCMF_SCANSSID_INC_RNR; + } + /* Adding mask to channel numbers */ +done: + *struct_size = length; + return params_le; +} + +static void * +brcmf_scan_param_get_prepped_struct_v4(struct brcmf_cfg80211_info *cfg, + u32 *struct_size, + struct cfg80211_scan_request *request) +{ + u32 n_ssids; + u32 n_channels; + u32 params_size = sizeof(struct brcmf_scan_params_v4_le); + u32 length; + struct brcmf_scan_params_v4_le *params_le = NULL; + u32 scan_type = BRCMF_SCANTYPE_ACTIVE; + + length = offsetof(struct brcmf_scan_params_v4_le, channel_list); + params_size = brcmf_scan_param_tail_size(request, params_size); + params_le = kzalloc(params_size, GFP_KERNEL); + if (!params_le) { + bphy_err(cfg, "Could not allocate scan params\n"); + return NULL; + } + params_le->version = cpu_to_le16(BRCMF_SCAN_PARAMS_VERSION_V4); + params_le->ssid_type = 0; + brcmf_scan_param_set_defaults(¶ms_le->bssid, + ¶ms_le->bss_type, ¶ms_le->channel_num, + ¶ms_le->nprobes, ¶ms_le->active_time, + ¶ms_le->passive_time, ¶ms_le->home_time); + + /* Scan abort */ + if (!request) { + length += sizeof(u16); + params_le->channel_num = cpu_to_le32(1); + params_le->channel_list[0] = cpu_to_le16(-1); + params_le->length = cpu_to_le16(length); + goto done; + } + + n_ssids = request->n_ssids; + n_channels = request->n_channels; + + /* Copy channel array if applicable */ + brcmf_dbg(SCAN, "### List of channelspecs to scan ### %d\n", + n_channels); + if (n_channels > 0) { + length += roundup(sizeof(u16) * n_channels, sizeof(u32)); + brcmf_scan_param_copy_chanspecs(cfg, ¶ms_le->channel_list, + request->channels, n_channels); + } else { + brcmf_dbg(SCAN, "Scanning all channels\n"); + } + + /* Copy ssid array if applicable */ + brcmf_dbg(SCAN, "### List of SSIDs to scan ### %d\n", n_ssids); + if (n_ssids > 0) { + s32 offset; + char *ptr; + + offset = + offsetof(struct brcmf_scan_params_v4_le, channel_list) + + n_channels * sizeof(u16); + offset = roundup(offset, sizeof(u32)); + length += sizeof(struct brcmf_ssid_le) * n_ssids; + ptr = (char *)params_le + offset; + brcmf_scan_param_copy_ssids(ptr, request->ssids, n_ssids); + } else { + brcmf_dbg(SCAN, "Performing passive scan\n"); + scan_type = BRCMF_SCANTYPE_PASSIVE; + } + scan_type |= brcmf_nl80211_scan_flags_to_scan_flags(request->flags); + params_le->scan_type = cpu_to_le32(scan_type); + params_le->length = cpu_to_le16(length); + /* Adding mask to channel numbers */ + params_le->channel_num = + cpu_to_le32((n_ssids << BRCMF_SCAN_PARAMS_NSSID_SHIFT) | + (n_channels & BRCMF_SCAN_PARAMS_COUNT_MASK)); + /* Include RNR results if requested */ + if (request->flags & NL80211_SCAN_FLAG_COLOCATED_6GHZ) { + params_le->ssid_type |= BRCMF_SCANSSID_INC_RNR; + } +done: + *struct_size = length; + return params_le; +} + +int brcmf_scan_param_setup_for_version(struct brcmf_pub *drvr, u8 version) +{ + drvr->scan_param_handler.version = version; + switch (version) { + case 1: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v1; + } break; + case 2: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v2; + } break; + case 3: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v3; + } break; + case 4: { + drvr->scan_param_handler.get_struct_for_request = + brcmf_scan_param_get_prepped_struct_v4; + + } break; + default: + return -EINVAL; + } + return 0; +} diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h new file mode 100644 index 00000000000000..577de083c6e3cd --- /dev/null +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/scan_param.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: ISC +/* + * Copyright (c) 2023 Daniel Berlin + */ + +#ifndef _BRCMF_SCAN_PARAM_H +#define _BRCMF_SCAN_PARAM_H + +struct brcmf_pub; + +/** + * brcmf_scan_param_setup_for_version() - Setup the driver to handle join structures + * + * There are a number of different structures and interface versions for scanning info + * This sets up the driver to handle a particular interface version. + * + * @drvr Driver structure to setup + * @ver Interface version + * Return: %0 if okay, error code otherwise + */ +int brcmf_scan_param_setup_for_version(struct brcmf_pub *, u8 ver); +#endif /* _BRCMF_SCAN_PARAM_H */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index 30f6fcb6863279..8fb595733b9c36 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -2476,8 +2476,9 @@ static void brcmf_sdio_bus_stop(struct device *dev) brcmf_dbg(TRACE, "Enter\n"); if (bus->watchdog_tsk) { + get_task_struct(bus->watchdog_tsk); send_sig(SIGTERM, bus->watchdog_tsk, 1); - kthread_stop(bus->watchdog_tsk); + kthread_stop_put(bus->watchdog_tsk); bus->watchdog_tsk = NULL; } @@ -4567,8 +4568,9 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus) if (bus) { /* Stop watchdog task */ if (bus->watchdog_tsk) { + get_task_struct(bus->watchdog_tsk); send_sig(SIGTERM, bus->watchdog_tsk, 1); - kthread_stop(bus->watchdog_tsk); + kthread_stop_put(bus->watchdog_tsk); bus->watchdog_tsk = NULL; } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c index 1e2b1e487eb76e..faf7eeeeb2d57e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmutil/d11.c @@ -87,10 +87,20 @@ static void brcmu_d11ac_encchspec(struct brcmu_chan *ch) 0, d11ac_bw(ch->bw)); ch->chspec &= ~BRCMU_CHSPEC_D11AC_BND_MASK; - if (ch->chnum <= CH_MAX_2G_CHANNEL) - ch->chspec |= BRCMU_CHSPEC_D11AC_BND_2G; - else + switch (ch->band) { + case BRCMU_CHAN_BAND_6G: + ch->chspec |= BRCMU_CHSPEC_D11AC_BND_6G; + break; + case BRCMU_CHAN_BAND_5G: ch->chspec |= BRCMU_CHSPEC_D11AC_BND_5G; + break; + case BRCMU_CHAN_BAND_2G: + ch->chspec |= BRCMU_CHSPEC_D11AC_BND_2G; + break; + default: + WARN_ONCE(1, "Invalid band 0x%04x\n", ch->band); + break; + } } static void brcmu_d11n_decchspec(struct brcmu_chan *ch) @@ -117,7 +127,9 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch) } break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11n bandwidth 0x%04x\n", + ch->chspec); break; } @@ -129,7 +141,8 @@ static void brcmu_d11n_decchspec(struct brcmu_chan *ch) ch->band = BRCMU_CHAN_BAND_2G; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, "Invalid chanspec - unknown 11n band 0x%04x\n", + ch->chspec); break; } } @@ -156,7 +169,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->sb = BRCMU_CHAN_SB_U; ch->control_ch_num += CH_10MHZ_APART; } else { - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); } break; case BRCMU_CHSPEC_D11AC_BW_80: @@ -177,7 +192,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->control_ch_num += CH_30MHZ_APART; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); break; } break; @@ -211,17 +228,24 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->control_ch_num += CH_70MHZ_APART; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel distance 0x%04x\n", + ch->chspec); break; } break; case BRCMU_CHSPEC_D11AC_BW_8080: default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel bandwidth 0x%04x\n", + ch->chspec); break; } switch (ch->chspec & BRCMU_CHSPEC_D11AC_BND_MASK) { + case BRCMU_CHSPEC_D11AC_BND_6G: + ch->band = BRCMU_CHAN_BAND_6G; + break; case BRCMU_CHSPEC_D11AC_BND_5G: ch->band = BRCMU_CHAN_BAND_5G; break; @@ -229,7 +253,9 @@ static void brcmu_d11ac_decchspec(struct brcmu_chan *ch) ch->band = BRCMU_CHAN_BAND_2G; break; default: - WARN_ONCE(1, "Invalid chanspec 0x%04x\n", ch->chspec); + WARN_ONCE(1, + "Invalid chanspec - unknown 11ac channel band 0x%04x\n", + ch->chspec); break; } } diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h index df3b67ba4db290..f749337a06942e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h @@ -57,6 +57,7 @@ #define BRCM_CC_4377_CHIP_ID 0x4377 #define BRCM_CC_4378_CHIP_ID 0x4378 #define BRCM_CC_4387_CHIP_ID 0x4387 +#define BRCM_CC_4388_CHIP_ID 0x4388 #define CY_CC_4373_CHIP_ID 0x4373 #define CY_CC_43012_CHIP_ID 43012 #define CY_CC_43439_CHIP_ID 43439 @@ -99,6 +100,7 @@ #define BRCM_PCIE_4377_DEVICE_ID 0x4488 #define BRCM_PCIE_4378_DEVICE_ID 0x4425 #define BRCM_PCIE_4387_DEVICE_ID 0x4433 +#define BRCM_PCIE_4388_DEVICE_ID 0x4434 #define CY_PCIE_54591_DEVICE_ID 0x4417 /* brcmsmac IDs */ diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h index f6344023855c36..bb48b744206223 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_d11.h @@ -69,24 +69,44 @@ #define BRCMU_CHSPEC_D11AC_SB_UU BRCMU_CHSPEC_D11AC_SB_LUU #define BRCMU_CHSPEC_D11AC_SB_L BRCMU_CHSPEC_D11AC_SB_LLL #define BRCMU_CHSPEC_D11AC_SB_U BRCMU_CHSPEC_D11AC_SB_LLU +/* channel sideband indication for frequency >= 240MHz */ +#define BRCMU_CHSPEC_D11AC_320_SB_MASK 0x0780 +#define BRCMU_CHSPEC_D11AC_320_SB_SHIFT 7 +#define BRCMU_CHSPEC_D11AC_SB_LLLL 0x0000 +#define BRCMU_CHSPEC_D11AC_SB_LLLU 0x0080 +#define BRCMU_CHSPEC_D11AC_SB_LLUL 0x0100 +#define BRCMU_CHSPEC_D11AC_SB_LLUU 0x0180 +#define BRCMU_CHSPEC_D11AC_SB_LULL 0x0200 +#define BRCMU_CHSPEC_D11AC_SB_LULU 0x0280 +#define BRCMU_CHSPEC_D11AC_SB_LUUL 0x0300 +#define BRCMU_CHSPEC_D11AC_SB_LUUU 0x0380 +#define BRCMU_CHSPEC_D11AC_SB_ULLL 0x0400 +#define BRCMU_CHSPEC_D11AC_SB_ULLU 0x0480 +#define BRCMU_CHSPEC_D11AC_SB_ULUL 0x0500 +#define BRCMU_CHSPEC_D11AC_SB_ULUU 0x0580 +#define BRCMU_CHSPEC_D11AC_SB_UULL 0x0600 +#define BRCMU_CHSPEC_D11AC_SB_UULU 0x0680 +#define BRCMU_CHSPEC_D11AC_SB_UUUL 0x0700 +#define BRCMU_CHSPEC_D11AC_SB_UUUU 0x0780 #define BRCMU_CHSPEC_D11AC_BW_MASK 0x3800 #define BRCMU_CHSPEC_D11AC_BW_SHIFT 11 -#define BRCMU_CHSPEC_D11AC_BW_5 0x0000 -#define BRCMU_CHSPEC_D11AC_BW_10 0x0800 -#define BRCMU_CHSPEC_D11AC_BW_20 0x1000 -#define BRCMU_CHSPEC_D11AC_BW_40 0x1800 -#define BRCMU_CHSPEC_D11AC_BW_80 0x2000 -#define BRCMU_CHSPEC_D11AC_BW_160 0x2800 -#define BRCMU_CHSPEC_D11AC_BW_8080 0x3000 -#define BRCMU_CHSPEC_D11AC_BND_MASK 0xc000 -#define BRCMU_CHSPEC_D11AC_BND_SHIFT 14 -#define BRCMU_CHSPEC_D11AC_BND_2G 0x0000 -#define BRCMU_CHSPEC_D11AC_BND_3G 0x4000 -#define BRCMU_CHSPEC_D11AC_BND_4G 0x8000 -#define BRCMU_CHSPEC_D11AC_BND_5G 0xc000 +#define BRCMU_CHSPEC_D11AC_BW_10 0x0800 +#define BRCMU_CHSPEC_D11AC_BW_20 0x1000 +#define BRCMU_CHSPEC_D11AC_BW_40 0x1800 +#define BRCMU_CHSPEC_D11AC_BW_80 0x2000 +#define BRCMU_CHSPEC_D11AC_BW_160 0x2800 +#define BRCMU_CHSPEC_D11AC_BW_320 0x0000 +#define BRCMU_CHSPEC_D11AC_BW_8080 0x3000 +#define BRCMU_CHSPEC_D11AC_BND_MASK 0xc000 +#define BRCMU_CHSPEC_D11AC_BND_SHIFT 14 +#define BRCMU_CHSPEC_D11AC_BND_2G 0x0000 +#define BRCMU_CHSPEC_D11AC_BND_4G 0x8000 +#define BRCMU_CHSPEC_D11AC_BND_5G 0xc000 +#define BRCMU_CHSPEC_D11AC_BND_6G 0x4000 #define BRCMU_CHAN_BAND_2G 0 #define BRCMU_CHAN_BAND_5G 1 +#define BRCMU_CHAN_BAND_6G 2 enum brcmu_chan_bw { BRCMU_CHAN_BW_20, diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h index 7552bdb91991ce..ef042beeb586f9 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/brcmu_wifi.h @@ -31,6 +31,7 @@ /* bandstate array indices */ #define BAND_2G_INDEX 0 /* wlc->bandstate[x] index */ #define BAND_5G_INDEX 1 /* wlc->bandstate[x] index */ +#define BAND_6G_INDEX 2 /* wlc->bandstate[x] index */ /* * max # supported channels. The max channel no is 216, this is that + 1 @@ -48,17 +49,22 @@ #define WL_CHANSPEC_CTL_SB_UPPER 0x0200 #define WL_CHANSPEC_CTL_SB_NONE 0x0300 -#define WL_CHANSPEC_BW_MASK 0x0C00 -#define WL_CHANSPEC_BW_SHIFT 10 +#define WL_CHANSPEC_BW_MASK 0x3800 +#define WL_CHANSPEC_BW_SHIFT 11 #define WL_CHANSPEC_BW_10 0x0400 #define WL_CHANSPEC_BW_20 0x0800 #define WL_CHANSPEC_BW_40 0x0C00 #define WL_CHANSPEC_BW_80 0x2000 - -#define WL_CHANSPEC_BAND_MASK 0xf000 -#define WL_CHANSPEC_BAND_SHIFT 12 -#define WL_CHANSPEC_BAND_5G 0x1000 -#define WL_CHANSPEC_BAND_2G 0x2000 +#define WL_CHANSPEC_BW_160 0x2800 +#define WL_CHANSPEC_BW_8080 0x3000 +#define WL_CHANSPEC_BW_320 0x0000 + +#define WL_CHANSPEC_BAND_MASK 0xc000 +#define WL_CHANSPEC_BAND_SHIFT 14 +#define WL_CHANSPEC_BAND_2G 0x0000 +#define WL_CHANSPEC_BAND_4G 0x8000 +#define WL_CHANSPEC_BAND_5G 0xc000 +#define WL_CHANSPEC_BAND_6G 0x4000 #define INVCHANSPEC 255 #define WL_CHAN_VALID_HW (1 << 0) /* valid with current HW */ @@ -93,6 +99,9 @@ #define WLC_BAND_5G 1 /* 5 Ghz */ #define WLC_BAND_2G 2 /* 2.4 Ghz */ #define WLC_BAND_ALL 3 /* all bands */ +#define WLC_BAND_6G 4 /* 6 Ghz */ + +#define WLC_AP_IOV_OP_MANUAL_AP_BSSCFG_CREATE 2 #define CHSPEC_CHANNEL(chspec) ((u8)((chspec) & WL_CHANSPEC_CHAN_MASK)) #define CHSPEC_BAND(chspec) ((chspec) & WL_CHANSPEC_BAND_MASK) @@ -112,6 +121,12 @@ #define CHSPEC_IS80(chspec) \ (((chspec) & WL_CHANSPEC_BW_MASK) == WL_CHANSPEC_BW_80) +#define CHSPEC_IS160(chspec) \ + (((chspec) & WL_CHANSPEC_BW_MASK) == WL_CHANSPEC_BW_160) + +#define CHSPEC_IS6G(chspec) \ + (((chspec) & WL_CHANSPEC_BAND_MASK) == WL_CHANSPEC_BAND_6G) + #define CHSPEC_IS5G(chspec) \ (((chspec) & WL_CHANSPEC_BAND_MASK) == WL_CHANSPEC_BAND_5G) @@ -200,6 +215,13 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define CRYPTO_ALGO_AES_RESERVED1 5 #define CRYPTO_ALGO_AES_RESERVED2 6 #define CRYPTO_ALGO_NALG 7 +#define CRYPTO_ALGO_AES_GCM 14 /* 128 bit GCM */ +#define CRYPTO_ALGO_AES_CCM256 15 /* 256 bit CCM */ +#define CRYPTO_ALGO_AES_GCM256 16 /* 256 bit GCM */ +#define CRYPTO_ALGO_BIP_CMAC256 17 /* 256 bit BIP CMAC */ +#define CRYPTO_ALGO_BIP_GMAC 18 /* 128 bit BIP GMAC */ +#define CRYPTO_ALGO_BIP_GMAC256 19 /* 256 bit BIP GMAC */ + /* wireless security bitvec */ @@ -232,6 +254,13 @@ static inline bool ac_bitmap_tst(u8 bitmap, int prec) #define WPA2_AUTH_PSK_SHA256 0x8000 /* PSK with SHA256 key derivation */ #define WPA3_AUTH_SAE_PSK 0x40000 /* SAE with 4-way handshake */ +#define WPA3_AUTH_OWE 0x100000 /* OWE */ +#define WFA_AUTH_DPP 0x200000 /* WFA DPP AUTH */ +#define WPA3_AUTH_1X_SUITE_B_SHA384 0x400000 /* Suite B-192 SHA384 */ + + +#define WFA_OUI "\x50\x6F\x9A" /* WFA OUI */ +#define DPP_VER 0x1A /* WFA DPP v1.0 */ #define DOT11_DEFAULT_RTS_LEN 2347 #define DOT11_DEFAULT_FRAG_LEN 2346 diff --git a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h index 0340bba968688f..5c3b8fb41194ae 100644 --- a/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h +++ b/drivers/net/wireless/broadcom/brcm80211/include/chipcommon.h @@ -302,6 +302,14 @@ struct chipcregs { #define PMU_RCTL_LOGIC_DISABLE_MASK (1 << 27) +/* watchdog */ +#define CC_WD_SSRESET_PCIE_F0_EN 0x10000000 +#define CC_WD_SSRESET_PCIE_F1_EN 0x20000000 +#define CC_WD_SSRESET_PCIE_F2_EN 0x40000000 +#define CC_WD_SSRESET_PCIE_ALL_FN_EN 0x80000000 +#define CC_WD_COUNTER_MASK 0x0fffffff +#define CC_WD_ENABLE_MASK 0xf0000000 + /* * Maximum delay for the PMU state transition in us. * This is an upper bound intended for spinwaits etc. diff --git a/drivers/net/wireless/marvell/libertas/if_usb.c b/drivers/net/wireless/marvell/libertas/if_usb.c index 8a6bf1365cfabb..d3b9f7619a1a03 100644 --- a/drivers/net/wireless/marvell/libertas/if_usb.c +++ b/drivers/net/wireless/marvell/libertas/if_usb.c @@ -114,8 +114,8 @@ static void if_usb_write_bulk_callback(struct urb *urb) static void if_usb_free(struct if_usb_card *cardp) { /* Unlink tx & rx urb */ - usb_kill_urb(cardp->tx_urb); - usb_kill_urb(cardp->rx_urb); + usb_kill_anchored_urbs(&cardp->tx_submitted); + usb_kill_anchored_urbs(&cardp->rx_submitted); usb_free_urb(cardp->tx_urb); cardp->tx_urb = NULL; @@ -221,6 +221,9 @@ static int if_usb_probe(struct usb_interface *intf, udev->descriptor.bDeviceSubClass, udev->descriptor.bDeviceProtocol); + init_usb_anchor(&cardp->rx_submitted); + init_usb_anchor(&cardp->tx_submitted); + for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) { endpoint = &iface_desc->endpoint[i].desc; if (usb_endpoint_is_bulk_in(endpoint)) { @@ -426,7 +429,12 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb goto tx_ret; } - usb_kill_urb(cardp->tx_urb); + /* check if there are pending URBs */ + if (!usb_anchor_empty(&cardp->tx_submitted)) { + lbs_deb_usbd(&cardp->udev->dev, "%s failed: pending URB\n", __func__); + ret = -EBUSY; + goto tx_ret; + } usb_fill_bulk_urb(cardp->tx_urb, cardp->udev, usb_sndbulkpipe(cardp->udev, @@ -435,8 +443,10 @@ static int usb_tx_block(struct if_usb_card *cardp, uint8_t *payload, uint16_t nb cardp->tx_urb->transfer_flags |= URB_ZERO_PACKET; + usb_anchor_urb(cardp->tx_urb, &cardp->tx_submitted); if ((ret = usb_submit_urb(cardp->tx_urb, GFP_ATOMIC))) { lbs_deb_usbd(&cardp->udev->dev, "usb_submit_urb failed: %d\n", ret); + usb_unanchor_urb(cardp->tx_urb); } else { lbs_deb_usb2(&cardp->udev->dev, "usb_submit_urb success\n"); ret = 0; @@ -467,8 +477,10 @@ static int __if_usb_submit_rx_urb(struct if_usb_card *cardp, cardp); lbs_deb_usb2(&cardp->udev->dev, "Pointer for rx_urb %p\n", cardp->rx_urb); + usb_anchor_urb(cardp->rx_urb, &cardp->rx_submitted); if ((ret = usb_submit_urb(cardp->rx_urb, GFP_ATOMIC))) { lbs_deb_usbd(&cardp->udev->dev, "Submit Rx URB failed: %d\n", ret); + usb_unanchor_urb(cardp->rx_urb); kfree_skb(skb); cardp->rx_skb = NULL; ret = -1; @@ -838,8 +850,8 @@ static void if_usb_prog_firmware(struct lbs_private *priv, int ret, } /* Cancel any pending usb business */ - usb_kill_urb(cardp->rx_urb); - usb_kill_urb(cardp->tx_urb); + usb_kill_anchored_urbs(&cardp->rx_submitted); + usb_kill_anchored_urbs(&cardp->tx_submitted); cardp->fwlastblksent = 0; cardp->fwdnldover = 0; @@ -869,8 +881,8 @@ static void if_usb_prog_firmware(struct lbs_private *priv, int ret, if (cardp->bootcmdresp == BOOT_CMD_RESP_NOT_SUPPORTED) { /* Return to normal operation */ ret = -EOPNOTSUPP; - usb_kill_urb(cardp->rx_urb); - usb_kill_urb(cardp->tx_urb); + usb_kill_anchored_urbs(&cardp->rx_submitted); + usb_kill_anchored_urbs(&cardp->tx_submitted); if (if_usb_submit_rx_urb(cardp) < 0) ret = -EIO; goto done; @@ -900,7 +912,7 @@ static void if_usb_prog_firmware(struct lbs_private *priv, int ret, wait_event_interruptible(cardp->fw_wq, cardp->surprise_removed || cardp->fwdnldover); timer_delete_sync(&cardp->fw_timeout); - usb_kill_urb(cardp->rx_urb); + usb_kill_anchored_urbs(&cardp->rx_submitted); if (!cardp->fwdnldover) { pr_info("failed to load fw, resetting device!\n"); @@ -960,8 +972,8 @@ static int if_usb_suspend(struct usb_interface *intf, pm_message_t message) goto out; /* Unlink tx & rx urb */ - usb_kill_urb(cardp->tx_urb); - usb_kill_urb(cardp->rx_urb); + usb_kill_anchored_urbs(&cardp->tx_submitted); + usb_kill_anchored_urbs(&cardp->rx_submitted); out: return ret; diff --git a/drivers/net/wireless/marvell/libertas/if_usb.h b/drivers/net/wireless/marvell/libertas/if_usb.h index 7d0daeb33c3f77..a0cd36197c2b0b 100644 --- a/drivers/net/wireless/marvell/libertas/if_usb.h +++ b/drivers/net/wireless/marvell/libertas/if_usb.h @@ -48,6 +48,9 @@ struct if_usb_card { struct urb *rx_urb, *tx_urb; struct lbs_private *priv; + struct usb_anchor rx_submitted; + struct usb_anchor tx_submitted; + struct sk_buff *rx_skb; uint8_t ep_in; diff --git a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c index 34b4b34276d6d3..042b1fe5f0d676 100644 --- a/drivers/net/wireless/marvell/mwifiex/11n_aggr.c +++ b/drivers/net/wireless/marvell/mwifiex/11n_aggr.c @@ -203,6 +203,7 @@ mwifiex_11n_aggregate_pkt(struct mwifiex_private *priv, if (!mwifiex_is_ralist_valid(priv, pra_list, ptrindex)) { spin_unlock_bh(&priv->wmm.ra_list_spinlock); + mwifiex_write_data_complete(adapter, skb_aggr, 1, -1); return -1; } diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c index 5c9a46e64d23c6..0c892501372480 100644 --- a/drivers/net/wireless/marvell/mwifiex/init.c +++ b/drivers/net/wireless/marvell/mwifiex/init.c @@ -391,7 +391,7 @@ static void mwifiex_invalidate_lists(struct mwifiex_adapter *adapter) static void mwifiex_adapter_cleanup(struct mwifiex_adapter *adapter) { - timer_delete(&adapter->wakeup_timer); + timer_delete_sync(&adapter->wakeup_timer); cancel_delayed_work_sync(&adapter->devdump_work); mwifiex_cancel_all_pending_cmd(adapter); wake_up_interruptible(&adapter->cmd_wait_q.wait); diff --git a/drivers/net/wireless/mediatek/mt76/channel.c b/drivers/net/wireless/mediatek/mt76/channel.c index 2b705bdb7993c4..d9f8529db7ed41 100644 --- a/drivers/net/wireless/mediatek/mt76/channel.c +++ b/drivers/net/wireless/mediatek/mt76/channel.c @@ -326,7 +326,7 @@ void mt76_roc_complete(struct mt76_phy *phy) mlink->mvif->roc_phy = NULL; if (phy->main_chandef.chan && !test_bit(MT76_MCU_RESET, &dev->phy.state)) - mt76_set_channel(phy, &phy->main_chandef, false); + __mt76_set_channel(phy, &phy->main_chandef, false); mt76_put_vif_phy_link(phy, phy->roc_vif, phy->roc_link); phy->roc_vif = NULL; phy->roc_link = NULL; @@ -370,6 +370,8 @@ int mt76_remain_on_channel(struct ieee80211_hw *hw, struct ieee80211_vif *vif, if (!phy) return -EINVAL; + cancel_delayed_work_sync(&phy->mac_work); + mutex_lock(&dev->mutex); if (phy->roc_vif || dev->scan.phy == phy || @@ -388,7 +390,14 @@ int mt76_remain_on_channel(struct ieee80211_hw *hw, struct ieee80211_vif *vif, phy->roc_vif = vif; phy->roc_link = mlink; cfg80211_chandef_create(&chandef, chan, NL80211_CHAN_HT20); - mt76_set_channel(phy, &chandef, true); + ret = __mt76_set_channel(phy, &chandef, true); + if (ret) { + mlink->mvif->roc_phy = NULL; + phy->roc_vif = NULL; + phy->roc_link = NULL; + mt76_put_vif_phy_link(phy, vif, mlink); + goto out; + } ieee80211_ready_on_channel(hw); ieee80211_queue_delayed_work(phy->hw, &phy->roc_work, msecs_to_jiffies(duration)); diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index f240016ed9f0ea..893ac14285cab3 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -874,7 +874,12 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) if (!buf) break; - if (!mt76_queue_is_wed_rro(q)) + if (mtk_wed_device_active(&dev->mmio.wed) && + mt76_queue_is_wed_rro(q)) + continue; + + if (!mt76_queue_is_wed_rro_rxdmad_c(q) && + !mt76_queue_is_wed_rro_ind(q)) mt76_put_page_pool_buf(buf, false); } while (1); @@ -1168,10 +1173,6 @@ void mt76_dma_cleanup(struct mt76_dev *dev) mt76_for_each_q_rx(dev, i) { struct mt76_queue *q = &dev->q_rx[i]; - if (mtk_wed_device_active(&dev->mmio.wed) && - mt76_queue_is_wed_rro(q)) - continue; - netif_napi_del(&dev->napi[i]); mt76_dma_rx_cleanup(dev, q); diff --git a/drivers/net/wireless/mediatek/mt76/eeprom.c b/drivers/net/wireless/mediatek/mt76/eeprom.c index 573400d57ce727..afdb73661866e9 100644 --- a/drivers/net/wireless/mediatek/mt76/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/eeprom.c @@ -9,6 +9,13 @@ #include #include #include "mt76.h" +#include "mt76_connac.h" + +enum mt76_sku_type { + MT76_SKU_RATE, + MT76_SKU_BACKOFF, + MT76_SKU_BACKOFF_BF_OFFSET, +}; static int mt76_get_of_eeprom_data(struct mt76_dev *dev, void *eep, int len) { @@ -292,7 +299,6 @@ mt76_find_channel_node(struct device_node *np, struct ieee80211_channel *chan) } EXPORT_SYMBOL_GPL(mt76_find_channel_node); - static s8 mt76_get_txs_delta(struct device_node *np, u8 nss) { @@ -306,9 +312,24 @@ mt76_get_txs_delta(struct device_node *np, u8 nss) return be32_to_cpu(val[nss - 1]); } +static inline u8 mt76_backoff_n_chains(struct mt76_dev *dev, u8 idx) +{ + /* 0:1T1ss, 1:2T1ss, ..., 14:5T5ss */ + static const u8 connac3_table[] = { + 1, 2, 3, 4, 5, 2, 3, 4, 5, 3, 4, 5, 4, 5, 5}; + static const u8 connac2_table[] = { + 1, 2, 3, 4, 2, 3, 4, 3, 4, 4, 0, 0, 0, 0, 0}; + + if (idx >= ARRAY_SIZE(connac3_table)) + return 0; + + return is_mt799x(dev) ? connac3_table[idx] : connac2_table[idx]; +} + static void -mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const s8 *data, - s8 target_power, s8 nss_delta, s8 *max_power) +mt76_apply_array_limit(struct mt76_dev *dev, s8 *pwr, size_t pwr_len, + const s8 *data, s8 target_power, s8 nss_delta, + s8 *max_power, int n_chains, enum mt76_sku_type type) { int i; @@ -316,18 +337,51 @@ mt76_apply_array_limit(s8 *pwr, size_t pwr_len, const s8 *data, return; for (i = 0; i < pwr_len; i++) { - pwr[i] = min_t(s8, target_power, data[i] + nss_delta); + u8 backoff_chain_idx = i; + int backoff_n_chains; + s8 backoff_delta; + s8 delta; + + switch (type) { + case MT76_SKU_RATE: + delta = 0; + backoff_delta = 0; + backoff_n_chains = 0; + break; + case MT76_SKU_BACKOFF_BF_OFFSET: + backoff_chain_idx += 1; + fallthrough; + case MT76_SKU_BACKOFF: + delta = mt76_tx_power_path_delta(n_chains); + backoff_n_chains = mt76_backoff_n_chains(dev, backoff_chain_idx); + backoff_delta = mt76_tx_power_path_delta(backoff_n_chains); + break; + default: + return; + } + + pwr[i] = min_t(s8, target_power + delta - backoff_delta, data[i] + nss_delta); + + /* used for padding, doesn't need to be considered */ + if (data[i] >= S8_MAX - 1) + continue; + + /* only consider backoff value for the configured chain number */ + if (type != MT76_SKU_RATE && n_chains != backoff_n_chains) + continue; + *max_power = max(*max_power, pwr[i]); } } static void -mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, - const s8 *data, size_t len, s8 target_power, - s8 nss_delta) +mt76_apply_multi_array_limit(struct mt76_dev *dev, s8 *pwr, size_t pwr_len, + s8 pwr_num, const s8 *data, size_t len, + s8 target_power, s8 nss_delta, s8 *max_power, + int n_chains, enum mt76_sku_type type) { + static const int connac2_backoff_ru_idx = 2; int i, cur; - s8 max_power = -128; if (!data) return; @@ -337,8 +391,26 @@ mt76_apply_multi_array_limit(s8 *pwr, size_t pwr_len, s8 pwr_num, if (len < pwr_len + 1) break; - mt76_apply_array_limit(pwr + pwr_len * i, pwr_len, data + 1, - target_power, nss_delta, &max_power); + /* Each RU entry (RU26, RU52, RU106, BW20, ...) in the DTS + * corresponds to 10 stream combinations (1T1ss, 2T1ss, 3T1ss, + * 4T1ss, 2T2ss, 3T2ss, 4T2ss, 3T3ss, 4T3ss, 4T4ss). + * + * For beamforming tables: + * - In connac2, beamforming entries for BW20~BW160 and OFDM + * do not include 1T1ss. + * - In connac3, beamforming entries for BW20~BW160 and RU + * include 1T1ss, but OFDM beamforming does not include 1T1ss. + * + * Non-beamforming and RU entries for both connac2 and connac3 + * include 1T1ss. + */ + if (!is_mt799x(dev) && type == MT76_SKU_BACKOFF && + i > connac2_backoff_ru_idx) + type = MT76_SKU_BACKOFF_BF_OFFSET; + + mt76_apply_array_limit(dev, pwr + pwr_len * i, pwr_len, data + 1, + target_power, nss_delta, max_power, + n_chains, type); if (--cur > 0) continue; @@ -360,18 +432,11 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, struct device_node *np; const s8 *val; char name[16]; - u32 mcs_rates = dev->drv->mcs_rates; - u32 ru_rates = ARRAY_SIZE(dest->ru[0]); char band; size_t len; - s8 max_power = 0; - s8 max_power_backoff = -127; + s8 max_power = -127; s8 txs_delta; int n_chains = hweight16(phy->chainmask); - s8 target_power_combine = target_power + mt76_tx_power_path_delta(n_chains); - - if (!mcs_rates) - mcs_rates = 10; memset(dest, target_power, sizeof(*dest) - sizeof(dest->path)); memset(&dest->path, 0, sizeof(dest->path)); @@ -409,46 +474,45 @@ s8 mt76_get_rate_power_limits(struct mt76_phy *phy, txs_delta = mt76_get_txs_delta(np, hweight16(phy->chainmask)); val = mt76_get_of_array_s8(np, "rates-cck", &len, ARRAY_SIZE(dest->cck)); - mt76_apply_array_limit(dest->cck, ARRAY_SIZE(dest->cck), val, - target_power, txs_delta, &max_power); + mt76_apply_array_limit(dev, dest->cck, ARRAY_SIZE(dest->cck), val, + target_power, txs_delta, &max_power, n_chains, MT76_SKU_RATE); - val = mt76_get_of_array_s8(np, "rates-ofdm", - &len, ARRAY_SIZE(dest->ofdm)); - mt76_apply_array_limit(dest->ofdm, ARRAY_SIZE(dest->ofdm), val, - target_power, txs_delta, &max_power); + val = mt76_get_of_array_s8(np, "rates-ofdm", &len, ARRAY_SIZE(dest->ofdm)); + mt76_apply_array_limit(dev, dest->ofdm, ARRAY_SIZE(dest->ofdm), val, + target_power, txs_delta, &max_power, n_chains, MT76_SKU_RATE); - val = mt76_get_of_array_s8(np, "rates-mcs", &len, mcs_rates + 1); - mt76_apply_multi_array_limit(dest->mcs[0], ARRAY_SIZE(dest->mcs[0]), - ARRAY_SIZE(dest->mcs), val, len, - target_power, txs_delta); + val = mt76_get_of_array_s8(np, "rates-mcs", &len, ARRAY_SIZE(dest->mcs[0]) + 1); + mt76_apply_multi_array_limit(dev, dest->mcs[0], ARRAY_SIZE(dest->mcs[0]), + ARRAY_SIZE(dest->mcs), val, len, target_power, + txs_delta, &max_power, n_chains, MT76_SKU_RATE); - val = mt76_get_of_array_s8(np, "rates-ru", &len, ru_rates + 1); - mt76_apply_multi_array_limit(dest->ru[0], ARRAY_SIZE(dest->ru[0]), - ARRAY_SIZE(dest->ru), val, len, - target_power, txs_delta); + val = mt76_get_of_array_s8(np, "rates-ru", &len, ARRAY_SIZE(dest->ru[0]) + 1); + mt76_apply_multi_array_limit(dev, dest->ru[0], ARRAY_SIZE(dest->ru[0]), + ARRAY_SIZE(dest->ru), val, len, target_power, + txs_delta, &max_power, n_chains, MT76_SKU_RATE); - max_power_backoff = max_power; val = mt76_get_of_array_s8(np, "paths-cck", &len, ARRAY_SIZE(dest->path.cck)); - mt76_apply_array_limit(dest->path.cck, ARRAY_SIZE(dest->path.cck), val, - target_power_combine, txs_delta, &max_power_backoff); + mt76_apply_array_limit(dev, dest->path.cck, ARRAY_SIZE(dest->path.cck), val, + target_power, txs_delta, &max_power, n_chains, MT76_SKU_BACKOFF); val = mt76_get_of_array_s8(np, "paths-ofdm", &len, ARRAY_SIZE(dest->path.ofdm)); - mt76_apply_array_limit(dest->path.ofdm, ARRAY_SIZE(dest->path.ofdm), val, - target_power_combine, txs_delta, &max_power_backoff); + mt76_apply_array_limit(dev, dest->path.ofdm, ARRAY_SIZE(dest->path.ofdm), val, + target_power, txs_delta, &max_power, n_chains, MT76_SKU_BACKOFF); val = mt76_get_of_array_s8(np, "paths-ofdm-bf", &len, ARRAY_SIZE(dest->path.ofdm_bf)); - mt76_apply_array_limit(dest->path.ofdm_bf, ARRAY_SIZE(dest->path.ofdm_bf), val, - target_power_combine, txs_delta, &max_power_backoff); + mt76_apply_array_limit(dev, dest->path.ofdm_bf, ARRAY_SIZE(dest->path.ofdm_bf), val, + target_power, txs_delta, &max_power, n_chains, + MT76_SKU_BACKOFF_BF_OFFSET); val = mt76_get_of_array_s8(np, "paths-ru", &len, ARRAY_SIZE(dest->path.ru[0]) + 1); - mt76_apply_multi_array_limit(dest->path.ru[0], ARRAY_SIZE(dest->path.ru[0]), - ARRAY_SIZE(dest->path.ru), val, len, - target_power_combine, txs_delta); + mt76_apply_multi_array_limit(dev, dest->path.ru[0], ARRAY_SIZE(dest->path.ru[0]), + ARRAY_SIZE(dest->path.ru), val, len, target_power, + txs_delta, &max_power, n_chains, MT76_SKU_BACKOFF); val = mt76_get_of_array_s8(np, "paths-ru-bf", &len, ARRAY_SIZE(dest->path.ru_bf[0]) + 1); - mt76_apply_multi_array_limit(dest->path.ru_bf[0], ARRAY_SIZE(dest->path.ru_bf[0]), - ARRAY_SIZE(dest->path.ru_bf), val, len, - target_power_combine, txs_delta); + mt76_apply_multi_array_limit(dev, dest->path.ru_bf[0], ARRAY_SIZE(dest->path.ru_bf[0]), + ARRAY_SIZE(dest->path.ru_bf), val, len, target_power, + txs_delta, &max_power, n_chains, MT76_SKU_BACKOFF); return max_power; } diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 75772979f438ea..4d041f88155c21 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -726,6 +726,7 @@ mt76_alloc_device(struct device *pdev, unsigned int size, INIT_LIST_HEAD(&dev->rxwi_cache); dev->token_size = dev->drv->token_size; INIT_DELAYED_WORK(&dev->scan_work, mt76_scan_work); + spin_lock_init(&dev->scan_lock); for (i = 0; i < ARRAY_SIZE(dev->q_rx); i++) skb_queue_head_init(&dev->rx_skb[i]); diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index d05e83ea1cacc4..df93ab79c5b48e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -540,7 +540,6 @@ struct mt76_driver_ops { u32 survey_flags; u16 txwi_size; u16 token_size; - u8 mcs_rates; unsigned int link_data_size; @@ -1002,6 +1001,7 @@ struct mt76_dev { u32 rxfilter; struct delayed_work scan_work; + spinlock_t scan_lock; struct { struct cfg80211_scan_request *req; struct ieee80211_channel *chan; @@ -1009,6 +1009,8 @@ struct mt76_dev { struct mt76_vif_link *mlink; struct mt76_phy *phy; int chan_idx; + bool beacon_wait; + bool beacon_received; } scan; #ifdef CONFIG_NL80211_TESTMODE @@ -1596,6 +1598,7 @@ int mt76_get_rate(struct mt76_dev *dev, int mt76_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_scan_request *hw_req); void mt76_cancel_hw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif); +void mt76_scan_rx_beacon(struct mt76_dev *dev, struct ieee80211_channel *chan); void mt76_sw_scan(struct ieee80211_hw *hw, struct ieee80211_vif *vif, const u8 *mac); void mt76_sw_scan_complete(struct ieee80211_hw *hw, diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index 45992fdcec60c3..ce005146850164 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -1167,21 +1167,6 @@ void mt7615_mac_set_rates(struct mt7615_phy *phy, struct mt7615_sta *sta, } EXPORT_SYMBOL_GPL(mt7615_mac_set_rates); -void mt7615_mac_enable_rtscts(struct mt7615_dev *dev, - struct ieee80211_vif *vif, bool enable) -{ - struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv; - u32 addr; - - addr = mt7615_mac_wtbl_addr(dev, mvif->sta.wcid.idx) + 3 * 4; - - if (enable) - mt76_set(dev, addr, MT_WTBL_W3_RTS); - else - mt76_clear(dev, addr, MT_WTBL_W3_RTS); -} -EXPORT_SYMBOL_GPL(mt7615_mac_enable_rtscts); - static int mt7615_mac_wtbl_update_key(struct mt7615_dev *dev, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c index 727266892c3df6..fc619acbb40d0b 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c @@ -583,9 +583,6 @@ static void mt7615_bss_info_changed(struct ieee80211_hw *hw, } } - if (changed & BSS_CHANGED_ERP_CTS_PROT) - mt7615_mac_enable_rtscts(dev, vif, info->use_cts_prot); - if (changed & BSS_CHANGED_BEACON_ENABLED && info->enable_beacon) { mt7615_mcu_add_bss_info(phy, vif, NULL, true); mt7615_mcu_sta_add(phy, vif, NULL, true); @@ -598,6 +595,10 @@ static void mt7615_bss_info_changed(struct ieee80211_hw *hw, BSS_CHANGED_BEACON_ENABLED)) mt7615_mcu_add_beacon(dev, hw, vif, info->enable_beacon); + if (changed & BSS_CHANGED_HT || changed & BSS_CHANGED_ERP_CTS_PROT) + mt7615_mcu_set_protection(phy, vif, info->ht_operation_mode, + info->use_cts_prot); + if (changed & BSS_CHANGED_PS) mt76_connac_mcu_set_vif_ps(&dev->mt76, vif); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c index fc0054f8bd602d..ff57ede87f7179 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mcu.c @@ -2564,3 +2564,50 @@ int mt7615_mcu_set_roc(struct mt7615_phy *phy, struct ieee80211_vif *vif, return mt76_mcu_send_msg(&dev->mt76, MCU_CE_CMD(SET_ROC), &req, sizeof(req), false); } + +int mt7615_mcu_set_protection(struct mt7615_phy *phy, struct ieee80211_vif *vif, + u8 ht_mode, bool use_cts_prot) +{ + struct mt7615_dev *dev = phy->dev; + struct { + u8 prot_idx; + u8 band; + u8 rsv[2]; + + bool long_nav; + bool prot_mm; + bool prot_gf; + bool prot_bw40; + bool prot_rifs; + bool prot_bw80; + bool prot_bw160; + u8 prot_erp_mask; + } __packed req = { + .prot_idx = 0x2, + .band = phy != &dev->phy, + }; + + switch (ht_mode & IEEE80211_HT_OP_MODE_PROTECTION) { + case IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER: + case IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED: + req.prot_mm = true; + req.prot_gf = true; + fallthrough; + case IEEE80211_HT_OP_MODE_PROTECTION_20MHZ: + req.prot_bw40 = true; + break; + } + + if (ht_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT) + req.prot_gf = true; + + if (use_cts_prot) { + struct mt7615_vif *mvif = (struct mt7615_vif *)vif->drv_priv; + u8 i = mvif->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0 : mvif->mt76.omac_idx; + + req.prot_erp_mask = BIT(i); + } + + return mt76_mcu_send_msg(&dev->mt76, MCU_EXT_CMD(PROTECT_CTRL), &req, + sizeof(req), true); +} diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h index c93fd245c90fb1..391928405f322d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mt7615.h @@ -467,8 +467,6 @@ void mt7615_mac_reset_counters(struct mt7615_phy *phy); void mt7615_mac_cca_stats_reset(struct mt7615_phy *phy); void mt7615_mac_set_scs(struct mt7615_phy *phy, bool enable); void mt7615_mac_enable_nf(struct mt7615_dev *dev, bool ext_phy); -void mt7615_mac_enable_rtscts(struct mt7615_dev *dev, - struct ieee80211_vif *vif, bool enable); void mt7615_mac_sta_poll(struct mt7615_dev *dev); int mt7615_mac_write_txwi(struct mt7615_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, @@ -523,7 +521,8 @@ int mt7615_mcu_set_sku_en(struct mt7615_phy *phy, bool enable); int mt7615_mcu_apply_rx_dcoc(struct mt7615_phy *phy); int mt7615_mcu_apply_tx_dpd(struct mt7615_phy *phy); int mt7615_dfs_init_radar_detector(struct mt7615_phy *phy); - +int mt7615_mcu_set_protection(struct mt7615_phy *phy, struct ieee80211_vif *vif, + u8 ht_mode, bool use_cts_prot); int mt7615_mcu_set_roc(struct mt7615_phy *phy, struct ieee80211_vif *vif, struct ieee80211_channel *chan, int duration); diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/regs.h b/drivers/net/wireless/mediatek/mt76/mt7615/regs.h index eb3c24d51987d1..e4133e9181d0f1 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7615/regs.h @@ -455,8 +455,6 @@ enum mt7615_reg_base { #define MT_WTBL_RIUCR3_RATE6 GENMASK(19, 8) #define MT_WTBL_RIUCR3_RATE7 GENMASK(31, 20) -#define MT_WTBL_W3_RTS BIT(22) - #define MT_WTBL_W5_CHANGE_BW_RATE GENMASK(7, 5) #define MT_WTBL_W5_SHORT_GI_20 BIT(8) #define MT_WTBL_W5_SHORT_GI_40 BIT(9) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c index b41ca1410da922..aab46305700930 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c @@ -1153,8 +1153,10 @@ void mt76_connac2_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi) return; wcid = (struct mt76_wcid *)sta->drv_priv; - if (!test_and_set_bit(tid, &wcid->ampdu_state)) - ieee80211_start_tx_ba_session(sta, tid, 0); + if (!test_and_set_bit(tid, &wcid->ampdu_state)) { + if (ieee80211_start_tx_ba_session(sta, tid, 0)) + clear_bit(tid, &wcid->ampdu_state); + } } EXPORT_SYMBOL_GPL(mt76_connac2_tx_check_aggr); diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index 0457712286d55c..3f583e2a1dc128 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -1295,8 +1295,10 @@ int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif_link *mvif, wtbl_hdr); ret = mt76_connac_mcu_sta_wed_update(dev, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } ret = mt76_mcu_skb_send_msg(dev, skb, cmd, true); if (ret) @@ -1309,8 +1311,10 @@ int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif_link *mvif, mt76_connac_mcu_sta_ba_tlv(skb, params, enable, tx); ret = mt76_connac_mcu_sta_wed_update(dev, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(dev, skb, cmd, true); } @@ -2764,12 +2768,16 @@ int mt76_connac_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, return PTR_ERR(skb); ret = mt76_connac_mcu_sta_key_tlv(sta_key_conf, skb, key, cmd); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } ret = mt76_connac_mcu_sta_wed_update(dev, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index 22443cbc74adc0..250c2d2479b0c6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -1294,6 +1294,7 @@ int mt7915_register_device(struct mt7915_dev *dev) void mt7915_unregister_device(struct mt7915_dev *dev) { + cancel_work_sync(&dev->dump_work); mt7915_unregister_ext_phy(dev); mt7915_coredump_unregister(dev); mt7915_unregister_thermal(&dev->phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index cefe56c05731dd..cec2c4208255fd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -232,19 +232,6 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev) rcu_read_unlock(); } -void mt7915_mac_enable_rtscts(struct mt7915_dev *dev, - struct ieee80211_vif *vif, bool enable) -{ - struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; - u32 addr; - - addr = mt7915_mac_wtbl_lmac_addr(dev, mvif->sta.wcid.idx, 5); - if (enable) - mt76_set(dev, addr, BIT(5)); - else - mt76_clear(dev, addr, BIT(5)); -} - static void mt7915_wed_check_ppe(struct mt7915_dev *dev, struct mt76_queue *q, struct mt7915_sta *msta, struct sk_buff *skb, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index 90d5e79fbf74dd..0892291616ead6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -68,7 +68,7 @@ int mt7915_run(struct ieee80211_hw *hw) if (ret) goto out; - ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, 0x92b, + ret = mt76_connac_mcu_set_rts_thresh(&dev->mt76, MT7915_RTS_LEN_THRES, phy->mt76->band_idx); if (ret) goto out; @@ -633,8 +633,9 @@ static void mt7915_bss_info_changed(struct ieee80211_hw *hw, if (set_sta == 1) mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_PORT_SECURE, false); - if (changed & BSS_CHANGED_ERP_CTS_PROT) - mt7915_mac_enable_rtscts(dev, vif, info->use_cts_prot); + if (changed & BSS_CHANGED_HT || changed & BSS_CHANGED_ERP_CTS_PROT) + mt7915_mcu_set_protection(phy, vif, info->ht_operation_mode, + info->use_cts_prot); if (changed & BSS_CHANGED_ERP_SLOT) { int slottime = 9; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index 00bff4d3aab806..023c92dac06484 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -1765,8 +1765,10 @@ int mt7915_mcu_add_sta(struct mt7915_dev *dev, struct ieee80211_vif *vif, } out: ret = mt76_connac_mcu_sta_wed_update(&dev->mt76, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_EXT_CMD(STA_REC_UPDATE), true); @@ -3954,6 +3956,68 @@ int mt7915_mcu_get_rx_rate(struct mt7915_phy *phy, struct ieee80211_vif *vif, return ret; } +int mt7915_mcu_set_protection(struct mt7915_phy *phy, struct ieee80211_vif *vif, + u8 ht_mode, bool use_cts_prot) +{ + struct mt7915_dev *dev = phy->dev; + int len = sizeof(struct sta_req_hdr) + sizeof(struct bss_info_prot); + struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; + struct bss_info_prot *prot; + struct sk_buff *skb; + struct tlv *tlv; + enum { + PROT_NONMEMBER = BIT(1), + PROT_20MHZ = BIT(2), + PROT_NONHT_MIXED = BIT(3), + PROT_LEGACY_ERP = BIT(5), + PROT_NONGF_STA = BIT(7), + }; + u32 rts_threshold; + + skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &mvif->mt76, + NULL, len); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + tlv = mt76_connac_mcu_add_tlv(skb, BSS_INFO_PROTECT_INFO, + sizeof(*prot)); + prot = (struct bss_info_prot *)tlv; + + switch (ht_mode & IEEE80211_HT_OP_MODE_PROTECTION) { + case IEEE80211_HT_OP_MODE_PROTECTION_NONMEMBER: + prot->prot_mode = cpu_to_le32(PROT_NONMEMBER); + break; + case IEEE80211_HT_OP_MODE_PROTECTION_20MHZ: + prot->prot_mode = cpu_to_le32(PROT_20MHZ); + break; + case IEEE80211_HT_OP_MODE_PROTECTION_NONHT_MIXED: + prot->prot_mode = cpu_to_le32(PROT_NONHT_MIXED); + break; + } + + if (ht_mode & IEEE80211_HT_OP_MODE_NON_GF_STA_PRSNT) + prot->prot_mode |= cpu_to_le32(PROT_NONGF_STA); + + if (use_cts_prot) + prot->prot_mode |= cpu_to_le32(PROT_LEGACY_ERP); + + /* reuse current RTS setting */ + rts_threshold = phy->mt76->hw->wiphy->rts_threshold; + if (rts_threshold == (u32)-1) + prot->rts_len_thres = cpu_to_le32(MT7915_RTS_LEN_THRES); + else + prot->rts_len_thres = cpu_to_le32(rts_threshold); + + prot->rts_pkt_thres = 0x2; + + prot->he_rts_thres = cpu_to_le16(vif->bss_conf.frame_time_rts_th); + if (!prot->he_rts_thres) + prot->he_rts_thres = cpu_to_le16(DEFAULT_HE_DURATION_RTS_THRES); + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_EXT_CMD(BSS_INFO_UPDATE), true); +} + int mt7915_mcu_update_bss_color(struct mt7915_dev *dev, struct ieee80211_vif *vif, struct cfg80211_he_bss_color *he_bss_color) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h index 3af11a075a2f44..22f73a5ed42590 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h @@ -399,6 +399,17 @@ struct bss_info_inband_discovery { __le16 prob_rsp_len; } __packed __aligned(4); +struct bss_info_prot { + __le16 tag; + __le16 len; + __le32 prot_type; + __le32 prot_mode; + __le32 rts_len_thres; + __le16 he_rts_thres; + u8 rts_pkt_thres; + u8 rsv[5]; +} __packed; + enum { BSS_INFO_BCN_CSA, BSS_INFO_BCN_BCC, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index b5c06201b7078d..bf1d915a3ca238 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -84,6 +84,8 @@ #define MT7915_CRIT_TEMP 110 #define MT7915_MAX_TEMP 120 +#define MT7915_RTS_LEN_THRES 0x92b + struct mt7915_vif; struct mt7915_sta; struct mt7915_dfs_pulse; @@ -473,6 +475,8 @@ int mt7915_mcu_add_inband_discov(struct mt7915_dev *dev, struct ieee80211_vif *v u32 changed); int mt7915_mcu_add_beacon(struct ieee80211_hw *hw, struct ieee80211_vif *vif, int enable, u32 changed); +int mt7915_mcu_set_protection(struct mt7915_phy *phy, struct ieee80211_vif *vif, + u8 ht_mode, bool use_cts_prot); int mt7915_mcu_add_obss_spr(struct mt7915_phy *phy, struct ieee80211_vif *vif, struct ieee80211_he_obss_pd *he_obss_pd); int mt7915_mcu_add_rate_ctrl(struct mt7915_dev *dev, struct ieee80211_vif *vif, diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c index 5fae9a6e273c02..3d74fabe74085e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c @@ -371,12 +371,15 @@ void mt7921_roc_abort_sync(struct mt792x_dev *dev) { struct mt792x_phy *phy = &dev->phy; + if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) + return; + timer_delete_sync(&phy->roc_timer); - cancel_work_sync(&phy->roc_work); - if (test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) - ieee80211_iterate_interfaces(mt76_hw(dev), - IEEE80211_IFACE_ITER_RESUME_ALL, - mt7921_roc_iter, (void *)phy); + cancel_work(&phy->roc_work); + + ieee80211_iterate_interfaces(mt76_hw(dev), + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7921_roc_iter, (void *)phy); } EXPORT_SYMBOL_GPL(mt7921_roc_abort_sync); @@ -387,10 +390,11 @@ void mt7921_roc_work(struct work_struct *work) phy = (struct mt792x_phy *)container_of(work, struct mt792x_phy, roc_work); - if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) - return; - mt792x_mutex_acquire(phy->dev); + if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) { + mt792x_mutex_release(phy->dev); + return; + } ieee80211_iterate_active_interfaces(phy->mt76->hw, IEEE80211_IFACE_ITER_RESUME_ALL, mt7921_roc_iter, phy); @@ -796,7 +800,8 @@ mt7921_regd_set_6ghz_power_type(struct ieee80211_vif *vif, bool is_add) } out: - mt7921_mcu_set_clc(dev, dev->mt76.alpha2, dev->country_ie_env); + if (vif->bss_conf.chanreq.oper.chan->band == NL80211_BAND_6GHZ) + mt7921_regd_update(dev); } int mt7921_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, @@ -807,6 +812,9 @@ int mt7921_mac_sta_add(struct mt76_dev *mdev, struct ieee80211_vif *vif, struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; int ret, idx; + if (sta->aid > MT7921_MAX_AID) + return -ENOENT; + idx = mt76_wcid_alloc(dev->mt76.wcid_mask, MT792x_WTBL_STA - 1); if (idx < 0) return -ENOSPC; @@ -850,6 +858,9 @@ int mt7921_mac_sta_event(struct mt76_dev *mdev, struct ieee80211_vif *vif, struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; struct mt792x_vif *mvif = (struct mt792x_vif *)vif->drv_priv; + if (sta->aid > MT7921_MAX_AID) + return -ENOENT; + if (ev != MT76_STA_EVENT_ASSOC) return 0; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index 833d0ab6423034..8442dbd2ee23f8 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -1353,6 +1353,9 @@ int __mt7921_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, u16 len = le16_to_cpu(rule->len); u16 offset = len + sizeof(*rule); + if (buf_len < offset) + break; + pos += offset; buf_len -= offset; if (rule->alpha2[0] != alpha2[0] || diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h index 83fc7f49ff8488..ad92af98e31453 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h @@ -7,6 +7,8 @@ #include "../mt792x.h" #include "regs.h" +#define MT7921_MAX_AID 20 + #define MT7921_TX_RING_SIZE 2048 #define MT7921_TX_MCU_RING_SIZE 256 #define MT7921_TX_FWDL_RING_SIZE 128 diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/init.c b/drivers/net/wireless/mediatek/mt76/mt7925/init.c index 3ce5d6fcc69df9..c0c5cb9aff75a4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/init.c @@ -91,6 +91,8 @@ int mt7925_mac_init(struct mt792x_dev *dev) mt7925_mac_init_basic_rates(dev); + memzero_explicit(&dev->mt76.alpha2, sizeof(dev->mt76.alpha2)); + return 0; } EXPORT_SYMBOL_GPL(mt7925_mac_init); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index 0d94359004233e..82eedd80f694d9 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -804,8 +804,8 @@ mt7925_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, txwi[5] = cpu_to_le32(val); val = MT_TXD6_DAS | FIELD_PREP(MT_TXD6_MSDU_CNT, 1); - if (!ieee80211_vif_is_mld(vif) || - (q_idx >= MT_LMAC_ALTX0 && q_idx <= MT_LMAC_BCN0)) + if (vif && (!ieee80211_vif_is_mld(vif) || + (q_idx >= MT_LMAC_ALTX0 && q_idx <= MT_LMAC_BCN0))) val |= MT_TXD6_DIS_MAT; txwi[6] = cpu_to_le32(val); txwi[7] = 0; @@ -846,11 +846,14 @@ static void mt7925_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb, bool is_8023; u16 fc, tid; + if (!sta) + return; + link_sta = rcu_dereference(sta->link[wcid->link_id]); if (!link_sta) return; - if (!sta || !(link_sta->ht_cap.ht_supported || link_sta->he_cap.has_he)) + if (!(link_sta->ht_cap.ht_supported || link_sta->he_cap.has_he)) return; tid = skb->priority & IEEE80211_QOS_CTL_TID_MASK; @@ -882,8 +885,10 @@ static void mt7925_tx_check_aggr(struct ieee80211_sta *sta, struct sk_buff *skb, else mlink = &msta->deflink; - if (!test_and_set_bit(tid, &mlink->wcid.ampdu_state)) - ieee80211_start_tx_ba_session(sta, tid, 0); + if (!test_and_set_bit(tid, &mlink->wcid.ampdu_state)) { + if (ieee80211_start_tx_ba_session(sta, tid, 0)) + clear_bit(tid, &mlink->wcid.ampdu_state); + } } static bool diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 2d358a96640c9d..fec54d5f4eaf1e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -457,12 +457,16 @@ void mt7925_roc_abort_sync(struct mt792x_dev *dev) { struct mt792x_phy *phy = &dev->phy; + if (!test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) + return; + timer_delete_sync(&phy->roc_timer); - cancel_work_sync(&phy->roc_work); - if (test_and_clear_bit(MT76_STATE_ROC, &phy->mt76->state)) - ieee80211_iterate_interfaces(mt76_hw(dev), - IEEE80211_IFACE_ITER_RESUME_ALL, - mt7925_roc_iter, (void *)phy); + + cancel_work(&phy->roc_work); + + ieee80211_iterate_interfaces(mt76_hw(dev), + IEEE80211_IFACE_ITER_RESUME_ALL, + mt7925_roc_iter, (void *)phy); } EXPORT_SYMBOL_GPL(mt7925_roc_abort_sync); @@ -541,7 +545,7 @@ static int mt7925_set_mlo_roc(struct mt792x_phy *phy, phy->roc_grant = false; - err = mt7925_mcu_set_mlo_roc(mconf, sel_links, 5, ++phy->roc_token_id); + err = mt7925_mcu_set_mlo_roc(phy, mconf, sel_links, 5, ++phy->roc_token_id); if (err < 0) { clear_bit(MT76_STATE_ROC, &phy->mt76->state); goto out; @@ -1894,10 +1898,8 @@ static void mt7925_link_info_changed(struct ieee80211_hw *hw, struct mt792x_phy *phy = mt792x_hw_phy(hw); struct mt792x_dev *dev = mt792x_hw_dev(hw); struct mt792x_bss_conf *mconf; - struct ieee80211_bss_conf *link_conf; mconf = mt792x_vif_to_link(mvif, info->link_id); - link_conf = mt792x_vif_to_bss_conf(vif, mconf->link_id); mt792x_mutex_acquire(dev); @@ -1939,10 +1941,6 @@ static void mt7925_link_info_changed(struct ieee80211_hw *hw, mvif->mlo_pm_state = MT792x_MLO_CHANGED_PS; } - if (changed & IEEE80211_CHANCTX_CHANGE_PUNCTURING) - mt7925_mcu_set_eht_pp(mvif->phy->mt76, &mconf->mt76, - link_conf, NULL); - if (changed & BSS_CHANGED_CQM) mt7925_mcu_set_rssimonitor(dev, vif); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index cf0fdea45cf735..abcdd0e0b3b5ae 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1288,14 +1288,16 @@ int mt7925_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, return PTR_ERR(skb); ret = mt7925_mcu_sta_key_tlv(wcid, sta_key_conf, skb, key, cmd, msta); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } -int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, - int duration, u8 token_id) +int mt7925_mcu_set_mlo_roc(struct mt792x_phy *phy, struct mt792x_bss_conf *mconf, + u16 sel_links, int duration, u8 token_id) { struct mt792x_vif *mvif = mconf->vif; struct ieee80211_vif *vif = container_of((void *)mvif, @@ -1330,6 +1332,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, .roc[1].len = cpu_to_le16(sizeof(struct roc_acquire_tlv)) }; + struct wiphy *wiphy = phy->mt76->hw->wiphy; + if (!mconf || hweight16(vif->valid_links) < 2 || hweight16(sel_links) != 2) return -EPERM; @@ -1352,7 +1356,8 @@ int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, is_AG_band |= links[i].chan->band == NL80211_BAND_2GHZ; } - if (vif->cfg.eml_cap & IEEE80211_EML_CAP_EMLSR_SUPP) + if (!(wiphy->iftype_ext_capab[0].mld_capa_and_ops & + IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS)) type = is_AG_band ? MT7925_ROC_REQ_MLSR_AG : MT7925_ROC_REQ_MLSR_AA; else @@ -3375,7 +3380,6 @@ __mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, u8 rsvd[64]; } __packed req = { .tag = cpu_to_le16(0x3), - .len = cpu_to_le16(sizeof(req) - 4), .idx = idx, .env = env_cap, @@ -3404,6 +3408,7 @@ __mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, memcpy(req.type, rule->type, 2); req.size = cpu_to_le16(seg->len); + req.len = cpu_to_le16(sizeof(req) + seg->len - 4); dev->phy.clc_chan_conf = clc->ver == 1 ? 0xff : rule->flag; skb = __mt76_mcu_msg_alloc(&dev->mt76, &req, le16_to_cpu(req.size) + sizeof(req), @@ -3727,7 +3732,7 @@ mt7925_mcu_rate_txpower_band(struct mt76_phy *phy, memcpy(tx_power_tlv->alpha2, dev->alpha2, sizeof(dev->alpha2)); tx_power_tlv->n_chan = num_ch; tx_power_tlv->tag = cpu_to_le16(0x1); - tx_power_tlv->len = cpu_to_le16(sizeof(*tx_power_tlv)); + tx_power_tlv->len = cpu_to_le16(msg_len); switch (band) { case NL80211_BAND_2GHZ: diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h b/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h index 6b9bf1b8903208..a1d902ccce6d24 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mt7925.h @@ -349,8 +349,8 @@ int mt7925_set_tx_sar_pwr(struct ieee80211_hw *hw, int mt7925_mcu_regval(struct mt792x_dev *dev, u32 regidx, u32 *val, bool set); int mt7925_mcu_set_clc(struct mt792x_dev *dev, u8 *alpha2, enum environment_cap env_cap); -int mt7925_mcu_set_mlo_roc(struct mt792x_bss_conf *mconf, u16 sel_links, - int duration, u8 token_id); +int mt7925_mcu_set_mlo_roc(struct mt792x_phy *phy, struct mt792x_bss_conf *mconf, + u16 sel_links, int duration, u8 token_id); int mt7925_mcu_set_roc(struct mt792x_phy *phy, struct mt792x_bss_conf *mconf, struct ieee80211_channel *chan, int duration, enum mt7925_roc_req type, u8 token_id); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/regd.c b/drivers/net/wireless/mediatek/mt76/mt7925/regd.c index 292087e882d1f1..16f56ee879d453 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/regd.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/regd.c @@ -232,7 +232,8 @@ int mt7925_regd_change(struct mt792x_phy *phy, char *alpha2) dev->regd_user) return -EINVAL; - if (mdev->alpha2[0] != '0' && mdev->alpha2[1] != '0') + if ((mdev->alpha2[0] && mdev->alpha2[0] != '0') && + (mdev->alpha2[1] && mdev->alpha2[1] != '0')) return 0; /* do not need to update the same country twice */ diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_regs.h b/drivers/net/wireless/mediatek/mt76/mt792x_regs.h index acf627aed609d6..699a15c44df022 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt792x_regs.h @@ -390,6 +390,10 @@ #define MT_CBTOP_RGU_WF_SUBSYS_RST MT_CBTOP_RGU(0x600) #define MT_CBTOP_RGU_WF_SUBSYS_RST_WF_WHOLE_PATH BIT(0) +#define MT7925_CBTOP_RGU_WF_SUBSYS_RST 0x70028600 +#define MT7925_WFSYS_INIT_DONE_ADDR 0x184c1604 +#define MT7925_WFSYS_INIT_DONE 0x00001d1e + #define MT_HW_BOUND 0x70010020 #define MT_HW_CHIPID 0x70010200 #define MT_HW_REV 0x70010204 diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_usb.c b/drivers/net/wireless/mediatek/mt76/mt792x_usb.c index 552808458138a6..47827d1c5ccb12 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_usb.c @@ -206,6 +206,33 @@ static void mt792xu_epctl_rst_opt(struct mt792x_dev *dev, bool reset) mt792xu_uhw_wr(&dev->mt76, MT_SSUSB_EPCTL_CSR_EP_RST_OPT, val); } +struct mt792xu_wfsys_desc { + u32 rst_reg; + u32 done_reg; + u32 done_mask; + u32 done_val; + u32 delay_ms; + bool need_status_sel; +}; + +static const struct mt792xu_wfsys_desc mt7921_wfsys_desc = { + .rst_reg = MT_CBTOP_RGU_WF_SUBSYS_RST, + .done_reg = MT_UDMA_CONN_INFRA_STATUS, + .done_mask = MT_UDMA_CONN_WFSYS_INIT_DONE, + .done_val = MT_UDMA_CONN_WFSYS_INIT_DONE, + .delay_ms = 0, + .need_status_sel = true, +}; + +static const struct mt792xu_wfsys_desc mt7925_wfsys_desc = { + .rst_reg = MT7925_CBTOP_RGU_WF_SUBSYS_RST, + .done_reg = MT7925_WFSYS_INIT_DONE_ADDR, + .done_mask = U32_MAX, + .done_val = MT7925_WFSYS_INIT_DONE, + .delay_ms = 20, + .need_status_sel = false, +}; + int mt792xu_dma_init(struct mt792x_dev *dev, bool resume) { int err; @@ -236,25 +263,33 @@ EXPORT_SYMBOL_GPL(mt792xu_dma_init); int mt792xu_wfsys_reset(struct mt792x_dev *dev) { + const struct mt792xu_wfsys_desc *desc = is_mt7925(&dev->mt76) ? + &mt7925_wfsys_desc : + &mt7921_wfsys_desc; u32 val; int i; mt792xu_epctl_rst_opt(dev, false); - val = mt792xu_uhw_rr(&dev->mt76, MT_CBTOP_RGU_WF_SUBSYS_RST); + val = mt792xu_uhw_rr(&dev->mt76, desc->rst_reg); val |= MT_CBTOP_RGU_WF_SUBSYS_RST_WF_WHOLE_PATH; - mt792xu_uhw_wr(&dev->mt76, MT_CBTOP_RGU_WF_SUBSYS_RST, val); + mt792xu_uhw_wr(&dev->mt76, desc->rst_reg, val); - usleep_range(10, 20); + if (desc->delay_ms) + msleep(desc->delay_ms); + else + usleep_range(10, 20); - val = mt792xu_uhw_rr(&dev->mt76, MT_CBTOP_RGU_WF_SUBSYS_RST); + val = mt792xu_uhw_rr(&dev->mt76, desc->rst_reg); val &= ~MT_CBTOP_RGU_WF_SUBSYS_RST_WF_WHOLE_PATH; - mt792xu_uhw_wr(&dev->mt76, MT_CBTOP_RGU_WF_SUBSYS_RST, val); + mt792xu_uhw_wr(&dev->mt76, desc->rst_reg, val); + + if (desc->need_status_sel) + mt792xu_uhw_wr(&dev->mt76, MT_UDMA_CONN_INFRA_STATUS_SEL, 0); - mt792xu_uhw_wr(&dev->mt76, MT_UDMA_CONN_INFRA_STATUS_SEL, 0); for (i = 0; i < MT792x_WFSYS_INIT_RETRY_COUNT; i++) { - val = mt792xu_uhw_rr(&dev->mt76, MT_UDMA_CONN_INFRA_STATUS); - if (val & MT_UDMA_CONN_WFSYS_INIT_DONE) + val = mt792xu_uhw_rr(&dev->mt76, desc->done_reg); + if ((val & desc->done_mask) == desc->done_val) break; msleep(100); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index 00a8286bd13686..fca2d84493b9b9 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -34,6 +34,20 @@ static const struct ieee80211_iface_combination if_comb_global = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), + .beacon_int_min_gcd = 100, +}; + +static const struct ieee80211_iface_combination if_comb_global_7992 = { + .limits = &if_limits_global, + .n_limits = 1, + .max_interfaces = 32, + .num_different_channels = MT7996_MAX_RADIOS - 1, + .radar_detect_widths = BIT(NL80211_CHAN_WIDTH_20_NOHT) | + BIT(NL80211_CHAN_WIDTH_20) | + BIT(NL80211_CHAN_WIDTH_40) | + BIT(NL80211_CHAN_WIDTH_80) | + BIT(NL80211_CHAN_WIDTH_160), + .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_limit if_limits[] = { @@ -485,7 +499,8 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed) hw->vif_data_size = sizeof(struct mt7996_vif); hw->chanctx_data_size = sizeof(struct mt76_chanctx); - wiphy->iface_combinations = &if_comb_global; + wiphy->iface_combinations = is_mt7996(&dev->mt76) ? &if_comb_global : + &if_comb_global_7992; wiphy->n_iface_combinations = 1; wiphy->radio = dev->radios; @@ -521,6 +536,7 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed) ieee80211_hw_set(hw, SUPPORTS_RX_DECAP_OFFLOAD); ieee80211_hw_set(hw, NO_VIRTUAL_MONITOR); ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); + ieee80211_hw_set(hw, CHANCTX_STA_CSA); hw->max_tx_fragments = 4; @@ -842,8 +858,7 @@ void mt7996_rro_hw_init(struct mt7996_dev *dev) } } else { /* set emul 3.0 function */ - mt76_wr(dev, MT_RRO_3_0_EMU_CONF, - MT_RRO_3_0_EMU_CONF_EN_MASK); + mt76_set(dev, MT_RRO_3_0_EMU_CONF, MT_RRO_3_0_EMU_CONF_EN_MASK); mt76_wr(dev, MT_RRO_ADDR_ARRAY_BASE0, dev->wed_rro.addr_elem[0].phy_addr); @@ -1726,6 +1741,7 @@ int mt7996_register_device(struct mt7996_dev *dev) void mt7996_unregister_device(struct mt7996_dev *dev) { + cancel_work_sync(&dev->dump_work); cancel_work_sync(&dev->wed_rro.work); mt7996_unregister_phy(mt7996_phy3(dev)); mt7996_unregister_phy(mt7996_phy2(dev)); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index d4f3ee943b4727..fc08ef94df6379 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -527,7 +527,7 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, !(csum_status & (BIT(0) | BIT(2) | BIT(3)))) skb->ip_summed = CHECKSUM_UNNECESSARY; - if (rxd1 & MT_RXD3_NORMAL_FCS_ERR) + if (rxd3 & MT_RXD3_NORMAL_FCS_ERR) status->flag |= RX_FLAG_FAILED_FCS_CRC; if (rxd1 & MT_RXD1_NORMAL_TKIP_MIC_ERR) @@ -554,6 +554,9 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, qos_ctl = FIELD_GET(MT_RXD10_QOS_CTL, v2); seq_ctrl = FIELD_GET(MT_RXD10_SEQ_CTRL, v2); + if (ieee80211_is_beacon(fc)) + mt76_scan_rx_beacon(&dev->mt76, mphy->chandef.chan); + rxd += 4; if ((u8 *)rxd - skb->data >= skb->len) return -EINVAL; @@ -1139,10 +1142,10 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, * req */ if (le32_to_cpu(ptr[7]) & MT_TXD7_MAC_TXD) { - u32 val; + u32 val, mac_txp_size = sizeof(struct mt76_connac_hw_txp); ptr = (__le32 *)(txwi + MT_TXD_SIZE); - memset((void *)ptr, 0, sizeof(struct mt76_connac_fw_txp)); + memset((void *)ptr, 0, mac_txp_size); val = FIELD_PREP(MT_TXP0_TOKEN_ID0, id) | MT_TXP0_TOKEN_ID0_VALID_MASK; @@ -1161,6 +1164,8 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, tx_info->buf[1].addr >> 32); #endif ptr[3] = cpu_to_le32(val); + + tx_info->buf[0].len = MT_TXD_SIZE + mac_txp_size; } else { struct mt76_connac_txp_common *txp; @@ -1270,8 +1275,9 @@ mt7996_tx_check_aggr(struct ieee80211_link_sta *link_sta, if (unlikely(fc != (IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA))) return; - if (!test_and_set_bit(tid, &wcid->ampdu_state)) - ieee80211_start_tx_ba_session(link_sta->sta, tid, 0); + if (!test_and_set_bit(tid, &wcid->ampdu_state) && + ieee80211_start_tx_ba_session(link_sta->sta, tid, 0)) + clear_bit(tid, &wcid->ampdu_state); } static void @@ -2397,6 +2403,7 @@ mt7996_mac_reset_sta_iter(void *data, struct ieee80211_sta *sta) for (i = 0; i < ARRAY_SIZE(msta->link); i++) { struct mt7996_sta_link *msta_link = NULL; + struct mt7996_phy *phy; msta_link = rcu_replace_pointer(msta->link[i], msta_link, lockdep_is_held(&dev->mt76.mutex)); @@ -2404,14 +2411,16 @@ mt7996_mac_reset_sta_iter(void *data, struct ieee80211_sta *sta) continue; mt7996_mac_sta_deinit_link(dev, msta_link); + phy = __mt7996_phy(dev, msta_link->wcid.phy_idx); + if (phy) + phy->mt76->num_sta--; - if (msta->deflink_id == i) { - msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; - continue; - } - - kfree_rcu(msta_link, rcu_head); + if (msta_link != &msta->deflink) + kfree_rcu(msta_link, rcu_head); } + + msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; + msta->seclink_id = msta->deflink_id; } static void @@ -2544,6 +2553,7 @@ void mt7996_mac_reset_work(struct work_struct *work) if (mtk_wed_device_active(&dev->mt76.mmio.wed)) mtk_wed_device_stop(&dev->mt76.mmio.wed); + mt7996_npu_hw_stop(dev); ieee80211_stop_queues(mt76_hw(dev)); set_bit(MT76_RESET, &dev->mphy.state); @@ -2570,8 +2580,6 @@ void mt7996_mac_reset_work(struct work_struct *work) mutex_lock(&dev->mt76.mutex); - mt7996_npu_hw_stop(dev); - mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { @@ -2591,7 +2599,7 @@ void mt7996_mac_reset_work(struct work_struct *work) mt7996_dma_start(dev, false, false); if (!is_mt7996(&dev->mt76) && dev->mt76.hwrro_mode == MT76_HWRRO_V3) - mt76_wr(dev, MT_RRO_3_0_EMU_CONF, MT_RRO_3_0_EMU_CONF_EN_MASK); + mt76_set(dev, MT_RRO_3_0_EMU_CONF, MT_RRO_3_0_EMU_CONF_EN_MASK); if (mtk_wed_device_active(&dev->mt76.mmio.wed)) { u32 wed_irq_mask = MT_INT_TX_DONE_BAND2 | @@ -2974,7 +2982,7 @@ static void mt7996_dfs_stop_radar_detector(struct mt7996_phy *phy) static int mt7996_dfs_start_rdd(struct mt7996_dev *dev, int rdd_idx) { - int err, region; + int region; switch (dev->mt76.region) { case NL80211_DFS_ETSI: @@ -2989,11 +2997,7 @@ static int mt7996_dfs_start_rdd(struct mt7996_dev *dev, int rdd_idx) break; } - err = mt7996_mcu_rdd_cmd(dev, RDD_START, rdd_idx, region); - if (err < 0) - return err; - - return mt7996_mcu_rdd_cmd(dev, RDD_DET_MODE, rdd_idx, 1); + return mt7996_mcu_rdd_cmd(dev, RDD_START, rdd_idx, region); } static int mt7996_dfs_start_radar_detector(struct mt7996_phy *phy) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index f16135f0b7f94c..ff3050c2344ab4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -79,6 +79,7 @@ static void mt7996_stop_phy(struct mt7996_phy *phy) mutex_lock(&dev->mt76.mutex); + mt7996_mcu_rdd_resume_tx(phy); mt7996_mcu_set_radio_en(phy, false); clear_bit(MT76_STATE_RUNNING, &phy->mt76->state); @@ -300,7 +301,6 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, .cmd = SET_KEY, .link_id = link_conf->link_id, }; - struct mt76_txq *mtxq; int mld_idx, idx, ret; mlink->idx = __ffs64(~dev->mt76.vif_mask); @@ -343,11 +343,6 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, mt7996_mac_wtbl_update(dev, idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); - if (vif->txq) { - mtxq = (struct mt76_txq *)vif->txq->drv_priv; - mtxq->wcid = idx; - } - if (vif->type != NL80211_IFTYPE_AP && (!mlink->omac_idx || mlink->omac_idx > 3)) vif->offload_flags = 0; @@ -370,9 +365,13 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); - if (!mlink->wcid->offchannel && - mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) + if (vif->txq && !mlink->wcid->offchannel && + mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) { + struct mt76_txq *mtxq = (struct mt76_txq *)vif->txq->drv_priv; + mvif->mt76.deflink_id = link_conf->link_id; + mtxq->wcid = idx; + } return 0; } @@ -403,17 +402,28 @@ void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif, rcu_assign_pointer(dev->mt76.wcid[idx], NULL); - if (!mlink->wcid->offchannel && + if (vif->txq && !mlink->wcid->offchannel && mvif->mt76.deflink_id == link_conf->link_id) { struct ieee80211_bss_conf *iter; + struct mt76_txq *mtxq; unsigned int link_id; mvif->mt76.deflink_id = IEEE80211_LINK_UNSPECIFIED; + mtxq = (struct mt76_txq *)vif->txq->drv_priv; + /* Primary link will be removed, look for a new one */ for_each_vif_active_link(vif, iter, link_id) { - if (link_id != IEEE80211_LINK_UNSPECIFIED) { - mvif->mt76.deflink_id = link_id; - break; - } + struct mt7996_vif_link *link; + + if (link_id == link_conf->link_id) + continue; + + link = mt7996_vif_link(dev, vif, link_id); + if (!link) + continue; + + mtxq->wcid = link->msta_link.wcid.idx; + mvif->mt76.deflink_id = link_id; + break; } } @@ -933,6 +943,40 @@ mt7996_channel_switch_beacon(struct ieee80211_hw *hw, mutex_unlock(&dev->mt76.mutex); } +static int +mt7996_post_channel_switch(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf) +{ + struct cfg80211_chan_def *chandef = &link_conf->chanreq.oper; + struct mt7996_dev *dev = mt7996_hw_dev(hw); + struct mt7996_phy *phy = mt7996_band_phy(dev, chandef->chan->band); + int ret; + + mutex_lock(&dev->mt76.mutex); + + ret = mt7996_mcu_rdd_resume_tx(phy); + + mutex_unlock(&dev->mt76.mutex); + + return ret; +} + +static void +mt7996_sta_init_txq_wcid(struct ieee80211_sta *sta, int idx) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { + struct mt76_txq *mtxq; + + if (!sta->txq[i]) + continue; + + mtxq = (struct mt76_txq *)sta->txq[i]->drv_priv; + mtxq->wcid = idx; + } +} + static int mt7996_mac_sta_init_link(struct mt7996_dev *dev, struct ieee80211_bss_conf *link_conf, @@ -950,21 +994,10 @@ mt7996_mac_sta_init_link(struct mt7996_dev *dev, return -ENOSPC; if (msta->deflink_id == IEEE80211_LINK_UNSPECIFIED) { - int i; - msta_link = &msta->deflink; msta->deflink_id = link_id; msta->seclink_id = msta->deflink_id; - - for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { - struct mt76_txq *mtxq; - - if (!sta->txq[i]) - continue; - - mtxq = (struct mt76_txq *)sta->txq[i]->drv_priv; - mtxq->wcid = idx; - } + mt7996_sta_init_txq_wcid(sta, idx); } else { msta_link = kzalloc_obj(*msta_link); if (!msta_link) @@ -1010,6 +1043,7 @@ void mt7996_mac_sta_deinit_link(struct mt7996_dev *dev, list_del_init(&msta_link->rc_list); spin_unlock_bh(&dev->mt76.sta_poll_lock); + rcu_assign_pointer(dev->mt76.wcid[msta_link->wcid.idx], NULL); mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid); mt76_wcid_mask_clear(dev->mt76.wcid_mask, msta_link->wcid.idx); } @@ -1024,8 +1058,7 @@ mt7996_mac_sta_remove_links(struct mt7996_dev *dev, struct ieee80211_vif *vif, for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) { struct mt7996_sta_link *msta_link = NULL; - struct mt7996_vif_link *link; - struct mt76_phy *mphy; + struct mt7996_phy *phy; msta_link = rcu_replace_pointer(msta->link[link_id], msta_link, lockdep_is_held(&mdev->mutex)); @@ -1034,25 +1067,36 @@ mt7996_mac_sta_remove_links(struct mt7996_dev *dev, struct ieee80211_vif *vif, mt7996_mac_wtbl_update(dev, msta_link->wcid.idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR); - mt7996_mac_sta_deinit_link(dev, msta_link); - link = mt7996_vif_link(dev, vif, link_id); - if (!link) - continue; - mphy = mt76_vif_link_phy(&link->mt76); - if (!mphy) - continue; + phy = __mt7996_phy(dev, msta_link->wcid.phy_idx); + if (phy) + phy->mt76->num_sta--; - mphy->num_sta--; if (msta->deflink_id == link_id) { msta->deflink_id = IEEE80211_LINK_UNSPECIFIED; - continue; + if (msta->seclink_id == link_id) { + /* no secondary link available */ + msta->seclink_id = msta->deflink_id; + } else { + struct mt7996_sta_link *msta_seclink; + + /* switch to the secondary link */ + msta_seclink = mt76_dereference( + msta->link[msta->seclink_id], + mdev); + if (msta_seclink) { + msta->deflink_id = msta->seclink_id; + mt7996_sta_init_txq_wcid(sta, + msta_seclink->wcid.idx); + } + } } else if (msta->seclink_id == link_id) { - msta->seclink_id = IEEE80211_LINK_UNSPECIFIED; + msta->seclink_id = msta->deflink_id; } - kfree_rcu(msta_link, rcu_head); + if (msta_link != &msta->deflink) + kfree_rcu(msta_link, rcu_head); } } @@ -2306,6 +2350,7 @@ const struct ieee80211_ops mt7996_ops = { .release_buffered_frames = mt76_release_buffered_frames, .get_txpower = mt7996_get_txpower, .channel_switch_beacon = mt7996_channel_switch_beacon, + .post_channel_switch = mt7996_post_channel_switch, .get_stats = mt7996_get_stats, .get_et_sset_count = mt7996_get_et_sset_count, .get_et_stats = mt7996_get_et_stats, diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index c0c042de477b87..20ade7ae7da953 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -233,7 +233,7 @@ mt7996_mcu_parse_response(struct mt76_dev *mdev, int cmd, event = (struct mt7996_mcu_uni_event *)skb->data; ret = le32_to_cpu(event->status); /* skip invalid event */ - if (mcu_cmd != event->cid) + if (mcu_cmd != le16_to_cpu(event->cid)) ret = -EAGAIN; } else { skb_pull(skb, sizeof(struct mt7996_mcu_rxd)); @@ -416,24 +416,32 @@ mt7996_mcu_rx_radar_detected(struct mt7996_dev *dev, struct sk_buff *skb) break; case MT_RDD_IDX_BACKGROUND: if (!dev->rdd2_phy) - return; + goto err; mphy = dev->rdd2_phy->mt76; break; default: - dev_err(dev->mt76.dev, "Unknown RDD idx %d\n", r->rdd_idx); - return; + goto err; } if (!mphy) - return; + goto err; - if (r->rdd_idx == MT_RDD_IDX_BACKGROUND) + if (r->rdd_idx == MT_RDD_IDX_BACKGROUND) { cfg80211_background_radar_event(mphy->hw->wiphy, &dev->rdd2_chandef, GFP_ATOMIC); - else + } else { + struct mt7996_phy *phy = mphy->priv; + + phy->rdd_tx_paused = true; ieee80211_radar_detected(mphy->hw, NULL); + } dev->hw_pattern++; + + return; + +err: + dev_err(dev->mt76.dev, "Invalid RDD idx %d\n", r->rdd_idx); } static void @@ -4557,6 +4565,35 @@ int mt7996_mcu_set_radio_en(struct mt7996_phy *phy, bool enable) &req, sizeof(req), true); } +int mt7996_mcu_rdd_resume_tx(struct mt7996_phy *phy) +{ + struct { + u8 band_idx; + u8 _rsv[3]; + + __le16 tag; + __le16 len; + u8 mac_enable; + u8 _rsv2[3]; + } __packed req = { + .band_idx = phy->mt76->band_idx, + .tag = cpu_to_le16(UNI_BAND_CONFIG_MAC_ENABLE_CTRL), + .len = cpu_to_le16(sizeof(req) - 4), + .mac_enable = 2, + }; + int ret; + + if (!phy->rdd_tx_paused) + return 0; + + ret = mt76_mcu_send_msg(&phy->dev->mt76, MCU_WM_UNI_CMD(BAND_CONFIG), + &req, sizeof(req), true); + if (!ret) + phy->rdd_tx_paused = false; + + return ret; +} + int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 rdd_idx, u8 val) { struct { diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h index e0b83ac9f5e2ae..f87a8d316f17d0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -25,8 +25,8 @@ struct mt7996_mcu_rxd { }; struct mt7996_mcu_uni_event { - u8 cid; - u8 __rsv[3]; + __le16 cid; + u8 __rsv[2]; __le32 status; /* 0: success, others: fail */ } __packed; @@ -837,6 +837,7 @@ enum { enum { UNI_BAND_CONFIG_RADIO_ENABLE, UNI_BAND_CONFIG_RTS_THRESHOLD = 0x08, + UNI_BAND_CONFIG_MAC_ENABLE_CTRL = 0x0c, }; enum { diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index 7a884311800ea8..d31864f973cceb 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -377,6 +377,7 @@ struct mt7996_phy { bool has_aux_rx; bool counter_reset; + bool rdd_tx_paused; }; struct mt7996_dev { @@ -726,6 +727,7 @@ int mt7996_mcu_get_temperature(struct mt7996_phy *phy); int mt7996_mcu_set_thermal_throttling(struct mt7996_phy *phy, u8 state); int mt7996_mcu_set_thermal_protect(struct mt7996_phy *phy, bool enable); int mt7996_mcu_set_txpower_sku(struct mt7996_phy *phy); +int mt7996_mcu_rdd_resume_tx(struct mt7996_phy *phy); int mt7996_mcu_rdd_cmd(struct mt7996_dev *dev, int cmd, u8 rdd_idx, u8 val); int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, struct cfg80211_chan_def *chandef); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/npu.c b/drivers/net/wireless/mediatek/mt76/mt7996/npu.c index 29bb735da4cb8f..067ef647e40407 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/npu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/npu.c @@ -320,33 +320,38 @@ int mt7996_npu_hw_init(struct mt7996_dev *dev) int mt7996_npu_hw_stop(struct mt7996_dev *dev) { struct airoha_npu *npu; - int i, err; + int i, err = 0; u32 info; + mutex_lock(&dev->mt76.mutex); + npu = rcu_dereference_protected(dev->mt76.mmio.npu, &dev->mt76.mutex); if (!npu) - return 0; + goto unlock; err = mt76_npu_send_msg(npu, 4, WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR, 0, GFP_KERNEL); if (err) - return err; + goto unlock; for (i = 0; i < 10; i++) { err = mt76_npu_get_msg(npu, 3, WLAN_FUNC_GET_WAIT_NPU_INFO, &info, GFP_KERNEL); - if (err) - continue; + if (!err && !info) + break; - if (info) { - err = -ETIMEDOUT; - continue; - } + err = -ETIMEDOUT; + usleep_range(10000, 15000); } if (!err) err = mt76_npu_send_msg(npu, 6, WLAN_FUNC_SET_WAIT_INODE_TXRX_REG_ADDR, 0, GFP_KERNEL); + else + dev_err(dev->mt76.dev, "npu stop failed\n"); +unlock: + mutex_unlock(&dev->mt76.mutex); + return err; } diff --git a/drivers/net/wireless/mediatek/mt76/npu.c b/drivers/net/wireless/mediatek/mt76/npu.c index ec36975f6dc946..9679237f739842 100644 --- a/drivers/net/wireless/mediatek/mt76/npu.c +++ b/drivers/net/wireless/mediatek/mt76/npu.c @@ -457,6 +457,7 @@ int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type) dev->mmio.npu_type = type; /* NPU offloading requires HW-RRO for RX packet reordering. */ dev->hwrro_mode = MT76_HWRRO_V3_1; + dev->rx_token_size = 32768; rcu_assign_pointer(dev->mmio.npu, npu); rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev); diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c index 63b0447e55c15b..ab153b4df04795 100644 --- a/drivers/net/wireless/mediatek/mt76/scan.c +++ b/drivers/net/wireless/mediatek/mt76/scan.c @@ -16,7 +16,7 @@ static void mt76_scan_complete(struct mt76_dev *dev, bool abort) clear_bit(MT76_SCANNING, &phy->state); - if (dev->scan.chan && phy->main_chandef.chan && + if (dev->scan.chan && phy->main_chandef.chan && phy->offchannel && !test_bit(MT76_MCU_RESET, &dev->phy.state)) mt76_set_channel(phy, &phy->main_chandef, false); mt76_put_vif_phy_link(phy, dev->scan.vif, dev->scan.mlink); @@ -27,6 +27,10 @@ static void mt76_scan_complete(struct mt76_dev *dev, bool abort) void mt76_abort_scan(struct mt76_dev *dev) { + spin_lock_bh(&dev->scan_lock); + dev->scan.beacon_wait = false; + spin_unlock_bh(&dev->scan_lock); + cancel_delayed_work_sync(&dev->scan_work); mt76_scan_complete(dev, true); } @@ -77,6 +81,28 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) rcu_read_unlock(); } +void mt76_scan_rx_beacon(struct mt76_dev *dev, struct ieee80211_channel *chan) +{ + struct mt76_phy *phy; + + spin_lock(&dev->scan_lock); + + if (!dev->scan.beacon_wait || dev->scan.beacon_received || + dev->scan.chan != chan) + goto out; + + phy = dev->scan.phy; + if (!phy) + goto out; + + dev->scan.beacon_received = true; + ieee80211_queue_delayed_work(phy->hw, &dev->scan_work, 0); + +out: + spin_unlock(&dev->scan_lock); +} +EXPORT_SYMBOL_GPL(mt76_scan_rx_beacon); + void mt76_scan_work(struct work_struct *work) { struct mt76_dev *dev = container_of(work, struct mt76_dev, @@ -85,14 +111,26 @@ void mt76_scan_work(struct work_struct *work) struct cfg80211_chan_def chandef = {}; struct mt76_phy *phy = dev->scan.phy; int duration = HZ / 9; /* ~110 ms */ + bool beacon_rx, offchannel = true; int i; + if (!phy || !req) + return; + + spin_lock_bh(&dev->scan_lock); + beacon_rx = dev->scan.beacon_wait && dev->scan.beacon_received; + dev->scan.beacon_wait = false; + spin_unlock_bh(&dev->scan_lock); + + if (beacon_rx) + goto probe; + if (dev->scan.chan_idx >= req->n_channels) { mt76_scan_complete(dev, false); return; } - if (dev->scan.chan && phy->num_sta) { + if (dev->scan.chan && phy->num_sta && phy->offchannel) { dev->scan.chan = NULL; mt76_set_channel(phy, &phy->main_chandef, false); goto out; @@ -100,20 +138,34 @@ void mt76_scan_work(struct work_struct *work) dev->scan.chan = req->channels[dev->scan.chan_idx++]; cfg80211_chandef_create(&chandef, dev->scan.chan, NL80211_CHAN_HT20); - mt76_set_channel(phy, &chandef, true); + if (phy->main_chandef.chan == dev->scan.chan) { + chandef = phy->main_chandef; + offchannel = false; + } + + mt76_set_channel(phy, &chandef, offchannel); - if (!req->n_ssids || - chandef.chan->flags & (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR)) + if (!req->n_ssids) goto out; - duration = HZ / 16; /* ~60 ms */ + if (chandef.chan->flags & (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR)) { + spin_lock_bh(&dev->scan_lock); + dev->scan.beacon_received = false; + dev->scan.beacon_wait = true; + spin_unlock_bh(&dev->scan_lock); + goto out; + } + +probe: + if (phy->offchannel) + duration = HZ / 16; /* ~60 ms */ local_bh_disable(); for (i = 0; i < req->n_ssids; i++) mt76_scan_send_probe(dev, &req->ssids[i]); local_bh_enable(); out: - if (dev->scan.chan) + if (dev->scan.chan && phy->offchannel) duration = max_t(int, duration, msecs_to_jiffies(req->duration + (req->duration >> 5))); diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index 794187d28caa09..2f872c4f2eeeed 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -4697,20 +4697,6 @@ static const struct ieee80211_rate rtl8xxxu_legacy_ratetable[] = { {.bitrate = 540, .hw_value = 0x0b,}, }; -static void rtl8xxxu_desc_to_mcsrate(u16 rate, u8 *mcs, u8 *nss) -{ - if (rate <= DESC_RATE_54M) - return; - - if (rate >= DESC_RATE_MCS0 && rate <= DESC_RATE_MCS15) { - if (rate < DESC_RATE_MCS8) - *nss = 1; - else - *nss = 2; - *mcs = rate - DESC_RATE_MCS0; - } -} - static void rtl8xxxu_set_basic_rates(struct rtl8xxxu_priv *priv, u32 rate_cfg) { struct ieee80211_hw *hw = priv->hw; @@ -4820,23 +4806,25 @@ static void rtl8xxxu_set_aifs(struct rtl8xxxu_priv *priv, u8 slot_time) void rtl8xxxu_update_ra_report(struct rtl8xxxu_ra_report *rarpt, u8 rate, u8 sgi, u8 bw) { - u8 mcs, nss; - rarpt->txrate.flags = 0; if (rate <= DESC_RATE_54M) { rarpt->txrate.legacy = rtl8xxxu_legacy_ratetable[rate].bitrate; - } else { - rtl8xxxu_desc_to_mcsrate(rate, &mcs, &nss); + } else if (rate >= DESC_RATE_MCS0 && rate <= DESC_RATE_MCS15) { rarpt->txrate.flags |= RATE_INFO_FLAGS_MCS; + if (rate < DESC_RATE_MCS8) + rarpt->txrate.nss = 1; + else + rarpt->txrate.nss = 2; - rarpt->txrate.mcs = mcs; - rarpt->txrate.nss = nss; + rarpt->txrate.mcs = rate - DESC_RATE_MCS0; if (sgi) rarpt->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI; rarpt->txrate.bw = bw; + } else { + return; } rarpt->bit_rate = cfg80211_calculate_bitrate(&rarpt->txrate); diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index d080469264cf89..f0010336e78c1b 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -1674,6 +1674,7 @@ static void rtl_pci_deinit(struct ieee80211_hw *hw) synchronize_irq(rtlpci->pdev->irq); tasklet_kill(&rtlpriv->works.irq_tasklet); + tasklet_kill(&rtlpriv->works.irq_prepare_bcn_tasklet); cancel_work_sync(&rtlpriv->works.lps_change_work); } diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index 56b16186d3aa4b..ec0a45bfb670eb 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -1804,7 +1804,8 @@ int rtw_pci_probe(struct pci_dev *pdev, } /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */ - if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL) + if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && + bridge && bridge->vendor == PCI_VENDOR_ID_INTEL) rtwpci->rx_no_aspm = true; rtw_pci_phy_cfg(rtwdev); diff --git a/drivers/net/wireless/realtek/rtw88/usb.c b/drivers/net/wireless/realtek/rtw88/usb.c index 433b06c8d8a625..718940ebba3161 100644 --- a/drivers/net/wireless/realtek/rtw88/usb.c +++ b/drivers/net/wireless/realtek/rtw88/usb.c @@ -1041,7 +1041,7 @@ static int rtw_usb_intf_init(struct rtw_dev *rtwdev, struct usb_interface *intf) { struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); - struct usb_device *udev = usb_get_dev(interface_to_usbdev(intf)); + struct usb_device *udev = interface_to_usbdev(intf); int ret; rtwusb->udev = udev; @@ -1067,7 +1067,6 @@ static void rtw_usb_intf_deinit(struct rtw_dev *rtwdev, { struct rtw_usb *rtwusb = rtw_get_usb_priv(rtwdev); - usb_put_dev(rtwusb->udev); kfree(rtwusb->usb_data); usb_set_intfdata(intf, NULL); } diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c index ee6ab2136b9a74..ee8a36003e5da4 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.c +++ b/drivers/net/wireless/realtek/rtw89/phy.c @@ -4860,7 +4860,7 @@ static void rtw89_phy_cfo_set_crystal_cap(struct rtw89_dev *rtwdev, { struct rtw89_cfo_tracking_info *cfo = &rtwdev->cfo_tracking; const struct rtw89_chip_info *chip = rtwdev->chip; - u8 sc_xi_val, sc_xo_val; + u8 sc_xi_val = 0, sc_xo_val = 0; if (!force && cfo->crystal_cap == crystal_cap) return; diff --git a/drivers/net/wireless/rsi/rsi_common.h b/drivers/net/wireless/rsi/rsi_common.h index 7aa5124575cfef..c40f8101febcb8 100644 --- a/drivers/net/wireless/rsi/rsi_common.h +++ b/drivers/net/wireless/rsi/rsi_common.h @@ -70,12 +70,11 @@ static inline int rsi_create_kthread(struct rsi_common *common, return 0; } -static inline int rsi_kill_thread(struct rsi_thread *handle) +static inline void rsi_kill_thread(struct rsi_thread *handle) { atomic_inc(&handle->thread_done); rsi_set_event(&handle->event); - - return kthread_stop(handle->task); + wait_for_completion(&handle->completion); } void rsi_mac80211_detach(struct rsi_hw *hw); diff --git a/drivers/net/wwan/t7xx/t7xx_modem_ops.c b/drivers/net/wwan/t7xx/t7xx_modem_ops.c index 7968e208dd37c1..adb29d30c63fe7 100644 --- a/drivers/net/wwan/t7xx/t7xx_modem_ops.c +++ b/drivers/net/wwan/t7xx/t7xx_modem_ops.c @@ -457,8 +457,20 @@ static int t7xx_parse_host_rt_data(struct t7xx_fsm_ctl *ctl, struct t7xx_sys_inf offset = sizeof(struct feature_query); for (i = 0; i < FEATURE_COUNT && offset < data_length; i++) { + size_t remaining = data_length - offset; + size_t feat_data_len, feat_total; + + if (remaining < sizeof(*rt_feature)) + break; + rt_feature = data + offset; - offset += sizeof(*rt_feature) + le32_to_cpu(rt_feature->data_len); + feat_data_len = le32_to_cpu(rt_feature->data_len); + + if (feat_data_len > remaining - sizeof(*rt_feature)) + break; + + feat_total = sizeof(*rt_feature) + feat_data_len; + offset += feat_total; ft_spt_cfg = FIELD_GET(FEATURE_MSK, core->feature_set[i]); if (ft_spt_cfg != MTK_FEATURE_MUST_BE_SUPPORTED) @@ -468,8 +480,10 @@ static int t7xx_parse_host_rt_data(struct t7xx_fsm_ctl *ctl, struct t7xx_sys_inf if (ft_spt_st != MTK_FEATURE_MUST_BE_SUPPORTED) return -EINVAL; - if (i == RT_ID_MD_PORT_ENUM || i == RT_ID_AP_PORT_ENUM) - t7xx_port_enum_msg_handler(ctl->md, rt_feature->data); + if (i == RT_ID_MD_PORT_ENUM || i == RT_ID_AP_PORT_ENUM) { + t7xx_port_enum_msg_handler(ctl->md, rt_feature->data, + feat_data_len); + } } return 0; diff --git a/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c b/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c index ae632ef966983e..f869e4ed9ee9a9 100644 --- a/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c +++ b/drivers/net/wwan/t7xx/t7xx_port_ctrl_msg.c @@ -117,6 +117,7 @@ static int fsm_ee_message_handler(struct t7xx_port *port, struct t7xx_fsm_ctl *c * t7xx_port_enum_msg_handler() - Parse the port enumeration message to create/remove nodes. * @md: Modem context. * @msg: Message. + * @msg_len: Length of @msg in bytes. * * Used to control create/remove device node. * @@ -124,12 +125,18 @@ static int fsm_ee_message_handler(struct t7xx_port *port, struct t7xx_fsm_ctl *c * * 0 - Success. * * -EFAULT - Message check failure. */ -int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg) +int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg, size_t msg_len) { struct device *dev = &md->t7xx_dev->pdev->dev; unsigned int version, port_count, i; struct port_msg *port_msg = msg; + if (msg_len < sizeof(*port_msg)) { + dev_err(dev, "Port enum msg too short for header: need %zu, have %zu\n", + sizeof(*port_msg), msg_len); + return -EINVAL; + } + version = FIELD_GET(PORT_MSG_VERSION, le32_to_cpu(port_msg->info)); if (version != PORT_ENUM_VER || le32_to_cpu(port_msg->head_pattern) != PORT_ENUM_HEAD_PATTERN || @@ -141,6 +148,13 @@ int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg) } port_count = FIELD_GET(PORT_MSG_PRT_CNT, le32_to_cpu(port_msg->info)); + + if (msg_len < struct_size(port_msg, data, port_count)) { + dev_err(dev, "Port enum msg too short: need %zu, have %zu\n", + struct_size(port_msg, data, port_count), msg_len); + return -EINVAL; + } + for (i = 0; i < port_count; i++) { u32 port_info = le32_to_cpu(port_msg->data[i]); unsigned int ch_id; @@ -191,7 +205,7 @@ static int control_msg_handler(struct t7xx_port *port, struct sk_buff *skb) case CTL_ID_PORT_ENUM: skb_pull(skb, sizeof(*ctrl_msg_h)); - ret = t7xx_port_enum_msg_handler(ctl->md, (struct port_msg *)skb->data); + ret = t7xx_port_enum_msg_handler(ctl->md, (struct port_msg *)skb->data, skb->len); if (!ret) ret = port_ctl_send_msg_to_md(port, CTL_ID_PORT_ENUM, 0); else diff --git a/drivers/net/wwan/t7xx/t7xx_port_proxy.h b/drivers/net/wwan/t7xx/t7xx_port_proxy.h index f0918b36e899bd..7c3190bf0fcf39 100644 --- a/drivers/net/wwan/t7xx/t7xx_port_proxy.h +++ b/drivers/net/wwan/t7xx/t7xx_port_proxy.h @@ -103,7 +103,7 @@ void t7xx_port_proxy_reset(struct port_proxy *port_prox); void t7xx_port_proxy_uninit(struct port_proxy *port_prox); int t7xx_port_proxy_init(struct t7xx_modem *md); void t7xx_port_proxy_md_status_notify(struct port_proxy *port_prox, unsigned int state); -int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg); +int t7xx_port_enum_msg_handler(struct t7xx_modem *md, void *msg, size_t msg_len); int t7xx_port_proxy_chl_enable_disable(struct port_proxy *port_prox, unsigned int ch_id, bool en_flag); void t7xx_port_proxy_set_cfg(struct t7xx_modem *md, enum port_cfg_id cfg_id); diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c index d17c701c7888b3..08c27bb438b597 100644 --- a/drivers/nfc/trf7970a.c +++ b/drivers/nfc/trf7970a.c @@ -317,6 +317,7 @@ #define TRF7970A_RSSI_OSC_STATUS_RSSI_MASK (BIT(2) | BIT(1) | BIT(0)) #define TRF7970A_RSSI_OSC_STATUS_RSSI_X_MASK (BIT(5) | BIT(4) | BIT(3)) #define TRF7970A_RSSI_OSC_STATUS_RSSI_OSC_OK BIT(6) +#define TRF7970A_RSSI_OSC_STATUS_RSSI_NOISE_LEVEL 1 #define TRF7970A_SPECIAL_FCN_REG1_COL_7_6 BIT(0) #define TRF7970A_SPECIAL_FCN_REG1_14_ANTICOLL BIT(1) @@ -1300,7 +1301,7 @@ static int trf7970a_is_rf_field(struct trf7970a *trf, bool *is_rf_field) if (ret) return ret; - if (rssi & TRF7970A_RSSI_OSC_STATUS_RSSI_MASK) + if ((rssi & TRF7970A_RSSI_OSC_STATUS_RSSI_MASK) > TRF7970A_RSSI_OSC_STATUS_RSSI_NOISE_LEVEL) *is_rf_field = true; else *is_rf_field = false; diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 78e02fe6caba0c..2e77b699be2a40 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1779,12 +1779,13 @@ static void ntb_tx_copy_callback(void *data, static void ntb_memcpy_tx_on_stack(struct ntb_queue_entry *entry, void __iomem *offset) { -#ifdef ARCH_HAS_NOCACHE_UACCESS +#ifdef copy_to_nontemporal /* * Using non-temporal mov to improve performance on non-cached - * writes, even though we aren't actually copying from user space. + * writes. This only works if __iomem is strictly memory-like, + * but that is the case on x86-64 */ - __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len); + copy_to_nontemporal(offset, entry->buf, entry->len); #else memcpy_toio(offset, entry->buf, entry->len); #endif diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index ed61b97fde59f7..6f07c9245096c6 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -203,8 +203,20 @@ struct apple_nvme { int irq; spinlock_t lock; + + /* + * Delayed cache flush handling state + */ + struct nvme_ns *flush_ns; + unsigned long flush_interval; + unsigned long last_flush; + struct delayed_work flush_dwork; }; +unsigned int flush_interval = 1000; +module_param(flush_interval, uint, 0644); +MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); + static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); @@ -762,6 +774,26 @@ static int apple_nvme_remove_sq(struct apple_nvme *anv) return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); } +static bool apple_nvme_delayed_flush(struct apple_nvme *anv, struct nvme_ns *ns, + struct request *req) +{ + if (!anv->flush_interval || req_op(req) != REQ_OP_FLUSH) + return false; + if (delayed_work_pending(&anv->flush_dwork)) + return true; + if (time_before(jiffies, anv->last_flush + anv->flush_interval)) { + kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &anv->flush_dwork, + anv->flush_interval); + if (WARN_ON_ONCE(anv->flush_ns && anv->flush_ns != ns)) + goto out; + anv->flush_ns = ns; + return true; + } +out: + anv->last_flush = jiffies; + return false; +} + static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -798,6 +830,11 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, nvme_start_request(req); + if (apple_nvme_delayed_flush(anv, ns, req)) { + blk_mq_complete_request(req); + return BLK_STS_OK; + } + if (anv->hw->has_lsq_nvmmu) apple_nvme_submit_cmd_t8103(q, cmnd); else @@ -1009,6 +1046,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) unsigned int depth = apple_nvme_queue_depth(q); struct apple_nvme *anv = queue_to_apple_nvme(q); + q->sq_tail = 0; q->cq_head = 0; q->cq_phase = 1; if (anv->hw->has_lsq_nvmmu) @@ -1267,11 +1305,7 @@ static int apple_nvme_get_address(struct nvme_ctrl *ctrl, char *buf, int size) static void apple_nvme_free_ctrl(struct nvme_ctrl *ctrl) { - struct apple_nvme *anv = ctrl_to_apple_nvme(ctrl); - - if (anv->ctrl.admin_q) - blk_put_queue(anv->ctrl.admin_q); - put_device(anv->dev); + put_device(ctrl->dev); } static const struct nvme_ctrl_ops nvme_ctrl_ops = { @@ -1453,6 +1487,28 @@ static void devm_apple_nvme_mempool_destroy(void *data) mempool_destroy(data); } +static void apple_nvme_flush_work(struct work_struct *work) +{ + struct nvme_command c = { }; + struct apple_nvme *anv; + struct nvme_ns *ns; + int err; + + anv = container_of(work, struct apple_nvme, flush_dwork.work); + ns = anv->flush_ns; + if (WARN_ON_ONCE(!ns)) + return; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(anv->flush_ns->head->ns_id); + err = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (err) { + dev_err(anv->dev, "Deferred flush failed: %d\n", err); + } else { + anv->last_flush = jiffies; + } +} + static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1621,6 +1677,14 @@ static int apple_nvme_probe(struct platform_device *pdev) goto out_uninit_ctrl; } + if (flush_interval) { + anv->flush_interval = msecs_to_jiffies(flush_interval); + anv->flush_ns = NULL; + anv->last_flush = jiffies - anv->flush_interval; + } + + INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush_work); + nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -1658,6 +1722,7 @@ static void apple_nvme_shutdown(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + flush_delayed_work(&anv->flush_dwork); apple_nvme_disable(anv, true); if (apple_rtkit_is_running(anv->rtk)) { apple_rtkit_shutdown(anv->rtk); diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 766e9cc4ffca5e..ce25c8a4e84bc7 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3388,7 +3388,7 @@ static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl) ctrl->dmrl = id->dmrl; ctrl->dmrsl = le32_to_cpu(id->dmrsl); - if (id->wzsl) + if (id->wzsl && !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES)) ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl); free_data: diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index b78ba239c8ea8e..4c052ed18cb8df 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2241,6 +2241,7 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled) static const struct blk_mq_ops nvme_mq_admin_ops = { .queue_rq = nvme_queue_rq, .complete = nvme_pci_complete_rq, + .commit_rqs = nvme_commit_rqs, .init_hctx = nvme_admin_init_hctx, .init_request = nvme_pci_init_request, .timeout = nvme_timeout, @@ -4178,6 +4179,8 @@ static const struct pci_device_id nvme_id_table[] = { .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x2646, 0x501E), /* KINGSTON OM3PGP4xxxxQ OS21011 NVMe SSD */ .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, + { PCI_DEVICE(0x2646, 0x502F), /* KINGSTON OM3SGP4xxxxK NVMe SSD */ + .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, }, { PCI_DEVICE(0x1f40, 0x1202), /* Netac Technologies Co. NV3000 NVMe SSD */ .driver_data = NVME_QUIRK_BOGUS_NID, }, { PCI_DEVICE(0x1f40, 0x5236), /* Netac Technologies Co. NV7000 NVMe SSD */ diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 9238e13bd480ac..ba4e1a5e0d4c8e 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1743,7 +1743,7 @@ static void nvmet_ctrl_free(struct kref *ref) nvmet_stop_keep_alive_timer(ctrl); - flush_work(&ctrl->async_event_work); + cancel_work_sync(&ctrl->async_event_work); cancel_work_sync(&ctrl->fatal_err_work); nvmet_destroy_auth(ctrl); diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index acc71a26733f90..dc65894696ad9b 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -351,7 +351,7 @@ static void nvmet_tcp_free_cmd_buffers(struct nvmet_tcp_cmd *cmd) static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue); -static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) +static int nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) { struct bio_vec *iov = cmd->iov; struct scatterlist *sg; @@ -364,22 +364,19 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) offset = cmd->rbytes_done; cmd->sg_idx = offset / PAGE_SIZE; sg_offset = offset % PAGE_SIZE; - if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt) { - nvmet_tcp_fatal_error(cmd->queue); - return; - } + if (!cmd->req.sg_cnt || cmd->sg_idx >= cmd->req.sg_cnt) + return -EPROTO; + sg = &cmd->req.sg[cmd->sg_idx]; sg_remaining = cmd->req.sg_cnt - cmd->sg_idx; while (length) { - if (!sg_remaining) { - nvmet_tcp_fatal_error(cmd->queue); - return; - } - if (!sg->length || sg->length <= sg_offset) { - nvmet_tcp_fatal_error(cmd->queue); - return; - } + if (!sg_remaining) + return -EPROTO; + + if (!sg->length || sg->length <= sg_offset) + return -EPROTO; + u32 iov_len = min_t(u32, length, sg->length - sg_offset); bvec_set_page(iov, sg_page(sg), iov_len, @@ -394,10 +391,24 @@ static void nvmet_tcp_build_pdu_iovec(struct nvmet_tcp_cmd *cmd) iov_iter_bvec(&cmd->recv_msg.msg_iter, ITER_DEST, cmd->iov, nr_pages, cmd->pdu_len); + return 0; } static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue) { + /* + * Keep rcv_state at RECV_ERR even for the internal -ESHUTDOWN path. + * nvmet_tcp_handle_icreq() can return -ESHUTDOWN after the ICReq has + * already been consumed and queue teardown has started. + * + * If nvmet_tcp_data_ready() or nvmet_tcp_write_space() queues + * nvmet_tcp_io_work() again before nvmet_tcp_release_queue_work() + * cancels it, the queue must not keep that old receive state. + * Otherwise the next nvmet_tcp_io_work() run can reach + * nvmet_tcp_done_recv_pdu() and try to handle the same ICReq again. + * + * That is why queue->rcv_state needs to be updated before we return. + */ queue->rcv_state = NVMET_TCP_RECV_ERR; if (queue->nvme_sq.ctrl) nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); @@ -922,16 +933,29 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue) iov.iov_len = sizeof(*icresp); ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); if (ret < 0) { + spin_lock_bh(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_DISCONNECTING) { + spin_unlock_bh(&queue->state_lock); + return -ESHUTDOWN; + } queue->state = NVMET_TCP_Q_FAILED; + spin_unlock_bh(&queue->state_lock); return ret; /* queue removal will cleanup */ } + spin_lock_bh(&queue->state_lock); + if (queue->state == NVMET_TCP_Q_DISCONNECTING) { + spin_unlock_bh(&queue->state_lock); + /* Tell nvmet_tcp_socket_error() teardown is in progress. */ + return -ESHUTDOWN; + } queue->state = NVMET_TCP_Q_LIVE; + spin_unlock_bh(&queue->state_lock); nvmet_prepare_receive_pdu(queue); return 0; } -static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue, +static int nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue, struct nvmet_tcp_cmd *cmd, struct nvmet_req *req) { size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length); @@ -947,19 +971,23 @@ static void nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue, if (!nvme_is_write(cmd->req.cmd) || !data_len || data_len > cmd->req.port->inline_data_size) { nvmet_prepare_receive_pdu(queue); - return; + return 0; } ret = nvmet_tcp_map_data(cmd); if (unlikely(ret)) { pr_err("queue %d: failed to map data\n", queue->idx); nvmet_tcp_fatal_error(queue); - return; + return -EPROTO; } queue->rcv_state = NVMET_TCP_RECV_DATA; - nvmet_tcp_build_pdu_iovec(cmd); cmd->flags |= NVMET_TCP_F_INIT_FAILED; + ret = nvmet_tcp_build_pdu_iovec(cmd); + if (unlikely(ret)) + pr_err("queue %d: failed to build PDU iovec\n", queue->idx); + + return ret; } static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) @@ -1011,7 +1039,10 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) goto err_proto; } cmd->pdu_recv = 0; - nvmet_tcp_build_pdu_iovec(cmd); + if (unlikely(nvmet_tcp_build_pdu_iovec(cmd))) { + pr_err("queue %d: failed to build PDU iovec\n", queue->idx); + goto err_proto; + } queue->cmd = cmd; queue->rcv_state = NVMET_TCP_RECV_DATA; @@ -1074,8 +1105,7 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) le32_to_cpu(req->cmd->common.dptr.sgl.length), le16_to_cpu(req->cqe->status)); - nvmet_tcp_handle_req_failure(queue, queue->cmd, req); - return 0; + return nvmet_tcp_handle_req_failure(queue, queue->cmd, req); } ret = nvmet_tcp_map_data(queue->cmd); @@ -1092,8 +1122,11 @@ static int nvmet_tcp_done_recv_pdu(struct nvmet_tcp_queue *queue) if (nvmet_tcp_need_data_in(queue->cmd)) { if (nvmet_tcp_has_inline_data(queue->cmd)) { queue->rcv_state = NVMET_TCP_RECV_DATA; - nvmet_tcp_build_pdu_iovec(queue->cmd); - return 0; + ret = nvmet_tcp_build_pdu_iovec(queue->cmd); + if (unlikely(ret)) + pr_err("queue %d: failed to build PDU iovec\n", + queue->idx); + return ret; } /* send back R2T */ nvmet_tcp_queue_response(&queue->cmd->req); diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index 311cb2e5a5c02d..877b867d4080a9 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -1615,12 +1615,14 @@ static void nvmem_shift_read_buffer_in_place(struct nvmem_cell_entry *cell, void *p = *b++ >> bit_offset; /* setup rest of the bytes if any */ - for (i = 1; i < cell->bytes; i++) { + for (i = 1; i < (cell->bytes - bytes_offset); i++) { /* Get bits from next byte and shift them towards msb */ *p++ |= *b << (BITS_PER_BYTE - bit_offset); *p = *b++ >> bit_offset; } + /* point to end of the buffer unused bits will be cleared */ + p = buf + cell->bytes - 1; } else if (p != b) { memmove(p, b, cell->bytes - bytes_offset); p += cell->bytes - 1; diff --git a/drivers/of/base.c b/drivers/of/base.c index 57420806c1a2b9..8d1972e1816106 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -1915,7 +1915,7 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)) if (name) of_stdout = of_find_node_opts_by_path(name, &of_stdout_options); if (of_stdout) - of_stdout->fwnode.flags |= FWNODE_FLAG_BEST_EFFORT; + fwnode_set_flag(&of_stdout->fwnode, FWNODE_FLAG_BEST_EFFORT); } if (!of_aliases) diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index 1a06175def3773..ade288372101bb 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -225,7 +225,7 @@ static void __of_attach_node(struct device_node *np) np->sibling = np->parent->child; np->parent->child = np; of_node_clear_flag(np, OF_DETACHED); - np->fwnode.flags |= FWNODE_FLAG_NOT_DEVICE; + fwnode_set_flag(&np->fwnode, FWNODE_FLAG_NOT_DEVICE); raw_spin_unlock_irqrestore(&devtree_lock, flags); diff --git a/drivers/of/platform.c b/drivers/of/platform.c index ba591fbceb5695..7eeaf8e27b5b21 100644 --- a/drivers/of/platform.c +++ b/drivers/of/platform.c @@ -742,7 +742,7 @@ static int of_platform_notify(struct notifier_block *nb, * Clear the flag before adding the device so that fw_devlink * doesn't skip adding consumers to this device. */ - rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE; + fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); /* pdev_parent may be NULL when no bus platform device */ pdev_parent = of_find_device_by_node(parent); pdev = of_platform_device_create(rd->dn, NULL, diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 2940295843e6fc..4078569a0f9674 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -896,8 +896,6 @@ static void __init of_unittest_changeset(void) unittest(!of_changeset_apply(&chgset), "apply failed\n"); - of_node_put(nchangeset); - /* Make sure node names are constructed correctly */ unittest((np = of_find_node_by_path("/testcase-data/changeset/n2/n21")), "'%pOF' not added\n", n21); @@ -919,6 +917,7 @@ static void __init of_unittest_changeset(void) if (!ret) unittest(strcmp(propstr, "hello") == 0, "original value not in updated property after revert"); + of_node_put(nchangeset); of_changeset_destroy(&chgset); of_node_put(n1); @@ -4318,7 +4317,6 @@ static int testdrv_probe(struct pci_dev *pdev, const struct pci_device_id *id) size = info->dtbo_end - info->dtbo_begin; ret = of_overlay_fdt_apply(info->dtbo_begin, size, &ovcs_id, dn); - of_node_put(dn); if (ret) return ret; diff --git a/drivers/opp/core.c b/drivers/opp/core.c index 866641666e410c..da3f5eba434197 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -2742,8 +2742,8 @@ struct dev_pm_opp *dev_pm_opp_xlate_required_opp(struct opp_table *src_table, break; } } - break; } + break; } if (IS_ERR(dest_opp)) { diff --git a/drivers/opp/debugfs.c b/drivers/opp/debugfs.c index 8fc6238b172841..61506d30d5ff00 100644 --- a/drivers/opp/debugfs.c +++ b/drivers/opp/debugfs.c @@ -130,22 +130,24 @@ void opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table) { struct dentry *pdentry = opp_table->dentry; struct dentry *d; - unsigned long id; - char name[25]; /* 20 chars for 64 bit value + 5 (opp:\0) */ + char name[36]; /* "opp:"(4) + u64(20) + "-" (1) + u32(10) + NULL(1) */ /* * Get directory name for OPP. * - * - Normally rate is unique to each OPP, use it to get unique opp-name. + * - Normally rate is unique to each OPP, use it to get unique opp-name, + * together with performance level if available. * - For some devices rate isn't available or there are multiple, use * index instead for them. */ - if (likely(opp_table->clk_count == 1 && opp->rates[0])) - id = opp->rates[0]; - else - id = _get_opp_count(opp_table); - - snprintf(name, sizeof(name), "opp:%lu", id); + if (likely(opp_table->clk_count == 1 && opp->rates[0])) { + if (opp->level == OPP_LEVEL_UNSET) + snprintf(name, sizeof(name), "opp:%lu", opp->rates[0]); + else + snprintf(name, sizeof(name), "opp:%lu-%u", opp->rates[0], opp->level); + } else { + snprintf(name, sizeof(name), "opp:%u", _get_opp_count(opp_table)); + } /* Create per-opp directory */ d = debugfs_create_dir(name, pdentry); diff --git a/drivers/parisc/lasi.c b/drivers/parisc/lasi.c index ef6125d838788b..a5b80cd5cc37d7 100644 --- a/drivers/parisc/lasi.c +++ b/drivers/parisc/lasi.c @@ -193,8 +193,7 @@ static int __init lasi_init_chip(struct parisc_device *dev) ret = request_irq(lasi->gsc_irq.irq, gsc_asic_intr, 0, "lasi", lasi); if (ret < 0) { - kfree(lasi); - return ret; + goto err_free; } /* enable IRQ's for devices below LASI */ @@ -203,8 +202,7 @@ static int __init lasi_init_chip(struct parisc_device *dev) /* Done init'ing, register this driver */ ret = gsc_common_setup(dev, lasi); if (ret) { - kfree(lasi); - return ret; + goto err_irq; } gsc_fixup_irqs(dev, lasi, lasi_choose_irq); @@ -214,6 +212,12 @@ static int __init lasi_init_chip(struct parisc_device *dev) SYS_OFF_PRIO_DEFAULT, lasi_power_off, lasi); return ret; + +err_irq: + free_irq(lasi->gsc_irq.irq, lasi); +err_free: + kfree(lasi); + return ret; } static struct parisc_device_id lasi_tbl[] __initdata = { diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 5aaed8ac6e443b..a08bca9aa086fe 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -45,6 +45,7 @@ config PCIE_APPLE depends on ARCH_APPLE || COMPILE_TEST depends on OF depends on PCI_MSI + depends on PAGE_SIZE_16KB || COMPILE_TEST select PCI_HOST_COMMON select IRQ_MSI_LIB help diff --git a/drivers/pci/controller/cadence/pci-sky1.c b/drivers/pci/controller/cadence/pci-sky1.c index d8c216dc120d6f..9853a9c82c0e67 100644 --- a/drivers/pci/controller/cadence/pci-sky1.c +++ b/drivers/pci/controller/cadence/pci-sky1.c @@ -176,8 +176,10 @@ static int sky1_pcie_probe(struct platform_device *pdev) cdns_pcie->is_rc = 1; reg_off = devm_kzalloc(dev, sizeof(*reg_off), GFP_KERNEL); - if (!reg_off) + if (!reg_off) { + pci_ecam_free(pcie->cfg); return -ENOMEM; + } reg_off->ip_reg_bank_offset = SKY1_IP_REG_BANK; reg_off->ip_cfg_ctrl_reg_offset = SKY1_IP_CFG_CTRL_REG_BANK; diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c index db3154c1eccbf8..0bc9e6e90e0e00 100644 --- a/drivers/pci/controller/cadence/pcie-cadence-host.c +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -147,6 +147,13 @@ static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0); cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); + value = cdns_pcie_rp_readl(pcie, CDNS_PCIE_RP_CAP_OFFSET + PCI_EXP_LNKCAP); + if (rc->quirk_broken_aspm_l0s) + value &= ~PCI_EXP_LNKCAP_ASPM_L0S; + if (rc->quirk_broken_aspm_l1) + value &= ~PCI_EXP_LNKCAP_ASPM_L1; + cdns_pcie_rp_writel(pcie, CDNS_PCIE_RP_CAP_OFFSET + PCI_EXP_LNKCAP, value); + return 0; } diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h index 443033c607d752..574e9cf4d003f4 100644 --- a/drivers/pci/controller/cadence/pcie-cadence.h +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -115,6 +115,8 @@ struct cdns_pcie { * @quirk_detect_quiet_flag: LTSSM Detect Quiet min delay set as quirk * @ecam_supported: Whether the ECAM is supported * @no_inbound_map: Whether inbound mapping is supported + * @quirk_broken_aspm_l0s: Disable ASPM L0s support as quirk + * @quirk_broken_aspm_l1: Disable ASPM L1 support as quirk */ struct cdns_pcie_rc { struct cdns_pcie pcie; @@ -127,6 +129,8 @@ struct cdns_pcie_rc { unsigned int quirk_detect_quiet_flag:1; unsigned int ecam_supported:1; unsigned int no_inbound_map:1; + unsigned int quirk_broken_aspm_l0s:1; + unsigned int quirk_broken_aspm_l1:1; }; /** @@ -249,37 +253,6 @@ static inline u32 cdns_pcie_hpa_readl(struct cdns_pcie *pcie, return readl(pcie->reg_base + reg); } -static inline u16 cdns_pcie_readw(struct cdns_pcie *pcie, u32 reg) -{ - return readw(pcie->reg_base + reg); -} - -static inline u8 cdns_pcie_readb(struct cdns_pcie *pcie, u32 reg) -{ - return readb(pcie->reg_base + reg); -} - -static inline int cdns_pcie_read_cfg_byte(struct cdns_pcie *pcie, int where, - u8 *val) -{ - *val = cdns_pcie_readb(pcie, where); - return PCIBIOS_SUCCESSFUL; -} - -static inline int cdns_pcie_read_cfg_word(struct cdns_pcie *pcie, int where, - u16 *val) -{ - *val = cdns_pcie_readw(pcie, where); - return PCIBIOS_SUCCESSFUL; -} - -static inline int cdns_pcie_read_cfg_dword(struct cdns_pcie *pcie, int where, - u32 *val) -{ - *val = cdns_pcie_readl(pcie, where); - return PCIBIOS_SUCCESSFUL; -} - static inline u32 cdns_pcie_read_sz(void __iomem *addr, int size) { void __iomem *aligned_addr = PTR_ALIGN_DOWN(addr, 0x4); @@ -320,6 +293,31 @@ static inline void cdns_pcie_write_sz(void __iomem *addr, int size, u32 value) writel(val, aligned_addr); } +static inline int cdns_pcie_read_cfg_byte(struct cdns_pcie *pcie, int where, + u8 *val) +{ + void __iomem *addr = pcie->reg_base + where; + + *val = cdns_pcie_read_sz(addr, 0x1); + return PCIBIOS_SUCCESSFUL; +} + +static inline int cdns_pcie_read_cfg_word(struct cdns_pcie *pcie, int where, + u16 *val) +{ + void __iomem *addr = pcie->reg_base + where; + + *val = cdns_pcie_read_sz(addr, 0x2); + return PCIBIOS_SUCCESSFUL; +} + +static inline int cdns_pcie_read_cfg_dword(struct cdns_pcie *pcie, int where, + u32 *val) +{ + *val = cdns_pcie_readl(pcie, where); + return PCIBIOS_SUCCESSFUL; +} + /* Root Port register access */ static inline void cdns_pcie_rp_writeb(struct cdns_pcie *pcie, u32 reg, u8 value) @@ -344,6 +342,21 @@ static inline u16 cdns_pcie_rp_readw(struct cdns_pcie *pcie, u32 reg) return cdns_pcie_read_sz(addr, 0x2); } +static inline void cdns_pcie_rp_writel(struct cdns_pcie *pcie, + u32 reg, u32 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + cdns_pcie_write_sz(addr, 0x4, value); +} + +static inline u32 cdns_pcie_rp_readl(struct cdns_pcie *pcie, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + return cdns_pcie_read_sz(addr, 0x4); +} + static inline void cdns_pcie_hpa_rp_writeb(struct cdns_pcie *pcie, u32 reg, u8 value) { diff --git a/drivers/pci/controller/cadence/pcie-sg2042.c b/drivers/pci/controller/cadence/pcie-sg2042.c index 0c50c74d03eeb2..4a2af4d0713e65 100644 --- a/drivers/pci/controller/cadence/pcie-sg2042.c +++ b/drivers/pci/controller/cadence/pcie-sg2042.c @@ -48,6 +48,8 @@ static int sg2042_pcie_probe(struct platform_device *pdev) bridge->child_ops = &sg2042_pcie_child_ops; rc = pci_host_bridge_priv(bridge); + rc->quirk_broken_aspm_l0s = 1; + rc->quirk_broken_aspm_l1 = 1; pcie = &rc->pcie; pcie->dev = dev; diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c index a5b8d0b71677e0..0a494c9dd6aa18 100644 --- a/drivers/pci/controller/dwc/pci-imx6.c +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -117,6 +117,8 @@ enum imx_pcie_variants { #define IMX_PCIE_FLAG_HAS_LUT BIT(10) #define IMX_PCIE_FLAG_8GT_ECN_ERR051586 BIT(11) #define IMX_PCIE_FLAG_SKIP_L23_READY BIT(12) +/* Preserve MSI capability for platforms that require it */ +#define IMX_PCIE_FLAG_KEEP_MSI_CAP BIT(13) #define imx_check_flag(pci, val) (pci->drvdata->flags & val) @@ -268,8 +270,8 @@ static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie) IMX95_PCIE_PHY_CR_PARA_SEL); regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_GEN_CTRL, - ext ? IMX95_PCIE_REF_USE_PAD : 0, - IMX95_PCIE_REF_USE_PAD); + IMX95_PCIE_REF_USE_PAD, + ext ? IMX95_PCIE_REF_USE_PAD : 0); regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, IMX95_PCIE_REF_CLKEN, ext ? 0 : IMX95_PCIE_REF_CLKEN); @@ -1647,7 +1649,6 @@ static int imx_pcie_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dw_pcie *pci; struct imx_pcie *imx_pcie; - struct device_node *np; struct device_node *node = dev->of_node; int i, ret, domain; u16 val; @@ -1674,7 +1675,8 @@ static int imx_pcie_probe(struct platform_device *pdev) pci->pp.ops = &imx_pcie_host_dw_pme_ops; /* Find the PHY if one is defined, only imx7d uses it */ - np = of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0); + struct device_node *np __free(device_node) = + of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0); if (np) { struct resource res; @@ -1830,6 +1832,8 @@ static int imx_pcie_probe(struct platform_device *pdev) } else { if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY)) pci->pp.skip_l23_ready = true; + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_KEEP_MSI_CAP)) + pci->pp.keep_rp_msi_en = true; pci->pp.use_atu_msg = true; ret = dw_pcie_host_init(&pci->pp); if (ret < 0) @@ -1876,6 +1880,7 @@ static const struct imx_pcie_drvdata drvdata[] = { .variant = IMX6SX, .flags = IMX_PCIE_FLAG_IMX_PHY | IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .gpr = "fsl,imx6q-iomuxc-gpr", .ltssm_off = IOMUXC_GPR12, @@ -1907,6 +1912,7 @@ static const struct imx_pcie_drvdata drvdata[] = { [IMX7D] = { .variant = IMX7D, .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP | IMX_PCIE_FLAG_HAS_APP_RESET | IMX_PCIE_FLAG_SKIP_L23_READY | IMX_PCIE_FLAG_HAS_PHY_RESET, @@ -1919,6 +1925,7 @@ static const struct imx_pcie_drvdata drvdata[] = { [IMX8MQ] = { .variant = IMX8MQ, .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_KEEP_MSI_CAP | IMX_PCIE_FLAG_HAS_PHY_RESET | IMX_PCIE_FLAG_SUPPORTS_SUSPEND, .gpr = "fsl,imx8mq-iomuxc-gpr", @@ -1933,6 +1940,7 @@ static const struct imx_pcie_drvdata drvdata[] = { [IMX8MM] = { .variant = IMX8MM, .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP | IMX_PCIE_FLAG_HAS_PHYDRV | IMX_PCIE_FLAG_HAS_APP_RESET, .gpr = "fsl,imx8mm-iomuxc-gpr", diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c index 0d1340c9b3642a..9461be0744907f 100644 --- a/drivers/pci/controller/dwc/pcie-designware-debugfs.c +++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c @@ -208,10 +208,11 @@ static ssize_t lane_detect_write(struct file *file, const char __user *buf, struct dw_pcie *pci = file->private_data; struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; u32 lane, val; + int ret; - val = kstrtou32_from_user(buf, count, 0, &lane); - if (val) - return val; + ret = kstrtou32_from_user(buf, count, 0, &lane); + if (ret) + return ret; val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG); val &= ~(LANE_SELECT); @@ -347,10 +348,11 @@ static ssize_t counter_enable_write(struct file *file, const char __user *buf, struct dw_pcie *pci = pdata->pci; struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; u32 val, enable; + int ret; - val = kstrtou32_from_user(buf, count, 0, &enable); - if (val) - return val; + ret = kstrtou32_from_user(buf, count, 0, &enable); + if (ret) + return ret; mutex_lock(&rinfo->reg_event_lock); set_event_number(pdata, pci, rinfo); @@ -408,10 +410,11 @@ static ssize_t counter_lane_write(struct file *file, const char __user *buf, struct dw_pcie *pci = pdata->pci; struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; u32 val, lane; + int ret; - val = kstrtou32_from_user(buf, count, 0, &lane); - if (val) - return val; + ret = kstrtou32_from_user(buf, count, 0, &lane); + if (ret) + return ret; mutex_lock(&rinfo->reg_event_lock); set_event_number(pdata, pci, rinfo); diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index c57ae4d6c5c0e0..ab2e7de5c55eba 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -754,7 +754,7 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, val = dw_pcie_ep_readw_dbi(ep, func_no, reg); val &= ~PCI_MSIX_FLAGS_QSIZE; val |= nr_irqs - 1; /* encoded as N-1 */ - dw_pcie_writew_dbi(pci, reg, val); + dw_pcie_ep_writew_dbi(ep, func_no, reg, val); reg = ep_func->msix_cap + PCI_MSIX_TABLE; val = offset | bir; @@ -1110,7 +1110,8 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) { struct dw_pcie_ep *ep = &pci->ep; u8 funcs = ep->epc->max_functions; - u8 func_no; + u32 func0_lnkcap, lnkcap; + u8 func_no, offset; dw_pcie_dbi_ro_wr_en(pci); @@ -1118,6 +1119,32 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) dw_pcie_ep_init_rebar_registers(ep, func_no); dw_pcie_setup(pci); + + /* + * PCIe r7.0, section 7.5.3.6 states that for multi-function + * endpoints, max link width and speed fields must report same + * values for all functions. However, dw_pcie_setup() programs + * these fields only for function 0. Hence, mirror these fields + * to all other functions as well. + */ + if (funcs > 1) { + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + func0_lnkcap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + func0_lnkcap = FIELD_GET(PCI_EXP_LNKCAP_MLW | + PCI_EXP_LNKCAP_SLS, func0_lnkcap); + + for (func_no = 1; func_no < funcs; func_no++) { + offset = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_EXP); + lnkcap = dw_pcie_ep_readl_dbi(ep, func_no, + offset + PCI_EXP_LNKCAP); + FIELD_MODIFY(PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS, + &lnkcap, func0_lnkcap); + dw_pcie_ep_writel_dbi(ep, func_no, + offset + PCI_EXP_LNKCAP, lnkcap); + } + } + dw_pcie_dbi_ro_wr_dis(pci); } diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c index 6ae6189e9b8a90..6adde3fc32be9d 100644 --- a/drivers/pci/controller/dwc/pcie-designware-host.c +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -1171,7 +1171,7 @@ int dw_pcie_setup_rc(struct dw_pcie_rp *pp) * the MSI and MSI-X capabilities of the Root Port to allow the drivers * to fall back to INTx instead. */ - if (pp->use_imsi_rx) { + if (pp->use_imsi_rx && !pp->keep_rp_msi_en) { dw_pcie_remove_capability(pci, PCI_CAP_ID_MSI); dw_pcie_remove_capability(pci, PCI_CAP_ID_MSIX); } @@ -1300,15 +1300,24 @@ int dw_pcie_resume_noirq(struct dw_pcie *pci) ret = dw_pcie_start_link(pci); if (ret) - return ret; + goto err_deinit; ret = dw_pcie_wait_for_link(pci); - if (ret) - return ret; + if (ret == -ETIMEDOUT) + goto err_stop_link; if (pci->pp.ops->post_init) pci->pp.ops->post_init(&pci->pp); + return 0; + +err_stop_link: + dw_pcie_stop_link(pci); + +err_deinit: + if (pci->pp.ops->deinit) + pci->pp.ops->deinit(&pci->pp); + return ret; } EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq); diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 5741c09dde7f40..bb4e82fbfd5c8c 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -487,13 +487,13 @@ static inline void dw_pcie_writel_atu_ob(struct dw_pcie *pci, u32 index, u32 reg static inline u32 dw_pcie_enable_ecrc(u32 val) { /* - * DesignWare core version 4.90A has a design issue where the 'TD' - * bit in the Control register-1 of the ATU outbound region acts - * like an override for the ECRC setting, i.e., the presence of TLP - * Digest (ECRC) in the outgoing TLPs is solely determined by this - * bit. This is contrary to the PCIe spec which says that the - * enablement of the ECRC is solely determined by the AER - * registers. + * DWC versions 0x3530302a and 0x3536322a have a design issue where + * the 'TD' bit in the Control register-1 of the ATU outbound + * region acts like an override for the ECRC setting, i.e., the + * presence of TLP Digest (ECRC) in the outgoing TLPs is solely + * determined by this bit. This is contrary to the PCIe spec which + * says that the enablement of the ECRC is solely determined by the + * AER registers. * * Because of this, even when the ECRC is enabled through AER * registers, the transactions going through ATU won't have TLP @@ -563,7 +563,7 @@ int dw_pcie_prog_outbound_atu(struct dw_pcie *pci, if (upper_32_bits(limit_addr) > upper_32_bits(parent_bus_addr) && dw_pcie_ver_is_ge(pci, 460A)) val |= PCIE_ATU_INCREASE_REGION_SIZE; - if (dw_pcie_ver_is(pci, 490A)) + if (dw_pcie_ver_is(pci, 490A) || dw_pcie_ver_is(pci, 500A)) val = dw_pcie_enable_ecrc(val); dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL1, val); diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index ae6389dd9caa5c..3e69ef60165b0e 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -34,8 +34,10 @@ #define DW_PCIE_VER_470A 0x3437302a #define DW_PCIE_VER_480A 0x3438302a #define DW_PCIE_VER_490A 0x3439302a +#define DW_PCIE_VER_500A 0x3530302a #define DW_PCIE_VER_520A 0x3532302a #define DW_PCIE_VER_540A 0x3534302a +#define DW_PCIE_VER_562A 0x3536322a #define __dw_pcie_ver_cmp(_pci, _ver, _op) \ ((_pci)->version _op DW_PCIE_VER_ ## _ver) @@ -421,6 +423,7 @@ struct dw_pcie_host_ops { struct dw_pcie_rp { bool use_imsi_rx:1; + bool keep_rp_msi_en:1; bool cfg0_io_shared:1; u64 cfg0_base; void __iomem *va_cfg0_base; diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c index 67a16af69ddc75..9fdfc88ac15120 100644 --- a/drivers/pci/controller/dwc/pcie-qcom.c +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -350,15 +350,20 @@ static void qcom_pcie_clear_aspm_l0s(struct dw_pcie *pci) dw_pcie_dbi_ro_wr_dis(pci); } -static void qcom_pcie_clear_hpc(struct dw_pcie *pci) +static void qcom_pcie_set_slot_nccs(struct dw_pcie *pci) { u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); u32 val; dw_pcie_dbi_ro_wr_en(pci); + /* + * Qcom PCIe Root Ports do not support generating command completion + * notifications for the Hot-Plug commands. So set the NCCS field to + * avoid waiting for the completions. + */ val = readl(pci->dbi_base + offset + PCI_EXP_SLTCAP); - val &= ~PCI_EXP_SLTCAP_HPC; + val |= PCI_EXP_SLTCAP_NCCS; writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP); dw_pcie_dbi_ro_wr_dis(pci); @@ -558,7 +563,7 @@ static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie) writel(CFG_BRIDGE_SB_INIT, pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL1); - qcom_pcie_clear_hpc(pcie->pci); + qcom_pcie_set_slot_nccs(pcie->pci); return 0; } @@ -638,7 +643,7 @@ static int qcom_pcie_post_init_1_0_0(struct qcom_pcie *pcie) writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT); } - qcom_pcie_clear_hpc(pcie->pci); + qcom_pcie_set_slot_nccs(pcie->pci); return 0; } @@ -731,7 +736,7 @@ static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie) val |= EN; writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); - qcom_pcie_clear_hpc(pcie->pci); + qcom_pcie_set_slot_nccs(pcie->pci); return 0; } @@ -1037,7 +1042,7 @@ static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) writel(WR_NO_SNOOP_OVERRIDE_EN | RD_NO_SNOOP_OVERRIDE_EN, pcie->parf + PARF_NO_SNOOP_OVERRIDE); - qcom_pcie_clear_hpc(pcie->pci); + qcom_pcie_set_slot_nccs(pcie->pci); return 0; } diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c index a6912e85e4ddc3..8c7bc0691e52c8 100644 --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -426,7 +426,7 @@ static const struct pci_epc_features rcar_gen4_pcie_epc_features = { .bar[BAR_3] = { .type = BAR_RESERVED, }, .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256 }, .bar[BAR_5] = { .type = BAR_RESERVED, }, - .align = SZ_1M, + .align = SZ_4K, }; static const struct pci_epc_features* diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 06571d806ab31f..336d3c759547ab 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -35,8 +35,8 @@ #include #include "../../pci.h" -#define TEGRA194_DWC_IP_VER 0x490A -#define TEGRA234_DWC_IP_VER 0x562A +#define TEGRA194_DWC_IP_VER DW_PCIE_VER_500A +#define TEGRA234_DWC_IP_VER DW_PCIE_VER_562A #define APPL_PINMUX 0x0 #define APPL_PINMUX_PEX_RST BIT(0) @@ -137,7 +137,11 @@ #define APPL_DEBUG_PM_LINKST_IN_L0 0x11 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) #define APPL_DEBUG_LTSSM_STATE_SHIFT 3 -#define LTSSM_STATE_PRE_DETECT 5 +#define LTSSM_STATE_DETECT_QUIET 0x00 +#define LTSSM_STATE_DETECT_ACT 0x08 +#define LTSSM_STATE_PRE_DETECT_QUIET 0x28 +#define LTSSM_STATE_DETECT_WAIT 0x30 +#define LTSSM_STATE_L2_IDLE 0xa8 #define APPL_RADM_STATUS 0xE4 #define APPL_PM_XMT_TURNOFF_STATE BIT(0) @@ -198,9 +202,8 @@ #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 -#define PME_ACK_TIMEOUT 10000 - -#define LTSSM_TIMEOUT 50000 /* 50ms */ +#define LTSSM_DELAY_US 10000 /* 10 ms */ +#define LTSSM_TIMEOUT_US 120000 /* 120 ms */ #define GEN3_GEN4_EQ_PRESET_INIT 5 @@ -231,6 +234,7 @@ struct tegra_pcie_dw_of_data { bool has_sbr_reset_fix; bool has_l1ss_exit_fix; bool has_ltr_req_fix; + bool disable_l1_2; u32 cdm_chk_int_en_bit; u32 gen4_preset_vec; u8 n_fts[2]; @@ -482,15 +486,6 @@ static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) if (val & PCI_COMMAND_MASTER) { ktime_t timeout; - /* 110us for both snoop and no-snoop */ - val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) | - FIELD_PREP(PCI_LTR_SCALE_MASK, 2) | - LTR_MSG_REQ | - FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) | - FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) | - LTR_NOSNOOP_MSG_REQ; - appl_writel(pcie, val, APPL_LTR_MSG_1); - /* Send LTR upstream */ val = appl_readl(pcie, APPL_LTR_MSG_2); val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; @@ -685,6 +680,23 @@ static void init_host_aspm(struct tegra_pcie_dw *pcie) if (pcie->supports_clkreq) pci->l1ss_support = true; + /* + * Disable L1.2 capability advertisement for Tegra234 Endpoint mode. + * Tegra234 has a hardware bug where during L1.2 exit, the UPHY PLL is + * powered up immediately without waiting for REFCLK to stabilize. This + * causes the PLL to fail to lock to the correct frequency, resulting in + * PCIe link loss. Since there is no hardware fix available, we prevent + * the Endpoint from advertising L1.2 support by clearing the L1.2 bits + * in the L1 PM Substates Capabilities register. This ensures the host + * will not attempt to enter L1.2 state with this Endpoint. + */ + if (pcie->of_data->disable_l1_2 && + pcie->of_data->mode == DW_PCIE_EP_TYPE) { + val = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); + val &= ~(PCI_L1SS_CAP_PCIPM_L1_2 | PCI_L1SS_CAP_ASPM_L1_2); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, val); + } + /* Program L0s and L1 entrance latencies */ val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; @@ -1022,7 +1034,8 @@ static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) { struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); - disable_irq(pcie->pex_rst_irq); + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + disable_irq(pcie->pex_rst_irq); } static const struct dw_pcie_ops tegra_dw_pcie_ops = { @@ -1163,9 +1176,9 @@ static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) return err; } - pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev, - "nvidia,refclk-select", - GPIOD_OUT_HIGH); + pcie->pex_refclk_sel_gpiod = devm_gpiod_get_optional(pcie->dev, + "nvidia,refclk-select", + GPIOD_OUT_HIGH); if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { int err = PTR_ERR(pcie->pex_refclk_sel_gpiod); const char *level = KERN_ERR; @@ -1255,44 +1268,6 @@ static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, return 0; } -static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) -{ - struct dw_pcie_rp *pp = &pcie->pci.pp; - struct pci_bus *child, *root_port_bus = NULL; - struct pci_dev *pdev; - - /* - * link doesn't go into L2 state with some of the endpoints with Tegra - * if they are not in D0 state. So, need to make sure that immediate - * downstream devices are in D0 state before sending PME_TurnOff to put - * link into L2 state. - * This is as per PCI Express Base r4.0 v1.0 September 27-2017, - * 5.2 Link State Power Management (Page #428). - */ - - list_for_each_entry(child, &pp->bridge->bus->children, node) { - if (child->parent == pp->bridge->bus) { - root_port_bus = child; - break; - } - } - - if (!root_port_bus) { - dev_err(pcie->dev, "Failed to find downstream bus of Root Port\n"); - return; - } - - /* Bring downstream devices to D0 if they are not already in */ - list_for_each_entry(pdev, &root_port_bus->devices, bus_list) { - if (PCI_SLOT(pdev->devfn) == 0) { - if (pci_set_power_state(pdev, PCI_D0)) - dev_err(pcie->dev, - "Failed to transition %s to D0 state\n", - dev_name(&pdev->dev)); - } - } -} - static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) { pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); @@ -1553,9 +1528,10 @@ static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) val |= APPL_PM_XMT_TURNOFF_STATE; appl_writel(pcie, val, APPL_RADM_STATUS); - return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val, - val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, - 1, PME_ACK_TIMEOUT); + return readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, + val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, + PCIE_PME_TO_L2_TIMEOUT_US/10, + PCIE_PME_TO_L2_TIMEOUT_US); } static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) @@ -1590,23 +1566,22 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) data &= ~APPL_PINMUX_PEX_RST; appl_writel(pcie, data, APPL_PINMUX); + err = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, data, + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_QUIET) || + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_ACT) || + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_PRE_DETECT_QUIET) || + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_WAIT), + LTSSM_DELAY_US, LTSSM_TIMEOUT_US); + if (err) + dev_info(pcie->dev, "LTSSM state: 0x%x detect timeout: %d\n", data, err); + /* - * Some cards do not go to detect state even after de-asserting - * PERST#. So, de-assert LTSSM to bring link to detect state. + * Deassert LTSSM state to stop the state toggling between + * Polling and Detect. */ data = readl(pcie->appl_base + APPL_CTRL); data &= ~APPL_CTRL_LTSSM_EN; writel(data, pcie->appl_base + APPL_CTRL); - - err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, - data, - ((data & - APPL_DEBUG_LTSSM_STATE_MASK) >> - APPL_DEBUG_LTSSM_STATE_SHIFT) == - LTSSM_STATE_PRE_DETECT, - 1, LTSSM_TIMEOUT); - if (err) - dev_info(pcie->dev, "Link didn't go to detect state\n"); } /* * DBI registers may not be accessible after this as PLL-E would be @@ -1622,7 +1597,6 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) { - tegra_pcie_downstream_dev_to_D0(pcie); dw_pcie_host_deinit(&pcie->pci.pp); tegra_pcie_dw_pme_turnoff(pcie); tegra_pcie_unconfig_controller(pcie); @@ -1680,19 +1654,24 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) if (pcie->ep_state == EP_STATE_DISABLED) return; - /* Disable LTSSM */ + ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_QUIET) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_ACT) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_PRE_DETECT_QUIET) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_WAIT) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_L2_IDLE), + LTSSM_DELAY_US, LTSSM_TIMEOUT_US); + if (ret) + dev_info(pcie->dev, "LTSSM state: 0x%x detect timeout: %d\n", val, ret); + + /* + * Deassert LTSSM state to stop the state toggling between + * Polling and Detect. + */ val = appl_readl(pcie, APPL_CTRL); val &= ~APPL_CTRL_LTSSM_EN; appl_writel(pcie, val, APPL_CTRL); - ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, - ((val & APPL_DEBUG_LTSSM_STATE_MASK) >> - APPL_DEBUG_LTSSM_STATE_SHIFT) == - LTSSM_STATE_PRE_DETECT, - 1, LTSSM_TIMEOUT); - if (ret) - dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); - reset_control_assert(pcie->core_rst); tegra_pcie_disable_phy(pcie); @@ -1771,10 +1750,6 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) goto fail_phy; } - /* Perform cleanup that requires refclk */ - pci_epc_deinit_notify(pcie->pci.ep.epc); - dw_pcie_ep_cleanup(&pcie->pci.ep); - /* Clear any stale interrupt statuses */ appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); @@ -1833,8 +1808,25 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); + /* 110us for both snoop and no-snoop */ + val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) | + FIELD_PREP(PCI_LTR_SCALE_MASK, 2) | + LTR_MSG_REQ | + FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) | + FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) | + LTR_NOSNOOP_MSG_REQ; + appl_writel(pcie, val, APPL_LTR_MSG_1); + reset_control_deassert(pcie->core_rst); + /* Perform cleanup that requires refclk and core reset deasserted */ + pci_epc_deinit_notify(pcie->pci.ep.epc); + dw_pcie_ep_cleanup(&pcie->pci.ep); + + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val &= ~PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + if (pcie->update_fc_fixup) { val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; @@ -2277,6 +2269,7 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) static void tegra_pcie_dw_remove(struct platform_device *pdev) { struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); + struct dw_pcie_ep *ep = &pcie->pci.ep; if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { if (!pcie->link_state) @@ -2288,6 +2281,7 @@ static void tegra_pcie_dw_remove(struct platform_device *pdev) } else { disable_irq(pcie->pex_rst_irq); pex_ep_event_pex_rst_assert(pcie); + dw_pcie_ep_deinit(ep); } pm_runtime_disable(pcie->dev); @@ -2296,16 +2290,28 @@ static void tegra_pcie_dw_remove(struct platform_device *pdev) gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); } -static int tegra_pcie_dw_suspend_late(struct device *dev) +static int tegra_pcie_dw_suspend(struct device *dev) { struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); - u32 val; if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { - dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n"); - return -EPERM; + if (pcie->ep_state == EP_STATE_ENABLED) { + dev_err(dev, "Tegra PCIe is in EP mode, suspend not allowed\n"); + return -EPERM; + } + + disable_irq(pcie->pex_rst_irq); + return 0; } + return 0; +} + +static int tegra_pcie_dw_suspend_late(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + u32 val; + if (!pcie->link_state) return 0; @@ -2325,10 +2331,12 @@ static int tegra_pcie_dw_suspend_noirq(struct device *dev) { struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + return 0; + if (!pcie->link_state) return 0; - tegra_pcie_downstream_dev_to_D0(pcie); tegra_pcie_dw_pme_turnoff(pcie); tegra_pcie_unconfig_controller(pcie); @@ -2340,6 +2348,9 @@ static int tegra_pcie_dw_resume_noirq(struct device *dev) struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); int ret; + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + return 0; + if (!pcie->link_state) return 0; @@ -2372,8 +2383,8 @@ static int tegra_pcie_dw_resume_early(struct device *dev) u32 val; if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { - dev_err(dev, "Suspend is not supported in EP mode"); - return -ENOTSUPP; + enable_irq(pcie->pex_rst_irq); + return 0; } if (!pcie->link_state) @@ -2402,7 +2413,6 @@ static void tegra_pcie_dw_shutdown(struct platform_device *pdev) return; debugfs_remove_recursive(pcie->debugfs); - tegra_pcie_downstream_dev_to_D0(pcie); disable_irq(pcie->pci.pp.irq); if (IS_ENABLED(CONFIG_PCI_MSI)) @@ -2452,6 +2462,7 @@ static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = { .mode = DW_PCIE_EP_TYPE, .has_l1ss_exit_fix = true, .has_ltr_req_fix = true, + .disable_l1_2 = true, .cdm_chk_int_en_bit = BIT(18), /* Gen4 - 6, 8 and 9 presets enabled */ .gen4_preset_vec = 0x340, @@ -2479,6 +2490,7 @@ static const struct of_device_id tegra_pcie_dw_of_match[] = { }; static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { + .suspend = tegra_pcie_dw_suspend, .suspend_late = tegra_pcie_dw_suspend_late, .suspend_noirq = tegra_pcie_dw_suspend_noirq, .resume_noirq = tegra_pcie_dw_resume_noirq, diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 2d92fc79f6ddfa..d962724750840d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -33,6 +33,10 @@ #include "pci-host-common.h" +static int link_up_timeout = 500; +module_param(link_up_timeout, int, 0644); +MODULE_PARM_DESC(link_up_timeout, "PCIe link training timeout in milliseconds"); + /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 #define CORE_RC_PHYIF_CTL_RUN BIT(0) @@ -550,22 +554,105 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return readl_relaxed(port_rid2sid_addr(port, idx)); } +static int apple_pcie_setup_link(struct apple_pcie *pcie, + struct apple_pcie_port *port, + struct device_node *np) +{ +#define MAX_AUX_PERST 3 + struct gpio_desc *aux_reset[MAX_AUX_PERST] = { NULL }; + u32 num_aux_resets = 0; + struct gpio_desc *reset, *pwren = NULL; + u32 stat; + int ret; + + /* + * Assert PERST# and configure the pin as output. + * The Aquantia AQC113 10GB nic used desktop macs is sensitive to + * deasserting it without prior clock setup. + * Observed on M1 Max/Ultra Mac Studios under m1n1's hypervisor. + */ + reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", + GPIOD_OUT_HIGH, "PERST#"); + if (IS_ERR(reset)) + return PTR_ERR(reset); + // HACK: use additional "reset-gpios" until pci-pwrctrl gains PERST# support. + for (u32 idx = 0; idx < MAX_AUX_PERST; idx++) { + aux_reset[idx] = devm_fwnode_gpiod_get_index(pcie->dev, + of_fwnode_handle(np), + "reset", idx + 1, + GPIOD_OUT_HIGH, + "PERST#"); + if (IS_ERR(aux_reset[idx])) { + if (PTR_ERR(aux_reset[idx]) == -ENOENT) + break; + else + return PTR_ERR(aux_reset[idx]); + } + num_aux_resets++; + } + dev_info(pcie->dev, "Using %u auxiliary PERST#\n", num_aux_resets); + + pwren = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "pwren", + GPIOD_ASIS, "PWREN"); + if (IS_ERR(pwren)) { + if (PTR_ERR(pwren) == -ENOENT) + pwren = NULL; + else + return PTR_ERR(pwren); + } + + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* Assert PERST# before setting up the clock */ + gpiod_set_value_cansleep(reset, 1); + for (u32 idx = 0; idx < num_aux_resets; idx++) + gpiod_set_value_cansleep(aux_reset[idx], 1); + + /* Power on the device if required */ + gpiod_set_value_cansleep(pwren, 1); + + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + + /* + * The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) + * If powering up, the minimal Tpvperl is 100ms + */ + if (pwren) + msleep(100); + else + usleep_range(100, 200); + + /* Deassert PERST# */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + gpiod_set_value_cansleep(reset, 0); + for (u32 idx = 0; idx < num_aux_resets; idx++) + gpiod_set_value_cansleep(aux_reset[idx], 0); + + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ + msleep(100); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, 100, 250000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); + return ret; + } + + return 0; +} + static int apple_pcie_setup_port(struct apple_pcie *pcie, struct device_node *np) { struct platform_device *platform = to_platform_device(pcie->dev); struct apple_pcie_port *port; - struct gpio_desc *reset; struct resource *res; char name[16]; - u32 stat, idx; + u32 link_stat, idx; int ret, i; - reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", - GPIOD_OUT_LOW, "PERST#"); - if (IS_ERR(reset)) - return PTR_ERR(reset); - port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; @@ -601,30 +688,12 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, else port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); - rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); - - /* Assert PERST# before setting up the clock */ - gpiod_set_value_cansleep(reset, 1); - - ret = apple_pcie_setup_refclk(pcie, port); - if (ret < 0) - return ret; - - /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ - usleep_range(100, 200); - - /* Deassert PERST# */ - rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); - gpiod_set_value_cansleep(reset, 0); - - /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ - msleep(100); - - ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, - stat & PORT_STATUS_READY, 100, 250000); - if (ret < 0) { - dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); - return ret; + /* link might be already brought up by u-boot, skip setup then */ + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + ret = apple_pcie_setup_link(pcie, port, np); + if (ret) + return ret; } if (pcie->hw->port_refclk) @@ -658,10 +727,21 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, ret = apple_pcie_port_register_irqs(port); WARN_ON(ret); - writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + link_stat = readl_relaxed(port->base + PORT_LINKSTS); + if (!(link_stat & PORT_LINKSTS_UP)) { + unsigned long timeout, left; + /* start link training */ + writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); - if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) - dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + timeout = link_up_timeout * HZ / 1000; + left = wait_for_completion_timeout(&pcie->event, timeout); + if (!left) + dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + else + dev_info(pcie->dev, "%pOF link up after %ldms\n", np, + (timeout - left) * 1000 / HZ); + + } return 0; } @@ -845,13 +925,47 @@ static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { } }; +static int apple_pcie_probe_port(struct device_node *np) +{ + struct gpio_desc *gd; + + /* check whether the GPPIO pin exists but leave it as is */ + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "reset", 0, + GPIOD_ASIS, "PERST#"); + if (IS_ERR(gd)) + return PTR_ERR(gd); + + gpiod_put(gd); + + gd = fwnode_gpiod_get_index(of_fwnode_handle(np), "pwren", 0, + GPIOD_ASIS, "PWREN"); + if (IS_ERR(gd)) { + if (PTR_ERR(gd) != -ENOENT) + return PTR_ERR(gd); + } else { + gpiod_put(gd); + } + + return 0; +} + static int apple_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct pci_host_bridge *bridge; + struct device_node *of_port; struct apple_pcie *pcie; int ret; + /* Check for probe dependencies for all ports first */ + for_each_available_child_of_node(dev->of_node, of_port) { + ret = apple_pcie_probe_port(of_port); + if (ret) { + of_node_put(of_port); + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", of_port); + } + } + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); if (!bridge) return -ENOMEM; diff --git a/drivers/pci/controller/pcie-aspeed.c b/drivers/pci/controller/pcie-aspeed.c index 3e1a39d1e64842..6acfae7d026e45 100644 --- a/drivers/pci/controller/pcie-aspeed.c +++ b/drivers/pci/controller/pcie-aspeed.c @@ -1052,14 +1052,14 @@ static int aspeed_pcie_probe(struct platform_device *pdev) if (ret) return ret; - irq = platform_get_irq(pdev, 0); - if (irq < 0) - return irq; - ret = devm_add_action_or_reset(dev, aspeed_pcie_irq_domain_free, pcie); if (ret) return ret; + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + ret = devm_request_irq(dev, irq, aspeed_pcie_intr_handler, IRQF_SHARED, dev_name(dev), pcie); if (ret) diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c index 75ddb8bee168f6..e45c43ccc84c2a 100644 --- a/drivers/pci/controller/pcie-mediatek-gen3.c +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -851,14 +851,14 @@ static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie) struct platform_device *pdev = to_platform_device(dev); int err; - err = mtk_pcie_init_irq_domains(pcie); - if (err) - return err; - pcie->irq = platform_get_irq(pdev, 0); if (pcie->irq < 0) return pcie->irq; + err = mtk_pcie_init_irq_domains(pcie); + if (err) + return err; + irq_set_chained_handler_and_data(pcie->irq, mtk_pcie_irq_handler, pcie); return 0; diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c index 2809112e631714..986f0a319b3ce1 100644 --- a/drivers/pci/controller/pcie-rzg3s-host.c +++ b/drivers/pci/controller/pcie-rzg3s-host.c @@ -1589,8 +1589,7 @@ static int rzg3s_pcie_probe(struct platform_device *pdev) host_probe_teardown: rzg3s_pcie_teardown_irqdomain(host); - reset_control_bulk_deassert(host->data->num_cfg_resets, - host->cfg_resets); + reset_control_bulk_assert(host->data->num_cfg_resets, host->cfg_resets); rpm_put: pm_runtime_put_sync(dev); rpm_disable: @@ -1625,31 +1624,31 @@ static int rzg3s_pcie_suspend_noirq(struct device *dev) clk_disable_unprepare(port->refclk); - ret = reset_control_bulk_assert(data->num_power_resets, - host->power_resets); + ret = reset_control_bulk_assert(data->num_cfg_resets, + host->cfg_resets); if (ret) goto refclk_restore; - ret = reset_control_bulk_assert(data->num_cfg_resets, - host->cfg_resets); + ret = reset_control_bulk_assert(data->num_power_resets, + host->power_resets); if (ret) - goto power_resets_restore; + goto cfg_resets_restore; ret = regmap_update_bits(sysc, RZG3S_SYS_PCIE_RST_RSM_B, RZG3S_SYS_PCIE_RST_RSM_B_MASK, FIELD_PREP(RZG3S_SYS_PCIE_RST_RSM_B_MASK, 0)); if (ret) - goto cfg_resets_restore; + goto power_resets_restore; return 0; /* Restore the previous state if any error happens */ -cfg_resets_restore: - reset_control_bulk_deassert(data->num_cfg_resets, - host->cfg_resets); power_resets_restore: reset_control_bulk_deassert(data->num_power_resets, host->power_resets); +cfg_resets_restore: + reset_control_bulk_deassert(data->num_cfg_resets, + host->cfg_resets); refclk_restore: clk_prepare_enable(port->refclk); pm_runtime_resume_and_get(dev); diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c index f9cf18aa5b3421..7f5326925ed54a 100644 --- a/drivers/pci/endpoint/functions/pci-epf-mhi.c +++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c @@ -367,6 +367,8 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl, dev_err(dev, "DMA transfer timeout\n"); dmaengine_terminate_sync(chan); ret = -ETIMEDOUT; + } else { + ret = 0; } err_unmap: @@ -438,6 +440,8 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl, dev_err(dev, "DMA transfer timeout\n"); dmaengine_terminate_sync(chan); ret = -ETIMEDOUT; + } else { + ret = 0; } err_unmap: diff --git a/drivers/pci/endpoint/functions/pci-epf-ntb.c b/drivers/pci/endpoint/functions/pci-epf-ntb.c index a3a588e522e715..2bdcc35b652cfb 100644 --- a/drivers/pci/endpoint/functions/pci-epf-ntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-ntb.c @@ -1494,47 +1494,6 @@ static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, return ret; } -/** - * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface - * @ntb: NTB device that facilitates communication between HOST1 and HOST2 - * @type: PRIMARY interface or SECONDARY interface - * - * Unbind NTB function device from EPC and relinquish reference to pci_epc - * for each of the interface. - */ -static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb, - enum pci_epc_interface_type type) -{ - struct epf_ntb_epc *ntb_epc; - struct pci_epc *epc; - struct pci_epf *epf; - - if (type < 0) - return; - - epf = ntb->epf; - ntb_epc = ntb->epc[type]; - if (!ntb_epc) - return; - epc = ntb_epc->epc; - pci_epc_remove_epf(epc, epf, type); - pci_epc_put(epc); -} - -/** - * epf_ntb_epc_destroy() - Cleanup NTB EPC interface - * @ntb: NTB device that facilitates communication between HOST1 and HOST2 - * - * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces - */ -static void epf_ntb_epc_destroy(struct epf_ntb *ntb) -{ - enum pci_epc_interface_type type; - - for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) - epf_ntb_epc_destroy_interface(ntb, type); -} - /** * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface * @ntb: NTB device that facilitates communication between HOST1 and HOST2 @@ -1614,15 +1573,8 @@ static int epf_ntb_epc_create(struct epf_ntb *ntb) ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, SECONDARY_INTERFACE); - if (ret) { + if (ret) dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n"); - goto err_epc_create; - } - - return 0; - -err_epc_create: - epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE); return ret; } @@ -1887,7 +1839,7 @@ static int epf_ntb_bind(struct pci_epf *epf) ret = epf_ntb_init_epc_bar(ntb); if (ret) { dev_err(dev, "Failed to create NTB EPC\n"); - goto err_bar_init; + return ret; } ret = epf_ntb_config_spad_bar_alloc_interface(ntb); @@ -1909,9 +1861,6 @@ static int epf_ntb_bind(struct pci_epf *epf) err_bar_alloc: epf_ntb_config_spad_bar_free(ntb); -err_bar_init: - epf_ntb_epc_destroy(ntb); - return ret; } @@ -1927,7 +1876,6 @@ static void epf_ntb_unbind(struct pci_epf *epf) epf_ntb_epc_cleanup(ntb); epf_ntb_config_spad_bar_free(ntb); - epf_ntb_epc_destroy(ntb); } #define EPF_NTB_R(_name) \ diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 33548935765e33..0e7cbcbebf0b45 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -715,7 +715,6 @@ static void pci_epf_test_doorbell_cleanup(struct pci_epf_test *epf_test) struct pci_epf_test_reg *reg = epf_test->reg[epf_test->test_reg_bar]; struct pci_epf *epf = epf_test->epf; - free_irq(epf->db_msg[0].virq, epf_test); reg->doorbell_bar = cpu_to_le32(NO_BAR); pci_epf_free_doorbell(epf); @@ -759,7 +758,7 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test, &epf_test->db_bar.phys_addr, &offset); if (ret) - goto err_doorbell_cleanup; + goto err_free_irq; reg->doorbell_offset = cpu_to_le32(offset); @@ -769,12 +768,14 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test, ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf_test->db_bar); if (ret) - goto err_doorbell_cleanup; + goto err_free_irq; status |= STATUS_DOORBELL_ENABLE_SUCCESS; reg->status = cpu_to_le32(status); return; +err_free_irq: + free_irq(epf->db_msg[0].virq, epf_test); err_doorbell_cleanup: pci_epf_test_doorbell_cleanup(epf_test); set_status_err: @@ -794,6 +795,7 @@ static void pci_epf_test_disable_doorbell(struct pci_epf_test *epf_test, if (bar < BAR_0) goto set_status_err; + free_irq(epf->db_msg[0].virq, epf_test); pci_epf_test_doorbell_cleanup(epf_test); /* diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c index 20a400e8343927..c9c7b50587dd21 100644 --- a/drivers/pci/endpoint/functions/pci-epf-vntb.c +++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c @@ -527,20 +527,20 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb, struct msi_msg *msg; size_t sz; int ret; - int i; + int i, req; ret = pci_epf_alloc_doorbell(epf, ntb->db_count); if (ret) return ret; - for (i = 0; i < ntb->db_count; i++) { - ret = request_irq(epf->db_msg[i].virq, epf_ntb_doorbell_handler, + for (req = 0; req < ntb->db_count; req++) { + ret = request_irq(epf->db_msg[req].virq, epf_ntb_doorbell_handler, 0, "pci_epf_vntb_db", ntb); if (ret) { dev_err(&epf->dev, "Failed to request doorbell IRQ: %d\n", - epf->db_msg[i].virq); + epf->db_msg[req].virq); goto err_free_irq; } } @@ -598,8 +598,8 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb, return 0; err_free_irq: - for (i--; i >= 0; i--) - free_irq(epf->db_msg[i].virq, ntb); + for (req--; req >= 0; req--) + free_irq(epf->db_msg[req].virq, ntb); pci_epf_free_doorbell(ntb->epf); return ret; @@ -763,19 +763,6 @@ static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws) } } -/** - * epf_ntb_epc_destroy() - Cleanup NTB EPC interface - * @ntb: NTB device that facilitates communication between HOST and VHOST - * - * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces - */ -static void epf_ntb_epc_destroy(struct epf_ntb *ntb) -{ - pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0); - pci_epc_put(ntb->epf->epc); -} - - /** * epf_ntb_is_bar_used() - Check if a bar is used in the ntb configuration * @ntb: NTB device that facilitates communication between HOST and VHOST @@ -955,6 +942,7 @@ static int epf_ntb_epc_init(struct epf_ntb *ntb) */ static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) { + disable_delayed_work_sync(&ntb->cmd_handler); epf_ntb_mw_bar_clear(ntb, ntb->num_mws); epf_ntb_db_bar_clear(ntb); epf_ntb_config_sspad_bar_clear(ntb); @@ -1525,7 +1513,7 @@ static int epf_ntb_bind(struct pci_epf *epf) ret = epf_ntb_init_epc_bar(ntb); if (ret) { dev_err(dev, "Failed to create NTB EPC\n"); - goto err_bar_init; + return ret; } ret = epf_ntb_config_spad_bar_alloc(ntb); @@ -1565,9 +1553,6 @@ static int epf_ntb_bind(struct pci_epf *epf) err_bar_alloc: epf_ntb_config_spad_bar_free(ntb); -err_bar_init: - epf_ntb_epc_destroy(ntb); - return ret; } @@ -1583,7 +1568,6 @@ static void epf_ntb_unbind(struct pci_epf *epf) epf_ntb_epc_cleanup(ntb); epf_ntb_config_spad_bar_free(ntb); - epf_ntb_epc_destroy(ntb); pci_unregister_driver(&vntb_pci_driver); } diff --git a/drivers/pci/endpoint/pci-ep-msi.c b/drivers/pci/endpoint/pci-ep-msi.c index 51c19942a81efa..1395919571f83f 100644 --- a/drivers/pci/endpoint/pci-ep-msi.c +++ b/drivers/pci/endpoint/pci-ep-msi.c @@ -50,6 +50,9 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db) return -EINVAL; } + if (epf->db_msg) + return -EBUSY; + domain = of_msi_map_get_device_domain(epc->dev.parent, 0, DOMAIN_BUS_PLATFORM_MSI); if (!domain) { @@ -79,6 +82,8 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db) if (ret) { dev_err(dev, "Failed to allocate MSI\n"); kfree(msg); + epf->db_msg = NULL; + epf->num_db = 0; return ret; } diff --git a/drivers/pci/npem.c b/drivers/pci/npem.c index ffeeedf6e31128..c51879fcd43863 100644 --- a/drivers/pci/npem.c +++ b/drivers/pci/npem.c @@ -504,7 +504,7 @@ static int pci_npem_set_led_classdev(struct npem *npem, struct npem_led *nled) led->brightness_get = brightness_get; led->max_brightness = 1; led->default_trigger = "none"; - led->flags = 0; + led->flags = LED_HW_PLUGGABLE; ret = led_classdev_register(&npem->dev->dev, led); if (ret) diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index dd9075403987d8..e3f59001785a17 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -138,9 +138,11 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, { struct pci_dynid *dynid; const struct pci_device_id *found_id = NULL, *ids; + int ret; /* When driver_override is set, only bind to the matching driver */ - if (dev->driver_override && strcmp(dev->driver_override, drv->name)) + ret = device_match_driver_override(&dev->dev, &drv->driver); + if (ret == 0) return NULL; /* Look at the dynamic ids first, before the static ones */ @@ -164,7 +166,7 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, * matching. */ if (found_id->override_only) { - if (dev->driver_override) + if (ret > 0) return found_id; } else { return found_id; @@ -172,11 +174,16 @@ static const struct pci_device_id *pci_match_device(struct pci_driver *drv, } /* driver_override will always match, send a dummy id */ - if (dev->driver_override) + if (ret > 0) return &pci_device_id_any; return NULL; } +static void _pci_free_device(struct device *dev) +{ + kfree(to_pci_dev(dev)); +} + /** * new_id_store - sysfs frontend to pci_add_dynid() * @driver: target device driver @@ -212,11 +219,13 @@ static ssize_t new_id_store(struct device_driver *driver, const char *buf, pdev->subsystem_vendor = subvendor; pdev->subsystem_device = subdevice; pdev->class = class; + pdev->dev.release = _pci_free_device; + device_initialize(&pdev->dev); if (pci_match_device(pdrv, pdev)) retval = -EEXIST; - kfree(pdev); + put_device(&pdev->dev); if (retval) return retval; @@ -452,7 +461,7 @@ static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev) static inline bool pci_device_can_probe(struct pci_dev *pdev) { return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe || - pdev->driver_override); + device_has_driver_override(&pdev->dev)); } #else static inline bool pci_device_can_probe(struct pci_dev *pdev) @@ -1722,6 +1731,7 @@ static const struct cpumask *pci_device_irq_get_affinity(struct device *dev, const struct bus_type pci_bus_type = { .name = "pci", + .driver_override = true, .match = pci_bus_match, .uevent = pci_uevent, .probe = pci_device_probe, diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 16eaaf749ba970..a9006cf4e9c850 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -615,33 +615,6 @@ static ssize_t devspec_show(struct device *dev, static DEVICE_ATTR_RO(devspec); #endif -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct pci_dev *pdev = to_pci_dev(dev); - int ret; - - ret = driver_set_override(dev, &pdev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct pci_dev *pdev = to_pci_dev(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", pdev->driver_override); - device_unlock(dev); - return len; -} -static DEVICE_ATTR_RW(driver_override); - static struct attribute *pci_dev_attrs[] = { &dev_attr_power_state.attr, &dev_attr_resource.attr, @@ -669,7 +642,6 @@ static struct attribute *pci_dev_attrs[] = { #ifdef CONFIG_OF &dev_attr_devspec.attr, #endif - &dev_attr_driver_override.attr, &dev_attr_ari_enabled.attr, NULL, }; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 8479c2e1f74f10..9298a461bd302a 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -2241,10 +2241,9 @@ EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); #ifdef CONFIG_PCIEAER void pcie_clear_device_status(struct pci_dev *dev) { - u16 sta; - - pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta); - pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta); + pcie_capability_write_word(dev, PCI_EXP_DEVSTA, + PCI_EXP_DEVSTA_CED | PCI_EXP_DEVSTA_NFED | + PCI_EXP_DEVSTA_FED | PCI_EXP_DEVSTA_URD); } #endif @@ -3675,8 +3674,7 @@ void pci_acs_init(struct pci_dev *dev) */ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) { - struct pci_bus *bus = dev->bus; - struct pci_dev *bridge; + struct pci_dev *root, *bridge; u32 cap, ctl2; /* @@ -3706,35 +3704,35 @@ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) return -EINVAL; } - while (bus->parent) { - bridge = bus->self; + root = pcie_find_root_port(dev); + if (!root) + return -EINVAL; - pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); + pcie_capability_read_dword(root, PCI_EXP_DEVCAP2, &cap); + if ((cap & cap_mask) != cap_mask) + return -EINVAL; + bridge = pci_upstream_bridge(dev); + while (bridge != root) { switch (pci_pcie_type(bridge)) { - /* Ensure switch ports support AtomicOp routing */ case PCI_EXP_TYPE_UPSTREAM: - case PCI_EXP_TYPE_DOWNSTREAM: - if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) - return -EINVAL; - break; - - /* Ensure root port supports all the sizes we care about */ - case PCI_EXP_TYPE_ROOT_PORT: - if ((cap & cap_mask) != cap_mask) - return -EINVAL; - break; - } - - /* Ensure upstream ports don't block AtomicOps on egress */ - if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) { + /* Upstream ports must not block AtomicOps on egress */ pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, &ctl2); if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK) return -EINVAL; + fallthrough; + + /* All switch ports need to route AtomicOps */ + case PCI_EXP_TYPE_DOWNSTREAM: + pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, + &cap); + if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) + return -EINVAL; + break; } - bus = bus->parent; + bridge = pci_upstream_bridge(bridge); } pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index d916378bc70729..c4fd9c0b2a5482 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -1041,8 +1041,6 @@ static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) * 3) There are multiple errors and prior ID comparing fails; * We check AER status registers to find possible reporter. */ - if (atomic_read(&dev->enable_cnt) == 0) - return false; /* Check if AER is enabled */ pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 21f5d23e0b61bd..925373b98dff0c 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -706,22 +706,29 @@ static void aspm_calc_l12_info(struct pcie_link_state *link, } /* Program T_POWER_ON times in both ports */ - pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); - pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); + pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, + PCI_L1SS_CTL2_T_PWR_ON_VALUE | + PCI_L1SS_CTL2_T_PWR_ON_SCALE, ctl2); + pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL2, + PCI_L1SS_CTL2_T_PWR_ON_VALUE | + PCI_L1SS_CTL2_T_PWR_ON_SCALE, ctl2); /* Program Common_Mode_Restore_Time in upstream device */ pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, - PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1); + PCI_L1SS_CTL1_CM_RESTORE_TIME, + ctl1 & PCI_L1SS_CTL1_CM_RESTORE_TIME); /* Program LTR_L1.2_THRESHOLD time in both ports */ pci_clear_and_set_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_LTR_L12_TH_VALUE | PCI_L1SS_CTL1_LTR_L12_TH_SCALE, - ctl1); + ctl1 & (PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)); pci_clear_and_set_config_dword(child, child->l1ss + PCI_L1SS_CTL1, PCI_L1SS_CTL1_LTR_L12_TH_VALUE | PCI_L1SS_CTL1_LTR_L12_TH_SCALE, - ctl1); + ctl1 & (PCI_L1SS_CTL1_LTR_L12_TH_VALUE | + PCI_L1SS_CTL1_LTR_L12_TH_SCALE)); if (pl1_2_enables || cl1_2_enables) { pci_clear_and_set_config_dword(parent, diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c index fc18349614d7cf..7605ddd9f0ba85 100644 --- a/drivers/pci/pcie/dpc.c +++ b/drivers/pci/pcie/dpc.c @@ -256,6 +256,7 @@ static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev, info->dev[0] = dev; info->error_dev_num = 1; + info->ratelimit_print[0] = 1; return 1; } diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index bccc7a4bdd7943..b4707640e1021e 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -2488,7 +2488,6 @@ static void pci_release_dev(struct device *dev) pci_release_of_node(pci_dev); pcibios_release_device(pci_dev); pci_bus_put(pci_dev->bus); - kfree(pci_dev->driver_override); bitmap_free(pci_dev->dma_alias_mask); dev_dbg(dev, "device released\n"); kfree(pci_dev); diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c index 61f769aaa2f6ca..8f2830c6d34f78 100644 --- a/drivers/pci/setup-bus.c +++ b/drivers/pci/setup-bus.c @@ -434,6 +434,10 @@ static void reassign_resources_sorted(struct list_head *realloc_head, dev = add_res->dev; idx = pci_resource_num(dev, res); + /* Skip this resource if not found in head list */ + if (!res_to_dev_res(head, res)) + continue; + /* * Skip resource that failed the earlier assignment and is * not optional as it would just fail again. @@ -442,10 +446,6 @@ static void reassign_resources_sorted(struct list_head *realloc_head, !pci_resource_is_optional(dev, idx)) goto out; - /* Skip this resource if not found in head list */ - if (!res_to_dev_res(head, res)) - continue; - res_name = pci_resource_name(dev, idx); add_size = add_res->add_size; align = add_res->min_align; @@ -1333,7 +1333,14 @@ static void pbus_size_mem(struct pci_bus *bus, struct resource *b_res, r_size = resource_size(r); size += max(r_size, align); - aligns[order] += align; + /* + * If resource's size is larger than its alignment, + * some configurations result in an unwanted gap in + * the head space that the larger resource cannot + * fill. + */ + if (r_size <= align) + aligns[order] += align; if (order > max_order) max_order = order; } @@ -1837,6 +1844,7 @@ static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res, resource_size_t new_size) { resource_size_t add_size, size = resource_size(res); + struct pci_dev_resource *dev_res; if (resource_assigned(res)) return; @@ -1849,9 +1857,46 @@ static void adjust_bridge_window(struct pci_dev *bridge, struct resource *res, pci_dbg(bridge, "bridge window %pR extended by %pa\n", res, &add_size); } else if (new_size < size) { + int idx = pci_resource_num(bridge, res); + + /* + * hpio/mmio/mmioprefsize hasn't been included at all? See the + * add_size param at the callsites of calculate_memsize(). + */ + if (!add_list) + return; + + /* Only shrink if the hotplug extra relates to window size. */ + switch (idx) { + case PCI_BRIDGE_IO_WINDOW: + if (size > pci_hotplug_io_size) + return; + break; + case PCI_BRIDGE_MEM_WINDOW: + if (size > pci_hotplug_mmio_size) + return; + break; + case PCI_BRIDGE_PREF_MEM_WINDOW: + if (size > pci_hotplug_mmio_pref_size) + return; + break; + default: + break; + } + + dev_res = res_to_dev_res(add_list, res); add_size = size - new_size; - pci_dbg(bridge, "bridge window %pR shrunken by %pa\n", res, - &add_size); + if (add_size < dev_res->add_size) { + dev_res->add_size -= add_size; + pci_dbg(bridge, "bridge window %pR optional size shrunken by %pa\n", + res, &add_size); + } else { + pci_dbg(bridge, "bridge window %pR optional size removed\n", + res); + pci_dev_res_remove_from_list(add_list, res); + } + return; + } else { return; } diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index bb2aef373d6fc0..d11babcb129005 100644 --- a/drivers/pci/setup-res.c +++ b/drivers/pci/setup-res.c @@ -102,6 +102,7 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno) } pci_write_config_dword(dev, reg, new); + dev->saved_config_space[reg / 4] = new; pci_read_config_dword(dev, reg, &check); if ((new ^ check) & mask) { @@ -112,6 +113,7 @@ static void pci_std_update_resource(struct pci_dev *dev, int resno) if (res->flags & IORESOURCE_MEM_64) { new = region.start >> 16 >> 16; pci_write_config_dword(dev, reg + 4, new); + dev->saved_config_space[(reg + 4) / 4] = new; pci_read_config_dword(dev, reg + 4, &check); if (check != new) { pci_err(dev, "%s: error updating (high %#010x != %#010x)\n", diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c index ca4f97be75389d..e896b395828185 100644 --- a/drivers/pci/tph.c +++ b/drivers/pci/tph.c @@ -407,10 +407,13 @@ int pcie_enable_tph(struct pci_dev *pdev, int mode) else pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY; - rp_req_type = get_rp_completer_type(pdev); + /* Check if the device is behind a Root Port */ + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_END) { + rp_req_type = get_rp_completer_type(pdev); - /* Final req_type is the smallest value of two */ - pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type); + /* Final req_type is the smallest value of two */ + pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type); + } if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE) return -EINVAL; diff --git a/drivers/pcmcia/rsrc_nonstatic.c b/drivers/pcmcia/rsrc_nonstatic.c index 0679dd434719d8..b28d754ba414f8 100644 --- a/drivers/pcmcia/rsrc_nonstatic.c +++ b/drivers/pcmcia/rsrc_nonstatic.c @@ -187,7 +187,7 @@ static void do_io_probe(struct pcmcia_socket *s, unsigned int base, int any; u_char *b, hole, most; - dev_info(&s->dev, "cs: IO port probe %#x-%#x:", base, base+num-1); + pr_info("%s: cs: IO port probe %#x-%#x:", dev_name(&s->dev), base, base+num-1); /* First, what does a floating port look like? */ b = kzalloc(256, GFP_KERNEL); @@ -409,8 +409,8 @@ static int do_mem_probe(struct pcmcia_socket *s, u_long base, u_long num, struct socket_data *s_data = s->resource_data; u_long i, j, bad, fail, step; - dev_info(&s->dev, "cs: memory probe 0x%06lx-0x%06lx:", - base, base+num-1); + pr_info("%s: cs: memory probe 0x%06lx-0x%06lx:", + dev_name(&s->dev), base, base+num-1); bad = fail = 0; step = (num < 0x20000) ? 0x2000 : ((num>>4) & ~0x1fff); /* don't allow too large steps */ diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index d82d6f291a7537..8409b67f6d1ecb 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -11,3 +11,13 @@ config PHY_APPLE_ATC USB3, USB4, Thunderbolt, and DisplayPort. If M is selected the module will be called 'phy-apple-atc'. + +config PHY_APPLE_DPTX + tristate "Apple DPTX PHY" + depends on ARCH_APPLE || COMPILE_TEST + select GENERIC_PHY + help + Enable this to add support for the Apple DPTX PHY found on Apple SoCs + such as the M2. + This driver provides support for DisplayPort and is used on the + Mac mini (M2 and M2 Pro, 2023). diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile index e02836a63df3b5..b9e7bf3e4ac170 100644 --- a/drivers/phy/apple/Makefile +++ b/drivers/phy/apple/Makefile @@ -2,3 +2,6 @@ obj-$(CONFIG_PHY_APPLE_ATC) += phy-apple-atc.o phy-apple-atc-y := atc.o + +obj-$(CONFIG_PHY_APPLE_DPTX) += phy-apple-dptx.o +phy-apple-dptx-y += dptx.o diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index dc867f368b6874..64d0c3dba1cbb9 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2202,14 +2202,16 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph { "pipehandler", &atcphy->regs.pipehandler, NULL }, }; struct resource *res; + void __iomem *addr; for (int i = 0; i < ARRAY_SIZE(resources); i++) { res = platform_get_resource_byname(pdev, IORESOURCE_MEM, resources[i].name); - *resources[i].addr = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(resources[i].addr)) - return dev_err_probe(atcphy->dev, PTR_ERR(resources[i].addr), + addr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(addr)) + return dev_err_probe(atcphy->dev, PTR_ERR(addr), "Unable to map %s regs", resources[i].name); + *resources[i].addr = addr; if (resources[i].res) *resources[i].res = res; } diff --git a/drivers/phy/apple/dptx.c b/drivers/phy/apple/dptx.c new file mode 100644 index 00000000000000..5e58f149ce96af --- /dev/null +++ b/drivers/phy/apple/dptx.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple dptx PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Janne Grunau + * + * based on drivers/phy/apple/atc.c + * + * Copyright (C) The Asahi Linux Contributors + * Author: Sven Peter + */ + +#include "dptx.h" + +#include +#include "linux/of.h" +#include +#include +#include +#include +#include +#include +#include + +#define DPTX_MAX_LANES 4 +#define DPTX_LANE0_OFFSET 0x5000 +#define DPTX_LANE_STRIDE 0x1000 +#define DPTX_LANE_END (DPTX_LANE0_OFFSET + DPTX_MAX_LANES * DPTX_LANE_STRIDE) + +enum apple_dptx_type { + DPTX_PHY_T8112, + DPTX_PHY_T6020, +}; + +struct apple_dptx_phy_hw { + enum apple_dptx_type type; +}; + +struct apple_dptx_phy { + struct device *dev; + + struct apple_dptx_phy_hw hw; + + int dp_link_rate; + + struct { + void __iomem *core; + void __iomem *dptx; + } regs; + + struct phy *phy_dp; + struct phy_provider *phy_provider; + + struct mutex lock; + + // TODO: m1n1 port things to clean up + u32 active_lanes; +}; + + +static inline void mask32(void __iomem *reg, u32 mask, u32 set) +{ + u32 value = readl(reg); + value &= ~mask; + value |= set; + writel(value, reg); +} + +static inline void set32(void __iomem *reg, u32 set) +{ + mask32(reg, 0, set); +} + +static inline void clear32(void __iomem *reg, u32 clear) +{ + mask32(reg, clear, 0); +} + + +static int dptx_phy_set_active_lane_count(struct apple_dptx_phy *phy, u32 num_lanes) +{ + u32 l, ctrl; + + dev_dbg(phy->dev, "set_active_lane_count(%u)\n", num_lanes); + + if (num_lanes == 3 || num_lanes > DPTX_MAX_LANES) + return -1; + + ctrl = readl(phy->regs.dptx + 0x4000); + writel(ctrl, phy->regs.dptx + 0x4000); + + for (l = 0; l < num_lanes; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x100, phy->regs.dptx + offset); + } + for (; l < DPTX_MAX_LANES; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x300, phy->regs.dptx + offset); + } + for (l = 0; l < num_lanes; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x0, phy->regs.dptx + offset); + } + for (; l < DPTX_MAX_LANES; l++) { + u64 offset = 0x5000 + 0x1000 * l; + readl(phy->regs.dptx + offset); + writel(0x300, phy->regs.dptx + offset); + } + + if (num_lanes > 0) { + // clear32(phy->regs.dptx + 0x4000, 0x4000000); + ctrl = readl(phy->regs.dptx + 0x4000); + ctrl &= ~0x4000000; + writel(ctrl, phy->regs.dptx + 0x4000); + } + phy->active_lanes = num_lanes; + + return 0; +} + +static int dptx_phy_activate(struct apple_dptx_phy *phy, u32 dcp_index) +{ + u32 val_2014; + u32 val_4008; + u32 val_4408; + + dev_dbg(phy->dev, "activate(dcp:%u)\n", dcp_index); + + // MMIO: R.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 + // MMIO: W.4 0x23c500010 (dptx-phy[1], offset 0x10) = 0x0 + readl(phy->regs.core + 0x10); + writel(dcp_index, phy->regs.core + 0x10); + + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x444 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x454 + set32(phy->regs.core + 0x48, 0x010); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x454 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x474 + set32(phy->regs.core + 0x48, 0x020); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x474 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x434 + clear32(phy->regs.core + 0x48, 0x040); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x434 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x534 + set32(phy->regs.core + 0x48, 0x100); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x534 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x734 + set32(phy->regs.core + 0x48, 0x200); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x734 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x334 + clear32(phy->regs.core + 0x48, 0x400); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x334 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x335 + set32(phy->regs.core + 0x48, 0x001); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x335 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x337 + set32(phy->regs.core + 0x48, 0x002); + // MMIO: R.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x337 + // MMIO: W.4 0x23c500048 (dptx-phy[1], offset 0x48) = 0x333 + clear32(phy->regs.core + 0x48, 0x004); + + // MMIO: R.4 0x23c542014 (dptx-phy[0], offset 0x2014) = 0x80a0c + val_2014 = readl(phy->regs.dptx + 0x2014); + // MMIO: W.4 0x23c542014 (dptx-phy[0], offset 0x2014) = 0x300a0c + writel((0x30 << 16) | (val_2014 & 0xffff), phy->regs.dptx + 0x2014); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x644800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + set32(phy->regs.dptx + 0x20b8, 0x010000); + + // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a2 + // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 + clear32(phy->regs.dptx + 0x2220, 0x0000002); + + // MMIO: R.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103003 + // MMIO: W.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103803 + set32(phy->regs.dptx + 0x222c, 0x000800); + // MMIO: R.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103803 + // MMIO: W.4 0x23c54222c (dptx-phy[0], offset 0x222c) = 0x103903 + set32(phy->regs.dptx + 0x222c, 0x000100); + + // MMIO: R.4 0x23c542230 (dptx-phy[0], offset 0x2230) = 0x2308804 + // MMIO: W.4 0x23c542230 (dptx-phy[0], offset 0x2230) = 0x2208804 + clear32(phy->regs.dptx + 0x2230, 0x0100000); + + // MMIO: R.4 0x23c542278 (dptx-phy[0], offset 0x2278) = 0x18300811 + // MMIO: W.4 0x23c542278 (dptx-phy[0], offset 0x2278) = 0x10300811 + clear32(phy->regs.dptx + 0x2278, 0x08000000); + + // MMIO: R.4 0x23c5422a4 (dptx-phy[0], offset 0x22a4) = 0x1044200 + // MMIO: W.4 0x23c5422a4 (dptx-phy[0], offset 0x22a4) = 0x1044201 + set32(phy->regs.dptx + 0x22a4, 0x0000001); + + // MMIO: R.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x18030 + val_4008 = readl(phy->regs.dptx + 0x4008); + // MMIO: W.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30030 + writel((0x6 << 15) | (val_4008 & 0x7fff), phy->regs.dptx + 0x4008); + // MMIO: R.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30030 + // MMIO: W.4 0x23c544008 (dptx-phy[0], offset 0x4008) = 0x30010 + clear32(phy->regs.dptx + 0x4008, 0x00020); + + // MMIO: R.4 0x23c54420c (dptx-phy[0], offset 0x420c) = 0x88e3 + // MMIO: W.4 0x23c54420c (dptx-phy[0], offset 0x420c) = 0x88c3 + clear32(phy->regs.dptx + 0x420c, 0x0020); + + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x0 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + set32(phy->regs.dptx + 0x4600, 0x8000000); + + // MMIO: R.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x21780 + // MMIO: W.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x221780 + // MMIO: R.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x21780 + // MMIO: W.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x221780 + // MMIO: R.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x21780 + // MMIO: W.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x221780 + // MMIO: R.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x21780 + // MMIO: W.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x221780 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x200000); + + // MMIO: R.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x221780 + // MMIO: W.4 0x23c545040 (dptx-phy[0], offset 0x5040) = 0x2a1780 + // MMIO: R.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x221780 + // MMIO: W.4 0x23c546040 (dptx-phy[0], offset 0x6040) = 0x2a1780 + // MMIO: R.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x221780 + // MMIO: W.4 0x23c547040 (dptx-phy[0], offset 0x7040) = 0x2a1780 + // MMIO: R.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x221780 + // MMIO: W.4 0x23c548040 (dptx-phy[0], offset 0x8040) = 0x2a1780 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + set32(phy->regs.dptx + loff + 0x40, 0x080000); + + // MMIO: R.4 0x23c545244 (dptx-phy[0], offset 0x5244) = 0x18 + // MMIO: W.4 0x23c545244 (dptx-phy[0], offset 0x5244) = 0x8 + // MMIO: R.4 0x23c546244 (dptx-phy[0], offset 0x6244) = 0x18 + // MMIO: W.4 0x23c546244 (dptx-phy[0], offset 0x6244) = 0x8 + // MMIO: R.4 0x23c547244 (dptx-phy[0], offset 0x7244) = 0x18 + // MMIO: W.4 0x23c547244 (dptx-phy[0], offset 0x7244) = 0x8 + // MMIO: R.4 0x23c548244 (dptx-phy[0], offset 0x8244) = 0x18 + // MMIO: W.4 0x23c548244 (dptx-phy[0], offset 0x8244) = 0x8 + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; + loff += DPTX_LANE_STRIDE) + clear32(phy->regs.dptx + loff + 0x244, 0x10); + + // MMIO: R.4 0x23c542214 (dptx-phy[0], offset 0x2214) = 0x1e0 + // MMIO: W.4 0x23c542214 (dptx-phy[0], offset 0x2214) = 0x1e1 + set32(phy->regs.dptx + 0x2214, 0x001); + + // MMIO: R.4 0x23c542224 (dptx-phy[0], offset 0x2224) = 0x20086001 + // MMIO: W.4 0x23c542224 (dptx-phy[0], offset 0x2224) = 0x20086000 + clear32(phy->regs.dptx + 0x2224, 0x00000001); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + set32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c541000 (dptx-phy[0], offset 0x1000) = 0xe0000003 + // MMIO: W.4 0x23c541000 (dptx-phy[0], offset 0x1000) = 0xe0000001 + clear32(phy->regs.dptx + 0x1000, 0x00000002); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + /* TODO: no idea what happens here, supposedly setting/clearing some bits */ + // MMIO: R.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + readl(phy->regs.dptx + 0x4404); + // MMIO: W.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + writel(0x555d444, phy->regs.dptx + 0x4404); + // MMIO: R.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + readl(phy->regs.dptx + 0x4404); + // MMIO: W.4 0x23c544404 (dptx-phy[0], offset 0x4404) = 0x555d444 + writel(0x555d444, phy->regs.dptx + 0x4404); + + dptx_phy_set_active_lane_count(phy, 0); + + // MMIO: R.4 0x23c544200 (dptx-phy[0], offset 0x4200) = 0x4002430 + // MMIO: W.4 0x23c544200 (dptx-phy[0], offset 0x4200) = 0x4002420 + clear32(phy->regs.dptx + 0x4200, 0x0000010); + + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + clear32(phy->regs.dptx + 0x4600, 0x0000001); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000000 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000001 + set32(phy->regs.dptx + 0x4600, 0x0000001); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000001 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000003 + set32(phy->regs.dptx + 0x4600, 0x0000002); + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000043 + // MMIO: R.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000043 + // MMIO: W.4 0x23c544600 (dptx-phy[0], offset 0x4600) = 0x8000041 + /* TODO: read first to check if the previous set(...,0x2) sticked? */ + readl(phy->regs.dptx + 0x4600); + clear32(phy->regs.dptx + 0x4600, 0x0000001); + + // MMIO: R.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + // MMIO: W.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + /* TODO: probably a set32 of an already set bit */ + val_4408 = readl(phy->regs.dptx + 0x4408); + if (val_4408 != 0x482 && val_4408 != 0x483) + dev_warn( + phy->dev, + "unexpected initial value at regs.dptx offset 0x4408: 0x%03x\n", + val_4408); + writel(val_4408, phy->regs.dptx + 0x4408); + // MMIO: R.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x482 + // MMIO: W.4 0x23c544408 (dptx-phy[0], offset 0x4408) = 0x483 + set32(phy->regs.dptx + 0x4408, 0x001); + + return 0; +} + +static int dptx_phy_deactivate(struct apple_dptx_phy *phy) +{ + return 0; +} + +static int dptx_phy_set_link_rate(struct apple_dptx_phy *phy, u32 link_rate) +{ + u32 sts_1008, sts_1014, val_100c, val_20b0, val_20b4; + + dev_dbg(phy->dev, "set_link_rate(%u)\n", link_rate); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + clear32(phy->regs.dptx + 0x4000, 0x0000040); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + clear32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + clear32(phy->regs.dptx + 0x4000, 0x2000000); + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + set32(phy->regs.dptx + 0x4000, 0x1000000); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + /* TODO: what is this read checking for? */ + readl(phy->regs.dptx + 0x2200); + clear32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf008 + /* TODO: what is the setting/clearing? */ + val_100c = readl(phy->regs.dptx + 0x100c); + writel(val_100c, phy->regs.dptx + 0x100c); + set32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541014 (dptx-phy[0], offset 0x1014) = 0x1 + sts_1014 = readl(phy->regs.dptx + 0x1014); + if (sts_1014 != 0x1) + dev_dbg(phy->dev, "unexpected?: dptx[0x1014]: %02x\n", sts_1014); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf008 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + clear32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541008 (dptx-phy[0], offset 0x1008) = 0x1 + sts_1008 = readl(phy->regs.dptx + 0x1008); + if (sts_1008 != 0x1) + dev_dbg(phy->dev, "unexpected?: dptx[0x1008]: %02x\n", sts_1008); + + // MMIO: R.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x11090a0 + // MMIO: W.4 0x23c542220 (dptx-phy[0], offset 0x2220) = 0x1109020 + clear32(phy->regs.dptx + 0x2220, 0x0000080); + + // MMIO: R.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 + // MMIO: W.4 0x23c5420b0 (dptx-phy[0], offset 0x20b0) = 0x1e0e01c2 + val_20b0 = readl(phy->regs.dptx + 0x20b0); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b0 = (val_20b0 & ~0x3ff) | 0x2a3; + writel(val_20b0, phy->regs.dptx + 0x20b0); + + // MMIO: R.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + // MMIO: W.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + val_20b4 = readl(phy->regs.dptx + 0x20b4); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b4 = (val_20b4 | 0x4000000) & ~0x0008000; + writel(val_20b4, phy->regs.dptx + 0x20b4); + + // MMIO: R.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + // MMIO: W.4 0x23c5420b4 (dptx-phy[0], offset 0x20b4) = 0x7fffffe + val_20b4 = readl(phy->regs.dptx + 0x20b4); + /* TODO: what happens on dptx-phy */ + if (phy->hw.type == DPTX_PHY_T6020) + val_20b4 = (val_20b4 | 0x0000001) & ~0x0000004; + writel(val_20b4, phy->regs.dptx + 0x20b4); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + /* TODO: unclear */ + if (phy->hw.type == DPTX_PHY_T6020) + set32(phy->regs.dptx + 0x20b8, 0x010000); + else + set32(phy->regs.dptx + 0x20b8, 0); + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x654800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + clear32(phy->regs.dptx + 0x20b8, 0x200000); + + // MMIO: R.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + // MMIO: W.4 0x23c5420b8 (dptx-phy[0], offset 0x20b8) = 0x454800 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x20b8, 0); + + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x0 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x4000c + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x4000c + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8000c + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8000c + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0xc + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: R.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x8 + // MMIO: W.4 0x23c5000a0 (dptx-phy[1], offset 0xa0) = 0x0 + set32(phy->regs.core + 0xa0, 0x8); + set32(phy->regs.core + 0xa0, 0x4); + set32(phy->regs.core + 0xa0, 0x40000); + clear32(phy->regs.core + 0xa0, 0x40000); + set32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x80000); + clear32(phy->regs.core + 0xa0, 0x4); + clear32(phy->regs.core + 0xa0, 0x8); + + // MMIO: R.4 0x23c542000 (dptx-phy[0], offset 0x2000) = 0x2 + // MMIO: W.4 0x23c542000 (dptx-phy[0], offset 0x2000) = 0x2 + /* TODO: unclear */ + set32(phy->regs.dptx + 0x2000, 0x0); + + // MMIO: R.4 0x23c542018 (dptx-phy[0], offset 0x2018) = 0x0 + // MMIO: W.4 0x23c542018 (dptx-phy[0], offset 0x2018) = 0x0 + clear32(phy->regs.dptx + 0x2018, 0x0); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf000 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + set32(phy->regs.dptx + 0x100c, 0x0007); + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf00f + set32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541014 (dptx-phy[0], offset 0x1014) = 0x38f + sts_1014 = readl(phy->regs.dptx + 0x1014); + if (sts_1014 != 0x38f) + dev_dbg(phy->dev, "unexpected?: dptx[0x1014]: %02x\n", sts_1014); + + // MMIO: R.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf00f + // MMIO: W.4 0x23c54100c (dptx-phy[0], offset 0x100c) = 0xf007 + clear32(phy->regs.dptx + 0x100c, 0x0008); + + // MMIO: R.4 0x23c541008 (dptx-phy[0], offset 0x1008) = 0x9 + sts_1008 = readl(phy->regs.dptx + 0x1008); + if (sts_1008 != 0x9) + dev_dbg(phy->dev, "unexpected?: dptx[0x1008]: %02x\n", sts_1008); + + // MMIO: R.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2000 + // MMIO: W.4 0x23c542200 (dptx-phy[0], offset 0x2200) = 0x2002 + set32(phy->regs.dptx + 0x2200, 0x0002); + + // MMIO: R.4 0x23c545010 (dptx-phy[0], offset 0x5010) = 0x18003000 + // MMIO: W.4 0x23c545010 (dptx-phy[0], offset 0x5010) = 0x18003000 + // MMIO: R.4 0x23c546010 (dptx-phy[0], offset 0x6010) = 0x18003000 + // MMIO: W.4 0x23c546010 (dptx-phy[0], offset 0x6010) = 0x18003000 + // MMIO: R.4 0x23c547010 (dptx-phy[0], offset 0x7010) = 0x18003000 + // MMIO: W.4 0x23c547010 (dptx-phy[0], offset 0x7010) = 0x18003000 + // MMIO: R.4 0x23c548010 (dptx-phy[0], offset 0x8010) = 0x18003000 + // MMIO: W.4 0x23c548010 (dptx-phy[0], offset 0x8010) = 0x18003000 + writel(0x18003000, phy->regs.dptx + 0x8010); + for (u32 loff = DPTX_LANE0_OFFSET; loff < DPTX_LANE_END; loff += DPTX_LANE_STRIDE) { + u32 val_l010 = readl(phy->regs.dptx + loff + 0x10); + writel(val_l010, phy->regs.dptx + loff + 0x10); + } + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x41021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x51021ac + set32(phy->regs.dptx + 0x4000, 0x1000000); + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x51021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ac + set32(phy->regs.dptx + 0x4000, 0x2000000); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x41 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + set32(phy->regs.dptx + 0x4004, 0x08); + + // MMIO: R.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ac + // MMIO: W.4 0x23c544000 (dptx-phy[0], offset 0x4000) = 0x71021ec + set32(phy->regs.dptx + 0x4000, 0x0000040); + + // MMIO: R.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x49 + // MMIO: W.4 0x23c544004 (dptx-phy[0], offset 0x4004) = 0x48 + clear32(phy->regs.dptx + 0x4004, 0x01); + + return 0; +} + +static int dptx_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode) +{ + struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + + switch (mode) { + case PHY_MODE_INVALID: + return dptx_phy_deactivate(dptx_phy); + case PHY_MODE_DP: + if (submode < 0 || submode > 5) + return -EINVAL; + return dptx_phy_activate(dptx_phy, submode); + default: + break; + } + + return -EINVAL; +} + +static int dptx_phy_validate(struct phy *phy, enum phy_mode mode, int submode, + union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + + if (mode == PHY_MODE_INVALID) { + memset(opts, 0, sizeof(*opts)); + return 0; + } + + if (mode != PHY_MODE_DP) + return -EINVAL; + if (submode < 0 || submode > 5) + return -EINVAL; + + opts->lanes = 4; + opts->link_rate = 8100; + + for (int i = 0; i < 4; ++i) { + opts->voltage[i] = 3; + opts->pre[i] = 3; + } + + return 0; +} + +static int dptx_phy_configure(struct phy *phy, union phy_configure_opts *opts_) +{ + struct phy_configure_opts_dp *opts = &opts_->dp; + struct apple_dptx_phy *dptx_phy = phy_get_drvdata(phy); + enum dptx_phy_link_rate link_rate; + int ret = 0; + + if (opts->set_lanes) { + mutex_lock(&dptx_phy->lock); + ret = dptx_phy_set_active_lane_count(dptx_phy, opts->lanes); + mutex_unlock(&dptx_phy->lock); + } + + if (opts->set_rate) { + switch (opts->link_rate) { + case 1620: + link_rate = DPTX_PHY_LINK_RATE_RBR; + break; + case 2700: + link_rate = DPTX_PHY_LINK_RATE_HBR; + break; + case 5400: + link_rate = DPTX_PHY_LINK_RATE_HBR2; + break; + case 8100: + link_rate = DPTX_PHY_LINK_RATE_HBR3; + break; + case 0: + // TODO: disable! + return 0; + break; + default: + dev_err(dptx_phy->dev, "Unsupported link rate: %d\n", + opts->link_rate); + return -EINVAL; + } + + mutex_lock(&dptx_phy->lock); + ret = dptx_phy_set_link_rate(dptx_phy, link_rate); + mutex_unlock(&dptx_phy->lock); + } + + return ret; +} + +static const struct phy_ops apple_atc_dp_phy_ops = { + .owner = THIS_MODULE, + .configure = dptx_phy_configure, + .validate = dptx_phy_validate, + .set_mode = dptx_phy_set_mode, +}; + +static int dptx_phy_probe(struct platform_device *pdev) +{ + struct apple_dptx_phy *dptx_phy; + struct device *dev = &pdev->dev; + + dptx_phy = devm_kzalloc(dev, sizeof(*dptx_phy), GFP_KERNEL); + if (!dptx_phy) + return -ENOMEM; + + dptx_phy->dev = dev; + dptx_phy->hw = + *(struct apple_dptx_phy_hw *)of_device_get_match_data(dev); + platform_set_drvdata(pdev, dptx_phy); + + mutex_init(&dptx_phy->lock); + + dptx_phy->regs.core = + devm_platform_ioremap_resource_byname(pdev, "core"); + if (IS_ERR(dptx_phy->regs.core)) + return PTR_ERR(dptx_phy->regs.core); + dptx_phy->regs.dptx = + devm_platform_ioremap_resource_byname(pdev, "dptx"); + if (IS_ERR(dptx_phy->regs.dptx)) + return PTR_ERR(dptx_phy->regs.dptx); + + /* create phy */ + dptx_phy->phy_dp = + devm_phy_create(dptx_phy->dev, NULL, &apple_atc_dp_phy_ops); + if (IS_ERR(dptx_phy->phy_dp)) + return PTR_ERR(dptx_phy->phy_dp); + phy_set_drvdata(dptx_phy->phy_dp, dptx_phy); + + dptx_phy->phy_provider = + devm_of_phy_provider_register(dev, of_phy_simple_xlate); + if (IS_ERR(dptx_phy->phy_provider)) + return PTR_ERR(dptx_phy->phy_provider); + + return 0; +} + +static const struct apple_dptx_phy_hw apple_dptx_hw_t6020 = { + .type = DPTX_PHY_T6020, +}; + +static const struct apple_dptx_phy_hw apple_dptx_hw_t8112 = { + .type = DPTX_PHY_T8112, +}; + +static const struct of_device_id dptx_phy_match[] = { + { .compatible = "apple,t6020-dptx-phy", .data = &apple_dptx_hw_t6020 }, + { .compatible = "apple,t8112-dptx-phy", .data = &apple_dptx_hw_t8112 }, + {}, +}; +MODULE_DEVICE_TABLE(of, dptx_phy_match); + +static struct platform_driver dptx_phy_driver = { + .driver = { + .name = "phy-apple-dptx", + .of_match_table = dptx_phy_match, + }, + .probe = dptx_phy_probe, +}; + +module_platform_driver(dptx_phy_driver); + +MODULE_AUTHOR("Janne Grunau "); +MODULE_DESCRIPTION("Apple DP TX PHY driver"); + +MODULE_LICENSE("GPL"); diff --git a/drivers/phy/apple/dptx.h b/drivers/phy/apple/dptx.h new file mode 100644 index 00000000000000..2dd36d753eb357 --- /dev/null +++ b/drivers/phy/apple/dptx.h @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +/* + * Apple DP TX PHY driver + * + * Copyright (C) The Asahi Linux Contributors + * Author: Janne Grunau + */ + +#ifndef PHY_APPLE_DPTX_H +#define PHY_APPLE_DPTX_H + +enum dptx_phy_link_rate { + DPTX_PHY_LINK_RATE_RBR, + DPTX_PHY_LINK_RATE_HBR, + DPTX_PHY_LINK_RATE_HBR2, + DPTX_PHY_LINK_RATE_HBR3, +}; +#endif /* PHY_APPLE_DPTX_H */ diff --git a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c index 95cd3175926d56..68f1ba8fec4ad0 100644 --- a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c @@ -83,7 +83,7 @@ static const struct m31_phy_tbl_entry m31_eusb2_setup_tbl[] = { M31_EUSB_PHY_INIT_CFG(USB_PHY_CFG0, UTMI_PHY_CMN_CTRL_OVERRIDE_EN, 1), M31_EUSB_PHY_INIT_CFG(USB_PHY_UTMI_CTRL5, POR, 1), M31_EUSB_PHY_INIT_CFG(USB_PHY_HS_PHY_CTRL_COMMON0, PHY_ENABLE, 1), - M31_EUSB_PHY_INIT_CFG(USB_PHY_CFG1, PLL_EN, 1), + M31_EUSB_PHY_INIT_CFG(USB_PHY_CFG1, PLL_EN, 0), M31_EUSB_PHY_INIT_CFG(USB_PHY_FSEL_SEL, FSEL_SEL, 1), }; diff --git a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c index 3d5ffd6cb14b66..15d73ea1028cfe 100644 --- a/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c +++ b/drivers/pinctrl/microchip/pinctrl-mpfs-mssio.c @@ -686,7 +686,7 @@ static int mpfs_pinctrl_probe(struct platform_device *pdev) pctrl->regmap = device_node_to_regmap(pdev->dev.parent->of_node); if (IS_ERR(pctrl->regmap)) - dev_err_probe(dev, PTR_ERR(pctrl->regmap), "Failed to find syscon regmap\n"); + return dev_err_probe(dev, PTR_ERR(pctrl->regmap), "Failed to find syscon regmap\n"); pctrl->sysreg_regmap = syscon_regmap_lookup_by_compatible("microchip,mpfs-sysreg-scb"); if (IS_ERR(pctrl->sysreg_regmap)) diff --git a/drivers/pinctrl/nomadik/pinctrl-abx500.c b/drivers/pinctrl/nomadik/pinctrl-abx500.c index fc7ebeda8440eb..858fbaebcf8e51 100644 --- a/drivers/pinctrl/nomadik/pinctrl-abx500.c +++ b/drivers/pinctrl/nomadik/pinctrl-abx500.c @@ -852,7 +852,7 @@ static int abx500_pin_config_set(struct pinctrl_dev *pctldev, int ret = -EINVAL; int i; enum pin_config_param param; - enum pin_config_param argument; + unsigned int argument; for (i = 0; i < num_configs; i++) { param = pinconf_to_config_param(configs[i]); diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c index 2b030bd0e6adca..6b4a794a362c04 100644 --- a/drivers/pinctrl/pinconf-generic.c +++ b/drivers/pinctrl/pinconf-generic.c @@ -287,12 +287,17 @@ int pinconf_generic_parse_dt_pinmux(struct device_node *np, struct device *dev, return -ENOENT; } + npins_t = prop->length / sizeof(u32); + if (npins_t == 0) { + dev_info(dev, "pinmux property doesn't have entries\n"); + return -ENODATA; + } + if (!pid || !pmux || !npins) { dev_err(dev, "parameters error\n"); return -EINVAL; } - npins_t = prop->length / sizeof(u32); pid_t = devm_kcalloc(dev, npins_t, sizeof(*pid_t), GFP_KERNEL); pmux_t = devm_kcalloc(dev, npins_t, sizeof(*pmux_t), GFP_KERNEL); if (!pid_t || !pmux_t) { diff --git a/drivers/pinctrl/pinctrl-cy8c95x0.c b/drivers/pinctrl/pinctrl-cy8c95x0.c index 5c055d344ac9d5..54b117f32f0eab 100644 --- a/drivers/pinctrl/pinctrl-cy8c95x0.c +++ b/drivers/pinctrl/pinctrl-cy8c95x0.c @@ -1310,6 +1310,7 @@ static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq) { struct gpio_irq_chip *girq = &chip->gpio_chip.irq; DECLARE_BITMAP(pending_irqs, MAX_LINE); + struct device *dev = chip->dev; int ret; mutex_init(&chip->irq_lock); @@ -1318,10 +1319,8 @@ static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq) /* Read IRQ status register to clear all pending interrupts */ ret = cy8c95x0_irq_pending(chip, pending_irqs); - if (ret) { - dev_err(chip->dev, "failed to clear irq status register\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, -EBUSY, "failed to clear irq status register\n"); /* Mask all interrupts */ bitmap_fill(chip->irq_mask, MAX_LINE); @@ -1336,17 +1335,9 @@ static int cy8c95x0_irq_setup(struct cy8c95x0_pinctrl *chip, int irq) girq->handler = handle_simple_irq; girq->threaded = true; - ret = devm_request_threaded_irq(chip->dev, irq, - NULL, cy8c95x0_irq_handler, - IRQF_ONESHOT | IRQF_SHARED, - dev_name(chip->dev), chip); - if (ret) { - dev_err(chip->dev, "failed to request irq %d\n", irq); - return ret; - } - dev_info(chip->dev, "Registered threaded IRQ\n"); - - return 0; + return devm_request_threaded_irq(dev, irq, NULL, cy8c95x0_irq_handler, + IRQF_ONESHOT | IRQF_SHARED, + dev_name(chip->dev), chip); } static int cy8c95x0_setup_pinctrl(struct cy8c95x0_pinctrl *chip) @@ -1362,11 +1353,7 @@ static int cy8c95x0_setup_pinctrl(struct cy8c95x0_pinctrl *chip) pd->owner = THIS_MODULE; chip->pctldev = devm_pinctrl_register(chip->dev, pd, chip); - if (IS_ERR(chip->pctldev)) - return dev_err_probe(chip->dev, PTR_ERR(chip->pctldev), - "can't register controller\n"); - - return 0; + return PTR_ERR_OR_ZERO(chip->pctldev); } static int cy8c95x0_detect(struct i2c_client *client, diff --git a/drivers/pinctrl/pinctrl-pic32.c b/drivers/pinctrl/pinctrl-pic32.c index 16bbbcf7206288..f61ab89bc0f7b9 100644 --- a/drivers/pinctrl/pinctrl-pic32.c +++ b/drivers/pinctrl/pinctrl-pic32.c @@ -2174,16 +2174,10 @@ static int pic32_pinctrl_probe(struct platform_device *pdev) if (IS_ERR(pctl->reg_base)) return PTR_ERR(pctl->reg_base); - pctl->clk = devm_clk_get(&pdev->dev, NULL); + pctl->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(pctl->clk)) { ret = PTR_ERR(pctl->clk); - dev_err(&pdev->dev, "clk get failed\n"); - return ret; - } - - ret = clk_prepare_enable(pctl->clk); - if (ret) { - dev_err(&pdev->dev, "clk enable failed\n"); + dev_err(&pdev->dev, "Failed to get and enable clock\n"); return ret; } @@ -2239,16 +2233,10 @@ static int pic32_gpio_probe(struct platform_device *pdev) if (irq < 0) return irq; - bank->clk = devm_clk_get(&pdev->dev, NULL); + bank->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(bank->clk)) { ret = PTR_ERR(bank->clk); - dev_err(&pdev->dev, "clk get failed\n"); - return ret; - } - - ret = clk_prepare_enable(bank->clk); - if (ret) { - dev_err(&pdev->dev, "clk enable failed\n"); + dev_err(&pdev->dev, "Failed to get and enable clock\n"); return ret; } diff --git a/drivers/pinctrl/realtek/pinctrl-rtd.c b/drivers/pinctrl/realtek/pinctrl-rtd.c index 2440604863327d..4c876d1f6ad59f 100644 --- a/drivers/pinctrl/realtek/pinctrl-rtd.c +++ b/drivers/pinctrl/realtek/pinctrl-rtd.c @@ -279,7 +279,7 @@ static const struct rtd_pin_sconfig_desc *rtd_pinctrl_find_sconfig(struct rtd_pi static int rtd_pconf_parse_conf(struct rtd_pinctrl *data, unsigned int pinnr, enum pin_config_param param, - enum pin_config_param arg) + unsigned int arg) { const struct rtd_pin_config_desc *config_desc; const struct rtd_pin_sconfig_desc *sconfig_desc; diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index 863e779dda0285..55e35f63343c71 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -3012,6 +3012,13 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen off = RZG2L_PIN_CFG_TO_PORT_OFFSET(cfg); pincnt = hweight8(FIELD_GET(PIN_CFG_PIN_MAP_MASK, cfg)); + if (cfg & RZG2L_VARIABLE_CFG) { + unsigned int pin = port * RZG2L_PINS_PER_PORT; + + for (unsigned int i = 0; i < RZG2L_PINS_PER_PORT; i++) + cfg |= *(u64 *)pctrl->desc.pins[pin + i].drv_data; + } + caps = FIELD_GET(PIN_CFG_MASK, cfg); has_iolh = !!(caps & (PIN_CFG_IOLH_A | PIN_CFG_IOLH_B | PIN_CFG_IOLH_C)); has_ien = !!(caps & PIN_CFG_IEN); diff --git a/drivers/pinctrl/sophgo/pinctrl-sg2042.c b/drivers/pinctrl/sophgo/pinctrl-sg2042.c index 185305ac897d9b..8dba12e122a45d 100644 --- a/drivers/pinctrl/sophgo/pinctrl-sg2042.c +++ b/drivers/pinctrl/sophgo/pinctrl-sg2042.c @@ -651,5 +651,5 @@ static struct platform_driver sg2042_pinctrl_driver = { }; module_platform_driver(sg2042_pinctrl_driver); -MODULE_DESCRIPTION("Pinctrl driver for the SG2002 series SoC"); +MODULE_DESCRIPTION("Pinctrl driver for the SG2042 series SoC"); MODULE_LICENSE("GPL"); diff --git a/drivers/pinctrl/sophgo/pinctrl-sg2044.c b/drivers/pinctrl/sophgo/pinctrl-sg2044.c index b0c46d8954ca1a..cf0b674c038f06 100644 --- a/drivers/pinctrl/sophgo/pinctrl-sg2044.c +++ b/drivers/pinctrl/sophgo/pinctrl-sg2044.c @@ -714,5 +714,5 @@ static struct platform_driver sg2044_pinctrl_driver = { }; module_platform_driver(sg2044_pinctrl_driver); -MODULE_DESCRIPTION("Pinctrl driver for the SG2002 series SoC"); +MODULE_DESCRIPTION("Pinctrl driver for the SG2044 series SoC"); MODULE_LICENSE("GPL"); diff --git a/drivers/platform/chrome/chromeos_tbmc.c b/drivers/platform/chrome/chromeos_tbmc.c index d1cf8f3463ce3b..e248567c0a1823 100644 --- a/drivers/platform/chrome/chromeos_tbmc.c +++ b/drivers/platform/chrome/chromeos_tbmc.c @@ -95,6 +95,11 @@ static int chromeos_tbmc_add(struct acpi_device *adev) return 0; } +static void chromeos_tbmc_remove(struct acpi_device *adev) +{ + device_init_wakeup(&adev->dev, false); +} + static const struct acpi_device_id chromeos_tbmc_acpi_device_ids[] = { { ACPI_DRV_NAME, 0 }, { } @@ -110,6 +115,7 @@ static struct acpi_driver chromeos_tbmc_driver = { .ids = chromeos_tbmc_acpi_device_ids, .ops = { .add = chromeos_tbmc_add, + .remove = chromeos_tbmc_remove, .notify = chromeos_tbmc_notify, }, .drv.pm = &chromeos_tbmc_pm_ops, diff --git a/drivers/platform/chrome/cros_typec_altmode.c b/drivers/platform/chrome/cros_typec_altmode.c index 557340b53af03b..66c546bf89b532 100644 --- a/drivers/platform/chrome/cros_typec_altmode.c +++ b/drivers/platform/chrome/cros_typec_altmode.c @@ -359,6 +359,7 @@ cros_typec_register_thunderbolt(struct cros_typec_port *port, } INIT_WORK(&adata->work, cros_typec_altmode_work); + mutex_init(&adata->lock); adata->alt = alt; adata->port = port; adata->ap_mode_entry = true; diff --git a/drivers/platform/surface/surfacepro3_button.c b/drivers/platform/surface/surfacepro3_button.c index 9bd39f09c7db48..a6c9d4d370bec5 100644 --- a/drivers/platform/surface/surfacepro3_button.c +++ b/drivers/platform/surface/surfacepro3_button.c @@ -242,6 +242,7 @@ static void surface_button_remove(struct acpi_device *device) { struct surface_button *button = acpi_driver_data(device); + device_init_wakeup(&device->dev, false); input_unregister_device(button->input); kfree(button); } diff --git a/drivers/platform/wmi/core.c b/drivers/platform/wmi/core.c index b8e6b9a421c627..750e3619724e09 100644 --- a/drivers/platform/wmi/core.c +++ b/drivers/platform/wmi/core.c @@ -842,39 +842,11 @@ static ssize_t expensive_show(struct device *dev, } static DEVICE_ATTR_RO(expensive); -static ssize_t driver_override_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct wmi_device *wdev = to_wmi_device(dev); - ssize_t ret; - - device_lock(dev); - ret = sysfs_emit(buf, "%s\n", wdev->driver_override); - device_unlock(dev); - - return ret; -} - -static ssize_t driver_override_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct wmi_device *wdev = to_wmi_device(dev); - int ret; - - ret = driver_set_override(dev, &wdev->driver_override, buf, count); - if (ret < 0) - return ret; - - return count; -} -static DEVICE_ATTR_RW(driver_override); - static struct attribute *wmi_attrs[] = { &dev_attr_modalias.attr, &dev_attr_guid.attr, &dev_attr_instance_count.attr, &dev_attr_expensive.attr, - &dev_attr_driver_override.attr, NULL }; ATTRIBUTE_GROUPS(wmi); @@ -943,7 +915,6 @@ static void wmi_dev_release(struct device *dev) { struct wmi_block *wblock = dev_to_wblock(dev); - kfree(wblock->dev.driver_override); kfree(wblock); } @@ -952,10 +923,12 @@ static int wmi_dev_match(struct device *dev, const struct device_driver *driver) const struct wmi_driver *wmi_driver = to_wmi_driver(driver); struct wmi_block *wblock = dev_to_wblock(dev); const struct wmi_device_id *id = wmi_driver->id_table; + int ret; /* When driver_override is set, only bind to the matching driver */ - if (wblock->dev.driver_override) - return !strcmp(wblock->dev.driver_override, driver->name); + ret = device_match_driver_override(dev, driver); + if (ret >= 0) + return ret; if (id == NULL) return 0; @@ -1076,6 +1049,7 @@ static struct class wmi_bus_class = { static const struct bus_type wmi_bus_type = { .name = "wmi", .dev_groups = wmi_groups, + .driver_override = true, .match = wmi_dev_match, .uevent = wmi_dev_uevent, .probe = wmi_dev_probe, diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c index 7c0915e097bae5..0f8eb5be7c7a1c 100644 --- a/drivers/platform/x86/asus-wmi.c +++ b/drivers/platform/x86/asus-wmi.c @@ -125,7 +125,6 @@ module_param(fnlock_default, bool, 0444); #define NVIDIA_TEMP_MIN 75 #define NVIDIA_TEMP_MAX 87 -#define ASUS_SCREENPAD_BRIGHT_MIN 20 #define ASUS_SCREENPAD_BRIGHT_MAX 255 #define ASUS_SCREENPAD_BRIGHT_DEFAULT 60 @@ -4408,43 +4407,35 @@ static int read_screenpad_brightness(struct backlight_device *bd) return err; /* The device brightness can only be read if powered, so return stored */ if (err == BACKLIGHT_POWER_OFF) - return asus->driver->screenpad_brightness - ASUS_SCREENPAD_BRIGHT_MIN; + return bd->props.brightness; err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_SCREENPAD_LIGHT, &retval); if (err < 0) return err; - return (retval & ASUS_WMI_DSTS_BRIGHTNESS_MASK) - ASUS_SCREENPAD_BRIGHT_MIN; + return retval & ASUS_WMI_DSTS_BRIGHTNESS_MASK; } static int update_screenpad_bl_status(struct backlight_device *bd) { - struct asus_wmi *asus = bl_get_data(bd); - int power, err = 0; - u32 ctrl_param; + u32 ctrl_param = bd->props.brightness; + int err = 0; - power = read_screenpad_backlight_power(asus); - if (power < 0) - return power; + if (bd->props.power) { + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 1, NULL); + if (err < 0) + return err; - if (bd->props.power != power) { - if (power != BACKLIGHT_POWER_ON) { - /* Only brightness > 0 can power it back on */ - ctrl_param = asus->driver->screenpad_brightness - ASUS_SCREENPAD_BRIGHT_MIN; - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_LIGHT, - ctrl_param, NULL); - } else { - err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 0, NULL); - } - } else if (power == BACKLIGHT_POWER_ON) { - /* Only set brightness if powered on or we get invalid/unsync state */ - ctrl_param = bd->props.brightness + ASUS_SCREENPAD_BRIGHT_MIN; err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_LIGHT, ctrl_param, NULL); + if (err < 0) + return err; } - /* Ensure brightness is stored to turn back on with */ - if (err == 0) - asus->driver->screenpad_brightness = bd->props.brightness + ASUS_SCREENPAD_BRIGHT_MIN; + if (!bd->props.power) { + err = asus_wmi_set_devstate(ASUS_WMI_DEVID_SCREENPAD_POWER, 0, NULL); + if (err < 0) + return err; + } return err; } @@ -4462,22 +4453,19 @@ static int asus_screenpad_init(struct asus_wmi *asus) int err, power; int brightness = 0; - power = read_screenpad_backlight_power(asus); + power = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_SCREENPAD_POWER); if (power < 0) return power; - if (power != BACKLIGHT_POWER_OFF) { + if (power) { err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_SCREENPAD_LIGHT, &brightness); if (err < 0) return err; } - /* default to an acceptable min brightness on boot if too low */ - if (brightness < ASUS_SCREENPAD_BRIGHT_MIN) - brightness = ASUS_SCREENPAD_BRIGHT_DEFAULT; memset(&props, 0, sizeof(struct backlight_properties)); props.type = BACKLIGHT_RAW; /* ensure this bd is last to be picked */ - props.max_brightness = ASUS_SCREENPAD_BRIGHT_MAX - ASUS_SCREENPAD_BRIGHT_MIN; + props.max_brightness = ASUS_SCREENPAD_BRIGHT_MAX; bd = backlight_device_register("asus_screenpad", &asus->platform_device->dev, asus, &asus_screenpad_bl_ops, &props); @@ -4488,7 +4476,7 @@ static int asus_screenpad_init(struct asus_wmi *asus) asus->screenpad_backlight_device = bd; asus->driver->screenpad_brightness = brightness; - bd->props.brightness = brightness - ASUS_SCREENPAD_BRIGHT_MIN; + bd->props.brightness = brightness; bd->props.power = power; backlight_update_status(bd); diff --git a/drivers/platform/x86/barco-p50-gpio.c b/drivers/platform/x86/barco-p50-gpio.c index 6f13e81f98fbb2..360ffd8505d6cd 100644 --- a/drivers/platform/x86/barco-p50-gpio.c +++ b/drivers/platform/x86/barco-p50-gpio.c @@ -275,8 +275,11 @@ static int p50_gpio_get(struct gpio_chip *gc, unsigned int offset) mutex_lock(&p50->lock); ret = p50_send_mbox_cmd(p50, P50_MBOX_CMD_READ_GPIO, gpio_params[offset], 0); - if (ret == 0) + if (ret == 0) { ret = p50_read_mbox_reg(p50, P50_MBOX_REG_DATA); + if (ret >= 0) + ret = !!ret; + } mutex_unlock(&p50->lock); diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c b/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c index 09996fbdc70740..a85639d8a076f2 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/enum-attributes.c @@ -6,10 +6,32 @@ * Copyright (c) 2020 Dell Inc. */ +#include + #include "dell-wmi-sysman.h" get_instance_id(enumeration); +static int append_enum_string(char *dest, const char *src) +{ + size_t dest_len = strlen(dest); + ssize_t copied; + + if (WARN_ON_ONCE(dest_len >= MAX_BUFF)) + return -EINVAL; + + copied = strscpy(dest + dest_len, src, MAX_BUFF - dest_len); + if (copied < 0) + return -EINVAL; + + dest_len += copied; + copied = strscpy(dest + dest_len, ";", MAX_BUFF - dest_len); + if (copied < 0) + return -EINVAL; + + return 0; +} + static ssize_t current_value_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { int instance_id = get_enumeration_instance_id(kobj); @@ -176,9 +198,9 @@ int populate_enum_data(union acpi_object *enumeration_obj, int instance_id, return -EINVAL; if (check_property_type(enumeration, next_obj, ACPI_TYPE_STRING)) return -EINVAL; - strcat(wmi_priv.enumeration_data[instance_id].dell_value_modifier, - enumeration_obj[next_obj++].string.pointer); - strcat(wmi_priv.enumeration_data[instance_id].dell_value_modifier, ";"); + if (append_enum_string(wmi_priv.enumeration_data[instance_id].dell_value_modifier, + enumeration_obj[next_obj++].string.pointer)) + return -EINVAL; } if (next_obj >= enum_property_count) @@ -193,9 +215,9 @@ int populate_enum_data(union acpi_object *enumeration_obj, int instance_id, return -EINVAL; if (check_property_type(enumeration, next_obj, ACPI_TYPE_STRING)) return -EINVAL; - strcat(wmi_priv.enumeration_data[instance_id].possible_values, - enumeration_obj[next_obj++].string.pointer); - strcat(wmi_priv.enumeration_data[instance_id].possible_values, ";"); + if (append_enum_string(wmi_priv.enumeration_data[instance_id].possible_values, + enumeration_obj[next_obj++].string.pointer)) + return -EINVAL; } return sysfs_create_group(attr_name_kobj, &enumeration_attr_group); diff --git a/drivers/platform/x86/dell/dell_rbu.c b/drivers/platform/x86/dell/dell_rbu.c index eb50f1d75d0c13..3fa9de9aa47b80 100644 --- a/drivers/platform/x86/dell/dell_rbu.c +++ b/drivers/platform/x86/dell/dell_rbu.c @@ -30,6 +30,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -619,9 +620,12 @@ static ssize_t packet_size_write(struct file *filp, struct kobject *kobj, char *buffer, loff_t pos, size_t count) { unsigned long temp; + + if (kstrtoul(buffer, 10, &temp)) + return -EINVAL; + spin_lock(&rbu_data.lock); packet_empty_list(); - sscanf(buffer, "%lu", &temp); if (temp < 0xffffffff) rbu_data.packetsize = temp; diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index 988a0acc9622b8..75682bb4cc52a1 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -399,6 +399,11 @@ static const struct key_entry hp_wmi_keymap[] = { { KE_KEY, 0x21a9, { KEY_TOUCHPAD_OFF } }, { KE_KEY, 0x121a9, { KEY_TOUCHPAD_ON } }, { KE_KEY, 0x231b, { KEY_HELP } }, + { KE_IGNORE, 0x21ab, }, /* FnLock on */ + { KE_IGNORE, 0x121ab, }, /* FnLock off */ + { KE_IGNORE, 0x30021aa, }, /* kbd backlight: level 2 -> off */ + { KE_IGNORE, 0x33221aa, }, /* kbd backlight: off -> level 1 */ + { KE_IGNORE, 0x36421aa, }, /* kbd backlight: level 1 -> level 2*/ { KE_END, 0 } }; @@ -453,23 +458,24 @@ enum pwm_modes { }; struct hp_wmi_hwmon_priv { + struct mutex lock; /* protects mode, pwm */ u8 min_rpm; u8 max_rpm; - u8 gpu_delta; + int gpu_delta; u8 mode; u8 pwm; struct delayed_work keep_alive_dwork; }; struct victus_s_fan_table_header { + u8 num_fans; u8 unknown; - u8 num_entries; } __packed; struct victus_s_fan_table_entry { u8 cpu_rpm; u8 gpu_rpm; - u8 unknown; + u8 noise_db; } __packed; struct victus_s_fan_table { @@ -2353,13 +2359,16 @@ static int hp_wmi_apply_fan_settings(struct hp_wmi_hwmon_priv *priv) switch (priv->mode) { case PWM_MODE_MAX: - if (is_victus_s_thermal_profile()) - hp_wmi_get_fan_count_userdefine_trigger(); + if (is_victus_s_thermal_profile()) { + ret = hp_wmi_get_fan_count_userdefine_trigger(); + if (ret < 0) + return ret; + } ret = hp_wmi_fan_speed_max_set(1); if (ret < 0) return ret; - schedule_delayed_work(&priv->keep_alive_dwork, - secs_to_jiffies(KEEP_ALIVE_DELAY_SECS)); + mod_delayed_work(system_wq, &priv->keep_alive_dwork, + secs_to_jiffies(KEEP_ALIVE_DELAY_SECS)); return 0; case PWM_MODE_MANUAL: if (!is_victus_s_thermal_profile()) @@ -2367,26 +2376,26 @@ static int hp_wmi_apply_fan_settings(struct hp_wmi_hwmon_priv *priv) ret = hp_wmi_fan_speed_set(priv, pwm_to_rpm(priv->pwm, priv)); if (ret < 0) return ret; - schedule_delayed_work(&priv->keep_alive_dwork, - secs_to_jiffies(KEEP_ALIVE_DELAY_SECS)); + mod_delayed_work(system_wq, &priv->keep_alive_dwork, + secs_to_jiffies(KEEP_ALIVE_DELAY_SECS)); return 0; case PWM_MODE_AUTO: if (is_victus_s_thermal_profile()) { - hp_wmi_get_fan_count_userdefine_trigger(); + ret = hp_wmi_get_fan_count_userdefine_trigger(); + if (ret < 0) + return ret; ret = hp_wmi_fan_speed_max_reset(priv); } else { ret = hp_wmi_fan_speed_max_set(0); } if (ret < 0) return ret; - cancel_delayed_work_sync(&priv->keep_alive_dwork); + cancel_delayed_work(&priv->keep_alive_dwork); return 0; default: /* shouldn't happen */ return -EINVAL; } - - return 0; } static umode_t hp_wmi_hwmon_is_visible(const void *data, @@ -2419,6 +2428,7 @@ static int hp_wmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type, { struct hp_wmi_hwmon_priv *priv; int rpm, ret; + u8 mode; priv = dev_get_drvdata(dev); switch (type) { @@ -2442,11 +2452,13 @@ static int hp_wmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type, *val = rpm_to_pwm(rpm / 100, priv); return 0; } - switch (priv->mode) { + scoped_guard(mutex, &priv->lock) + mode = priv->mode; + switch (mode) { case PWM_MODE_MAX: case PWM_MODE_MANUAL: case PWM_MODE_AUTO: - *val = priv->mode; + *val = mode; return 0; default: /* shouldn't happen */ @@ -2464,6 +2476,7 @@ static int hp_wmi_hwmon_write(struct device *dev, enum hwmon_sensor_types type, int rpm; priv = dev_get_drvdata(dev); + guard(mutex)(&priv->lock); switch (type) { case hwmon_pwm: if (attr == hwmon_pwm_input) { @@ -2528,22 +2541,30 @@ static void hp_wmi_hwmon_keep_alive_handler(struct work_struct *work) { struct delayed_work *dwork; struct hp_wmi_hwmon_priv *priv; + int ret; dwork = to_delayed_work(work); priv = container_of(dwork, struct hp_wmi_hwmon_priv, keep_alive_dwork); + + guard(mutex)(&priv->lock); /* * Re-apply the current hwmon context settings. * NOTE: hp_wmi_apply_fan_settings will handle the re-scheduling. */ - hp_wmi_apply_fan_settings(priv); + ret = hp_wmi_apply_fan_settings(priv); + if (ret) + pr_warn_ratelimited("keep-alive failed to refresh fan settings: %d\n", + ret); } static int hp_wmi_setup_fan_settings(struct hp_wmi_hwmon_priv *priv) { u8 fan_data[128] = { 0 }; struct victus_s_fan_table *fan_table; - u8 min_rpm, max_rpm, gpu_delta; - int ret; + u8 min_rpm, max_rpm; + u8 cpu_rpm, gpu_rpm, noise_db; + int gpu_delta, i, num_entries, ret; + size_t header_size, entry_size; /* Default behaviour on hwmon init is automatic mode */ priv->mode = PWM_MODE_AUTO; @@ -2558,13 +2579,36 @@ static int hp_wmi_setup_fan_settings(struct hp_wmi_hwmon_priv *priv) return ret; fan_table = (struct victus_s_fan_table *)fan_data; - if (fan_table->header.num_entries == 0 || - sizeof(struct victus_s_fan_table_header) + - sizeof(struct victus_s_fan_table_entry) * fan_table->header.num_entries > sizeof(fan_data)) + if (fan_table->header.num_fans == 0) + return -EINVAL; + + header_size = sizeof(struct victus_s_fan_table_header); + entry_size = sizeof(struct victus_s_fan_table_entry); + num_entries = (sizeof(fan_data) - header_size) / entry_size; + min_rpm = U8_MAX; + max_rpm = 0; + + for (i = 0 ; i < num_entries ; i++) { + cpu_rpm = fan_table->entries[i].cpu_rpm; + gpu_rpm = fan_table->entries[i].gpu_rpm; + noise_db = fan_table->entries[i].noise_db; + + /* + * On some devices, the fan table is truncated with an all-zero row, + * hence we stop parsing here. + */ + if (cpu_rpm == 0 && gpu_rpm == 0 && noise_db == 0) + break; + + if (cpu_rpm < min_rpm) + min_rpm = cpu_rpm; + if (cpu_rpm > max_rpm) + max_rpm = cpu_rpm; + } + + if (min_rpm == U8_MAX || max_rpm == 0) return -EINVAL; - min_rpm = fan_table->entries[0].cpu_rpm; - max_rpm = fan_table->entries[fan_table->header.num_entries - 1].cpu_rpm; gpu_delta = fan_table->entries[0].gpu_rpm - fan_table->entries[0].cpu_rpm; priv->min_rpm = min_rpm; priv->max_rpm = max_rpm; @@ -2584,6 +2628,10 @@ static int hp_wmi_hwmon_init(void) if (!priv) return -ENOMEM; + ret = devm_mutex_init(dev, &priv->lock); + if (ret) + return ret; + ret = hp_wmi_setup_fan_settings(priv); if (ret) return ret; @@ -2597,7 +2645,9 @@ static int hp_wmi_hwmon_init(void) INIT_DELAYED_WORK(&priv->keep_alive_dwork, hp_wmi_hwmon_keep_alive_handler); platform_set_drvdata(hp_wmi_platform_dev, priv); - hp_wmi_apply_fan_settings(priv); + ret = hp_wmi_apply_fan_settings(priv); + if (ret) + dev_warn(dev, "Failed to apply initial fan settings: %d\n", ret); return 0; } diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c index 98846e88d3d0ce..58ed0ed85e9e82 100644 --- a/drivers/platform/x86/intel/vsec_tpmi.c +++ b/drivers/platform/x86/intel/vsec_tpmi.c @@ -813,10 +813,6 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev) auxiliary_set_drvdata(auxdev, tpmi_info); - ret = tpmi_create_devices(tpmi_info); - if (ret) - return ret; - /* * Allow debugfs when security policy allows. Everything this debugfs * interface provides, can also be done via /dev/mem access. If @@ -826,6 +822,12 @@ static int intel_vsec_tpmi_init(struct auxiliary_device *auxdev) if (!security_locked_down(LOCKDOWN_DEV_MEM) && capable(CAP_SYS_RAWIO)) tpmi_dbgfs_register(tpmi_info); + ret = tpmi_create_devices(tpmi_info); + if (ret) { + debugfs_remove_recursive(tpmi_info->dbgfs_dir); + return ret; + } + return 0; } diff --git a/drivers/platform/x86/lenovo/wmi-capdata.c b/drivers/platform/x86/lenovo/wmi-capdata.c index ee1fb02d8e31ea..169665be4dcf53 100644 --- a/drivers/platform/x86/lenovo/wmi-capdata.c +++ b/drivers/platform/x86/lenovo/wmi-capdata.c @@ -27,7 +27,6 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include -#include #include #include #include @@ -48,6 +47,7 @@ #include #include "wmi-capdata.h" +#include "wmi-helpers.h" #define LENOVO_CAPABILITY_DATA_00_GUID "362A3AFE-3D96-4665-8530-96DAD5BB300E" #define LENOVO_CAPABILITY_DATA_01_GUID "7A8F5407-CB67-4D6E-B547-39B3BE018154" @@ -58,9 +58,9 @@ #define LWMI_FEATURE_ID_FAN_TEST 0x05 -#define LWMI_ATTR_ID_FAN_TEST \ - (FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, LWMI_DEVICE_ID_FAN) | \ - FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, LWMI_FEATURE_ID_FAN_TEST)) +#define LWMI_ATTR_ID_FAN_TEST \ + lwmi_attr_id(LWMI_DEVICE_ID_FAN, LWMI_FEATURE_ID_FAN_TEST, \ + LWMI_GZ_THERMAL_MODE_NONE, LWMI_TYPE_ID_NONE) enum lwmi_cd_type { LENOVO_CAPABILITY_DATA_00, diff --git a/drivers/platform/x86/lenovo/wmi-capdata.h b/drivers/platform/x86/lenovo/wmi-capdata.h index 8c1df3efcc5533..c3e760b8c3c3df 100644 --- a/drivers/platform/x86/lenovo/wmi-capdata.h +++ b/drivers/platform/x86/lenovo/wmi-capdata.h @@ -6,6 +6,7 @@ #define _LENOVO_WMI_CAPDATA_H_ #include +#include #include #define LWMI_SUPP_VALID BIT(0) @@ -19,6 +20,8 @@ #define LWMI_DEVICE_ID_FAN 0x04 +#define LWMI_TYPE_ID_NONE 0x00 + struct component_match; struct device; struct cd_list; @@ -57,6 +60,23 @@ struct lwmi_cd_binder { cd_list_cb_t cd_fan_list_cb; }; +/** + * lwmi_attr_id() - Formats a capability data attribute ID + * @dev_id: The u8 corresponding to the device ID. + * @feat_id: The u8 corresponding to the feature ID on the device. + * @mode_id: The u8 corresponding to the wmi-gamezone mode for set/get. + * @type_id: The u8 corresponding to the sub-device. + * + * Return: encoded capability data attribute ID. + */ +static inline u32 lwmi_attr_id(u8 dev_id, u8 feat_id, u8 mode_id, u8 type_id) +{ + return (FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, dev_id) | + FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, feat_id) | + FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, mode_id) | + FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, type_id)); +} + void lwmi_cd_match_add_all(struct device *master, struct component_match **matchptr); int lwmi_cd00_get_data(struct cd_list *list, u32 attribute_id, struct capdata00 *output); int lwmi_cd01_get_data(struct cd_list *list, u32 attribute_id, struct capdata01 *output); diff --git a/drivers/platform/x86/lenovo/wmi-events.c b/drivers/platform/x86/lenovo/wmi-events.c index 0994cd7dd504cd..9e9f2e82e04dce 100644 --- a/drivers/platform/x86/lenovo/wmi-events.c +++ b/drivers/platform/x86/lenovo/wmi-events.c @@ -17,7 +17,7 @@ #include #include "wmi-events.h" -#include "wmi-gamezone.h" +#include "wmi-helpers.h" #define THERMAL_MODE_EVENT_GUID "D320289E-8FEA-41E0-86F9-911D83151B5F" diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.c b/drivers/platform/x86/lenovo/wmi-gamezone.c index c7fe7e3c9f1791..686d9128b0f1b0 100644 --- a/drivers/platform/x86/lenovo/wmi-gamezone.c +++ b/drivers/platform/x86/lenovo/wmi-gamezone.c @@ -21,7 +21,6 @@ #include #include "wmi-events.h" -#include "wmi-gamezone.h" #include "wmi-helpers.h" #include "wmi-other.h" @@ -201,7 +200,7 @@ static int lwmi_gz_profile_set(struct device *dev, enum platform_profile_option profile) { struct lwmi_gz_priv *priv = dev_get_drvdata(dev); - struct wmi_method_args_32 args; + struct wmi_method_args_32 args = {}; enum thermal_mode mode; int ret; diff --git a/drivers/platform/x86/lenovo/wmi-gamezone.h b/drivers/platform/x86/lenovo/wmi-gamezone.h deleted file mode 100644 index 6b163a5eeb959d..00000000000000 --- a/drivers/platform/x86/lenovo/wmi-gamezone.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ - -/* Copyright (C) 2025 Derek J. Clark */ - -#ifndef _LENOVO_WMI_GAMEZONE_H_ -#define _LENOVO_WMI_GAMEZONE_H_ - -enum gamezone_events_type { - LWMI_GZ_GET_THERMAL_MODE = 1, -}; - -enum thermal_mode { - LWMI_GZ_THERMAL_MODE_QUIET = 0x01, - LWMI_GZ_THERMAL_MODE_BALANCED = 0x02, - LWMI_GZ_THERMAL_MODE_PERFORMANCE = 0x03, - LWMI_GZ_THERMAL_MODE_EXTREME = 0xE0, /* Ver 6+ */ - LWMI_GZ_THERMAL_MODE_CUSTOM = 0xFF, -}; - -#endif /* !_LENOVO_WMI_GAMEZONE_H_ */ diff --git a/drivers/platform/x86/lenovo/wmi-helpers.c b/drivers/platform/x86/lenovo/wmi-helpers.c index 7379defac50028..018d7642e2bd5b 100644 --- a/drivers/platform/x86/lenovo/wmi-helpers.c +++ b/drivers/platform/x86/lenovo/wmi-helpers.c @@ -46,7 +46,6 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id, unsigned char *buf, size_t size, u32 *retval) { struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; - union acpi_object *ret_obj __free(kfree) = NULL; struct acpi_buffer input = { size, buf }; acpi_status status; @@ -55,8 +54,9 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id, if (ACPI_FAILURE(status)) return -EIO; + union acpi_object *ret_obj __free(kfree) = output.pointer; + if (retval) { - ret_obj = output.pointer; if (!ret_obj) return -ENODATA; diff --git a/drivers/platform/x86/lenovo/wmi-helpers.h b/drivers/platform/x86/lenovo/wmi-helpers.h index 20fd217498035d..3364d8e152ca1f 100644 --- a/drivers/platform/x86/lenovo/wmi-helpers.h +++ b/drivers/platform/x86/lenovo/wmi-helpers.h @@ -14,6 +14,19 @@ struct wmi_method_args_32 { u32 arg1; }; +enum lwmi_event_type { + LWMI_GZ_GET_THERMAL_MODE = 0x01, +}; + +enum thermal_mode { + LWMI_GZ_THERMAL_MODE_NONE = 0x00, + LWMI_GZ_THERMAL_MODE_QUIET = 0x01, + LWMI_GZ_THERMAL_MODE_BALANCED = 0x02, + LWMI_GZ_THERMAL_MODE_PERFORMANCE = 0x03, + LWMI_GZ_THERMAL_MODE_EXTREME = 0xE0, /* Ver 6+ */ + LWMI_GZ_THERMAL_MODE_CUSTOM = 0xFF, +}; + int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id, unsigned char *buf, size_t size, u32 *retval); diff --git a/drivers/platform/x86/lenovo/wmi-other.c b/drivers/platform/x86/lenovo/wmi-other.c index 6040f45aa2b0d6..d1cab4d5787aff 100644 --- a/drivers/platform/x86/lenovo/wmi-other.c +++ b/drivers/platform/x86/lenovo/wmi-other.c @@ -47,7 +47,6 @@ #include "wmi-capdata.h" #include "wmi-events.h" -#include "wmi-gamezone.h" #include "wmi-helpers.h" #include "wmi-other.h" #include "../firmware_attributes_class.h" @@ -62,8 +61,6 @@ #define LWMI_FEATURE_ID_FAN_RPM 0x03 -#define LWMI_TYPE_ID_NONE 0x00 - #define LWMI_FEATURE_VALUE_GET 17 #define LWMI_FEATURE_VALUE_SET 18 @@ -71,13 +68,12 @@ #define LWMI_FAN_NR 4 #define LWMI_FAN_ID(x) ((x) + LWMI_FAN_ID_BASE) -#define LWMI_ATTR_ID_FAN_RPM(x) \ - (FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, LWMI_DEVICE_ID_FAN) | \ - FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, LWMI_FEATURE_ID_FAN_RPM) | \ - FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, LWMI_FAN_ID(x))) - #define LWMI_FAN_DIV 100 +#define LWMI_ATTR_ID_FAN_RPM(x) \ + lwmi_attr_id(LWMI_DEVICE_ID_FAN, LWMI_FEATURE_ID_FAN_RPM, \ + LWMI_GZ_THERMAL_MODE_NONE, LWMI_FAN_ID(x)) + #define LWMI_OM_FW_ATTR_BASE_PATH "lenovo-wmi-other" #define LWMI_OM_HWMON_NAME "lenovo_wmi_other" @@ -166,7 +162,7 @@ MODULE_PARM_DESC(relax_fan_constraint, */ static int lwmi_om_fan_get_set(struct lwmi_om_priv *priv, int channel, u32 *val, bool set) { - struct wmi_method_args_32 args; + struct wmi_method_args_32 args = {}; u32 method_id, retval; int err; @@ -546,13 +542,26 @@ static void lwmi_om_fan_info_collect_cd_fan(struct device *dev, struct cd_list * /* ======== fw_attributes (component: lenovo-wmi-capdata 01) ======== */ struct tunable_attr_01 { - struct capdata01 *capdata; struct device *dev; - u32 feature_id; - u32 device_id; - u32 type_id; + u8 feature_id; + u8 device_id; + u8 type_id; + u8 cd_mode_id; /* mode arg for searching capdata */ + u8 cv_mode_id; /* mode arg for set/get current_value */ }; +/** + * tunable_attr_01_id() - Formats a tunable_attr_01 to a capdata attribute ID + * @attr: The tunable_attr_01 to format. + * @mode: The u8 corresponding to the wmi-gamezone mode for set/get. + * + * Return: encoded capability data attribute ID. + */ +static u32 tunable_attr_01_id(struct tunable_attr_01 *attr, u8 mode) +{ + return lwmi_attr_id(attr->device_id, attr->feature_id, mode, attr->type_id); +} + static struct tunable_attr_01 ppt_pl1_spl = { .device_id = LWMI_DEVICE_ID_CPU, .feature_id = LWMI_FEATURE_ID_CPU_SPL, @@ -716,12 +725,7 @@ static ssize_t attr_capdata01_show(struct kobject *kobj, u32 attribute_id; int value, ret; - attribute_id = - FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, tunable_attr->device_id) | - FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, tunable_attr->feature_id) | - FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, - LWMI_GZ_THERMAL_MODE_CUSTOM) | - FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, tunable_attr->type_id); + attribute_id = tunable_attr_01_id(tunable_attr, tunable_attr->cd_mode_id); ret = lwmi_cd01_get_data(priv->cd01_list, attribute_id, &capdata); if (ret) @@ -773,10 +777,9 @@ static ssize_t attr_current_value_store(struct kobject *kobj, struct tunable_attr_01 *tunable_attr) { struct lwmi_om_priv *priv = dev_get_drvdata(tunable_attr->dev); - struct wmi_method_args_32 args; + struct wmi_method_args_32 args = {}; struct capdata01 capdata; enum thermal_mode mode; - u32 attribute_id; u32 value; int ret; @@ -787,13 +790,9 @@ static ssize_t attr_current_value_store(struct kobject *kobj, if (mode != LWMI_GZ_THERMAL_MODE_CUSTOM) return -EBUSY; - attribute_id = - FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, tunable_attr->device_id) | - FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, tunable_attr->feature_id) | - FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, mode) | - FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, tunable_attr->type_id); + args.arg0 = tunable_attr_01_id(tunable_attr, tunable_attr->cd_mode_id); - ret = lwmi_cd01_get_data(priv->cd01_list, attribute_id, &capdata); + ret = lwmi_cd01_get_data(priv->cd01_list, args.arg0, &capdata); if (ret) return ret; @@ -804,7 +803,7 @@ static ssize_t attr_current_value_store(struct kobject *kobj, if (value < capdata.min_value || value > capdata.max_value) return -EINVAL; - args.arg0 = attribute_id; + args.arg0 = tunable_attr_01_id(tunable_attr, tunable_attr->cv_mode_id); args.arg1 = value; ret = lwmi_dev_evaluate_int(priv->wdev, 0x0, LWMI_FEATURE_VALUE_SET, @@ -836,9 +835,8 @@ static ssize_t attr_current_value_show(struct kobject *kobj, struct tunable_attr_01 *tunable_attr) { struct lwmi_om_priv *priv = dev_get_drvdata(tunable_attr->dev); - struct wmi_method_args_32 args; + struct wmi_method_args_32 args = {}; enum thermal_mode mode; - u32 attribute_id; int retval; int ret; @@ -846,13 +844,11 @@ static ssize_t attr_current_value_show(struct kobject *kobj, if (ret) return ret; - attribute_id = - FIELD_PREP(LWMI_ATTR_DEV_ID_MASK, tunable_attr->device_id) | - FIELD_PREP(LWMI_ATTR_FEAT_ID_MASK, tunable_attr->feature_id) | - FIELD_PREP(LWMI_ATTR_MODE_ID_MASK, mode) | - FIELD_PREP(LWMI_ATTR_TYPE_ID_MASK, tunable_attr->type_id); + /* If "no-mode" is the supported mode, ensure we never send current mode */ + if (tunable_attr->cv_mode_id == LWMI_GZ_THERMAL_MODE_NONE) + mode = tunable_attr->cv_mode_id; - args.arg0 = attribute_id; + args.arg0 = tunable_attr_01_id(tunable_attr, mode); ret = lwmi_dev_evaluate_int(priv->wdev, 0x0, LWMI_FEATURE_VALUE_GET, (unsigned char *)&args, sizeof(args), @@ -863,6 +859,81 @@ static ssize_t attr_current_value_show(struct kobject *kobj, return sysfs_emit(buf, "%d\n", retval); } +/** + * lwmi_attr_01_is_supported() - Determine if the given attribute is supported. + * @tunable_attr: The attribute to verify. + * + * For an attribute to be supported it must have a functional get/set method, + * as well as associated capability data stored in the capdata01 table. + * + * First check if the attribute has a corresponding data table under custom mode + * (0xff), then under no mode (0x00). If either of those passes, check if the + * supported field of the capdata struct is > 0. If it is supported, store the + * successful mode in the cd_mode_id field of tunable_attr. + * + * If the attribute capdata shows it is supported, attempt to determine the mode + * for the current value property get/set methods using a similar pattern to the + * capdata table check. If the value returned by either mode is 0 or an error, + * assume that mode is not supported. Otherwise, store the successful mode in the + * cv_mode_id field of tunable_attr. + * + * If any of the above checks fail then the attribute is not fully supported. + * + * Return: true if capdata and set/get modes are found, otherwise false. + */ +static bool lwmi_attr_01_is_supported(struct tunable_attr_01 *tunable_attr) +{ + u8 modes[2] = { LWMI_GZ_THERMAL_MODE_CUSTOM, LWMI_GZ_THERMAL_MODE_NONE }; + struct lwmi_om_priv *priv = dev_get_drvdata(tunable_attr->dev); + struct wmi_method_args_32 args = {}; + bool cd_mode_found = false; + bool cv_mode_found = false; + struct capdata01 capdata; + int retval, ret, i; + + /* Determine tunable_attr->cd_mode_id */ + for (i = 0; i < ARRAY_SIZE(modes); i++) { + args.arg0 = tunable_attr_01_id(tunable_attr, modes[i]); + + ret = lwmi_cd01_get_data(priv->cd01_list, args.arg0, &capdata); + if (ret || !capdata.supported) + continue; + + tunable_attr->cd_mode_id = modes[i]; + cd_mode_found = true; + break; + } + + if (!cd_mode_found) + return cd_mode_found; + + dev_dbg(tunable_attr->dev, + "cd_mode_id: %#010x\n", args.arg0); + + /* Determine tunable_attr->cv_mode_id, returns 1 if supported */ + for (i = 0; i < ARRAY_SIZE(modes); i++) { + args.arg0 = tunable_attr_01_id(tunable_attr, modes[i]); + + ret = lwmi_dev_evaluate_int(priv->wdev, 0x0, LWMI_FEATURE_VALUE_GET, + (u8 *)&args, sizeof(args), + &retval); + if (ret || !retval) + continue; + + tunable_attr->cv_mode_id = modes[i]; + cv_mode_found = true; + break; + } + + if (!cv_mode_found) + return cv_mode_found; + + dev_dbg(tunable_attr->dev, "cv_mode_id: %#010x, attribute support level: %#010x\n", + args.arg0, capdata.supported); + + return capdata.supported > 0; +} + /* Lenovo WMI Other Mode Attribute macros */ #define __LWMI_ATTR_RO(_func, _name) \ { \ @@ -957,17 +1028,17 @@ static struct capdata01_attr_group cd01_attr_groups[] = { /** * lwmi_om_fw_attr_add() - Register all firmware_attributes_class members * @priv: The Other Mode driver data. - * - * Return: Either 0, or an error code. */ -static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv) +static void lwmi_om_fw_attr_add(struct lwmi_om_priv *priv) { unsigned int i; int err; - priv->ida_id = ida_alloc(&lwmi_om_ida, GFP_KERNEL); - if (priv->ida_id < 0) - return priv->ida_id; + err = ida_alloc(&lwmi_om_ida, GFP_KERNEL); + if (err < 0) + goto err_no_ida; + + priv->ida_id = err; priv->fw_attr_dev = device_create(&firmware_attributes_class, NULL, MKDEV(0, 0), NULL, "%s-%u", @@ -986,14 +1057,16 @@ static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv) } for (i = 0; i < ARRAY_SIZE(cd01_attr_groups) - 1; i++) { + cd01_attr_groups[i].tunable_attr->dev = &priv->wdev->dev; + if (!lwmi_attr_01_is_supported(cd01_attr_groups[i].tunable_attr)) + continue; + err = sysfs_create_group(&priv->fw_attr_kset->kobj, cd01_attr_groups[i].attr_group); if (err) goto err_remove_groups; - - cd01_attr_groups[i].tunable_attr->dev = &priv->wdev->dev; } - return 0; + return; err_remove_groups: while (i--) @@ -1007,7 +1080,12 @@ static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv) err_free_ida: ida_free(&lwmi_om_ida, priv->ida_id); - return err; + +err_no_ida: + priv->ida_id = -EIDRM; + + dev_warn(&priv->wdev->dev, + "failed to register firmware-attributes device: %d\n", err); } /** @@ -1016,12 +1094,17 @@ static int lwmi_om_fw_attr_add(struct lwmi_om_priv *priv) */ static void lwmi_om_fw_attr_remove(struct lwmi_om_priv *priv) { + if (priv->ida_id < 0) + return; + for (unsigned int i = 0; i < ARRAY_SIZE(cd01_attr_groups) - 1; i++) sysfs_remove_group(&priv->fw_attr_kset->kobj, cd01_attr_groups[i].attr_group); kset_unregister(priv->fw_attr_kset); device_unregister(priv->fw_attr_dev); + ida_free(&lwmi_om_ida, priv->ida_id); + priv->ida_id = -EIDRM; } /* ======== Self (master: lenovo-wmi-other) ======== */ @@ -1058,12 +1141,17 @@ static int lwmi_om_master_bind(struct device *dev) priv->cd00_list = binder.cd00_list; priv->cd01_list = binder.cd01_list; - if (!priv->cd00_list || !priv->cd01_list) + if (!priv->cd00_list || !priv->cd01_list) { + component_unbind_all(dev, NULL); + return -ENODEV; + } lwmi_om_fan_info_collect_cd00(priv); - return lwmi_om_fw_attr_add(priv); + lwmi_om_fw_attr_add(priv); + + return 0; } /** @@ -1115,13 +1203,7 @@ static int lwmi_other_probe(struct wmi_device *wdev, const void *context) static void lwmi_other_remove(struct wmi_device *wdev) { - struct lwmi_om_priv *priv = dev_get_drvdata(&wdev->dev); - component_master_del(&wdev->dev, &lwmi_om_master_ops); - - /* No IDA to free if the driver is never bound to its components. */ - if (priv->ida_id >= 0) - ida_free(&lwmi_om_ida, priv->ida_id); } static const struct wmi_device_id lwmi_other_id_table[] = { diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c index d923ddaa48496d..97c0c51b5a4c3f 100644 --- a/drivers/platform/x86/panasonic-laptop.c +++ b/drivers/platform/x86/panasonic-laptop.c @@ -1093,9 +1093,10 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device) } result = device_create_file(&pcc->platform->dev, &dev_attr_cdpower); - pcc_register_optd_notifier(pcc, "\\_SB.PCI0.EHCI.ERHB.OPTD"); if (result) goto out_platform; + + pcc_register_optd_notifier(pcc, "\\_SB.PCI0.EHCI.ERHB.OPTD"); } else { pcc->platform = NULL; } @@ -1129,10 +1130,10 @@ static void acpi_pcc_hotkey_remove(struct acpi_device *device) i8042_remove_filter(panasonic_i8042_filter); if (pcc->platform) { + pcc_unregister_optd_notifier(pcc, "\\_SB.PCI0.EHCI.ERHB.OPTD"); device_remove_file(&pcc->platform->dev, &dev_attr_cdpower); platform_device_unregister(pcc->platform); } - pcc_unregister_optd_notifier(pcc, "\\_SB.PCI0.EHCI.ERHB.OPTD"); sysfs_remove_group(&device->dev.kobj, &pcc_attr_group); diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index a8973f8057fba7..6e9deb9de0739c 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -14,4 +14,10 @@ config APPLE_PMGR_PWRSTATE controls for SoC devices. This driver manages them through the generic power domain framework, and also provides reset support. +config APPLE_PMP_REPORT + bool "Apple PMP report control" + depends on OF + depends on PM + select PM_GENERIC_DOMAINS + endif diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index 53665af630be22..b098b204d9868a 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_APPLE_PMGR_PWRSTATE) += pmgr-pwrstate.o +obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 82c33cf727a825..a92db8cf88bb90 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -21,7 +21,8 @@ #define APPLE_PMGR_AUTO_ENABLE BIT(28) #define APPLE_PMGR_PS_AUTO GENMASK(27, 24) #define APPLE_PMGR_PS_MIN GENMASK(19, 16) -#define APPLE_PMGR_PARENT_OFF BIT(11) +#define APPLE_PMGR_PS_RESET BIT(12) +#define APPLE_PMGR_BUSY BIT(11) #define APPLE_PMGR_DEV_DISABLE BIT(10) #define APPLE_PMGR_WAS_CLKGATED BIT(9) #define APPLE_PMGR_WAS_PWRGATED BIT(8) @@ -44,6 +45,9 @@ struct apple_pmgr_ps { struct regmap *regmap; u32 offset; u32 min_state; + bool force_disable; + bool force_reset; + bool externally_clocked; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -53,7 +57,7 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a { int ret; struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); - u32 reg; + u32 reg, cur; ret = regmap_read(ps->regmap, ps->offset, ®); if (ret < 0) @@ -64,24 +68,57 @@ static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool a dev_err(ps->dev, "PS %s: powering off with RESET active\n", genpd->name); - reg &= ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); + if (pstate != APPLE_PMGR_PS_ACTIVE && (ps->force_disable || ps->force_reset)) { + u32 reg_pre = reg & ~(APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS); + + if (ps->force_disable) + reg_pre |= APPLE_PMGR_DEV_DISABLE; + if (ps->force_reset) + reg_pre |= APPLE_PMGR_PS_RESET; + + regmap_write(ps->regmap, ps->offset, reg_pre); + + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + (cur & (APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET)) == + (reg_pre & (APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET)), 1, + APPLE_PMGR_PS_SET_TIMEOUT); + + if (ret < 0) + dev_err(ps->dev, "PS %s: Failed to set reset/disable bits (now: 0x%x)\n", + genpd->name, reg); + } + + reg &= ~(APPLE_PMGR_DEV_DISABLE | APPLE_PMGR_PS_RESET | + APPLE_PMGR_AUTO_ENABLE | APPLE_PMGR_FLAGS | APPLE_PMGR_PS_TARGET); reg |= FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate); dev_dbg(ps->dev, "PS %s: pwrstate = 0x%x: 0x%x\n", genpd->name, pstate, reg); regmap_write(ps->regmap, ps->offset, reg); - ret = regmap_read_poll_timeout_atomic( - ps->regmap, ps->offset, reg, - (FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == pstate), 1, - APPLE_PMGR_PS_SET_TIMEOUT); + if (ps->externally_clocked && pstate == APPLE_PMGR_PS_ACTIVE) { + /* + * If this clock domain requires an external clock, then + * consider the "clock gated" state to be good enough. + */ + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) >= APPLE_PMGR_PS_CLKGATE, 1, + APPLE_PMGR_PS_SET_TIMEOUT); + } else { + ret = regmap_read_poll_timeout_atomic( + ps->regmap, ps->offset, cur, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, cur) == pstate, 1, + APPLE_PMGR_PS_SET_TIMEOUT); + } + if (ret < 0) dev_err(ps->dev, "PS %s: Failed to reach power state 0x%x (now: 0x%x)\n", genpd->name, pstate, reg); if (auto_enable) { /* Not all devices implement this; this is a no-op where not implemented. */ - reg &= ~APPLE_PMGR_FLAGS; reg |= APPLE_PMGR_AUTO_ENABLE; regmap_write(ps->regmap, ps->offset, reg); } @@ -234,6 +271,15 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); + if (of_property_read_bool(node, "apple,force-disable")) + ps->force_disable = true; + + if (of_property_read_bool(node, "apple,force-reset")) + ps->force_reset = true; + + if (of_property_read_bool(node, "apple,externally-clocked")) + ps->externally_clocked = true; + active = apple_pmgr_ps_is_active(ps); if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; @@ -242,6 +288,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) /* Turn it on so pm_genpd_init does not fail */ active = apple_pmgr_ps_power_on(&ps->genpd) == 0; } + } else if (active) { + ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; } /* Turn on auto-PM if the domain is already on */ diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c new file mode 100644 index 00000000000000..8050ade5ef8d08 --- /dev/null +++ b/drivers/pmdomain/apple/pmp-report.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMP power state reporting driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define PMP_REPORT_READY 0x1 + +struct apple_pmp_report_offsets { + u32 tgt_read; + u32 tgt_write; + u32 actual; + u32 status; +}; + +struct apple_pmp_report { + struct device *dev; + const struct apple_pmp_report_offsets *offsets; + void __iomem *base; + spinlock_t lock; +}; + +static int apple_pmp_report_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct apple_pmp_report *rep; + int ret; + + rep = devm_kzalloc(dev, sizeof(*rep), GFP_KERNEL); + if (!rep) + return -ENOMEM; + + rep->dev = dev; + rep->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(rep->base)) + return PTR_ERR(rep->base); + rep->offsets = of_device_get_match_data(dev); + dev_set_drvdata(dev, rep); + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); + + return 0; +} + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t600x = { + .tgt_read = 0xf80, + .tgt_write = 0x107c0, + .actual = 0x1000, + .status = 0x10, +}; + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t602x = { + .tgt_read = 0x2000, + .tgt_write = 0x11000, + .actual = 0x2080, + .status = 0x10, +}; + +static const struct apple_pmp_report_offsets apple_pmp_offsets_t8112 = { + .tgt_read = 0xa00, + .tgt_write = 0x10500, + .actual = 0xa40, + .status = 0x10, +}; + +static const struct of_device_id apple_pmp_report_of_match[] = { + { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, + { .compatible = "apple,t6020-pmp-v2-report", .data = &apple_pmp_offsets_t602x }, + { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, + {} +}; + +static struct platform_driver apple_pmp_report_driver = { + .probe = apple_pmp_report_probe, + .driver = { + .name = "apple-pmp-report", + .of_match_table = apple_pmp_report_of_match, + }, +}; + +struct apple_pmp_report_entry { + struct device *dev; + struct generic_pm_domain genpd; + u32 id; +}; + +#define genpd_to_apple_pmp_report_entry(_genpd) \ + container_of(_genpd, struct apple_pmp_report_entry, genpd) + +static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); + u64 bit_val = 1 << ent->id; + u64 val; + unsigned long flags; + + spin_lock_irqsave(&rep->lock, flags); + val = readq(rep->base + rep->offsets->tgt_read); + val &= ~bit_val; + if (enable) + val |= bit_val; + writeq(val, rep->base + rep->offsets->tgt_write); + spin_unlock_irqrestore(&rep->lock, flags); + val = readq(rep->base + rep->offsets->status); + if ((val & PMP_REPORT_READY) == 0) + return 0; + return readq_poll_timeout_atomic( + rep->base + rep->offsets->actual, + val, + !!(val & bit_val) == !!enable, + 100, + 50000); +} + +static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, true); +} + +static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, false); +} + +static int apple_pmp_report_entry_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct apple_pmp_report_entry *ent; + int ret; + const char *name; + struct of_phandle_iterator it; + + ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + if (!ent) + return -ENOMEM; + + ent->dev = dev; + + ret = of_property_read_u32(node, "reg", &ent->id); + if (ret) + return dev_err_probe(dev, ret, "missing reg property\n"); + + ret = of_property_read_string(node, "label", &name); + if (ret < 0) + return dev_err_probe(dev, ret, "missing label property\n"); + + if (of_property_read_bool(node, "apple,always-on")) { + ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; + apple_pmp_report_set_state(&ent->genpd, true); + } + + ent->genpd.name = name; + ent->genpd.power_on = apple_pmp_report_entry_power_on; + ent->genpd.power_off = apple_pmp_report_entry_power_off; + + ret = pm_genpd_init(&ent->genpd, NULL, true); + if (ret) + return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + + ret = of_genpd_add_provider_simple(node, &ent->genpd); + if (ret) + return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + + of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { + struct of_phandle_args parent, child; + + parent.np = it.node; + parent.args_count = of_phandle_iterator_args(&it, parent.args, MAX_PHANDLE_ARGS); + child.np = node; + child.args_count = 0; + ret = of_genpd_add_subdomain(&parent, &child); + + if (ret == -EPROBE_DEFER) { + of_node_put(parent.np); + goto err_remove; + } else if (ret < 0) { + dev_err(dev, "failed to add to parent domain: %d (%s -> %s)\n", + ret, it.node->name, node->name); + of_node_put(parent.np); + goto err_remove; + } + } + + pm_genpd_remove_device(dev); + + return 0; +err_remove: + of_genpd_del_provider(node); + pm_genpd_remove(&ent->genpd); + return ret; +} + +static const struct of_device_id apple_pmp_report_entry_of_match[] = { + { .compatible = "apple,t6000-pmp-v2-report-entry" }, + {} +}; + +static struct platform_driver apple_pmp_report_entry_driver = { + .probe = apple_pmp_report_entry_probe, + .driver = { + .name = "apple-pmp-report-entry", + .of_match_table = apple_pmp_report_entry_of_match, + }, +}; + +MODULE_DEVICE_TABLE(of, apple_pmp_report_of_match); +MODULE_DEVICE_TABLE(of, apple_pmp_report_entry_of_match); + +static int __init apple_pmp_report_init(void) +{ + platform_driver_register(&apple_pmp_report_entry_driver); + platform_driver_register(&apple_pmp_report_driver); + return 0; +} + +static void __exit apple_pmp_report_exit(void) +{ + platform_driver_unregister(&apple_pmp_report_entry_driver); + platform_driver_unregister(&apple_pmp_report_driver); +} + +module_init(apple_pmp_report_init); +module_exit(apple_pmp_report_exit); + +MODULE_DESCRIPTION("PMP power state reporting driver for Apple SoCs"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 52ea84e548ff6d..810c55ba3ce866 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -7,6 +7,7 @@ #define pr_fmt(fmt) "PM: " fmt #include +#include #include #include #include @@ -188,6 +189,7 @@ static const struct genpd_lock_ops genpd_raw_spin_ops = { #define genpd_is_dev_name_fw(genpd) (genpd->flags & GENPD_FLAG_DEV_NAME_FW) #define genpd_is_no_sync_state(genpd) (genpd->flags & GENPD_FLAG_NO_SYNC_STATE) #define genpd_is_no_stay_on(genpd) (genpd->flags & GENPD_FLAG_NO_STAY_ON) +#define genpd_is_defer_off(genpd) (genpd->flags & GENPD_FLAG_DEFER_OFF) static inline bool irq_safe_dev_in_sleep_domain(struct device *dev, const struct generic_pm_domain *genpd) @@ -941,6 +943,27 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) queue_work(pm_wq, &genpd->power_off_work); } +/** + * genpd_must_defer - Check whether the genpd cannot be safely powered off. + * @genpd: PM domain about to be powered down. + * @one_dev_probing: True if we are being called from RPM callbacks on a device that + * is probing, to allow poweroff if that device is the sole remaining consumer probing. + * + * Returns true if the @genpd has the GENPD_FLAG_DEFER_OFF flag and there + * are any consumer devices which either do not exist yet (only represented + * by fwlinks) or whose drivers have not probed yet. + */ +static bool genpd_must_defer(struct generic_pm_domain *genpd, bool one_dev_probing) +{ + if (genpd_is_defer_off(genpd) && genpd->has_provider) { + int absent = fw_devlink_count_absent_consumers(genpd->provider); + + if (absent > (one_dev_probing ? 1 : 0)) + return true; + } + return false; +} + /** * genpd_power_off - Remove power from a given PM domain. * @genpd: PM domain to power down. @@ -954,7 +977,7 @@ static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) * have been powered down, remove power from @genpd. */ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, - unsigned int depth) + bool one_dev_probing, unsigned int depth) { struct pm_domain_data *pdd; struct gpd_link *link; @@ -1002,6 +1025,14 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, if (not_suspended > 1 || (not_suspended == 1 && !one_dev_on)) return; + /* + * Do not allow PM domain to be powered off if it is marked + * as GENPD_FLAG_DEFER_OFF and there are consumer devices + * which have not probed yet. + */ + if (genpd_must_defer(genpd, one_dev_probing)) + return; + if (genpd->gov && genpd->gov->power_down_ok) { if (!genpd->gov->power_down_ok(&genpd->domain)) return; @@ -1027,7 +1058,7 @@ static void genpd_power_off(struct generic_pm_domain *genpd, bool one_dev_on, list_for_each_entry(link, &genpd->child_links, child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } } @@ -1086,7 +1117,7 @@ static int genpd_power_on(struct generic_pm_domain *genpd, unsigned int depth) child_node) { genpd_sd_counter_dec(link->parent); genpd_lock_nested(link->parent, depth + 1); - genpd_power_off(link->parent, false, depth + 1); + genpd_power_off(link->parent, false, false, depth + 1); genpd_unlock(link->parent); } @@ -1153,7 +1184,7 @@ static void genpd_power_off_work_fn(struct work_struct *work) genpd = container_of(work, struct generic_pm_domain, power_off_work); genpd_lock(genpd); - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } @@ -1218,6 +1249,7 @@ static int genpd_runtime_suspend(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool runtime_pm = pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1272,7 +1304,7 @@ static int genpd_runtime_suspend(struct device *dev) return 0; genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); @@ -1293,6 +1325,7 @@ static int genpd_runtime_resume(struct device *dev) struct generic_pm_domain_data *gpd_data = dev_gpd_data(dev); struct gpd_timing_data *td = gpd_data->td; bool timed = td && pm_runtime_enabled(dev); + bool probing = dev->links.status != DL_DEV_DRIVER_BOUND; ktime_t time_start = 0; s64 elapsed_ns; int ret; @@ -1350,7 +1383,7 @@ static int genpd_runtime_resume(struct device *dev) err_poweroff: if (!pm_runtime_is_irq_safe(dev) || genpd_is_irq_safe(genpd)) { genpd_lock(genpd); - genpd_power_off(genpd, true, 0); + genpd_power_off(genpd, true, probing, 0); gpd_data->rpm_pstate = genpd_drop_performance_state(dev); genpd_unlock(genpd); } @@ -1418,6 +1451,9 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, || atomic_read(&genpd->sd_count) > 0) return; + if (genpd_must_defer(genpd, false)) + return; + /* Check that the children are in their deepest (powered-off) state. */ list_for_each_entry(link, &genpd->parent_links, parent_node) { struct generic_pm_domain *child = link->child; @@ -2445,6 +2481,12 @@ int pm_genpd_init(struct generic_pm_domain *genpd, return -EINVAL; } + /* Deferred-off power domains should be powered on at initialization. */ + if (genpd_is_defer_off(genpd) && !genpd_status_on(genpd)) { + pr_warn("deferred-off PM domain %s is not on at init\n", genpd->name); + genpd->flags &= ~GENPD_FLAG_DEFER_OFF; + } + /* Multiple states but no governor doesn't make sense. */ if (!gov && genpd->state_count > 1) pr_warn("%s: no governor for states\n", genpd->name); @@ -3082,6 +3124,7 @@ static const struct bus_type genpd_bus_type = { static void genpd_dev_pm_detach(struct device *dev, bool power_off) { struct generic_pm_domain *pd; + bool is_virt_dev; unsigned int i; int ret = 0; @@ -3091,6 +3134,13 @@ static void genpd_dev_pm_detach(struct device *dev, bool power_off) dev_dbg(dev, "removing from PM domain %s\n", pd->name); + /* Check if the device was created by genpd at attach. */ + is_virt_dev = dev->bus == &genpd_bus_type; + + /* Disable runtime PM if we enabled it at attach. */ + if (is_virt_dev) + pm_runtime_disable(dev); + /* Drop the default performance state */ if (dev_gpd_data(dev)->default_pstate) { dev_pm_genpd_set_performance_state(dev, 0); @@ -3116,7 +3166,7 @@ static void genpd_dev_pm_detach(struct device *dev, bool power_off) genpd_queue_power_off_work(pd); /* Unregister the device if it was created by genpd. */ - if (dev->bus == &genpd_bus_type) + if (is_virt_dev) device_unregister(dev); } @@ -3511,7 +3561,7 @@ void of_genpd_sync_state(struct device_node *np) if (genpd->provider == of_fwnode_handle(np)) { genpd_lock(genpd); genpd->stay_on = false; - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); } } @@ -3539,7 +3589,7 @@ static void genpd_provider_sync_state(struct device *dev) case GENPD_SYNC_STATE_SIMPLE: genpd_lock(genpd); genpd->stay_on = false; - genpd_power_off(genpd, false, 0); + genpd_power_off(genpd, false, false, 0); genpd_unlock(genpd); break; diff --git a/drivers/pmdomain/imx/scu-pd.c b/drivers/pmdomain/imx/scu-pd.c index 01d465d88f60dc..3ec33667a308c3 100644 --- a/drivers/pmdomain/imx/scu-pd.c +++ b/drivers/pmdomain/imx/scu-pd.c @@ -326,6 +326,7 @@ static void imx_sc_pd_get_console_rsrc(void) return; imx_con_rsrc = specs.args[0]; + of_node_put(specs.np); } static int imx_sc_get_pd_power(struct device *dev, u32 rsrc) diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c index e2800aa1bc597f..d3b36f32417c7a 100644 --- a/drivers/pmdomain/mediatek/mtk-pm-domains.c +++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c @@ -993,6 +993,7 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s struct device_node *node, *smi_np; int num_regmaps = 0, i, j; struct regmap *regmap[3]; + int ret = 0; /* * Legacy code retrieves a maximum of three bus protection handles: @@ -1043,11 +1044,14 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s if (node) { regmap[2] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg-nao"); num_regmaps++; - of_node_put(node); - if (IS_ERR(regmap[2])) - return dev_err_probe(dev, PTR_ERR(regmap[2]), + if (IS_ERR(regmap[2])) { + ret = dev_err_probe(dev, PTR_ERR(regmap[2]), "%pOF: failed to get infracfg regmap\n", node); + of_node_put(node); + return ret; + } + of_node_put(node); } else { regmap[2] = NULL; } diff --git a/drivers/pmdomain/ti/omap_prm.c b/drivers/pmdomain/ti/omap_prm.c index 5142f064bf5cdc..64a187f79a1a70 100644 --- a/drivers/pmdomain/ti/omap_prm.c +++ b/drivers/pmdomain/ti/omap_prm.c @@ -655,6 +655,7 @@ static int omap_prm_domain_attach_dev(struct generic_pm_domain *domain, if (pd_args.args_count != 0) dev_warn(dev, "%s: unusupported #power-domain-cells: %i\n", prmd->pd.name, pd_args.args_count); + of_node_put(pd_args.np); genpd_data = dev_gpd_data(dev); genpd_data->data = NULL; diff --git a/drivers/power/reset/macsmc-reboot.c b/drivers/power/reset/macsmc-reboot.c index e9702acdd366b0..94fcbf12fe3b93 100644 --- a/drivers/power/reset/macsmc-reboot.c +++ b/drivers/power/reset/macsmc-reboot.c @@ -205,6 +205,14 @@ static int macsmc_reboot_probe(struct platform_device *pdev) struct macsmc_reboot *reboot; int ret, i; + /* + * MFD will probe this device even without a node in the device tree, + * thus bail out early if the SMC on the current machines does not + * support reboot and has no node in the device tree. + */ + if (!pdev->dev.of_node) + return -ENODEV; + reboot = devm_kzalloc(&pdev->dev, sizeof(*reboot), GFP_KERNEL); if (!reboot) return -ENOMEM; diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index 92f9f7aae92f24..e1018b864d8cd2 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -1132,4 +1132,11 @@ config FUEL_GAUGE_MM8013 the state of charge, temperature, cycle count, actual and design capacity, etc. +config CHARGER_MACSMC + tristate "Apple SMC Charger / Battery support" + depends on MFD_MACSMC + help + Say Y here to enable support for the charger and battery controls on + Apple SMC controllers, as used on Apple Silicon Macs. + endif # POWER_SUPPLY diff --git a/drivers/power/supply/Makefile b/drivers/power/supply/Makefile index 4b79d5abc49a7f..7c09106ab23004 100644 --- a/drivers/power/supply/Makefile +++ b/drivers/power/supply/Makefile @@ -81,6 +81,7 @@ obj-$(CONFIG_CHARGER_GPIO) += gpio-charger.o obj-$(CONFIG_CHARGER_MANAGER) += charger-manager.o obj-$(CONFIG_CHARGER_LT3651) += lt3651-charger.o obj-$(CONFIG_CHARGER_LTC4162L) += ltc4162-l-charger.o +obj-$(CONFIG_CHARGER_MACSMC) += macsmc-power.o obj-$(CONFIG_CHARGER_MAX14577) += max14577_charger.o obj-$(CONFIG_CHARGER_DETECTOR_MAX14656) += max14656_charger_detector.o obj-$(CONFIG_CHARGER_MAX77650) += max77650-charger.o diff --git a/drivers/power/supply/axp288_charger.c b/drivers/power/supply/axp288_charger.c index ac05942e4e6ac1..ca52c2c82b2cf2 100644 --- a/drivers/power/supply/axp288_charger.c +++ b/drivers/power/supply/axp288_charger.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -821,14 +822,6 @@ static int charger_init_hw_regs(struct axp288_chrg_info *info) return 0; } -static void axp288_charger_cancel_work(void *data) -{ - struct axp288_chrg_info *info = data; - - cancel_work_sync(&info->otg.work); - cancel_work_sync(&info->cable.work); -} - static int axp288_charger_probe(struct platform_device *pdev) { int ret, i, pirq; @@ -911,12 +904,12 @@ static int axp288_charger_probe(struct platform_device *pdev) } /* Cancel our work on cleanup, register this before the notifiers */ - ret = devm_add_action(dev, axp288_charger_cancel_work, info); + ret = devm_work_autocancel(dev, &info->cable.work, + axp288_charger_extcon_evt_worker); if (ret) return ret; /* Register for extcon notification */ - INIT_WORK(&info->cable.work, axp288_charger_extcon_evt_worker); info->cable.nb.notifier_call = axp288_charger_handle_cable_evt; ret = devm_extcon_register_notifier_all(dev, info->cable.edev, &info->cable.nb); @@ -926,8 +919,12 @@ static int axp288_charger_probe(struct platform_device *pdev) } schedule_work(&info->cable.work); + ret = devm_work_autocancel(dev, &info->otg.work, + axp288_charger_otg_evt_worker); + if (ret) + return ret; + /* Register for OTG notification */ - INIT_WORK(&info->otg.work, axp288_charger_otg_evt_worker); info->otg.id_nb.notifier_call = axp288_charger_handle_otg_evt; if (info->otg.cable) { ret = devm_extcon_register_notifier(dev, info->otg.cable, diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c new file mode 100644 index 00000000000000..dc3ec5ef2b81cf --- /dev/null +++ b/drivers/power/supply/macsmc-power.c @@ -0,0 +1,1053 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SMC Power/Battery Management + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_STRING_LENGTH 256 + +/* + * This number is not reported anywhere by SMC, but seems to be a good + * conversion factor for charge to energy across machines. We need this + * to convert in the driver, since if we don't userspace will try to do + * the conversion with a randomly guessed voltage and get it wrong. + * + * Ideally there would be a power supply prop to inform userspace of this + * number, but there isn't, only min/max. + */ +#define MACSMC_NOMINAL_CELL_VOLTAGE_MV 3800 + +struct macsmc_power { + struct device *dev; + struct apple_smc *smc; + struct power_supply_desc ac_desc; + struct power_supply_desc batt_desc; + + struct power_supply *batt; + char model_name[MAX_STRING_LENGTH]; + char serial_number[MAX_STRING_LENGTH]; + char mfg_date[MAX_STRING_LENGTH]; + + bool has_chwa; + bool has_chls; + bool has_ch0i; + bool has_ch0c; + bool has_chte; + + u8 num_cells; + int nominal_voltage_mv; + + struct power_supply *ac; + + struct notifier_block nb; + + struct work_struct critical_work; + bool shutdown_started; + + struct delayed_work dbg_log_work; +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp); + +static const struct kernel_param_ops macsmc_log_power_ops = { + .set = macsmc_log_power_set, + .get = param_get_bool, +}; + +static bool log_power = false; +module_param_cb(log_power, &macsmc_log_power_ops, &log_power, 0644); +MODULE_PARM_DESC(log_power, "Periodically log power consumption for debugging"); + +#define POWER_LOG_INTERVAL (HZ) + +static struct macsmc_power *g_power; + +#define CHNC_BATTERY_FULL BIT(0) +#define CHNC_NO_CHARGER BIT(7) +#define CHNC_NOCHG_CH0C BIT(14) +#define CHNC_NOCHG_CH0B_CH0K BIT(15) +#define CHNC_BATTERY_FULL_2 BIT(18) +#define CHNC_BMS_BUSY BIT(23) +#define CHNC_CHLS_LIMIT BIT(24) +#define CHNC_NOAC_CH0J BIT(53) +#define CHNC_NOAC_CH0I BIT(54) + +#define CH0R_LOWER_FLAGS GENMASK(15, 0) +#define CH0R_NOAC_CH0I BIT(0) +#define CH0R_NOAC_DISCONNECTED BIT(4) +#define CH0R_NOAC_CH0J BIT(5) +#define CH0R_BMS_BUSY BIT(8) +#define CH0R_NOAC_CH0K BIT(9) +#define CH0R_NOAC_CHWA BIT(11) + +#define CH0X_CH0C BIT(0) +#define CH0X_CH0B BIT(1) + +#define ACSt_CAN_BOOT_AP BIT(2) +#define ACSt_CAN_BOOT_IBOOT BIT(1) + +#define CHWA_CHLS_FIXED_START_OFFSET 5 +#define CHLS_MIN_END_THRESHOLD 10 +#define CHLS_FORCE_DISCHARGE 0x100 +#define CHWA_FIXED_END_THRESHOLD 80 +#define CHWA_PROP_WRITE_THRESHOLD 95 + +#define FLT_EXP_BIAS 127 +#define FLT_EXP_MASK GENMASK(30, 23) +#define FLT_MANT_BIAS 23 +#define FLT_MANT_MASK GENMASK(22, 0) +#define FLT_SIGN_MASK BIT(31) +/* + * Many sensors report their data as IEEE-754 floats. No other SMC function uses + * them. + */ +static int apple_smc_read_f32_scaled(struct apple_smc *smc, smc_key key, + int *p, int scale) +{ + u32 fval; + u64 val; + int ret, exp; + + BUILD_BUG_ON(scale <= 0); + + ret = apple_smc_read_u32(smc, key, &fval); + if (ret < 0) + return ret; + + val = ((u64)((fval & FLT_MANT_MASK) | BIT(23))); + exp = ((fval >> 23) & 0xff) - FLT_EXP_BIAS - FLT_MANT_BIAS; + val *= scale; + + if (exp > 63) + val = U64_MAX; + else if (exp < -63) + val = 0; + else if (exp < 0) + val >>= -exp; + else if (exp != 0 && (val & ~((1UL << (64 - exp)) - 1))) /* overflow */ + val = U64_MAX; + else + val <<= exp; + + if (fval & FLT_SIGN_MASK) { + if (val > (-(s64)INT_MIN)) + *p = INT_MIN; + else + *p = -val; + } else { + if (val > INT_MAX) + *p = INT_MAX; + else + *p = val; + } + + return 0; +} + +static void macsmc_do_dbg(struct macsmc_power *power) +{ + int p_in = 0, p_sys = 0, p_3v8 = 0, p_mpmu = 0, p_spmu = 0, p_clvr = 0, p_cpu = 0; + s32 p_bat = 0; + s16 t_full = 0, t_empty = 0; + u8 charge = 0; + + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PDTR), &p_in, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSTR), &p_sys, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PMVR), &p_3v8, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PHPC), &p_cpu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PSVR), &p_clvr, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPMC), &p_mpmu, 1000); + apple_smc_read_f32_scaled(power->smc, SMC_KEY(PPSC), &p_spmu, 1000); + apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &p_bat); + apple_smc_read_s16(power->smc, SMC_KEY(B0TE), &t_empty); + apple_smc_read_s16(power->smc, SMC_KEY(B0TF), &t_full); + apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &charge); + +#define FD3(x) ((x) / 1000), abs((x) % 1000) + dev_info(power->dev, + "In %2d.%03dW Sys %2d.%03dW 3V8 %2d.%03dW MPMU %2d.%03dW SPMU %2d.%03dW " + "CLVR %2d.%03dW CPU %2d.%03dW Batt %2d.%03dW %d%% T%s %dm\n", + FD3(p_in), FD3(p_sys), FD3(p_3v8), FD3(p_mpmu), FD3(p_spmu), FD3(p_clvr), + FD3(p_cpu), FD3(p_bat), charge, + t_full >= 0 ? "full" : "empty", + t_full >= 0 ? t_full : t_empty); +#undef FD3 +} + +static int macsmc_battery_get_status(struct macsmc_power *power) +{ + u64 nocharge_flags; + u32 nopower_flags; + u16 ac_current; + int charge_limit = 0; + bool limited = false; + bool flag; + int ret; + + /* + * Note: there are fallbacks in case some of these SMC keys disappear in the future + * or are not present on some machines. We treat the absence of the CHCE/CHCC/BSFC/CHSC + * flags as an error, since they are quite fundamental and simple booleans. + */ + + /* + * If power input is inhibited, we are definitely discharging. + * However, if the only reason is the BMS is doing a balancing cycle, + * go ahead and ignore that one to avoid spooking users. + */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(CH0R), &nopower_flags); + if (!ret && (nopower_flags & CH0R_LOWER_FLAGS & ~CH0R_BMS_BUSY)) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If no charger is present, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCE), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If AC is not charge capable, we are definitely discharging. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHCC), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* + * If the AC input current limit is tiny or 0, we are discharging no matter + * how much the BMS believes it can charge. + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &ac_current); + if (!ret && ac_current < 100) + return POWER_SUPPLY_STATUS_DISCHARGING; + + /* If the battery is full, report it as such. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag); + if (ret < 0) + return ret; + if (flag) + return POWER_SUPPLY_STATUS_FULL; + + /* + * If we have charge limits supported and enabled and the SoC is above + * the start threshold, that means we are not charging for that reason + * (if not charging). + */ + if (power->has_chls) { + u16 vu16; + + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + if (ret == sizeof(vu16) && (vu16 & 0xff) >= CHLS_MIN_END_THRESHOLD) + charge_limit = (vu16 & 0xff) - CHWA_CHLS_FIXED_START_OFFSET; + } else if (power->has_chwa) { + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag); + if (ret == 0 && flag) + charge_limit = CHWA_FIXED_END_THRESHOLD - CHWA_CHLS_FIXED_START_OFFSET; + } + + if (charge_limit > 0) { + u8 buic = 0; + + if (apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &buic) >= 0 && + buic >= charge_limit) + limited = true; + } + + /* If there are reasons we aren't charging... */ + ret = apple_smc_read_u64(power->smc, SMC_KEY(CHNC), &nocharge_flags); + if (!ret) { + /* Perhaps the battery is full after all */ + if (nocharge_flags & CHNC_BATTERY_FULL) + return POWER_SUPPLY_STATUS_FULL; + /* + * Or maybe the BMS is just busy doing something, if so call it charging anyway. + * But CHWA limits show up as this, so exclude those. + */ + else if (nocharge_flags == CHNC_BMS_BUSY && !limited) + return POWER_SUPPLY_STATUS_CHARGING; + /* If we have other reasons we aren't charging, say we aren't */ + else if (nocharge_flags) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + /* Else we're either charging or about to charge */ + else + return POWER_SUPPLY_STATUS_CHARGING; + } + + /* As a fallback, use the system charging flag. */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHSC), &flag); + if (ret < 0) + return ret; + if (!flag) + return POWER_SUPPLY_STATUS_NOT_CHARGING; + else + return POWER_SUPPLY_STATUS_CHARGING; +} + +static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) +{ + int ret; + u8 val8; + u8 chte_buf[4]; + + if (power->has_ch0i) { + /* CH0I returns a bitmask like the low byte of CH0R */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8); + if (ret) + return ret; + if (val8 & CH0R_NOAC_CH0I) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE; + } + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) { + ret = apple_smc_read(power->smc, SMC_KEY(CHTE), chte_buf, 4); + if (ret < 0) + return ret; + + if (chte_buf[0] == 0x01) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; + + } else if (power->has_ch0c) { + /* CH0C returns a bitmask containing CH0B/CH0C flags */ + ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8); + if (ret) + return ret; + if (val8 & CH0X_CH0C) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; + } + + return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; +} + +static int macsmc_battery_set_charge_behaviour(struct macsmc_power *power, int val) +{ + int ret; + + /* + * apple_smc_write_u32 does weird things with endianess, + * so we write raw bytes to ensure correctness of CHTE + */ + u8 chte_inhibit[4] = {0x01, 0x00, 0x00, 0x00}; + u8 chte_auto[4] = {0x00, 0x00, 0x00, 0x00}; + + /* + * CH0I/CH0C/CHTE are "hard" controls that will allow the battery to run down to 0. + * CH0K/CH0B are "soft" controls that are reset to 0 when SOC drops below 50%; + * we don't expose these yet. + */ + + switch (val) { + case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO: + if (power->has_ch0i) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + if (ret) + return ret; + } + + if (power->has_chte) { + ret = apple_smc_write(power->smc, SMC_KEY(CHTE), chte_auto, 4); + if (ret) + return ret; + } else if (power->has_ch0c) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + if (ret) + return ret; + } + break; + + case POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE: + if (power->has_ch0i) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + if (ret) + return ret; + } + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) + return apple_smc_write(power->smc, SMC_KEY(CHTE), chte_inhibit, 4); + else if (power->has_ch0c) + return apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 1); + else + return -EINVAL; + break; + + case POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE: + if (!power->has_ch0i) + return -EINVAL; + + /* Prefer CHTE available in newer firmwares */ + if (power->has_chte) { + ret = apple_smc_write(power->smc, SMC_KEY(CHTE), chte_auto, 4); + if (ret) + return ret; + } else if (power->has_ch0c) { + ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + if (ret) + return ret; + } + + return apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 1); + + default: + return -EINVAL; + } + + return 0; +} + +static int macsmc_battery_get_date(const char *s, int *out) +{ + if (!isdigit(s[0]) || !isdigit(s[1])) + return -ENOTSUPP; + + *out = (s[0] - '0') * 10 + s[1] - '0'; + return 0; +} + +static int macsmc_battery_get_capacity_level(struct macsmc_power *power) +{ + bool flag; + u32 val; + int ret; + + /* Check for emergency shutdown condition */ + if (apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val) >= 0 && val) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + /* Check AC status for whether we could boot in this state */ + if (apple_smc_read_u32(power->smc, SMC_KEY(ACSt), &val) >= 0) { + if (!(val & ACSt_CAN_BOOT_IBOOT)) + return POWER_SUPPLY_CAPACITY_LEVEL_CRITICAL; + + if (!(val & ACSt_CAN_BOOT_AP)) + return POWER_SUPPLY_CAPACITY_LEVEL_LOW; + } + + /* Check battery full flag */ + ret = apple_smc_read_flag(power->smc, SMC_KEY(BSFC), &flag); + if (ret < 0) + return POWER_SUPPLY_CAPACITY_LEVEL_UNKNOWN; + if (flag) + return POWER_SUPPLY_CAPACITY_LEVEL_FULL; + else + return POWER_SUPPLY_CAPACITY_LEVEL_NORMAL; +} + +static int macsmc_battery_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u8 vu8; + u16 vu16; + s16 vs16; + s32 vs32; + s64 vs64; + bool flag; + + switch (psp) { + case POWER_SUPPLY_PROP_STATUS: + val->intval = macsmc_battery_get_status(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_PRESENT: + val->intval = 1; + break; + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + val->intval = macsmc_battery_get_charge_behaviour(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TE), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_TIME_TO_FULL_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0TF), &vu16); + val->intval = vu16 == 0xffff ? 0 : vu16 * 60; + break; + case POWER_SUPPLY_PROP_CAPACITY: + ret = apple_smc_read_u8(power->smc, SMC_KEY(BUIC), &vu8); + val->intval = vu8; + break; + case POWER_SUPPLY_PROP_CAPACITY_LEVEL: + val->intval = macsmc_battery_get_capacity_level(power); + ret = val->intval < 0 ? val->intval : 0; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CURRENT_NOW: + ret = apple_smc_read_s16(power->smc, SMC_KEY(B0AC), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_POWER_NOW: + ret = apple_smc_read_s32(power->smc, SMC_KEY(B0AP), &vs32); + val->intval = vs32 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(BITV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN: + /* + * Battery cell max voltage? BVV* seem to return per-cell voltages, + * BVV[NOP] are probably the max voltages for the 3 cells but we don't + * know what will happen if they ever change the number of cells. + * So go with BVVN and multiply by the cell count (BNCB). + * BVVL seems to be the per-cell limit adjusted dynamically. + * Guess: BVVL = Limit, BVVN = Nominal, and the other cells got filled + * in around nearby letters? + */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(BVVN), &vu16); + val->intval = vu16 * 1000 * power->num_cells; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MIN: + /* Lifetime min */ + ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPM), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_VOLTAGE_MAX: + /* Lifetime max */ + ret = apple_smc_read_s16(power->smc, SMC_KEY(BLPX), &vs16); + val->intval = vs16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RI), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RV), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_CHARGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * 1000; + break; + case POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0DC), &vu16); + val->intval = vu16 * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_ENERGY_FULL: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0FC), &vu16); + val->intval = vu16 * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_ENERGY_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0RM), &vu16); + val->intval = swab16(vu16) * power->nominal_voltage_mv; + break; + case POWER_SUPPLY_PROP_TEMP: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0AT), &vu16); + val->intval = vu16 - 2732; + break; + case POWER_SUPPLY_PROP_CHARGE_COUNTER: + ret = apple_smc_read_s64(power->smc, SMC_KEY(BAAC), &vs64); + val->intval = vs64; + break; + case POWER_SUPPLY_PROP_CYCLE_COUNT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(B0CT), &vu16); + val->intval = vu16; + break; + case POWER_SUPPLY_PROP_SCOPE: + val->intval = POWER_SUPPLY_SCOPE_SYSTEM; + break; + case POWER_SUPPLY_PROP_HEALTH: + flag = false; + ret = apple_smc_read_flag(power->smc, SMC_KEY(BBAD), &flag); + val->intval = flag ? POWER_SUPPLY_HEALTH_DEAD : POWER_SUPPLY_HEALTH_GOOD; + break; + case POWER_SUPPLY_PROP_MODEL_NAME: + val->strval = power->model_name; + break; + case POWER_SUPPLY_PROP_SERIAL_NUMBER: + val->strval = power->serial_number; + break; + case POWER_SUPPLY_PROP_MANUFACTURE_YEAR: + ret = macsmc_battery_get_date(&power->mfg_date[0], &val->intval); + val->intval += 2000 - 8; /* -8 is a fixup for a firmware bug... */ + break; + case POWER_SUPPLY_PROP_MANUFACTURE_MONTH: + ret = macsmc_battery_get_date(&power->mfg_date[2], &val->intval); + break; + case POWER_SUPPLY_PROP_MANUFACTURE_DAY: + ret = macsmc_battery_get_date(&power->mfg_date[4], &val->intval); + break; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + if (power->has_chls) { + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + val->intval = vu16 & 0xff; + if (val->intval < CHLS_MIN_END_THRESHOLD || val->intval >= 100) + val->intval = 100; + } else if (power->has_chwa) { + flag = false; + ret = apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag); + val->intval = flag ? CHWA_FIXED_END_THRESHOLD : 100; + } else { + return -EINVAL; + } + if (psp == POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD && + ret >= 0 && val->intval < 100 && val->intval >= CHLS_MIN_END_THRESHOLD) + val->intval -= CHWA_CHLS_FIXED_START_OFFSET; + break; + default: + return -EINVAL; + } + + return ret; +} + +static int macsmc_battery_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return macsmc_battery_set_charge_behaviour(power, val->intval); + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + /* + * Ignore, we allow writes so userspace isn't confused but this is + * not configurable independently, it always is end - 5 or 100 depending + * on the end_threshold setting. + */ + return 0; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + if (power->has_chls) { + u16 kval = 0; + /* TODO: Make CHLS_FORCE_DISCHARGE configurable */ + if (val->intval < CHLS_MIN_END_THRESHOLD) + kval = CHLS_FORCE_DISCHARGE | CHLS_MIN_END_THRESHOLD; + else if (val->intval < 100) + kval = CHLS_FORCE_DISCHARGE | (val->intval & 0xff); + return apple_smc_write_u16(power->smc, SMC_KEY(CHLS), kval); + } else if (power->has_chwa) { + return apple_smc_write_flag(power->smc, SMC_KEY(CHWA), + val->intval <= CHWA_PROP_WRITE_THRESHOLD); + } else { + return -EINVAL; + } + default: + return -EINVAL; + } +} + +static int macsmc_battery_property_is_writeable(struct power_supply *psy, + enum power_supply_property psp) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + + switch (psp) { + case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: + return true; + case POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD: + case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: + return power->has_chwa || power->has_chls; + default: + return false; + } +} + +static const enum power_supply_property macsmc_battery_props[] = { + POWER_SUPPLY_PROP_STATUS, + POWER_SUPPLY_PROP_PRESENT, + POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_PROP_TIME_TO_EMPTY_NOW, + POWER_SUPPLY_PROP_TIME_TO_FULL_NOW, + POWER_SUPPLY_PROP_CAPACITY, + POWER_SUPPLY_PROP_CAPACITY_LEVEL, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_CURRENT_NOW, + POWER_SUPPLY_PROP_POWER_NOW, + POWER_SUPPLY_PROP_VOLTAGE_MIN_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MAX_DESIGN, + POWER_SUPPLY_PROP_VOLTAGE_MIN, + POWER_SUPPLY_PROP_VOLTAGE_MAX, + POWER_SUPPLY_PROP_CHARGE_TERM_CURRENT, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_CURRENT_MAX, + POWER_SUPPLY_PROP_CONSTANT_CHARGE_VOLTAGE, + POWER_SUPPLY_PROP_CHARGE_FULL_DESIGN, + POWER_SUPPLY_PROP_CHARGE_FULL, + POWER_SUPPLY_PROP_CHARGE_NOW, + POWER_SUPPLY_PROP_ENERGY_FULL_DESIGN, + POWER_SUPPLY_PROP_ENERGY_FULL, + POWER_SUPPLY_PROP_ENERGY_NOW, + POWER_SUPPLY_PROP_TEMP, + POWER_SUPPLY_PROP_CHARGE_COUNTER, + POWER_SUPPLY_PROP_CYCLE_COUNT, + POWER_SUPPLY_PROP_SCOPE, + POWER_SUPPLY_PROP_HEALTH, + POWER_SUPPLY_PROP_MODEL_NAME, + POWER_SUPPLY_PROP_SERIAL_NUMBER, + POWER_SUPPLY_PROP_MANUFACTURE_YEAR, + POWER_SUPPLY_PROP_MANUFACTURE_MONTH, + POWER_SUPPLY_PROP_MANUFACTURE_DAY, + POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD, + POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD +}; + +static const struct power_supply_desc macsmc_battery_desc = { + .name = "macsmc-battery", + .type = POWER_SUPPLY_TYPE_BATTERY, + .get_property = macsmc_battery_get_property, + .set_property = macsmc_battery_set_property, + .property_is_writeable = macsmc_battery_property_is_writeable, + .properties = macsmc_battery_props, + .num_properties = ARRAY_SIZE(macsmc_battery_props), + .charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE) + | BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE), +}; + +static int macsmc_ac_get_property(struct power_supply *psy, + enum power_supply_property psp, + union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret = 0; + u16 vu16; + u32 vu32; + + switch (psp) { + case POWER_SUPPLY_PROP_ONLINE: + ret = apple_smc_read_u32(power->smc, SMC_KEY(CHIS), &vu32); + val->intval = !!vu32; + break; + case POWER_SUPPLY_PROP_VOLTAGE_NOW: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT: + ret = apple_smc_read_u16(power->smc, SMC_KEY(AC-i), &vu16); + val->intval = vu16 * 1000; + break; + case POWER_SUPPLY_PROP_INPUT_POWER_LIMIT: + ret = apple_smc_read_u32(power->smc, SMC_KEY(ACPW), &vu32); + val->intval = vu32 * 1000; + break; + default: + return -EINVAL; + } + + return ret; +} + +static enum power_supply_property macsmc_ac_props[] = { + POWER_SUPPLY_PROP_ONLINE, + POWER_SUPPLY_PROP_INPUT_POWER_LIMIT, + POWER_SUPPLY_PROP_VOLTAGE_NOW, + POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, +}; + +static const struct power_supply_desc macsmc_ac_desc = { + .name = "macsmc-ac", + .type = POWER_SUPPLY_TYPE_MAINS, + .get_property = macsmc_ac_get_property, + .properties = macsmc_ac_props, + .num_properties = ARRAY_SIZE(macsmc_ac_props), +}; + +static int macsmc_log_power_set(const char *val, const struct kernel_param *kp) +{ + int ret = param_set_bool(val, kp); + + if (ret < 0) + return ret; + + if (log_power && g_power) + schedule_delayed_work(&g_power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_dbg_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(to_delayed_work(wrk), + struct macsmc_power, dbg_log_work); + + macsmc_do_dbg(power); + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, POWER_LOG_INTERVAL); +} + +static void macsmc_power_critical_work(struct work_struct *wrk) +{ + struct macsmc_power *power = container_of(wrk, struct macsmc_power, critical_work); + int ret; + u32 bcf0; + u16 bitv, b0av; + + /* + * Check if the battery voltage is below the design voltage. If it is, + * we have a few seconds until the machine dies. Explicitly shut down, + * which at least gets the NVMe controller to flush its cache. + */ + if (apple_smc_read_u16(power->smc, SMC_KEY(BITV), &bitv) >= 0 && + apple_smc_read_u16(power->smc, SMC_KEY(B0AV), &b0av) >= 0 && + b0av < bitv) { + dev_crit(power->dev, "Emergency notification: Battery is critical\n"); + if (kernel_can_power_off()) + kernel_power_off(); + else /* Missing macsmc-reboot driver? In this state, this will not boot anyway. */ + kernel_restart("Battery is critical"); + } + + /* This spams once per second, so make sure we only trigger shutdown once. */ + if (power->shutdown_started) + return; + + /* Check for battery empty condition */ + ret = apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &bcf0); + if (ret < 0) { + dev_err(power->dev, + "Emergency notification: Failed to read battery status\n"); + } else if (bcf0 == 0) { + dev_warn(power->dev, "Emergency notification: Battery status is OK?\n"); + return; + } else { + dev_warn(power->dev, "Emergency notification: Battery is empty\n"); + } + + power->shutdown_started = true; + + /* + * Attempt to trigger an orderly shutdown. At this point, we should have a few + * minutes of reserve capacity left, enough to do a clean shutdown. + */ + dev_warn(power->dev, "Shutting down in 10 seconds\n"); + ssleep(10); + + /* + * Don't force it; if this stalls or fails, the last-resort check above will + * trigger a hard shutdown when shutdown is truly imminent. + */ + orderly_poweroff(false); +} + +static int macsmc_power_event(struct notifier_block *nb, unsigned long event, void *data) +{ + struct macsmc_power *power = container_of(nb, struct macsmc_power, nb); + + if ((event & 0xffffff00) == 0x71010100) { + bool charging = (event & 0xff) != 0; + + dev_info(power->dev, "Charging: %d\n", charging); + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if (event == 0x71020000) { + schedule_work(&power->critical_work); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71060000) { + u8 changed_port = event >> 8; + u8 cur_port; + + /* Port charging state change? */ + if (apple_smc_read_u8(power->smc, SMC_KEY(AC-W), &cur_port) >= 0) { + dev_info(power->dev, "Port %d state change (charge port: %d)\n", + changed_port + 1, cur_port); + } + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x71130000) { + u8 port_index = (event >> 8) & 0xff; + u8 status = event & 0xff; + + if (port_index == 0xff) + dev_info(power->dev, "Connector event: Disconnect (status 0x%02x)\n", + status); + else + dev_info(power->dev, "Connector event: Port %d (status 0x%02x)\n", + port_index + 1, status); + + power_supply_changed(power->batt); + power_supply_changed(power->ac); + + return NOTIFY_OK; + } else if ((event & 0xff000000) == 0x71000000) { + dev_info(power->dev, "Unknown charger event 0x%lx\n", event); + + return NOTIFY_OK; + } else if ((event & 0xffff0000) == 0x72010000) { + /* Button event handled by macsmc-hid, but let's do a debug print */ + if (log_power) + macsmc_do_dbg(power); + + return NOTIFY_OK; + } + + return NOTIFY_DONE; +} + +static int macsmc_power_probe(struct platform_device *pdev) +{ + struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); + struct power_supply_config psy_cfg = {}; + struct macsmc_power *power; + bool flag; + u8 val8; + u16 vu16; + u32 val32; + int ret; + + power = devm_kzalloc(&pdev->dev, sizeof(*power), GFP_KERNEL); + if (!power) + return -ENOMEM; + + power->dev = &pdev->dev; + power->smc = smc; + power->ac_desc = macsmc_ac_desc; + power->batt_desc = macsmc_battery_desc; + dev_set_drvdata(&pdev->dev, power); + + /* Ignore devices without a charger/battery */ + if (macsmc_battery_get_status(power) <= POWER_SUPPLY_STATUS_UNKNOWN) + return -ENODEV; + + /* Fetch string properties */ + apple_smc_read(smc, SMC_KEY(BMDN), power->model_name, sizeof(power->model_name) - 1); + apple_smc_read(smc, SMC_KEY(BMSN), power->serial_number, sizeof(power->serial_number) - 1); + apple_smc_read(smc, SMC_KEY(BMDT), power->mfg_date, sizeof(power->mfg_date) - 1); + + if (apple_smc_read_u32(power->smc, SMC_KEY(CHTE), &val32) >= 0) + power->has_chte = true; + + if (apple_smc_read_u8(power->smc, SMC_KEY(CH0C), &val8) >= 0) + power->has_ch0c = true; + + if (apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8) >= 0) + power->has_ch0i = true; + + /* Turn off the "optimized battery charging" flags, in case macOS left them on */ + if (power->has_chte) + apple_smc_write_u32(power->smc, SMC_KEY(CHTE), 0); + else if (power->has_ch0c) + apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); + + if (power->has_ch0i) + apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); + + apple_smc_write_u8(power->smc, SMC_KEY(CH0K), 0); + apple_smc_write_u8(power->smc, SMC_KEY(CH0B), 0); + + power->batt_desc.charge_behaviours = BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + + /* Newer firmwares do not have force discharge, so check if it's supported */ + if (power->has_ch0i) + power->batt_desc.charge_behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE); + + /* Older firmware uses CH0C, and newer firmware uses CHTE, so check if at least one is present*/ + if (power->has_chte || power->has_ch0c) + power->batt_desc.charge_behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); + + /* + * Prefer CHWA as the SMC firmware from iBoot-10151.1.1 is not compatible with + * this CHLS usage. + */ + if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0) { + power->has_chwa = true; + } else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0) { + power->has_chls = true; + } else { + /* Remove the last 2 properties that control the charge threshold */ + power->batt_desc.num_properties -= 2; + } + + apple_smc_read_u8(power->smc, SMC_KEY(BNCB), &power->num_cells); + power->nominal_voltage_mv = MACSMC_NOMINAL_CELL_VOLTAGE_MV * power->num_cells; + + /* Doing one read of this flag enables critical shutdown notifications */ + apple_smc_read_u32(power->smc, SMC_KEY(BCF0), &val32); + + psy_cfg.drv_data = power; + power->batt = devm_power_supply_register(&pdev->dev, &power->batt_desc, &psy_cfg); + if (IS_ERR(power->batt)) { + dev_err(&pdev->dev, "Failed to register battery\n"); + ret = PTR_ERR(power->batt); + return ret; + } + + /* SMC firmware in macOS 15.4 dropped "AC-i" and "AC-n" (and all keys + * with lower case last letter) without obvious replacement. */ + if (apple_smc_read_u16(power->smc, SMC_KEY(AC-n), &vu16) < 0) + power->ac_desc.num_properties -= 2; + + power->ac = devm_power_supply_register(&pdev->dev, &power->ac_desc, &psy_cfg); + if (IS_ERR(power->ac)) { + dev_err(&pdev->dev, "Failed to register AC adapter\n"); + ret = PTR_ERR(power->ac); + return ret; + } + + power->nb.notifier_call = macsmc_power_event; + blocking_notifier_chain_register(&smc->event_handlers, &power->nb); + + INIT_WORK(&power->critical_work, macsmc_power_critical_work); + INIT_DELAYED_WORK(&power->dbg_log_work, macsmc_dbg_work); + + g_power = power; + + if (log_power) + schedule_delayed_work(&power->dbg_log_work, 0); + + return 0; +} + +static void macsmc_power_remove(struct platform_device *pdev) +{ + struct macsmc_power *power = dev_get_drvdata(&pdev->dev); + + cancel_work(&power->critical_work); + cancel_delayed_work(&power->dbg_log_work); + + g_power = NULL; + + blocking_notifier_chain_unregister(&power->smc->event_handlers, &power->nb); +} + +static struct platform_driver macsmc_power_driver = { + .driver = { + .name = "macsmc-power", + .owner = THIS_MODULE, + }, + .probe = macsmc_power_probe, + .remove = macsmc_power_remove, +}; +module_platform_driver(macsmc_power_driver); + +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple SMC battery and power management driver"); +MODULE_AUTHOR("Hector Martin "); +MODULE_ALIAS("platform:macsmc-power"); diff --git a/drivers/power/supply/max17042_battery.c b/drivers/power/supply/max17042_battery.c index acea176101fa88..a32ff503f45f79 100644 --- a/drivers/power/supply/max17042_battery.c +++ b/drivers/power/supply/max17042_battery.c @@ -201,7 +201,7 @@ static int max17042_get_battery_health(struct max17042_chip *chip, int *health) goto out; } - if (vbatt > chip->pdata->vmax + MAX17042_VMAX_TOLERANCE) { + if (vbatt > size_add(chip->pdata->vmax, MAX17042_VMAX_TOLERANCE)) { *health = POWER_SUPPLY_HEALTH_OVERVOLTAGE; goto out; } diff --git a/drivers/power/supply/max77705_charger.c b/drivers/power/supply/max77705_charger.c index 5dd02f658f5bd3..63b0b4f0cd217f 100644 --- a/drivers/power/supply/max77705_charger.c +++ b/drivers/power/supply/max77705_charger.c @@ -646,51 +646,37 @@ static int max77705_charger_probe(struct i2c_client *i2c) if (ret) return dev_err_probe(dev, ret, "failed to add irq chip\n"); - chg->wqueue = create_singlethread_workqueue(dev_name(dev)); + chg->wqueue = devm_alloc_ordered_workqueue(dev, "%s", 0, dev_name(dev)); if (!chg->wqueue) return -ENOMEM; ret = devm_work_autocancel(dev, &chg->chgin_work, max77705_chgin_isr_work); - if (ret) { - dev_err_probe(dev, ret, "failed to initialize interrupt work\n"); - goto destroy_wq; - } + if (ret) + return dev_err_probe(dev, ret, "failed to initialize interrupt work\n"); ret = max77705_charger_initialize(chg); - if (ret) { - dev_err_probe(dev, ret, "failed to initialize charger IC\n"); - goto destroy_wq; - } + if (ret) + return dev_err_probe(dev, ret, "failed to initialize charger IC\n"); ret = devm_request_threaded_irq(dev, regmap_irq_get_virq(irq_data, MAX77705_CHGIN_I), NULL, max77705_chgin_irq, IRQF_TRIGGER_NONE, "chgin-irq", chg); - if (ret) { - dev_err_probe(dev, ret, "Failed to Request chgin IRQ\n"); - goto destroy_wq; - } + if (ret) + return ret; ret = devm_request_threaded_irq(dev, regmap_irq_get_virq(irq_data, MAX77705_AICL_I), NULL, max77705_aicl_irq, IRQF_TRIGGER_NONE, "aicl-irq", chg); - if (ret) { - dev_err_probe(dev, ret, "Failed to Request aicl IRQ\n"); - goto destroy_wq; - } + if (ret) + return ret; ret = max77705_charger_enable(chg); - if (ret) { - dev_err_probe(dev, ret, "failed to enable charge\n"); - goto destroy_wq; - } + if (ret) + return dev_err_probe(dev, ret, "failed to enable charge\n"); return devm_add_action_or_reset(dev, max77705_charger_disable, chg); - -destroy_wq: - destroy_workqueue(chg->wqueue); - return ret; } static const struct of_device_id max77705_charger_of_match[] = { diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c index f9ff78ba122d49..3d30aeab507e0e 100644 --- a/drivers/pwm/pwm-atmel-tcb.c +++ b/drivers/pwm/pwm-atmel-tcb.c @@ -50,6 +50,8 @@ struct atmel_tcb_pwm_chip { spinlock_t lock; u8 channel; u8 width; + unsigned long rate; + unsigned long slow_rate; struct regmap *regmap; struct clk *clk; struct clk *gclk; @@ -266,7 +268,7 @@ static int atmel_tcb_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, int slowclk = 0; unsigned period; unsigned duty; - unsigned rate = clk_get_rate(tcbpwmc->clk); + unsigned long rate = tcbpwmc->rate; unsigned long long min; unsigned long long max; @@ -294,7 +296,7 @@ static int atmel_tcb_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, */ if (i == ARRAY_SIZE(atmel_tcb_divisors)) { i = slowclk; - rate = clk_get_rate(tcbpwmc->slow_clk); + rate = tcbpwmc->slow_rate; min = div_u64(NSEC_PER_SEC, rate); max = min << tcbpwmc->width; @@ -431,24 +433,49 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev) } chip->ops = &atmel_tcb_pwm_ops; + chip->atomic = true; tcbpwmc->channel = channel; tcbpwmc->width = config->counter_width; - err = clk_prepare_enable(tcbpwmc->slow_clk); + err = clk_prepare_enable(tcbpwmc->clk); if (err) goto err_gclk; + err = clk_prepare_enable(tcbpwmc->slow_clk); + if (err) + goto err_disable_clk;; + + err = clk_rate_exclusive_get(tcbpwmc->clk); + if (err) + goto err_disable_slow_clk; + + err = clk_rate_exclusive_get(tcbpwmc->slow_clk); + if (err) + goto err_clk_unlock; + + tcbpwmc->rate = clk_get_rate(tcbpwmc->clk); + tcbpwmc->slow_rate = clk_get_rate(tcbpwmc->slow_clk); + spin_lock_init(&tcbpwmc->lock); err = pwmchip_add(chip); if (err < 0) - goto err_disable_clk; + goto err_slow_clk_unlock; platform_set_drvdata(pdev, chip); return 0; +err_slow_clk_unlock: + clk_rate_exclusive_put(tcbpwmc->slow_clk); + +err_clk_unlock: + clk_rate_exclusive_put(tcbpwmc->clk); + err_disable_clk: + clk_disable_unprepare(tcbpwmc->clk); + +err_disable_slow_clk: clk_disable_unprepare(tcbpwmc->slow_clk); err_gclk: @@ -470,6 +497,9 @@ static void atmel_tcb_pwm_remove(struct platform_device *pdev) pwmchip_remove(chip); + clk_rate_exclusive_put(tcbpwmc->slow_clk); + clk_rate_exclusive_put(tcbpwmc->clk); + clk_disable_unprepare(tcbpwmc->clk); clk_disable_unprepare(tcbpwmc->slow_clk); clk_put(tcbpwmc->gclk); clk_put(tcbpwmc->clk); diff --git a/drivers/pwm/pwm-imx-tpm.c b/drivers/pwm/pwm-imx-tpm.c index 5b399de16d6040..80fdb3303400f8 100644 --- a/drivers/pwm/pwm-imx-tpm.c +++ b/drivers/pwm/pwm-imx-tpm.c @@ -352,7 +352,7 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev) struct clk *clk; void __iomem *base; int ret; - unsigned int npwm; + unsigned int i, npwm; u32 val; base = devm_platform_ioremap_resource(pdev, 0); @@ -382,6 +382,13 @@ static int pwm_imx_tpm_probe(struct platform_device *pdev) mutex_init(&tpm->lock); + /* count the enabled channels */ + for (i = 0; i < npwm; ++i) { + val = readl(base + PWM_IMX_TPM_CnSC(i)); + if (FIELD_GET(PWM_IMX_TPM_CnSC_ELS, val)) + ++tpm->enable_count; + } + ret = devm_pwmchip_add(&pdev->dev, chip); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to add PWM chip\n"); diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c index 2594fb771b04ad..935257a890b067 100644 --- a/drivers/pwm/pwm-stm32.c +++ b/drivers/pwm/pwm-stm32.c @@ -68,7 +68,7 @@ static int stm32_pwm_round_waveform_tohw(struct pwm_chip *chip, struct stm32_pwm *priv = to_stm32_pwm_dev(chip); unsigned int ch = pwm->hwpwm; unsigned long rate; - u64 ccr, duty; + u64 duty_ticks, offset_ticks; int ret; if (wf->period_length_ns == 0) { @@ -164,23 +164,25 @@ static int stm32_pwm_round_waveform_tohw(struct pwm_chip *chip, wfhw->arr = min_t(u64, arr, priv->max_arr) - 1; } - duty = mul_u64_u64_div_u64(wf->duty_length_ns, rate, - (u64)NSEC_PER_SEC * (wfhw->psc + 1)); - duty = min_t(u64, duty, wfhw->arr + 1); + duty_ticks = mul_u64_u64_div_u64(wf->duty_length_ns, rate, + (u64)NSEC_PER_SEC * (wfhw->psc + 1)); + duty_ticks = min_t(u64, duty_ticks, wfhw->arr + 1); - if (wf->duty_length_ns && wf->duty_offset_ns && - wf->duty_length_ns + wf->duty_offset_ns >= wf->period_length_ns) { + offset_ticks = mul_u64_u64_div_u64(wf->duty_offset_ns, rate, + (u64)NSEC_PER_SEC * (wfhw->psc + 1)); + offset_ticks = min_t(u64, offset_ticks, wfhw->arr + 1); + + if (duty_ticks && offset_ticks && + duty_ticks + offset_ticks >= wfhw->arr + 1) { wfhw->ccer |= TIM_CCER_CCxP(ch + 1); if (priv->have_complementary_output) wfhw->ccer |= TIM_CCER_CCxNP(ch + 1); - ccr = wfhw->arr + 1 - duty; + wfhw->ccr = wfhw->arr + 1 - duty_ticks; } else { - ccr = duty; + wfhw->ccr = duty_ticks; } - wfhw->ccr = min_t(u64, ccr, wfhw->arr + 1); - out: dev_dbg(&chip->dev, "pwm#%u: %lld/%lld [+%lld] @%lu -> CCER: %08x, PSC: %08x, ARR: %08x, CCR: %08x\n", pwm->hwpwm, wf->duty_length_ns, wf->period_length_ns, wf->duty_offset_ns, diff --git a/drivers/pwm/pwm_th1520.rs b/drivers/pwm/pwm_th1520.rs index b0e24ee724e450..36567fc17dcc8c 100644 --- a/drivers/pwm/pwm_th1520.rs +++ b/drivers/pwm/pwm_th1520.rs @@ -64,10 +64,7 @@ const TH1520_PWM_REG_SIZE: usize = 0xB0; fn ns_to_cycles(ns: u64, rate_hz: u64) -> u64 { const NSEC_PER_SEC_U64: u64 = time::NSEC_PER_SEC as u64; - (match ns.checked_mul(rate_hz) { - Some(product) => product, - None => u64::MAX, - }) / NSEC_PER_SEC_U64 + ns.saturating_mul(rate_hz) / NSEC_PER_SEC_U64 } fn cycles_to_ns(cycles: u64, rate_hz: u64) -> u64 { diff --git a/drivers/regulator/act8945a-regulator.c b/drivers/regulator/act8945a-regulator.c index 24cbdd8338630b..5bbe2bce740eaf 100644 --- a/drivers/regulator/act8945a-regulator.c +++ b/drivers/regulator/act8945a-regulator.c @@ -302,8 +302,9 @@ static int act8945a_pmic_probe(struct platform_device *pdev) num_regulators = ARRAY_SIZE(act8945a_regulators); } + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); + config.dev = &pdev->dev; - config.dev->of_node = pdev->dev.parent->of_node; config.driver_data = act8945a; for (i = 0; i < num_regulators; i++) { rdev = devm_regulator_register(&pdev->dev, ®ulators[i], diff --git a/drivers/regulator/bd9571mwv-regulator.c b/drivers/regulator/bd9571mwv-regulator.c index 209beabb5c37c6..f4de24a281b123 100644 --- a/drivers/regulator/bd9571mwv-regulator.c +++ b/drivers/regulator/bd9571mwv-regulator.c @@ -287,8 +287,9 @@ static int bd9571mwv_regulator_probe(struct platform_device *pdev) platform_set_drvdata(pdev, bdreg); + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); + config.dev = &pdev->dev; - config.dev->of_node = pdev->dev.parent->of_node; config.driver_data = bdreg; config.regmap = bdreg->regmap; diff --git a/drivers/regulator/bq257xx-regulator.c b/drivers/regulator/bq257xx-regulator.c index dab8f1ab44503e..711dbe045383c4 100644 --- a/drivers/regulator/bq257xx-regulator.c +++ b/drivers/regulator/bq257xx-regulator.c @@ -142,8 +142,7 @@ static int bq257xx_regulator_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; struct regulator_config cfg = {}; - pdev->dev.of_node = pdev->dev.parent->of_node; - pdev->dev.of_node_reused = true; + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); pdata = devm_kzalloc(&pdev->dev, sizeof(struct bq257xx_reg_data), GFP_KERNEL); if (!pdata) diff --git a/drivers/regulator/fp9931.c b/drivers/regulator/fp9931.c index abea3b69d8a085..002b41f53eff89 100644 --- a/drivers/regulator/fp9931.c +++ b/drivers/regulator/fp9931.c @@ -446,7 +446,7 @@ static int fp9931_probe(struct i2c_client *client) return dev_err_probe(&client->dev, PTR_ERR(data->regmap), "failed to allocate regmap!\n"); - data->vin_reg = devm_regulator_get_optional(&client->dev, "vin"); + data->vin_reg = devm_regulator_get(&client->dev, "vin"); if (IS_ERR(data->vin_reg)) return dev_err_probe(&client->dev, PTR_ERR(data->vin_reg), "failed to get vin regulator\n"); diff --git a/drivers/regulator/max77650-regulator.c b/drivers/regulator/max77650-regulator.c index a809264c77fc8e..11b04a13f889a2 100644 --- a/drivers/regulator/max77650-regulator.c +++ b/drivers/regulator/max77650-regulator.c @@ -337,7 +337,7 @@ static int max77650_regulator_probe(struct platform_device *pdev) parent = dev->parent; if (!dev->of_node) - dev->of_node = parent->of_node; + device_set_of_node_from_dev(dev, parent); rdescs = devm_kcalloc(dev, MAX77650_REGULATOR_NUM_REGULATORS, sizeof(*rdescs), GFP_KERNEL); diff --git a/drivers/regulator/mt6357-regulator.c b/drivers/regulator/mt6357-regulator.c index 1eb69c7a6acb38..09feb454ab6b84 100644 --- a/drivers/regulator/mt6357-regulator.c +++ b/drivers/regulator/mt6357-regulator.c @@ -410,7 +410,7 @@ static int mt6357_regulator_probe(struct platform_device *pdev) struct regulator_dev *rdev; int i; - pdev->dev.of_node = pdev->dev.parent->of_node; + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); for (i = 0; i < MT6357_MAX_REGULATOR; i++) { config.dev = &pdev->dev; diff --git a/drivers/regulator/rk808-regulator.c b/drivers/regulator/rk808-regulator.c index e66408f23bb64c..1e956153427e54 100644 --- a/drivers/regulator/rk808-regulator.c +++ b/drivers/regulator/rk808-regulator.c @@ -2114,8 +2114,7 @@ static int rk808_regulator_probe(struct platform_device *pdev) struct regmap *regmap; int ret, i, nregulators; - pdev->dev.of_node = pdev->dev.parent->of_node; - pdev->dev.of_node_reused = true; + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); regmap = dev_get_regmap(pdev->dev.parent, NULL); if (!regmap) diff --git a/drivers/regulator/s2dos05-regulator.c b/drivers/regulator/s2dos05-regulator.c index 1463585c456520..a1c394ddbaff6e 100644 --- a/drivers/regulator/s2dos05-regulator.c +++ b/drivers/regulator/s2dos05-regulator.c @@ -126,7 +126,7 @@ static int s2dos05_pmic_probe(struct platform_device *pdev) s2dos05->regmap = iodev->regmap_pmic; s2dos05->dev = dev; if (!dev->of_node) - dev->of_node = dev->parent->of_node; + device_set_of_node_from_dev(dev, dev->parent); config.dev = dev; config.driver_data = s2dos05; diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index 8c8ddbf995a46b..0dd80e688b0ea3 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -812,7 +812,7 @@ static int imx_rproc_addr_init(struct imx_rproc *priv, /* Not use resource version, because we might share region */ priv->mem[b].cpu_addr = devm_ioremap_resource_wc(&pdev->dev, &res); - if (!priv->mem[b].cpu_addr) { + if (IS_ERR(priv->mem[b].cpu_addr)) { dev_err(dev, "failed to remap %pr\n", &res); return -ENOMEM; } @@ -1007,7 +1007,11 @@ static int imx_rproc_mmio_detect_mode(struct rproc *rproc) } priv->regmap = regmap; - regmap_attach_dev(dev, regmap, &config); + ret = regmap_attach_dev(dev, regmap, &config); + if (ret) { + dev_err(dev, "regmap attach failed\n"); + return ret; + } if (priv->gpr) { ret = regmap_read(priv->gpr, dcfg->gpr_reg, &val); diff --git a/drivers/remoteproc/ti_k3_common.c b/drivers/remoteproc/ti_k3_common.c index 32aa954dc5be67..3cb8ae5d72f674 100644 --- a/drivers/remoteproc/ti_k3_common.c +++ b/drivers/remoteproc/ti_k3_common.c @@ -513,7 +513,7 @@ int k3_reserved_mem_init(struct k3_rproc *kproc) kproc->rmem[i].dev_addr = (u32)res.start; kproc->rmem[i].size = resource_size(&res); kproc->rmem[i].cpu_addr = devm_ioremap_resource_wc(dev, &res); - if (!kproc->rmem[i].cpu_addr) { + if (IS_ERR(kproc->rmem[i].cpu_addr)) { dev_err(dev, "failed to map reserved memory#%d at %pR\n", i + 1, &res); return -ENOMEM; diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c index b71ce69afe9f41..eb5f714de2bff8 100644 --- a/drivers/remoteproc/xlnx_r5_remoteproc.c +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c @@ -232,17 +232,19 @@ static void zynqmp_r5_mb_rx_cb(struct mbox_client *cl, void *msg) ipi = container_of(cl, struct mbox_info, mbox_cl); - /* copy data from ipi buffer to r5_core */ + /* copy data from ipi buffer to r5_core if IPI is buffered. */ ipi_msg = (struct zynqmp_ipi_message *)msg; - buf_msg = (struct zynqmp_ipi_message *)ipi->rx_mc_buf; - len = ipi_msg->len; - if (len > IPI_BUF_LEN_MAX) { - dev_warn(cl->dev, "msg size exceeded than %d\n", - IPI_BUF_LEN_MAX); - len = IPI_BUF_LEN_MAX; + if (ipi_msg) { + buf_msg = (struct zynqmp_ipi_message *)ipi->rx_mc_buf; + len = ipi_msg->len; + if (len > IPI_BUF_LEN_MAX) { + dev_warn(cl->dev, "msg size exceeded than %d\n", + IPI_BUF_LEN_MAX); + len = IPI_BUF_LEN_MAX; + } + buf_msg->len = len; + memcpy(buf_msg->data, ipi_msg->data, len); } - buf_msg->len = len; - memcpy(buf_msg->data, ipi_msg->data, len); /* received and processed interrupt ack */ if (mbox_send_message(ipi->rx_chan, NULL) < 0) @@ -1005,7 +1007,7 @@ static int zynqmp_r5_get_sram_banks(struct zynqmp_r5_core *r5_core) } /* Get SRAM device address */ - ret = of_property_read_reg(sram_np, i, &abs_addr, &size); + ret = of_property_read_reg(sram_np, 0, &abs_addr, &size); if (ret) { dev_err(dev, "failed to get reg property\n"); goto fail_sram_get; diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 0666be6b0e88d3..fd4f107bd00e35 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -148,11 +148,17 @@ static void mpam_free_garbage(void) /* * Once mpam is enabled, new requestors cannot further reduce the available * partid. Assert that the size is fixed, and new requestors will be turned - * away. + * away. This is needed when walking over structures sized by PARTID. + * + * During mpam_disable() these structures are not fixed, but the MSC state + * is still reset using whatever sizes have been discovered so far. As only + * PARTID 0 will be used after mpam_disable(), any race would be benign. + * Skip the check if a mpam_disable_reason has been set. */ static void mpam_assert_partid_sizes_fixed(void) { - WARN_ON_ONCE(!partid_max_published); + if (!mpam_disable_reason) + WARN_ON_ONCE(!partid_max_published); } static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg) @@ -632,10 +638,9 @@ static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, * Try and see what values stick in this bit. If we can write either value, * its probably not implemented by hardware. */ -static bool _mpam_ris_hw_probe_hw_nrdy(struct mpam_msc_ris *ris, u32 mon_reg) +static bool mpam_ris_hw_probe_csu_nrdy(struct mpam_msc_ris *ris) { - u32 now; - u64 mon_sel; + u32 now, mon_sel, ctl_val; bool can_set, can_clear; struct mpam_msc *msc = ris->vmsc->msc; @@ -644,23 +649,30 @@ static bool _mpam_ris_hw_probe_hw_nrdy(struct mpam_msc_ris *ris, u32 mon_reg) mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) | FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); - _mpam_write_monsel_reg(msc, mon_reg, mon_sel); + mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); + + /* Hardware might ignore nrdy if it's not enabled */ + ctl_val = MSMON_CFG_CSU_CTL_TYPE_CSU; + ctl_val |= MSMON_CFG_x_CTL_MATCH_PARTID; + ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG; + ctl_val |= MSMON_CFG_x_CTL_EN; + mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0); + mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val); - _mpam_write_monsel_reg(msc, mon_reg, MSMON___NRDY); - now = _mpam_read_monsel_reg(msc, mon_reg); + _mpam_write_monsel_reg(msc, MSMON_CSU, MSMON___NRDY); + now = _mpam_read_monsel_reg(msc, MSMON_CSU); can_set = now & MSMON___NRDY; - _mpam_write_monsel_reg(msc, mon_reg, 0); - now = _mpam_read_monsel_reg(msc, mon_reg); + _mpam_write_monsel_reg(msc, MSMON_CSU, 0); + /* Configuration change to try and coax hardware into setting nrdy */ + mpam_write_monsel_reg(msc, CFG_CSU_FLT, 0x1); + now = _mpam_read_monsel_reg(msc, MSMON_CSU); can_clear = !(now & MSMON___NRDY); mpam_mon_sel_unlock(msc); return (!can_set || !can_clear); } -#define mpam_ris_hw_probe_hw_nrdy(_ris, _mon_reg) \ - _mpam_ris_hw_probe_hw_nrdy(_ris, MSMON_##_mon_reg) - static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) { int err; @@ -770,20 +782,18 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) mpam_set_feature(mpam_feat_msmon_csu_xcl, props); /* Is NRDY hardware managed? */ - hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, CSU); - if (hw_managed) - mpam_set_feature(mpam_feat_msmon_csu_hw_nrdy, props); - } + hw_managed = mpam_ris_hw_probe_csu_nrdy(ris); - /* - * Accept the missing firmware property if NRDY appears - * un-implemented. - */ - if (err && mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, props)) - dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware."); + /* + * Accept the missing firmware property if NRDY appears + * un-implemented. + */ + if (err && hw_managed) + dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware."); + } } if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) { - bool has_long, hw_managed; + bool has_long; u32 mbwumon_idr = mpam_read_partsel_reg(msc, MBWUMON_IDR); props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr); @@ -802,16 +812,6 @@ static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) } else { mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props); } - - /* Is NRDY hardware managed? */ - hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, MBWU); - if (hw_managed) - mpam_set_feature(mpam_feat_msmon_mbwu_hw_nrdy, props); - - /* - * Don't warn about any missing firmware property for - * MBWU NRDY - it doesn't make any sense! - */ } } } @@ -1078,7 +1078,6 @@ static void __ris_msmon_read(void *arg) bool reset_on_next_read = false; struct mpam_msc_ris *ris = m->ris; struct msmon_mbwu_state *mbwu_state; - struct mpam_props *rprops = &ris->props; struct mpam_msc *msc = m->ris->vmsc->msc; u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt; @@ -1134,8 +1133,7 @@ static void __ris_msmon_read(void *arg) switch (m->type) { case mpam_feat_msmon_csu: now = mpam_read_monsel_reg(msc, CSU); - if (mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, rprops)) - nrdy = now & MSMON___NRDY; + nrdy = now & MSMON___NRDY; now = FIELD_GET(MSMON___VALUE, now); break; case mpam_feat_msmon_mbwu_31counter: @@ -1143,8 +1141,7 @@ static void __ris_msmon_read(void *arg) case mpam_feat_msmon_mbwu_63counter: if (m->type != mpam_feat_msmon_mbwu_31counter) { now = mpam_msc_read_mbwu_l(msc); - if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops)) - nrdy = now & MSMON___L_NRDY; + nrdy = now & MSMON___L_NRDY; if (m->type == mpam_feat_msmon_mbwu_63counter) now = FIELD_GET(MSMON___LWD_VALUE, now); @@ -1152,8 +1149,7 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___L_VALUE, now); } else { now = mpam_read_monsel_reg(msc, MBWU); - if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops)) - nrdy = now & MSMON___NRDY; + nrdy = now & MSMON___NRDY; now = FIELD_GET(MSMON___VALUE, now); } @@ -1364,17 +1360,15 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, __mpam_intpart_sel(ris->ris_idx, partid, msc); } - if (mpam_has_feature(mpam_feat_cpor_part, rprops) && - mpam_has_feature(mpam_feat_cpor_part, cfg)) { - if (cfg->reset_cpbm) - mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); - else + if (mpam_has_feature(mpam_feat_cpor_part, rprops)) { + if (mpam_has_feature(mpam_feat_cpor_part, cfg)) mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); + else + mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); } - if (mpam_has_feature(mpam_feat_mbw_part, rprops) && - mpam_has_feature(mpam_feat_mbw_part, cfg)) { - if (cfg->reset_mbw_pbm) + if (mpam_has_feature(mpam_feat_mbw_part, rprops)) { + if (mpam_has_feature(mpam_feat_mbw_part, cfg)) mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); else mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); @@ -1384,16 +1378,14 @@ static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, mpam_has_feature(mpam_feat_mbw_min, cfg)) mpam_write_partsel_reg(msc, MBW_MIN, 0); - if (mpam_has_feature(mpam_feat_mbw_max, rprops) && - mpam_has_feature(mpam_feat_mbw_max, cfg)) { - if (cfg->reset_mbw_max) - mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); - else + if (mpam_has_feature(mpam_feat_mbw_max, rprops)) { + if (mpam_has_feature(mpam_feat_mbw_max, cfg)) mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); + else + mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); } - if (mpam_has_feature(mpam_feat_mbw_prop, rprops) && - mpam_has_feature(mpam_feat_mbw_prop, cfg)) + if (mpam_has_feature(mpam_feat_mbw_prop, rprops)) mpam_write_partsel_reg(msc, MBW_PROP, 0); if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) @@ -1493,16 +1485,6 @@ static int mpam_save_mbwu_state(void *arg) return 0; } -static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) -{ - *reset_cfg = (struct mpam_config) { - .reset_cpbm = true, - .reset_mbw_pbm = true, - .reset_mbw_max = true, - }; - bitmap_fill(reset_cfg->features, MPAM_FEATURE_LAST); -} - /* * Called via smp_call_on_cpu() to prevent migration, while still being * pre-emptible. Caller must hold mpam_srcu. @@ -1510,14 +1492,12 @@ static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) static int mpam_reset_ris(void *arg) { u16 partid, partid_max; - struct mpam_config reset_cfg; + struct mpam_config reset_cfg = {}; struct mpam_msc_ris *ris = arg; if (ris->in_reset_state) return 0; - mpam_init_reset_cfg(&reset_cfg); - spin_lock(&partid_max_lock); partid_max = mpam_partid_max; spin_unlock(&partid_max_lock); @@ -2379,6 +2359,9 @@ static void __destroy_component_cfg(struct mpam_component *comp) lockdep_assert_held(&mpam_list_lock); + if (!comp->cfg) + return; + add_to_garbage(comp->cfg); list_for_each_entry(vmsc, &comp->vmsc, comp_list) { msc = vmsc->msc; @@ -2694,6 +2677,7 @@ int mpam_apply_config(struct mpam_component *comp, u16 partid, srcu_read_lock_held(&mpam_srcu)) { arg.ris = ris; mpam_touch_msc(msc, __write_config, &arg); + ris->in_reset_state = false; } mutex_unlock(&msc->cfg_lock); } diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index e8971842b124f2..5f1fca64ce8b96 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -167,14 +167,12 @@ enum mpam_device_features { mpam_feat_msmon_csu, mpam_feat_msmon_csu_capture, mpam_feat_msmon_csu_xcl, - mpam_feat_msmon_csu_hw_nrdy, mpam_feat_msmon_mbwu, mpam_feat_msmon_mbwu_31counter, mpam_feat_msmon_mbwu_44counter, mpam_feat_msmon_mbwu_63counter, mpam_feat_msmon_mbwu_capture, mpam_feat_msmon_mbwu_rwbw, - mpam_feat_msmon_mbwu_hw_nrdy, mpam_feat_partid_nrw, MPAM_FEATURE_LAST }; @@ -266,10 +264,6 @@ struct mpam_config { u32 mbw_pbm; u16 mbw_max; - bool reset_cpbm; - bool reset_mbw_pbm; - bool reset_mbw_max; - struct mpam_garbage garbage; }; diff --git a/drivers/reset/amlogic/reset-meson.c b/drivers/reset/amlogic/reset-meson.c index 84610365a823ca..c303e8590dd68b 100644 --- a/drivers/reset/amlogic/reset-meson.c +++ b/drivers/reset/amlogic/reset-meson.c @@ -42,6 +42,7 @@ static const struct meson_reset_param meson_s4_param = { }; static const struct meson_reset_param t7_param = { + .reset_ops = &meson_reset_ops, .reset_num = 224, .reset_offset = 0x0, .level_offset = 0x40, diff --git a/drivers/reset/reset-rzv2h-usb2phy.c b/drivers/reset/reset-rzv2h-usb2phy.c index ae643575b067ce..5bdd3927461278 100644 --- a/drivers/reset/reset-rzv2h-usb2phy.c +++ b/drivers/reset/reset-rzv2h-usb2phy.c @@ -49,9 +49,10 @@ static inline struct rzv2h_usb2phy_reset_priv return container_of(rcdev, struct rzv2h_usb2phy_reset_priv, rcdev); } -/* This function must be called only after pm_runtime_resume_and_get() has been called */ -static void rzv2h_usbphy_assert_helper(struct rzv2h_usb2phy_reset_priv *priv) +static int rzv2h_usbphy_reset_assert(struct reset_controller_dev *rcdev, + unsigned long id) { + struct rzv2h_usb2phy_reset_priv *priv = rzv2h_usbphy_rcdev_to_priv(rcdev); const struct rzv2h_usb2phy_reset_of_data *data = priv->data; scoped_guard(spinlock, &priv->lock) { @@ -60,24 +61,6 @@ static void rzv2h_usbphy_assert_helper(struct rzv2h_usb2phy_reset_priv *priv) } usleep_range(11, 20); -} - -static int rzv2h_usbphy_reset_assert(struct reset_controller_dev *rcdev, - unsigned long id) -{ - struct rzv2h_usb2phy_reset_priv *priv = rzv2h_usbphy_rcdev_to_priv(rcdev); - struct device *dev = priv->dev; - int ret; - - ret = pm_runtime_resume_and_get(dev); - if (ret) { - dev_err(dev, "pm_runtime_resume_and_get failed\n"); - return ret; - } - - rzv2h_usbphy_assert_helper(priv); - - pm_runtime_put(dev); return 0; } @@ -87,14 +70,6 @@ static int rzv2h_usbphy_reset_deassert(struct reset_controller_dev *rcdev, { struct rzv2h_usb2phy_reset_priv *priv = rzv2h_usbphy_rcdev_to_priv(rcdev); const struct rzv2h_usb2phy_reset_of_data *data = priv->data; - struct device *dev = priv->dev; - int ret; - - ret = pm_runtime_resume_and_get(dev); - if (ret) { - dev_err(dev, "pm_runtime_resume_and_get failed\n"); - return ret; - } scoped_guard(spinlock, &priv->lock) { writel(data->reset_deassert_val, priv->base + data->reset_reg); @@ -102,8 +77,6 @@ static int rzv2h_usbphy_reset_deassert(struct reset_controller_dev *rcdev, writel(data->reset_release_val, priv->base + data->reset_reg); } - pm_runtime_put(dev); - return 0; } @@ -111,20 +84,10 @@ static int rzv2h_usbphy_reset_status(struct reset_controller_dev *rcdev, unsigned long id) { struct rzv2h_usb2phy_reset_priv *priv = rzv2h_usbphy_rcdev_to_priv(rcdev); - struct device *dev = priv->dev; - int ret; u32 reg; - ret = pm_runtime_resume_and_get(dev); - if (ret) { - dev_err(dev, "pm_runtime_resume_and_get failed\n"); - return ret; - } - reg = readl(priv->base + priv->data->reset_reg); - pm_runtime_put(dev); - return (reg & priv->data->reset_status_bits) == priv->data->reset_status_bits; } @@ -141,6 +104,11 @@ static int rzv2h_usb2phy_reset_of_xlate(struct reset_controller_dev *rcdev, return 0; } +static void rzv2h_usb2phy_reset_pm_runtime_put(void *data) +{ + pm_runtime_put(data); +} + static int rzv2h_usb2phy_reset_probe(struct platform_device *pdev) { const struct rzv2h_usb2phy_reset_of_data *data; @@ -175,14 +143,14 @@ static int rzv2h_usb2phy_reset_probe(struct platform_device *pdev) if (error) return dev_err_probe(dev, error, "pm_runtime_resume_and_get failed\n"); + error = devm_add_action_or_reset(dev, rzv2h_usb2phy_reset_pm_runtime_put, + dev); + if (error) + return dev_err_probe(dev, error, "unable to register cleanup action\n"); + for (unsigned int i = 0; i < data->init_val_count; i++) writel(data->init_vals[i].val, priv->base + data->init_vals[i].reg); - /* keep usb2phy in asserted state */ - rzv2h_usbphy_assert_helper(priv); - - pm_runtime_put(dev); - priv->rcdev.ops = &rzv2h_usbphy_reset_ops; priv->rcdev.of_reset_n_cells = 0; priv->rcdev.nr_resets = 1; @@ -190,7 +158,11 @@ static int rzv2h_usb2phy_reset_probe(struct platform_device *pdev) priv->rcdev.of_node = dev->of_node; priv->rcdev.dev = dev; - return devm_reset_controller_register(dev, &priv->rcdev); + error = devm_reset_controller_register(dev, &priv->rcdev); + if (error) + return dev_err_probe(dev, error, "could not register reset controller\n"); + + return 0; } /* diff --git a/drivers/rtc/rtc-abx80x.c b/drivers/rtc/rtc-abx80x.c index 3fee27914ba805..5f3a3e60a19d09 100644 --- a/drivers/rtc/rtc-abx80x.c +++ b/drivers/rtc/rtc-abx80x.c @@ -933,6 +933,8 @@ static int abx80x_probe(struct i2c_client *client) client->irq = 0; } } + if (client->irq <= 0) + clear_bit(RTC_FEATURE_ALARM, priv->rtc->features); err = rtc_add_group(priv->rtc, &rtc_calib_attr_group); if (err) { diff --git a/drivers/rtc/rtc-ntxec.c b/drivers/rtc/rtc-ntxec.c index 850ca49186fdcf..d28ddb34e19e77 100644 --- a/drivers/rtc/rtc-ntxec.c +++ b/drivers/rtc/rtc-ntxec.c @@ -110,7 +110,7 @@ static int ntxec_rtc_probe(struct platform_device *pdev) struct rtc_device *dev; struct ntxec_rtc *rtc; - pdev->dev.of_node = pdev->dev.parent->of_node; + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); rtc = devm_kzalloc(&pdev->dev, sizeof(*rtc), GFP_KERNEL); if (!rtc) diff --git a/drivers/s390/cio/cio.h b/drivers/s390/cio/cio.h index 08a5e9380e75a9..bad142c536e1e6 100644 --- a/drivers/s390/cio/cio.h +++ b/drivers/s390/cio/cio.h @@ -103,11 +103,6 @@ struct subchannel { struct work_struct todo_work; struct schib_config config; u64 dma_mask; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; } __attribute__ ((aligned(8))); DECLARE_PER_CPU_ALIGNED(struct irb, cio_irb); diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 5ab239f38588f8..e5a0ec6b4e3e74 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -159,7 +159,6 @@ static void css_subchannel_release(struct device *dev) sch->config.intparm = 0; cio_commit_config(sch); - kfree(sch->driver_override); kfree(sch); } @@ -323,37 +322,9 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, static DEVICE_ATTR_RO(modalias); -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct subchannel *sch = to_subchannel(dev); - int ret; - - ret = driver_set_override(dev, &sch->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct subchannel *sch = to_subchannel(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", sch->driver_override); - device_unlock(dev); - return len; -} -static DEVICE_ATTR_RW(driver_override); - static struct attribute *subch_attrs[] = { &dev_attr_type.attr, &dev_attr_modalias.attr, - &dev_attr_driver_override.attr, NULL, }; @@ -1356,9 +1327,11 @@ static int css_bus_match(struct device *dev, const struct device_driver *drv) struct subchannel *sch = to_subchannel(dev); const struct css_driver *driver = to_cssdriver(drv); struct css_device_id *id; + int ret; /* When driver_override is set, only bind to the matching driver */ - if (sch->driver_override && strcmp(sch->driver_override, drv->name)) + ret = device_match_driver_override(dev, drv); + if (ret == 0) return 0; for (id = driver->subchannel_type; id->match_flags; id++) { @@ -1415,6 +1388,7 @@ static int css_uevent(const struct device *dev, struct kobj_uevent_env *env) static const struct bus_type css_bus_type = { .name = "css", + .driver_override = true, .match = css_bus_match, .probe = css_probe, .remove = css_remove, diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index d652df96a50784..f24e27add721d1 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -859,25 +859,24 @@ static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) static int __ap_revise_reserved(struct device *dev, void *dummy) { - int rc, card, queue, devres, drvres; + int rc, card, queue, devres, drvres, ovrd; if (is_queue_dev(dev)) { struct ap_driver *ap_drv = to_ap_drv(dev->driver); struct ap_queue *aq = to_ap_queue(dev); - struct ap_device *ap_dev = &aq->ap_dev; card = AP_QID_CARD(aq->qid); queue = AP_QID_QUEUE(aq->qid); - if (ap_dev->driver_override) { - if (strcmp(ap_dev->driver_override, - ap_drv->driver.name)) { - pr_debug("reprobing queue=%02x.%04x\n", card, queue); - rc = device_reprobe(dev); - if (rc) { - AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", - __func__, card, queue); - } + ovrd = device_match_driver_override(dev, &ap_drv->driver); + if (ovrd > 0) { + /* override set and matches, nothing to do */ + } else if (ovrd == 0) { + pr_debug("reprobing queue=%02x.%04x\n", card, queue); + rc = device_reprobe(dev); + if (rc) { + AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", + __func__, card, queue); } } else { mutex_lock(&ap_attr_mutex); @@ -928,7 +927,7 @@ int ap_owned_by_def_drv(int card, int queue) if (aq) { const struct device_driver *drv = aq->ap_dev.device.driver; const struct ap_driver *ap_drv = to_ap_drv(drv); - bool override = !!aq->ap_dev.driver_override; + bool override = device_has_driver_override(&aq->ap_dev.device); if (override && drv && ap_drv->flags & AP_DRIVER_FLAG_DEFAULT) rc = 1; @@ -977,7 +976,7 @@ static int ap_device_probe(struct device *dev) { struct ap_device *ap_dev = to_ap_dev(dev); struct ap_driver *ap_drv = to_ap_drv(dev->driver); - int card, queue, devres, drvres, rc = -ENODEV; + int card, queue, devres, drvres, rc = -ENODEV, ovrd; if (!get_device(dev)) return rc; @@ -991,10 +990,11 @@ static int ap_device_probe(struct device *dev) */ card = AP_QID_CARD(to_ap_queue(dev)->qid); queue = AP_QID_QUEUE(to_ap_queue(dev)->qid); - if (ap_dev->driver_override) { - if (strcmp(ap_dev->driver_override, - ap_drv->driver.name)) - goto out; + ovrd = device_match_driver_override(dev, &ap_drv->driver); + if (ovrd > 0) { + /* override set and matches, nothing to do */ + } else if (ovrd == 0) { + goto out; } else { mutex_lock(&ap_attr_mutex); devres = test_bit_inv(card, ap_perms.apm) && diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 51e08f27bd75e8..04ea256ecf9196 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -166,7 +166,6 @@ void ap_driver_unregister(struct ap_driver *); struct ap_device { struct device device; int device_type; /* AP device type. */ - const char *driver_override; }; #define to_ap_dev(x) container_of((x), struct ap_device, device) diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 3fe2e41c5c6b18..ca9819e6f7e761 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -734,26 +734,14 @@ static ssize_t driver_override_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct ap_queue *aq = to_ap_queue(dev); - struct ap_device *ap_dev = &aq->ap_dev; - int rc; - - device_lock(dev); - if (ap_dev->driver_override) - rc = sysfs_emit(buf, "%s\n", ap_dev->driver_override); - else - rc = sysfs_emit(buf, "\n"); - device_unlock(dev); - - return rc; + guard(spinlock)(&dev->driver_override.lock); + return sysfs_emit(buf, "%s\n", dev->driver_override.name ?: ""); } static ssize_t driver_override_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct ap_queue *aq = to_ap_queue(dev); - struct ap_device *ap_dev = &aq->ap_dev; int rc = -EINVAL; bool old_value; @@ -764,13 +752,13 @@ static ssize_t driver_override_store(struct device *dev, if (ap_apmask_aqmask_in_use) goto out; - old_value = ap_dev->driver_override ? true : false; - rc = driver_set_override(dev, &ap_dev->driver_override, buf, count); + old_value = device_has_driver_override(dev); + rc = __device_set_driver_override(dev, buf, count); if (rc) goto out; - if (old_value && !ap_dev->driver_override) + if (old_value && !device_has_driver_override(dev)) --ap_driver_override_ctr; - else if (!old_value && ap_dev->driver_override) + else if (!old_value && device_has_driver_override(dev)) ++ap_driver_override_ctr; rc = count; diff --git a/drivers/scsi/hpsa.h b/drivers/scsi/hpsa.h index 99b0750850b2ba..f6bfe75dd696d9 100644 --- a/drivers/scsi/hpsa.h +++ b/drivers/scsi/hpsa.h @@ -164,7 +164,7 @@ struct bmic_controller_parameters { struct ctlr_info { unsigned int *reply_map; int ctlr; - char devname[8]; + char devname[16]; char *product_name; struct pci_dev *pdev; u32 board_id; @@ -255,7 +255,7 @@ struct ctlr_info { int remove_in_progress; /* Address of h->q[x] is passed to intr handler to know which queue */ u8 q[MAX_REPLY_QUEUES]; - char intrname[MAX_REPLY_QUEUES][16]; /* "hpsa0-msix00" names */ + char intrname[MAX_REPLY_QUEUES][32]; /* controller and IRQ names */ u32 TMFSupportFlags; /* cache what task mgmt funcs are supported. */ #define HPSATMF_BITS_SUPPORTED (1 << 0) #define HPSATMF_PHYS_LUN_RESET (1 << 1) diff --git a/drivers/scsi/mpt3sas/mpt3sas_scsih.c b/drivers/scsi/mpt3sas/mpt3sas_scsih.c index 6ff78855729424..12caffeed3a0d2 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_scsih.c +++ b/drivers/scsi/mpt3sas/mpt3sas_scsih.c @@ -2738,8 +2738,20 @@ scsih_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim) pcie_device->enclosure_level, pcie_device->connector_name); + /* + * The HBA firmware passes the NVMe drive's MDTS + * (Maximum Data Transfer Size) up to the driver. However, + * the driver hardcodes a 4K buffer size for the PRP list, + * accommodating at most 512 entries. This strictly limits + * the maximum supported NVMe I/O transfer to 2 MiB. + * + * Cap max_hw_sectors to the smaller of the drive's reported + * MDTS or the 2 MiB driver limit to prevent kernel oopses. + */ + lim->max_hw_sectors = SZ_2M >> SECTOR_SHIFT; if (pcie_device->nvme_mdts) - lim->max_hw_sectors = pcie_device->nvme_mdts / 512; + lim->max_hw_sectors = min(lim->max_hw_sectors, + pcie_device->nvme_mdts >> SECTOR_SHIFT); pcie_device_put(pcie_device); spin_unlock_irqrestore(&ioc->pcie_device_lock, flags); diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index 2e584a8bf66b25..6a05ce195aa05b 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -1638,7 +1638,7 @@ qla2x00_fw_state_show(struct device *dev, struct device_attribute *attr, { scsi_qla_host_t *vha = shost_priv(class_to_shost(dev)); int rval = QLA_FUNCTION_FAILED; - uint16_t state[6]; + uint16_t state[16]; uint32_t pstate; if (IS_QLAFX00(vha->hw)) { @@ -2402,6 +2402,63 @@ qla2x00_dport_diagnostics_show(struct device *dev, vha->dport_data[0], vha->dport_data[1], vha->dport_data[2], vha->dport_data[3]); } + +static ssize_t +qla2x00_mpi_fw_state_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + scsi_qla_host_t *vha = shost_priv(class_to_shost(dev)); + int rval = QLA_FUNCTION_FAILED; + u16 state[16]; + u16 mpi_state; + struct qla_hw_data *ha = vha->hw; + + if (!(IS_QLA27XX(ha) || IS_QLA28XX(ha))) + return scnprintf(buf, PAGE_SIZE, + "MPI state reporting is not supported for this HBA.\n"); + + memset(state, 0, sizeof(state)); + + mutex_lock(&vha->hw->optrom_mutex); + if (qla2x00_chip_is_down(vha)) { + mutex_unlock(&vha->hw->optrom_mutex); + ql_dbg(ql_dbg_user, vha, 0x70df, + "ISP reset is in progress, failing mpi_fw_state.\n"); + return -EBUSY; + } else if (vha->hw->flags.eeh_busy) { + mutex_unlock(&vha->hw->optrom_mutex); + ql_dbg(ql_dbg_user, vha, 0x70ea, + "HBA in PCI error state, failing mpi_fw_state.\n"); + return -EBUSY; + } + + rval = qla2x00_get_firmware_state(vha, state); + mutex_unlock(&vha->hw->optrom_mutex); + if (rval != QLA_SUCCESS) { + ql_dbg(ql_dbg_user, vha, 0x70eb, + "MB Command to retrieve MPI state failed (%d), failing mpi_fw_state.\n", + rval); + return -EIO; + } + + mpi_state = state[11]; + + if (!(mpi_state & BIT_15)) + return scnprintf(buf, PAGE_SIZE, + "MPI firmware state reporting is not supported by this firmware. (0x%02x)\n", + mpi_state); + + if (!(mpi_state & BIT_8)) + return scnprintf(buf, PAGE_SIZE, + "MPI firmware is disabled. (0x%02x)\n", + mpi_state); + + return scnprintf(buf, PAGE_SIZE, + "MPI firmware is enabled, state is %s. (0x%02x)\n", + mpi_state & BIT_9 ? "active" : "inactive", + mpi_state); +} + static DEVICE_ATTR(dport_diagnostics, 0444, qla2x00_dport_diagnostics_show, NULL); @@ -2469,6 +2526,8 @@ static DEVICE_ATTR(port_speed, 0644, qla2x00_port_speed_show, qla2x00_port_speed_store); static DEVICE_ATTR(port_no, 0444, qla2x00_port_no_show, NULL); static DEVICE_ATTR(fw_attr, 0444, qla2x00_fw_attr_show, NULL); +static DEVICE_ATTR(mpi_fw_state, 0444, qla2x00_mpi_fw_state_show, NULL); + static struct attribute *qla2x00_host_attrs[] = { &dev_attr_driver_version.attr.attr, @@ -2517,6 +2576,7 @@ static struct attribute *qla2x00_host_attrs[] = { &dev_attr_qlini_mode.attr, &dev_attr_ql2xiniexchg.attr, &dev_attr_ql2xexchoffld.attr, + &dev_attr_mpi_fw_state.attr, NULL, }; diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index 730c42b1a7b9d7..e746c9274cdeda 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -4914,7 +4914,7 @@ qla2x00_fw_ready(scsi_qla_host_t *vha) unsigned long wtime, mtime, cs84xx_time; uint16_t min_wait; /* Minimum wait time if loop is down */ uint16_t wait_time; /* Wait time if loop is coming ready */ - uint16_t state[6]; + uint16_t state[16]; struct qla_hw_data *ha = vha->hw; if (IS_QLAFX00(vha->hw)) diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c index 0d598be6f3eabc..44e310f1a3708f 100644 --- a/drivers/scsi/qla2xxx/qla_mbx.c +++ b/drivers/scsi/qla2xxx/qla_mbx.c @@ -2268,6 +2268,13 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states) mcp->in_mb = MBX_6|MBX_5|MBX_4|MBX_3|MBX_2|MBX_1|MBX_0; else mcp->in_mb = MBX_1|MBX_0; + + if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) { + mcp->mb[12] = 0; + mcp->out_mb |= MBX_12; + mcp->in_mb |= MBX_12; + } + mcp->tov = MBX_TOV_SECONDS; mcp->flags = 0; rval = qla2x00_mailbox_command(vha, mcp); @@ -2280,6 +2287,8 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states) states[3] = mcp->mb[4]; states[4] = mcp->mb[5]; states[5] = mcp->mb[6]; /* DPORT status */ + if (IS_QLA27XX(ha) || IS_QLA28XX(ha)) + states[11] = mcp->mb[12]; /* MPI state. */ } if (rval != QLA_SUCCESS) { diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index 628a1d0a74bac2..aba22060fcd503 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -4018,6 +4018,7 @@ static int sd_probe(struct scsi_device *sdp) error = device_add(&sdkp->disk_dev); if (error) { put_device(&sdkp->disk_dev); + put_disk(gd); goto out; } diff --git a/drivers/scsi/sg.c b/drivers/scsi/sg.c index 37bac49f30f0db..f38d36fbeef371 100644 --- a/drivers/scsi/sg.c +++ b/drivers/scsi/sg.c @@ -1623,10 +1623,35 @@ sg_remove_device(struct device *cl_dev) } module_param_named(scatter_elem_sz, scatter_elem_sz, int, S_IRUGO | S_IWUSR); -module_param_named(def_reserved_size, def_reserved_size, int, - S_IRUGO | S_IWUSR); module_param_named(allow_dio, sg_allow_dio, int, S_IRUGO | S_IWUSR); +static int def_reserved_size_set(const char *val, const struct kernel_param *kp) +{ + int size, ret; + + if (!val) + return -EINVAL; + + ret = kstrtoint(val, 0, &size); + if (ret) + return ret; + + /* limit to 1 MB */ + if (size < 0 || size > 1048576) + return -ERANGE; + + def_reserved_size = size; + return 0; +} + +static const struct kernel_param_ops def_reserved_size_ops = { + .set = def_reserved_size_set, + .get = param_get_int, +}; + +module_param_cb(def_reserved_size, &def_reserved_size_ops, &def_reserved_size, + S_IRUGO | S_IWUSR); + MODULE_AUTHOR("Douglas Gilbert"); MODULE_DESCRIPTION("SCSI generic (sg) driver"); MODULE_LICENSE("GPL"); @@ -1691,13 +1716,13 @@ init_sg(void) sg_sysfs_valid = 1; rc = scsi_register_interface(&sg_interface); if (0 == rc) { + register_sg_sysctls(); #ifdef CONFIG_SCSI_PROC_FS sg_proc_init(); #endif /* CONFIG_SCSI_PROC_FS */ return 0; } class_unregister(&sg_sysfs_class); - register_sg_sysctls(); err_out: unregister_chrdev_region(MKDEV(SCSI_GENERIC_MAJOR, 0), SG_MAX_DEVS); return rc; diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 7adb2573f50d42..c36c54ecd354b4 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -395,7 +395,7 @@ static blk_status_t sr_init_command(struct scsi_cmnd *SCpnt) switch (req_op(rq)) { case REQ_OP_WRITE: - if (!cd->writeable) + if (get_disk_ro(cd->disk)) goto out; SCpnt->cmnd[0] = WRITE_10; cd->cdi.media_written = 1; @@ -681,6 +681,7 @@ static int sr_probe(struct scsi_device *sdev) error = -ENOMEM; if (get_capabilities(cd)) goto fail_minor; + cdrom_probe_write_features(&cd->cdi); sr_vendor_init(cd); set_capacity(disk, cd->capacity); @@ -899,14 +900,6 @@ static int get_capabilities(struct scsi_cd *cd) /*else I don't think it can close its tray cd->cdi.mask |= CDC_CLOSE_TRAY; */ - /* - * if DVD-RAM, MRW-W or CD-RW, we are randomly writable - */ - if ((cd->cdi.mask & (CDC_DVD_RAM | CDC_MRW_W | CDC_RAM | CDC_CD_RW)) != - (CDC_DVD_RAM | CDC_MRW_W | CDC_RAM | CDC_CD_RW)) { - cd->writeable = 1; - } - kfree(buffer); return 0; } diff --git a/drivers/scsi/sr.h b/drivers/scsi/sr.h index dc899277b3a441..2d92f9cb6fec7c 100644 --- a/drivers/scsi/sr.h +++ b/drivers/scsi/sr.h @@ -35,7 +35,6 @@ typedef struct scsi_cd { struct scsi_device *device; unsigned int vendor; /* vendor code, see sr_vendor.c */ unsigned long ms_offset; /* for reading multisession-CD's */ - unsigned writeable : 1; unsigned use:1; /* is this device still supportable */ unsigned xa_flag:1; /* CD has XA sectors ? */ unsigned readcd_known:1; /* drive supports READ_CD (0xbe) */ diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index d0ff32182a2b4a..41bd8fbc87aab3 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -4,6 +4,15 @@ if ARCH_APPLE || COMPILE_TEST menu "Apple SoC drivers" +config APPLE_DOCKCHANNEL + tristate "Apple DockChannel FIFO" + depends on ARCH_APPLE || COMPILE_TEST + help + DockChannel is a simple FIFO used on Apple SoCs for debug and inter-processor + communications. + + Say 'y' here if you have an Apple SoC. + config APPLE_MAILBOX tristate "Apple SoC mailboxes" depends on PM @@ -16,6 +25,14 @@ config APPLE_MAILBOX Say Y here if you have an Apple SoC. +config APPLE_PMGR_MISC + bool "Apple SoC PMGR miscellaneous support" + depends on PM + help + The PMGR block in Apple SoCs provides high-level power state + controls for SoC devices. This driver manages miscellaneous + power controls. + config APPLE_RTKIT tristate "Apple RTKit co-processor IPC protocol" depends on APPLE_MAILBOX @@ -28,6 +45,19 @@ config APPLE_RTKIT Say 'y' here if you have an Apple SoC. +config APPLE_RTKIT_HELPER + tristate "Apple Generic RTKit helper co-processor" + depends on APPLE_RTKIT + depends on ARCH_APPLE || COMPILE_TEST + help + Apple SoCs such as the M1 come with various co-processors running + their proprietary RTKit operating system. This option enables support + for a generic co-processor that does not implement any additional + in-band communications. It can be used for testing purposes, or for + coprocessors such as MTP that communicate over a different interface. + + Say 'y' here if you have an Apple SoC. + config APPLE_SART tristate "Apple SART DMA address filter" depends on ARCH_APPLE || COMPILE_TEST @@ -42,6 +72,56 @@ config APPLE_TUNABLE tristate depends on ARCH_APPLE || COMPILE_TEST +config RUST_APPLE_MAILBOX + bool + depends on PM + depends on RUST + select APPLE_MAILBOX + +config RUST_APPLE_RTKIT + bool + depends on PM + depends on RUST + select APPLE_RTKIT + +config APPLE_AOP + tristate "Apple \"Always-on\" Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on PM + depends on RUST + select RUST_APPLE_RTKIT + help + A co-processor persent on certain Apple SoCs controlling accelerometers, + gyros, ambient light sensors and microphones. Is not actually always on. + + Say 'y' here if you have an Apple laptop. + +config APPLE_SEP + tristate "Apple Secure Element Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on PM + depends on RUST + select RUST_APPLE_RTKIT + select RUST_APPLE_MAILBOX + help + A security co-processor persent on Apple SoCs, controlling transparent + disk encryption, secure boot, HDCP, biometric auth and probably more. + + Say 'y' here if you have an Apple SoC. + +config APPLE_PMP + tristate "Apple Power Management Processor" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select APPLE_PMP_REPORT + select RUST_APPLE_RTKIT + default y if ARCH_APPLE + help + A co-processor present on Apple SoCs, controlling power states of + fabric and other uncore components. + + Say 'y' here if you have an Apple SoC. + endmenu endif diff --git a/drivers/soc/apple/Makefile b/drivers/soc/apple/Makefile index 0b85ab61aefe13..1baa6de9449589 100644 --- a/drivers/soc/apple/Makefile +++ b/drivers/soc/apple/Makefile @@ -1,13 +1,27 @@ # SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_DOCKCHANNEL) += apple-dockchannel.o +apple-dockchannel-y = dockchannel.o + obj-$(CONFIG_APPLE_MAILBOX) += apple-mailbox.o apple-mailbox-y = mailbox.o +obj-$(CONFIG_APPLE_PMGR_MISC) += apple-pmgr-misc.o + obj-$(CONFIG_APPLE_RTKIT) += apple-rtkit.o apple-rtkit-y = rtkit.o rtkit-crashlog.o +obj-$(CONFIG_APPLE_RTKIT_HELPER) += apple-rtkit-helper.o +apple-rtkit-helper-y = rtkit-helper.o + obj-$(CONFIG_APPLE_SART) += apple-sart.o apple-sart-y = sart.o obj-$(CONFIG_APPLE_TUNABLE) += apple-tunable.o apple-tunable-y = tunable.o + +obj-$(CONFIG_APPLE_AOP) += aop.o + +obj-$(CONFIG_APPLE_SEP) += sep.o + +obj-$(CONFIG_APPLE_PMP) += pmp.o diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs new file mode 100644 index 00000000000000..d846d6ed8f4bcf --- /dev/null +++ b/drivers/soc/apple/aop.rs @@ -0,0 +1,1001 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple AOP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::{arch::asm, cmp, mem, ptr, slice}; + +use kernel::{ + bindings, c_str, device, + device::Core, + dma::{CoherentAllocation, Device, DmaMask}, + error::from_err_ptr, + io::{ + mem::IoMem, + Io, // + }, + iosys_map::IoSysMapRef, + module_platform_driver, new_condvar, new_mutex, of, platform, + prelude::*, + soc::apple::aop::{from_fourcc, EPICService, FakehidListener, AOP}, + soc::apple::rtkit, + sync::{Arc, ArcBorrow, CondVar, Mutex}, + types::{ARef, ForeignOwnable}, + workqueue::{self, impl_has_work, new_work, Work, WorkItem}, +}; + +const AOP_MAX_CALLS: usize = 8; +const AOP_MMIO_SIZE: usize = 0x1e0000; +const ASC_MMIO_SIZE: usize = 0x4000; +const BOOTARGS_OFFSET: usize = 0x22c; +const BOOTARGS_SIZE: usize = 0x230; +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; +const AFK_ENDPOINT_START: u8 = 0x20; +const AFK_ENDPOINT_COUNT: u8 = 0xf; +const AFK_OPC_GET_BUF: u64 = 0x89; +const AFK_OPC_INIT: u64 = 0x80; +const AFK_OPC_INIT_RX: u64 = 0x8b; +const AFK_OPC_INIT_TX: u64 = 0x8a; +const AFK_OPC_INIT_UNK: u64 = 0x8c; +const AFK_OPC_SEND: u64 = 0xa2; +const AFK_OPC_START_ACK: u64 = 0x86; +const AFK_OPC_SHUTDOWN_ACK: u64 = 0xc1; +const AFK_OPC_RECV: u64 = 0x85; +const AFK_MSG_GET_BUF_ACK: u64 = 0xa1 << 48; +const AFK_MSG_INIT: u64 = AFK_OPC_INIT << 48; +const AFK_MSG_INIT_ACK: u64 = 0xa0 << 48; +const AFK_MSG_START: u64 = 0xa3 << 48; +const AFK_MSG_SHUTDOWN: u64 = 0xc0 << 48; +const AFK_RB_BLOCK_STEP: usize = 0x40; +const EPIC_TYPE_NOTIFY: u32 = 0; +const EPIC_CATEGORY_REPORT: u8 = 0x00; +const EPIC_CATEGORY_NOTIFY: u8 = 0x10; +const EPIC_CATEGORY_REPLY: u8 = 0x20; +const EPIC_SUBTYPE_STD_SERVICE: u16 = 0xc0; +const EPIC_SUBTYPE_FAKEHID_REPORT: u16 = 0xc4; +const EPIC_SUBTYPE_RETCODE: u16 = 0x84; +const EPIC_SUBTYPE_RETCODE_PAYLOAD: u16 = 0xa0; +const EPIC_SUBTYPE_STRING: u16 = 0x8a; +const QE_MAGIC1: u32 = from_fourcc(b" POI"); +const QE_MAGIC2: u32 = from_fourcc(b" POA"); + +fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +#[inline(always)] +fn mem_sync() { + unsafe { + asm!("dsb sy"); + } +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct QEHeader { + magic: u32, + size: u32, + channel: u32, + ty: u32, +} + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct EPICHeader { + version: u8, + seq: u16, + _pad0: u8, + _unk0: u32, + timestamp: u64, + // Subheader + length: u32, + sub_version: u8, + category: u8, + subtype: u16, + tag: u16, + _unk1: u16, + _pad1: u64, + inline_len: u32, +} + +#[repr(C, packed)] +struct EPICServiceAnnounce { + name: [u8; 20], + _unk0: u32, + retcode: u32, + _unk1: u32, + channel: u32, + _unk2: u32, + _unk3: u32, +} + +#[pin_data] +struct FutureValue { + #[pin] + val: Mutex>, + #[pin] + completion: CondVar, +} + +impl FutureValue { + fn pin_init() -> impl PinInit> { + pin_init!( + FutureValue { + val <- new_mutex!(None), + completion <- new_condvar!() + } + ) + } + fn complete(&self, val: T) { + *self.val.lock() = Some(val); + self.completion.notify_all(); + } + fn wait(&self) -> T { + let mut ret_guard = self.val.lock(); + while ret_guard.is_none() { + self.completion.wait(&mut ret_guard); + } + ret_guard.take().unwrap() + } + fn reset(&self) { + *self.val.lock() = None; + } +} + +struct AFKRingBuffer { + offset: usize, + block_size: usize, + buf_size: usize, +} + +struct CallResult { + retcode: u32, + extra_data: Option>, +} + +struct AFKEndpoint { + index: u8, + iomem: Option>, + txbuf: Option, + rxbuf: Option, + seq: u16, + calls: [Option>>; AOP_MAX_CALLS], + call_returns: [Option>; AOP_MAX_CALLS], +} + +unsafe impl Send for AFKEndpoint {} + +impl AFKEndpoint { + fn new(index: u8) -> AFKEndpoint { + AFKEndpoint { + index, + iomem: None, + txbuf: None, + rxbuf: None, + seq: 0, + calls: [const { None }; AOP_MAX_CALLS], + call_returns: [const { None }; AOP_MAX_CALLS], + } + } + + fn start(&self, rtkit: Pin<&mut rtkit::RtKit>) -> Result<()> { + rtkit.send_message(self.index, AFK_MSG_INIT) + } + + fn stop(&self, rtkit: Pin<&mut rtkit::RtKit>) -> Result<()> { + rtkit.send_message(self.index, AFK_MSG_SHUTDOWN) + } + + fn recv_message( + &mut self, + client: ArcBorrow<'_, AopData>, + rtkit: Pin<&mut rtkit::RtKit>, + msg: u64, + ) -> Result<()> { + let opc = msg >> 48; + match opc { + AFK_OPC_INIT => { + rtkit.send_message(self.index, AFK_MSG_INIT_ACK)?; + } + AFK_OPC_GET_BUF => { + self.recv_get_buf(client.dev.clone(), rtkit, msg)?; + } + AFK_OPC_INIT_UNK => {} // no-op + AFK_OPC_START_ACK => {} + AFK_OPC_INIT_RX => { + if self.rxbuf.is_some() { + dev_err!( + client.dev, + "Got InitRX message with existing rxbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + self.rxbuf = Some(self.parse_ring_buf(msg)?); + if self.txbuf.is_some() { + rtkit.send_message(self.index, AFK_MSG_START)?; + } + } + AFK_OPC_INIT_TX => { + if self.txbuf.is_some() { + dev_err!( + client.dev, + "Got InitTX message with existing txbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + self.txbuf = Some(self.parse_ring_buf(msg)?); + if self.rxbuf.is_some() { + rtkit.send_message(self.index, AFK_MSG_START)?; + } + } + AFK_OPC_RECV => { + self.recv_rb(client)?; + } + AFK_OPC_SHUTDOWN_ACK => { + client.shutdown_complete(); + } + _ => dev_err!( + client.dev, + "AFK endpoint {} got unknown message {}", + self.index, + msg + ), + } + Ok(()) + } + + fn parse_ring_buf(&self, msg: u64) -> Result { + let msg = msg as usize; + let size = ((msg >> 16) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let offset = ((msg >> 32) & 0xFFFF) * AFK_RB_BLOCK_STEP; + let buf_size = self.iomem_read32(offset)? as usize; + let block_size = (size - buf_size) / 3; + Ok(AFKRingBuffer { + offset, + block_size, + buf_size, + }) + } + fn iomem_write32(&mut self, off: usize, data: u32) -> Result<()> { + let size = core::mem::size_of::(); + let data = data.to_le_bytes(); + let buf = unsafe { self.iomem.as_mut().ok_or(ENXIO)?.as_slice_mut(off, size)? }; + buf.copy_from_slice(&data); + Ok(()) + } + + fn iomem_read32(&self, off: usize) -> Result { + let size = core::mem::size_of::(); + let buf = unsafe { self.iomem.as_ref().ok_or(ENXIO)?.as_slice(off, size)? }; + Ok(u32::from_le_bytes(buf.try_into().unwrap())) + } + + fn memcpy_from_iomem(&self, off: usize, target: &mut [u8]) -> Result<()> { + // SAFETY: + // as_slice() checks that off and target.len() are whithin iomem's limits. + unsafe { + let src = self + .iomem + .as_ref() + .ok_or(ENXIO)? + .as_slice(off, target.len())?; + target.copy_from_slice(src); + } + Ok(()) + } + + fn memcpy_to_iomem(&mut self, off: usize, src: &[u8]) -> Result<()> { + // SAFETY: + // as_slice_mut() checks that off and src.len() are whithin iomem's limits. + unsafe { + let target = self + .iomem + .as_mut() + .ok_or(ENXIO)? + .as_slice_mut(off, src.len())?; + target.copy_from_slice(src); + } + Ok(()) + } + + fn recv_get_buf( + &mut self, + dev: ARef, + rtkit: Pin<&mut rtkit::RtKit>, + msg: u64, + ) -> Result<()> { + let size = ((msg & 0xFFFF0000) >> 16) as usize * AFK_RB_BLOCK_STEP; + if self.iomem.is_some() { + dev_err!( + dev, + "Got GetBuf message with existing buffer on endpoint {}", + self.index + ); + return Err(EIO); + } + let iomem = dev.while_bound_with(|bound_dev| { + CoherentAllocation::::alloc_coherent(bound_dev, size, GFP_KERNEL) + })?; + rtkit.send_message(self.index, AFK_MSG_GET_BUF_ACK | iomem.dma_handle())?; + self.iomem = Some(iomem); + Ok(()) + } + + fn recv_rb(&mut self, client: ArcBorrow<'_, AopData>) -> Result<()> { + let (buf_offset, block_size, buf_size) = match self.rxbuf.as_ref() { + Some(b) => (b.offset, b.block_size, b.buf_size), + None => { + dev_err!( + client.dev, + "Got Recv message with no rxbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + }; + let mut rptr = self.iomem_read32(buf_offset + block_size)? as usize; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)?; + mem_sync(); + let base = buf_offset + block_size * 3; + let mut msg_buf = KVec::new(); + const QEH_SIZE: usize = mem::size_of::(); + while wptr as usize != rptr { + let mut qeh_bytes = [0; QEH_SIZE]; + self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; + let mut qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { + let magic = qeh.magic; + dev_err!( + client.dev, + "Invalid magic on ep {}, got {:x}", + self.index, + magic + ); + return Err(EIO); + } + if qeh.size as usize > (buf_size - rptr - QEH_SIZE) { + rptr = 0; + self.memcpy_from_iomem(base + rptr, &mut qeh_bytes)?; + qeh = unsafe { &*(qeh_bytes.as_ptr() as *const QEHeader) }; + + if qeh.magic != QE_MAGIC1 && qeh.magic != QE_MAGIC2 { + let magic = qeh.magic; + dev_err!( + client.dev, + "Invalid magic on ep {}, got {:x}", + self.index, + magic + ); + return Err(EIO); + } + } + msg_buf.resize(qeh.size as usize, 0, GFP_KERNEL)?; + self.memcpy_from_iomem(base + rptr + QEH_SIZE, &mut msg_buf)?; + let (hdr_bytes, msg) = msg_buf.split_at(mem::size_of::()); + let header = unsafe { &*(hdr_bytes.as_ptr() as *const EPICHeader) }; + self.handle_ipc(client, qeh, header, msg)?; + rptr = align_up(rptr + QEH_SIZE + qeh.size as usize, block_size) % buf_size; + mem_sync(); + self.iomem_write32(buf_offset + block_size, rptr as u32)?; + wptr = self.iomem_read32(buf_offset + block_size * 2)?; + mem_sync(); + } + Ok(()) + } + fn handle_ipc( + &mut self, + client: ArcBorrow<'_, AopData>, + qhdr: &QEHeader, + ehdr: &EPICHeader, + data: &[u8], + ) -> Result<()> { + let subtype = ehdr.subtype; + if ehdr.category == EPIC_CATEGORY_REPORT { + if subtype == EPIC_SUBTYPE_STD_SERVICE { + let announce = unsafe { &*(data.as_ptr() as *const EPICServiceAnnounce) }; + let chan = announce.channel; + let name_len = announce + .name + .iter() + .position(|x| *x == 0) + .unwrap_or(announce.name.len()); + return Into::>::into(client).register_service( + self, + chan, + &announce.name[..name_len], + ); + } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { + return client.process_fakehid_report(self, qhdr.channel, data); + } else { + dev_err!( + client.dev, + "Unexpected EPIC report subtype {:x} on endpoint {}", + subtype, + self.index + ); + return Err(EIO); + } + } else if ehdr.category == EPIC_CATEGORY_REPLY { + if subtype == EPIC_SUBTYPE_RETCODE_PAYLOAD + || subtype == EPIC_SUBTYPE_RETCODE + || subtype == EPIC_SUBTYPE_STRING + { + if data.len() < mem::size_of::() { + dev_err!( + client.dev, + "Retcode data too short on endpoint {}", + self.index + ); + return Err(EIO); + } + let retcode = u32::from_ne_bytes(data[..4].try_into().unwrap()); + let tag = ehdr.tag as usize; + if tag == 0 || tag - 1 > self.calls.len() || self.calls[tag - 1].is_none() { + dev_err!( + client.dev, + "Got a retcode with invalid tag {:?} on endpoint {}", + tag, + self.index + ); + return Err(EIO); + } + let future = self.calls[tag - 1].take().unwrap(); + let extra_data = if let Some(mut ret) = self.call_returns[tag - 1].take() { + let len = cmp::min(data.len() - 4, ret.len()); + ret[..len].copy_from_slice(&data[4..(len + 4)]); + ret.truncate(len); + Some(ret) + } else { + None + }; + future.complete(CallResult { + retcode, + extra_data, + }); + + return Ok(()); + } else { + dev_err!( + client.dev, + "Unexpected EPIC reply subtype {:x} on endpoint {}", + subtype, + self.index + ); + return Err(EIO); + } + } + dev_err!( + client.dev, + "Unexpected EPIC category {:x} on endpoint {}", + ehdr.category, + self.index + ); + Err(EIO) + } + fn send_rb( + &mut self, + client: &AopData, + rtkit: Pin<&mut rtkit::RtKit>, + channel: u32, + ty: u32, + header: &[u8], + data: &[u8], + ) -> Result<()> { + let (buf_offset, block_size, buf_size) = match self.txbuf.as_ref() { + Some(b) => (b.offset, b.block_size, b.buf_size), + None => { + dev_err!( + client.dev, + "Attempting to send message with no txbuf at endpoint {}", + self.index + ); + return Err(EIO); + } + }; + let base = buf_offset + block_size * 3; + mem_sync(); + let rptr = self.iomem_read32(buf_offset + block_size)? as usize; + let mut wptr = self.iomem_read32(buf_offset + block_size * 2)? as usize; + const QEH_SIZE: usize = mem::size_of::(); + if wptr < rptr && wptr + QEH_SIZE >= rptr { + dev_err!(client.dev, "Tx buffer full at endpoint {}", self.index); + return Err(EIO); + } + let payload_len = header.len() + data.len(); + let qeh = QEHeader { + magic: QE_MAGIC1, + size: payload_len as u32, + channel, + ty, + }; + let qeh_bytes = unsafe { + slice::from_raw_parts( + &qeh as *const QEHeader as *const u8, + mem::size_of::(), + ) + }; + self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + if payload_len > buf_size - wptr - QEH_SIZE { + wptr = 0; + self.memcpy_to_iomem(base + wptr, qeh_bytes)?; + } + self.memcpy_to_iomem(base + wptr + QEH_SIZE, header)?; + self.memcpy_to_iomem(base + wptr + QEH_SIZE + header.len(), data)?; + wptr = align_up(wptr + QEH_SIZE + payload_len, block_size) % buf_size; + self.iomem_write32(buf_offset + block_size * 2, wptr as u32)?; + let msg = wptr as u64 | (AFK_OPC_SEND << 48); + rtkit.send_message(self.index, msg) + } + fn epic_notify( + &mut self, + client: &AopData, + rtkit: Pin<&mut rtkit::RtKit>, + channel: u32, + subtype: u16, + data: &[u8], + ret: Option>, + ) -> Result>> { + let mut tag = 0; + for i in 0..self.calls.len() { + if self.calls[i].is_none() { + tag = i + 1; + break; + } + } + if tag == 0 { + dev_err!( + client.dev, + "Too many inflight calls on endpoint {}", + self.index + ); + return Err(EIO); + } + let call = Arc::pin_init(FutureValue::pin_init(), GFP_KERNEL)?; + let hdr = EPICHeader { + version: 2, + seq: self.seq, + length: data.len() as u32, + sub_version: 2, + category: EPIC_CATEGORY_NOTIFY, + subtype, + tag: tag as u16, + ..EPICHeader::default() + }; + self.call_returns[tag - 1] = ret; + self.send_rb( + client, + rtkit, + channel, + EPIC_TYPE_NOTIFY, + unsafe { + slice::from_raw_parts( + &hdr as *const EPICHeader as *const u8, + mem::size_of::(), + ) + }, + data, + )?; + self.seq = self.seq.wrapping_add(1); + self.calls[tag - 1] = Some(call.clone()); + Ok(call) + } +} + +struct ListenerEntry { + svc: EPICService, + listener: Arc, +} + +unsafe impl Send for ListenerEntry {} + +#[pin_data] +struct AopData { + dev: ARef, + #[pin] + rtkit: Mutex>>, + #[pin] + endpoints: [Mutex; AFK_ENDPOINT_COUNT as usize], + #[pin] + ep_shutdown: FutureValue<()>, + #[pin] + hid_listeners: Mutex>, + #[pin] + subdevices: Mutex>, +} + +unsafe impl Send for AopData {} +unsafe impl Sync for AopData {} + +#[pin_data] +struct AopServiceRegisterWork { + name: &'static CStr, + data: Arc, + service: EPICService, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for AopServiceRegisterWork { self.work } +} + +impl AopServiceRegisterWork { + fn new( + name: &'static CStr, + data: Arc, + service: EPICService, + ) -> Result>> { + KBox::pin_init( + pin_init!(AopServiceRegisterWork { + name, data, service, + work <- new_work!("AopServiceRegisterWork::work"), + }), + GFP_KERNEL, + ) + } +} + +impl WorkItem for AopServiceRegisterWork { + type Pointer = Pin>; + + fn run(this: Pin>) { + let fwnode = this + .data + .dev + .fwnode() + .and_then(|x| x.get_child_by_name(this.name)); + let info = bindings::platform_device_info { + parent: this.data.dev.as_raw(), + name: this.name.as_ptr() as *const _, + id: bindings::PLATFORM_DEVID_AUTO, + res: ptr::null_mut(), + num_res: 0, + data: &this.service as *const EPICService as *const _, + size_data: mem::size_of::(), + dma_mask: 0, + fwnode: fwnode.map(|x| x.as_raw()).unwrap_or(ptr::null_mut()), + properties: ptr::null_mut(), + of_node_reused: false, + }; + let pdev = unsafe { from_err_ptr(bindings::platform_device_register_full(&info)) }; + match pdev { + Err(e) => { + dev_err!( + this.data.dev, + "Failed to create device for service {:?}: {:?}", + this.name, + e + ); + } + Ok(pdev) => { + let res = this.data.subdevices.lock().push(pdev, GFP_KERNEL); + if res.is_err() { + dev_err!(this.data.dev, "Failed to store subdevice"); + } + } + } + } +} + +impl AopData { + fn new(dev: &platform::Device) -> Result> { + Arc::pin_init( + pin_init!( + AopData { + dev: dev.as_ref().into(), + rtkit <- new_mutex!(None), + endpoints <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(AFKEndpoint::new(AFK_ENDPOINT_START + i as u8)) + }), + ep_shutdown <- FutureValue::pin_init(), + hid_listeners <- new_mutex!(KVec::new()), + subdevices <- new_mutex!(KVec::new()), + } + ), + GFP_KERNEL, + ) + } + fn start(&self) -> Result<()> { + { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.as_mut().wake()?; + } + for ep in 0..AFK_ENDPOINT_COUNT { + let rtk_ep_num = AFK_ENDPOINT_START + ep; + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + if !rtk.as_mut().has_endpoint(rtk_ep_num) { + continue; + } + rtk.as_mut().start_endpoint(rtk_ep_num)?; + let ep_guard = self.endpoints[ep as usize].lock(); + ep_guard.start(rtk.as_mut())?; + } + Ok(()) + } + fn register_service( + self: Arc, + ep: &mut AFKEndpoint, + channel: u32, + name: &[u8], + ) -> Result<()> { + let svc = EPICService { + channel, + endpoint: ep.index, + }; + let dev_name = match name { + b"aop-audio" => c_str!("audio"), + b"las" => c_str!("las"), + b"als" => c_str!("als"), + _ => { + return Ok(()); + } + }; + // probe can call back into us, run it with locks dropped. + let work = AopServiceRegisterWork::new(dev_name, self, svc)?; + workqueue::system().enqueue(work); + Ok(()) + } + + fn process_fakehid_report(&self, ep: &AFKEndpoint, ch: u32, data: &[u8]) -> Result<()> { + let guard = self.hid_listeners.lock(); + for entry in &*guard { + if entry.svc.endpoint == ep.index && entry.svc.channel == ch { + return entry.listener.process_fakehid_report(data); + } + } + Ok(()) + } + + fn shutdown_complete(&self) { + self.ep_shutdown.complete(()); + } + + fn stop(&self) -> Result<()> { + for ep in 0..AFK_ENDPOINT_COUNT { + { + let rtk_ep_num = AFK_ENDPOINT_START + ep; + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + if !rtk.as_mut().has_endpoint(rtk_ep_num) { + continue; + } + let ep_guard = self.endpoints[ep as usize].lock(); + ep_guard.stop(rtk.as_mut())?; + } + self.ep_shutdown.wait(); + self.ep_shutdown.reset(); + } + Ok(()) + } + + fn patch_bootargs( + &self, + aop_mmio: &IoMem, + patches: &[(u32, u64)], + ) -> Result<()> { + let offset = aop_mmio.read32_relaxed(BOOTARGS_OFFSET) as usize; + let size = aop_mmio.read32_relaxed(BOOTARGS_SIZE) as usize; + let mut arg_bytes = KVec::::from_elem(0, size, GFP_KERNEL)?; + aop_mmio.try_memcpy_fromio(&mut arg_bytes, offset)?; + let mut idx = 0; + while idx < size { + let key = u32::from_le_bytes(arg_bytes[idx..idx + 4].try_into().unwrap()); + let size = u32::from_le_bytes(arg_bytes[idx + 4..idx + 8].try_into().unwrap()) as usize; + idx += 8; + for (k, v) in patches.iter() { + if *k != key { + continue; + } + arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + break; + } + idx += size; + } + aop_mmio.try_memcpy_toio(offset, &arg_bytes) + } + + fn start_cpu(&self, asc_mmio: &IoMem) -> Result<()> { + let val = asc_mmio.read32_relaxed(CPU_CONTROL); + asc_mmio.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } +} + +impl AOP for AopData { + fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result { + let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let call = { + let mut rtk_guard = self.rtkit.lock(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + ep_guard.epic_notify(self, rtk.as_mut(), svc.channel, subtype, msg_bytes, None)? + }; + Ok(call.wait().retcode) + } + fn epic_call_ret( + &self, + svc: &EPICService, + subtype: u16, + msg_bytes: &[u8], + ret_len: usize, + ) -> Result<(u32, KVec)> { + let ep_idx = svc.endpoint - AFK_ENDPOINT_START; + let call = { + let mut rtk_guard = self.rtkit.lock(); + let mut rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = self.endpoints[ep_idx as usize].lock(); + let mut ret_buf = KVec::new(); + ret_buf.resize(ret_len, 0, GFP_KERNEL)?; + ep_guard.epic_notify( + self, + rtk.as_mut(), + svc.channel, + subtype, + msg_bytes, + Some(ret_buf), + )? + }; + let res = call.wait(); + Ok((res.retcode, res.extra_data.unwrap())) + } + fn add_fakehid_listener( + &self, + svc: EPICService, + listener: Arc, + ) -> Result<()> { + let mut guard = self.hid_listeners.lock(); + Ok(guard.push(ListenerEntry { svc, listener }, GFP_KERNEL)?) + } + fn remove_fakehid_listener(&self, svc: &EPICService) -> bool { + let mut guard = self.hid_listeners.lock(); + for i in 0..guard.len() { + if guard[i].svc == *svc { + guard.swap_remove(i); + return true; + } + } + false + } + fn remove(&self) { + if let Err(e) = self.stop() { + dev_err!(self.dev, "Failed to stop AOP {:?}", e); + } + *self.rtkit.lock() = None; + let guard = self.subdevices.lock(); + for pdev in &*guard { + unsafe { + bindings::platform_device_unregister(*pdev); + } + } + } +} + +struct NoBuffer; +impl rtkit::Buffer for NoBuffer { + fn iova(&self) -> Result { + unreachable!() + } + fn buf(&mut self) -> Result> { + unreachable!() + } +} + +#[vtable] +impl rtkit::Operations for AopData { + type Data = Arc; + type Buffer = NoBuffer; + + fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { + let mut guard = data.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + let mut ep_guard = data.endpoints[(ep - AFK_ENDPOINT_START) as usize].lock(); + let ret = ep_guard.recv_message(data, rtk.as_mut(), msg); + if let Err(e) = ret { + dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); + } + } + + fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + dev_err!(data.dev, "AOP firmware crashed"); + } +} + +#[repr(transparent)] +struct AopDriver(Arc); + +struct AopHwConfig { + ec0p: u64, + alig: u64, + aopt: u64, +} + +const HW_CFG_T8103: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 1, + alig: 128, +}; +const HW_CFG_T8112: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 0, + alig: 128, +}; +const HW_CFG_T6000: AopHwConfig = AopHwConfig { + ec0p: 0x020000, + aopt: 0, + alig: 64, +}; +const HW_CFG_T6020: AopHwConfig = AopHwConfig { + ec0p: 0x0100_00000000, + aopt: 0, + alig: 64, +}; + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + ::IdInfo, + [ + (of::DeviceId::new(c_str!("apple,t8103-aop")), &HW_CFG_T8103), + (of::DeviceId::new(c_str!("apple,t8112-aop")), &HW_CFG_T8112), + (of::DeviceId::new(c_str!("apple,t6000-aop")), &HW_CFG_T6000), + (of::DeviceId::new(c_str!("apple,t6020-aop")), &HW_CFG_T6020), + ] +); + +impl platform::Driver for AopDriver { + type IdInfo = &'static AopHwConfig; + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + info: Option<&Self::IdInfo>, + ) -> impl PinInit { + let cfg = info.ok_or(ENODEV)?; + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; + let aop_req = pdev.io_request_by_index(0).ok_or(EINVAL)?; + let aop_mmio = KBox::pin_init(aop_req.iomap_sized::(), GFP_KERNEL)?; + let asc_req = pdev.io_request_by_index(1).ok_or(EINVAL)?; + let asc_mmio = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + let data = AopData::new(pdev)?; + let aop_mmio = aop_mmio.access(pdev.as_ref())?; + data.patch_bootargs( + aop_mmio, + &[ + (from_fourcc(b"EC0p"), cfg.ec0p), + (from_fourcc(b"nCal"), 0x0), + (from_fourcc(b"alig"), cfg.alig), + (from_fourcc(b"AOPt"), cfg.aopt), + ], + )?; + let rtkit = rtkit::RtKit::::new(pdev.as_ref(), None, 0, data.clone())?; + *data.rtkit.lock() = Some(rtkit); + let asc_mmio = asc_mmio.access(pdev.as_ref())?; + let _ = data.start_cpu(asc_mmio); + data.start()?; + let data = data as Arc; + Ok(Self(data)) + } +} + +impl Drop for AopDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + +unsafe impl Send for AopDriver {} + +module_platform_driver! { + type: AopDriver, + name: "apple_aop", + description: "AOP driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/apple-pmgr-misc.c b/drivers/soc/apple/apple-pmgr-misc.c new file mode 100644 index 00000000000000..e768f34aacc586 --- /dev/null +++ b/drivers/soc/apple/apple-pmgr-misc.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple SoC PMGR device power state driver + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_CLKGEN_PSTATE 0 +#define APPLE_CLKGEN_PSTATE_DESIRED GENMASK(3, 0) + +#define SYS_DEV_PSTATE_SUSPEND 1 + +enum sys_device { + DEV_FABRIC, + DEV_DCS, + DEV_MAX, +}; + +struct apple_pmgr_sys_device { + void __iomem *base; + u32 active_state; + u32 suspend_state; +}; + +struct apple_pmgr_misc { + struct device *dev; + struct apple_pmgr_sys_device devices[DEV_MAX]; +}; + +static void apple_pmgr_sys_dev_set_pstate(struct apple_pmgr_misc *misc, + enum sys_device dev, bool active) +{ + u32 pstate; + u32 val; + + if (!misc->devices[dev].base) + return; + + if (active) + pstate = misc->devices[dev].active_state; + else + pstate = misc->devices[dev].suspend_state; + + printk("set %d ps to pstate %d\n", dev, pstate); + + val = readl_relaxed(misc->devices[dev].base + APPLE_CLKGEN_PSTATE); + val &= ~APPLE_CLKGEN_PSTATE_DESIRED; + val |= FIELD_PREP(APPLE_CLKGEN_PSTATE_DESIRED, pstate); + writel_relaxed(val, misc->devices[dev].base); +} + +static int __maybe_unused apple_pmgr_misc_suspend_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, false); + + return 0; +} + +static int __maybe_unused apple_pmgr_misc_resume_noirq(struct device *dev) +{ + struct apple_pmgr_misc *misc = dev_get_drvdata(dev); + int i; + + for (i = 0; i < DEV_MAX; i++) + apple_pmgr_sys_dev_set_pstate(misc, i, true); + + return 0; +} + +static bool apple_pmgr_init_device(struct apple_pmgr_misc *misc, + enum sys_device dev, const char *device_name) +{ + void __iomem *base; + char name[32]; + u32 val; + + snprintf(name, sizeof(name), "%s-ps", device_name); + + base = devm_platform_ioremap_resource_byname( + to_platform_device(misc->dev), name); + if (!base) + return false; + + val = readl_relaxed(base + APPLE_CLKGEN_PSTATE); + + misc->devices[dev].base = base; + misc->devices[dev].active_state = + FIELD_GET(APPLE_CLKGEN_PSTATE_DESIRED, val); + misc->devices[dev].suspend_state = SYS_DEV_PSTATE_SUSPEND; + + snprintf(name, sizeof(name), "apple,%s-min-ps", device_name); + of_property_read_u32(misc->dev->of_node, name, + &misc->devices[dev].suspend_state); + + return true; +} + +static int apple_pmgr_misc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_pmgr_misc *misc; + int ret = -ENODEV; + + misc = devm_kzalloc(dev, sizeof(*misc), GFP_KERNEL); + if (!misc) + return -ENOMEM; + + misc->dev = dev; + + if (apple_pmgr_init_device(misc, DEV_FABRIC, "fabric")) + ret = 0; + + if (apple_pmgr_init_device(misc, DEV_DCS, "dcs")) + ret = 0; + + platform_set_drvdata(pdev, misc); + + return ret; +} + +static const struct of_device_id apple_pmgr_misc_of_match[] = { + { .compatible = "apple,t6000-pmgr-misc" }, + {} +}; + +MODULE_DEVICE_TABLE(of, apple_pmgr_misc_of_match); + +static const struct dev_pm_ops apple_pmgr_misc_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(apple_pmgr_misc_suspend_noirq, + apple_pmgr_misc_resume_noirq) +}; + +static struct platform_driver apple_pmgr_misc_driver = { + .probe = apple_pmgr_misc_probe, + .driver = { + .name = "apple-pmgr-misc", + .of_match_table = apple_pmgr_misc_of_match, + .pm = pm_ptr(&apple_pmgr_misc_pm_ops), + }, +}; + +MODULE_AUTHOR("Hector Martin "); +MODULE_DESCRIPTION("PMGR misc driver for Apple SoCs"); +MODULE_LICENSE("GPL v2"); + +module_platform_driver(apple_pmgr_misc_driver); diff --git a/drivers/soc/apple/dockchannel.c b/drivers/soc/apple/dockchannel.c new file mode 100644 index 00000000000000..3557b1e4e05585 --- /dev/null +++ b/drivers/soc/apple/dockchannel.c @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple DockChannel FIFO driver + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DOCKCHANNEL_MAX_IRQ 32 + +#define DOCKCHANNEL_TX_TIMEOUT_MS 1000 +#define DOCKCHANNEL_RX_TIMEOUT_MS 1000 + +#define IRQ_MASK 0x0 +#define IRQ_FLAG 0x4 + +#define IRQ_TX BIT(0) +#define IRQ_RX BIT(1) + +#define CONFIG_TX_THRESH 0x0 +#define CONFIG_RX_THRESH 0x4 + +#define DATA_TX8 0x4 +#define DATA_TX16 0x8 +#define DATA_TX24 0xc +#define DATA_TX32 0x10 +#define DATA_TX_FREE 0x14 +#define DATA_RX8 0x1c +#define DATA_RX16 0x20 +#define DATA_RX24 0x24 +#define DATA_RX32 0x28 +#define DATA_RX_COUNT 0x2c + +struct dockchannel { + struct device *dev; + int tx_irq; + int rx_irq; + + void __iomem *config_base; + void __iomem *data_base; + + u32 fifo_size; + bool awaiting; + struct completion tx_comp; + struct completion rx_comp; + + void *cookie; + void (*data_available)(void *cookie, size_t avail); +}; + +struct dockchannel_common { + struct device *dev; + struct irq_domain *domain; + int irq; + + void __iomem *irq_base; +}; + +/* Dockchannel FIFO functions */ + +static irqreturn_t dockchannel_tx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + complete(&dockchannel->tx_comp); + + return IRQ_HANDLED; +} + +static irqreturn_t dockchannel_rx_irq(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + + disable_irq_nosync(irq); + + if (dockchannel->awaiting) { + return IRQ_WAKE_THREAD; + } else { + complete(&dockchannel->rx_comp); + return IRQ_HANDLED; + } +} + +static irqreturn_t dockchannel_rx_irq_thread(int irq, void *data) +{ + struct dockchannel *dockchannel = data; + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + + dockchannel->awaiting = false; + dockchannel->data_available(dockchannel->cookie, avail); + + return IRQ_HANDLED; +} + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count) +{ + size_t left = count; + const u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_TX_FREE); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_TX_THRESH); + reinit_completion(&dockchannel->tx_comp); + enable_irq(dockchannel->tx_irq); + + if (!wait_for_completion_timeout(&dockchannel->tx_comp, + msecs_to_jiffies(DOCKCHANNEL_TX_TIMEOUT_MS))) { + disable_irq(dockchannel->tx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + writel_relaxed(get_unaligned_le32(p), dockchannel->data_base + DATA_TX32); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + writeb_relaxed(*p++, dockchannel->data_base + DATA_TX8); + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_send); + +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count) +{ + size_t left = count; + u8 *p = buf; + + while (left > 0) { + size_t avail = readl_relaxed(dockchannel->data_base + DATA_RX_COUNT); + size_t block = min(left, avail); + + if (avail == 0) { + size_t threshold = min((size_t)(dockchannel->fifo_size / 2), left); + + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + reinit_completion(&dockchannel->rx_comp); + enable_irq(dockchannel->rx_irq); + + if (!wait_for_completion_timeout(&dockchannel->rx_comp, + msecs_to_jiffies(DOCKCHANNEL_RX_TIMEOUT_MS))) { + disable_irq(dockchannel->rx_irq); + return -ETIMEDOUT; + } + + continue; + } + + while (block >= 4) { + put_unaligned_le32(readl_relaxed(dockchannel->data_base + DATA_RX32), p); + p += 4; + left -= 4; + block -= 4; + } + while (block > 0) { + *p++ = readl_relaxed(dockchannel->data_base + DATA_RX8) >> 8; + left--; + block--; + } + } + + return count; +} +EXPORT_SYMBOL(dockchannel_recv); + +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count) +{ + size_t threshold = min((size_t)dockchannel->fifo_size, count); + + if (!count) { + dockchannel->awaiting = false; + disable_irq(dockchannel->rx_irq); + return 0; + } + + dockchannel->data_available = callback; + dockchannel->cookie = cookie; + dockchannel->awaiting = true; + writel_relaxed(threshold, dockchannel->config_base + CONFIG_RX_THRESH); + enable_irq(dockchannel->rx_irq); + + return threshold; +} +EXPORT_SYMBOL(dockchannel_await); + +struct dockchannel *dockchannel_init(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel *dockchannel; + int ret; + + dockchannel = devm_kzalloc(dev, sizeof(*dockchannel), GFP_KERNEL); + if (!dockchannel) + return ERR_PTR(-ENOMEM); + + dockchannel->dev = dev; + dockchannel->config_base = devm_platform_ioremap_resource_byname(pdev, "config"); + if (IS_ERR(dockchannel->config_base)) + return (__force void *)dockchannel->config_base; + + dockchannel->data_base = devm_platform_ioremap_resource_byname(pdev, "data"); + if (IS_ERR(dockchannel->data_base)) + return (__force void *)dockchannel->data_base; + + ret = of_property_read_u32(dev->of_node, "apple,fifo-size", &dockchannel->fifo_size); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Missing apple,fifo-size property")); + + init_completion(&dockchannel->tx_comp); + init_completion(&dockchannel->rx_comp); + + dockchannel->tx_irq = platform_get_irq_byname(pdev, "tx"); + if (dockchannel->tx_irq < 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->tx_irq, + "Failed to get TX IRQ")); + } + + dockchannel->rx_irq = platform_get_irq_byname(pdev, "rx"); + if (dockchannel->rx_irq < 0) { + return ERR_PTR(dev_err_probe(dev, dockchannel->rx_irq, + "Failed to get RX IRQ")); + } + + ret = devm_request_irq(dev, dockchannel->tx_irq, dockchannel_tx_irq, IRQF_NO_AUTOEN, + "apple-dockchannel-tx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request TX IRQ")); + + ret = devm_request_threaded_irq(dev, dockchannel->rx_irq, dockchannel_rx_irq, + dockchannel_rx_irq_thread, IRQF_NO_AUTOEN, + "apple-dockchannel-rx", dockchannel); + if (ret) + return ERR_PTR(dev_err_probe(dev, ret, "Failed to request RX IRQ")); + + return dockchannel; +} +EXPORT_SYMBOL(dockchannel_init); + + +/* Dockchannel IRQchip */ + +static void dockchannel_irq(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dockchannel_common *dcc = irq_get_handler_data(irq); + unsigned long flags = readl_relaxed(dcc->irq_base + IRQ_FLAG); + int bit; + + chained_irq_enter(chip, desc); + + for_each_set_bit(bit, &flags, DOCKCHANNEL_MAX_IRQ) + generic_handle_domain_irq(dcc->domain, bit); + + chained_irq_exit(chip, desc); +} + +static void dockchannel_irq_ack(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + + writel_relaxed(BIT(hwirq), dcc->irq_base + IRQ_FLAG); +} + +static void dockchannel_irq_mask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val & ~BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static void dockchannel_irq_unmask(struct irq_data *data) +{ + struct dockchannel_common *dcc = irq_data_get_irq_chip_data(data); + unsigned int hwirq = data->hwirq; + u32 val = readl_relaxed(dcc->irq_base + IRQ_MASK); + + writel_relaxed(val | BIT(hwirq), dcc->irq_base + IRQ_MASK); +} + +static const struct irq_chip dockchannel_irqchip = { + .name = "dockchannel-irqc", + .irq_ack = dockchannel_irq_ack, + .irq_mask = dockchannel_irq_mask, + .irq_unmask = dockchannel_irq_unmask, +}; + +static int dockchannel_irq_domain_map(struct irq_domain *d, unsigned int virq, + irq_hw_number_t hw) +{ + irq_set_chip_data(virq, d->host_data); + irq_set_chip_and_handler(virq, &dockchannel_irqchip, handle_level_irq); + + return 0; +} + +static const struct irq_domain_ops dockchannel_irq_domain_ops = { + .xlate = irq_domain_xlate_twocell, + .map = dockchannel_irq_domain_map, +}; + +static int dockchannel_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dockchannel_common *dcc; + + dcc = devm_kzalloc(dev, sizeof(*dcc), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->dev = dev; + platform_set_drvdata(pdev, dcc); + + dcc->irq_base = devm_platform_ioremap_resource_byname(pdev, "irq"); + if (IS_ERR(dcc->irq_base)) + return PTR_ERR(dcc->irq_base); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); + + dcc->domain = irq_domain_add_linear(dev->of_node, DOCKCHANNEL_MAX_IRQ, + &dockchannel_irq_domain_ops, dcc); + if (!dcc->domain) + return -ENOMEM; + + dcc->irq = platform_get_irq(pdev, 0); + if (dcc->irq < 0) + return dev_err_probe(dev, dcc->irq, "Failed to get IRQ"); + + irq_set_handler_data(dcc->irq, dcc); + irq_set_chained_handler(dcc->irq, dockchannel_irq); + + devm_of_platform_populate(dev); + + return 0; +} + +static void dockchannel_remove(struct platform_device *pdev) +{ + struct dockchannel_common *dcc = platform_get_drvdata(pdev); + int hwirq; + + device_for_each_child(&pdev->dev, NULL, of_platform_device_destroy); + + irq_set_chained_handler_and_data(dcc->irq, NULL, NULL); + + for (hwirq = 0; hwirq < DOCKCHANNEL_MAX_IRQ; hwirq++) + irq_dispose_mapping(irq_find_mapping(dcc->domain, hwirq)); + + irq_domain_remove(dcc->domain); + + writel_relaxed(0, dcc->irq_base + IRQ_MASK); + writel_relaxed(~0, dcc->irq_base + IRQ_FLAG); +} + +static const struct of_device_id dockchannel_of_match[] = { + { .compatible = "apple,dockchannel" }, + {}, +}; +MODULE_DEVICE_TABLE(of, dockchannel_of_match); + +static struct platform_driver dockchannel_driver = { + .driver = { + .name = "dockchannel", + .of_match_table = dockchannel_of_match, + }, + .probe = dockchannel_probe, + .remove = dockchannel_remove, +}; +module_platform_driver(dockchannel_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple DockChannel driver"); diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 5c48455185c9ba..330015eea2ebe4 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -28,9 +28,9 @@ #include #include #include +#include #include #include -#include "mailbox.h" #define APPLE_ASC_MBOX_CONTROL_FULL BIT(16) #define APPLE_ASC_MBOX_CONTROL_EMPTY BIT(17) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs new file mode 100644 index 00000000000000..b5148e5eb2adec --- /dev/null +++ b/drivers/soc/apple/pmp.rs @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple PMP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::{ + mem, + slice, // +}; + +use kernel::{ + bindings, + device::{ + self, + Core, // + }, + devres::Devres, + dma::CoherentAllocation, + io::{ + mem::IoMem, + Io, // + }, + iosys_map::IoSysMapRef, + kvec, + module_platform_driver, + new_mutex, + of, + platform, + prelude::*, + soc::apple::rtkit, + str::CString, + sync::{ + Arc, + Mutex, // + }, + types::{ + ARef, + ForeignOwnable, // + }, // +}; + +const PMP_MMIO_SIZE: usize = 0x80000; +const ASC_MMIO_SIZE: usize = 0x4000; +const BOOTARGS_OFFSET: usize = 0x22c; +const BOOTARGS_SIZE: usize = 0x230; +const CPU_CONTROL: usize = 0x44; +const CPU_RUN: u32 = 0x1 << 4; +const PMP_ENDPOINT: u8 = 0x20; +const OPC_GET_IOVA_TABLE: u64 = 0x10; +const OPC_MALLOC: u64 = 0x12; +const OPC_FREE: u64 = 0x14; +const OPC_SET_BUF: u64 = 0x30; +const OPC_REGISTER_IOREG: u64 = 0x32; +const OPC_SET_IOREG: u64 = 0x34; +const OPC_ACK_MASK: u64 = 0x1; +const OPC_SHIFT: u32 = 48; +const MALLOC_SIZE_MASK: u64 = 0xFFFFFF; +const MSG_IOVA_MASK: u64 = 0xFFFFFFFFFFFF; +const SET_IOREG_INDEX_MASK: u64 = 0xFFFF; +const PIO_VM_BASE: u64 = 0xc0000000; +const PIO_GRANULARITY: u64 = 0x1000000; + +const fn from_fourcc(b: &[u8; 4]) -> u32 { + b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 +} + +struct PmpAllocation { + addr: u64, + alloc: CoherentAllocation, +} + +struct PmpState { + iova_table: Option>, + allocs: KVec, + value_buf: Option, + ioreg_entries: KVec, +} + +impl PmpState { + fn new() -> Result { + Ok(PmpState { + iova_table: None, + allocs: KVec::with_capacity(10, GFP_KERNEL)?, + value_buf: None, + ioreg_entries: KVec::with_capacity(340, GFP_KERNEL)?, + }) + } + fn find_alloc(&self, addr: u64) -> Option { + // Due to how pmp manages memory, iterating in reverse will + // usually result in us getting the right one on the first try + for (i, e) in self.allocs.iter().enumerate().rev() { + if e.addr == addr { + return Some(i); + } + } + None + } + fn get_buf(&mut self, addr: u64) -> Option<&mut CoherentAllocation> { + let idx = self.find_alloc(addr)?; + Some(&mut self.allocs[idx].alloc) + } +} + +#[pin_data] +struct PmpData { + dev: ARef, + pmp_mmio: Pin>>>, + asc_mmio: Pin>>>, + #[pin] + rtkit: Mutex>>, + #[pin] + state: Mutex, +} + +impl PmpData { + fn new(dev: &platform::Device) -> Result> { + let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; + let pmp_mmio = KBox::pin_init(pmp_req.iomap_sized::(), GFP_KERNEL)?; + let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; + let asc_mmio = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; + Arc::pin_init( + try_pin_init!( + PmpData { + dev: dev.as_ref().into(), + pmp_mmio, + asc_mmio, + rtkit <- new_mutex!(None), + state <- new_mutex!(PmpState::new()?) + } + ), + GFP_KERNEL, + ) + } + fn start_cpu(&self, dev: &platform::Device) -> Result<()> { + let asc_mmio = self.asc_mmio.access(dev.as_ref())?; + let val = asc_mmio.read32_relaxed(CPU_CONTROL); + asc_mmio.write32_relaxed(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + fn start(&self) -> Result<()> { + let mut guard = self.rtkit.lock(); + let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + rtk.as_mut().wake()?; + rtk.start_endpoint(PMP_ENDPOINT) + } + fn patch_bootargs(&self, dev: &platform::Device, patches: &[(u32, u32)]) -> Result<()> { + let io = self.pmp_mmio.access(dev.as_ref())?; + let offset = io.read32_relaxed(BOOTARGS_OFFSET) as usize; + let size = io.read32_relaxed(BOOTARGS_SIZE) as usize; + let mut arg_bytes = kvec![0u8; size]?; + io.try_memcpy_fromio(&mut arg_bytes, offset)?; + let mut idx = 0; + while idx < size { + let key = u32::from_le_bytes(arg_bytes[idx..idx + 4].try_into().unwrap()); + let size = u32::from_le_bytes(arg_bytes[idx + 4..idx + 8].try_into().unwrap()) as usize; + idx += 8; + for (k, v) in patches.iter() { + if *k != key { + continue; + } + arg_bytes[idx..idx + size].copy_from_slice(&(*v as u64).to_le_bytes()[..size]); + break; + } + idx += size; + } + io.try_memcpy_toio(offset, &arg_bytes) + } + fn get_iova_table(&self) -> Result { + let mut state = self.state.lock(); + if state.iova_table.is_some() { + dev_err!(self.dev, "Asked for iova table with existing buffer"); + return Err(EIO); + } + let node = self.dev.fwnode().ok_or(EIO)?; + let mut pio_base = PIO_VM_BASE; + let prop_name = c"apple,pio-ranges"; + if !node.property_present(prop_name) { + return Ok((OPC_GET_IOVA_TABLE | OPC_ACK_MASK) << OPC_SHIFT); + } + let n_entries = node.property_count_elem::(prop_name)? / 2; + let ranges = node + .property_read_array_vec::(prop_name, n_entries * 2)? + .required_by(&self.dev)?; + let mut table = self.dev.while_bound_with(|bound_dev| { + CoherentAllocation::alloc_coherent(bound_dev, 512, GFP_KERNEL) + })?; + for i in 0..table.count() { + unsafe { table.write(&[0], i)? }; + } + + let domain = unsafe { bindings::iommu_get_domain_for_dev(self.dev.as_raw()) }; + for i in 0..n_entries { + let host_addr = ranges[i * 2]; + let size = ranges[i * 2 + 1]; + unsafe { + let err = bindings::iommu_map( + domain, + pio_base as usize, + host_addr, + size as usize, + (bindings::IOMMU_READ | bindings::IOMMU_WRITE | bindings::IOMMU_MMIO) as i32, + bindings::GFP_KERNEL, + ); + if err != 0 { + return Err(Error::from_errno(err)); + } + } + unsafe { table.write(&[host_addr, pio_base, size], i * 3)? }; + pio_base += PIO_GRANULARITY; + } + let msg = (OPC_GET_IOVA_TABLE | OPC_ACK_MASK) << OPC_SHIFT | table.dma_handle(); + state.iova_table = Some(table); + Ok(msg) + } + fn malloc(&self, size: u64) -> Result { + let iomem = self.dev.while_bound_with(|bound_dev| { + CoherentAllocation::alloc_coherent(bound_dev, size as usize, GFP_KERNEL) + })?; + let mut state = self.state.lock(); + let addr = iomem.dma_handle(); + let msg = (OPC_MALLOC | OPC_ACK_MASK) << OPC_SHIFT | addr; + state.allocs.push( + PmpAllocation { + addr: addr, + alloc: iomem, + }, + GFP_KERNEL, + )?; + Ok(msg) + } + fn free(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + if let Some(idx) = state.find_alloc(addr) { + state.allocs.swap_remove(idx); + } else { + dev_err!( + self.dev, + "Attempted to free memory that was not allocated {}", + addr + ); + return Err(EIO); + } + let msg = (OPC_FREE | OPC_ACK_MASK) << OPC_SHIFT; + Ok(msg) + } + fn set_buf(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + if state.value_buf.is_some() { + dev_err!(self.dev, "Setting a buffer when one exists"); + return Err(EIO); + } + let ptr_buf = if let Some(s) = state.get_buf(addr) { + s + } else { + dev_err!(self.dev, "Unable to find buffer"); + return Err(EIO); + }; + if ptr_buf.count() < mem::size_of::() { + dev_err!(self.dev, "Buffer too small"); + return Err(EIO); + } + let ptr = unsafe { *(ptr_buf.start_ptr() as *const u64) }; + state.value_buf = Some(ptr); + let msg = (OPC_SET_BUF | OPC_ACK_MASK) << OPC_SHIFT; + Ok(msg) + } + fn register_ioreg(&self, addr: u64) -> Result { + let mut state = self.state.lock(); + let msg_buf = if let Some(s) = state.get_buf(addr) { + s + } else { + dev_err!(self.dev, "Unable to find buffer"); + return Err(EIO); + }; + if msg_buf.count() < 0x44 { + dev_err!(self.dev, "Buffer too small"); + return Err(EIO); + } + let mut size = unsafe { *(msg_buf.start_ptr().offset(0x40) as *const u32) }; + if size == 0 { + let mut name_vec = KVec::with_capacity(0x31, GFP_KERNEL)?; + name_vec + .extend_from_slice( + unsafe { slice::from_raw_parts(msg_buf.start_ptr(), 0x30) }, + GFP_KERNEL, + ) + .unwrap(); + name_vec.push(0, GFP_KERNEL).unwrap(); + let name_str = CStr::from_bytes_until_nul(&name_vec).unwrap(); + let name_str = CString::try_from_fmt(fmt!("apple,tunable-{name_str}"))?; + let node = self.dev.fwnode().ok_or(EIO)?; + if state.value_buf.is_none() { + dev_err!(self.dev, "Value buf not set"); + return Err(EIO); + } + let val_buf_addr = state.value_buf.unwrap(); + let val_buf = if let Some(s) = state.get_buf(val_buf_addr) { + s + } else { + dev_err!(self.dev, "Unable to find value buffer"); + return Err(EIO); + }; + if node.property_present(&name_str) { + let len = node.property_count_elem::(&name_str)?; + let data = node + .property_read_array_vec::(&name_str, len)? + .required_by(&self.dev)?; + unsafe { + slice::from_raw_parts_mut(val_buf.start_ptr_mut(), len).copy_from_slice(&data); + } + size = len as u32; + } else { + dev_info!(self.dev, "unknown property {:?}", name_str); + } + } + state.ioreg_entries.push(size, GFP_KERNEL)?; + let index = state.ioreg_entries.len() as u64; + let msg = (OPC_REGISTER_IOREG | OPC_ACK_MASK) << OPC_SHIFT | (index << 32) | size as u64; + Ok(msg) + } + fn set_ioreg(&self, index: u64) -> Result { + let len = *self + .state + .lock() + .ioreg_entries + .get(index as usize) + .ok_or(EIO)? as u64; + let msg = (OPC_SET_IOREG | OPC_ACK_MASK) << OPC_SHIFT | len; + Ok(msg) + } + fn recv_message(&self, msg: u64) -> Result<()> { + let opc = (msg >> OPC_SHIFT) & 0xFF; + let reply = match opc { + OPC_GET_IOVA_TABLE => self.get_iova_table()?, + OPC_MALLOC => self.malloc(msg & MALLOC_SIZE_MASK)?, + OPC_FREE => self.free(msg & MSG_IOVA_MASK)?, + OPC_SET_BUF => self.set_buf(msg & MSG_IOVA_MASK)?, + OPC_REGISTER_IOREG => self.register_ioreg(msg & MSG_IOVA_MASK)?, + OPC_SET_IOREG => self.set_ioreg(msg & SET_IOREG_INDEX_MASK)?, + _ => { + dev_err!(self.dev, "Got unknown message {}", msg); + return Err(EIO); + } + }; + let mut rtk_guard = self.rtkit.lock(); + let rtk = rtk_guard.as_mut().as_pin_mut().unwrap(); + rtk.send_message(PMP_ENDPOINT, reply)?; + Ok(()) + } +} + +unsafe impl Send for PmpData {} +unsafe impl Sync for PmpData {} + +struct NoBuffer; +impl rtkit::Buffer for NoBuffer { + fn iova(&self) -> Result { + unreachable!() + } + fn buf(&mut self) -> Result> { + unreachable!() + } +} + +#[vtable] +impl rtkit::Operations for PmpData { + type Data = Arc; + type Buffer = NoBuffer; + + fn recv_message(data: ::Borrowed<'_>, _ep: u8, msg: u64) { + let ret = data.recv_message(msg); + if let Err(e) = ret { + dev_err!(data.dev, "Failed to handle rtkit message, error: {:?}", e); + } + } + + fn crashed(data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) { + dev_err!(data.dev, "PMP firmware crashed"); + } +} + +#[allow(dead_code)] +struct PmpDriver(Arc); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c"apple,t6000-pmp-v2"), ())] +); + +impl platform::Driver for PmpDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev: ARef = pdev.as_ref().into(); + let data = PmpData::new(pdev)?; + let node = dev.fwnode().ok_or(EIO)?; + let dvid = node + .property_read(c"apple,dram-vendor-id") + .required_by(&dev)?; + let bdid = node.property_read(c"apple,board-id").required_by(&dev)?; + match node.property_read(c"apple,dram-capacity").optional() { + Some(dcap) => data.patch_bootargs( + pdev, + &[ + (from_fourcc(b"BDID"), bdid), + (from_fourcc(b"DCAP"), dcap), + (from_fourcc(b"DVID"), dvid), + ], + )?, + None => data.patch_bootargs( + pdev, + &[(from_fourcc(b"BDID"), bdid), (from_fourcc(b"DVID"), dvid)], + )?, + }; + let rtkit = rtkit::RtKit::::new(&dev, None, 0, data.clone())?; + *data.rtkit.lock() = Some(rtkit); + data.start_cpu(pdev)?; + data.start()?; + Ok(PmpDriver(data)) + } +} + +module_platform_driver! { + type: PmpDriver, + name: "apple_pmp", + description: "Apple Power Management Processor", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c new file mode 100644 index 00000000000000..080d083ed9bd2f --- /dev/null +++ b/drivers/soc/apple/rtkit-helper.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple Generic RTKit helper coprocessor + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#define APPLE_ASC_CPU_CONTROL 0x44 +#define APPLE_ASC_CPU_CONTROL_RUN BIT(4) + +struct apple_rtkit_helper { + struct device *dev; + struct apple_rtkit *rtk; + + void __iomem *asc_base; + + struct resource *sram; + void __iomem *sram_base; +}; + +static int apple_rtkit_helper_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +{ + struct apple_rtkit_helper *helper = cookie; + struct resource res = { + .start = bfr->iova, + .end = bfr->iova + bfr->size - 1, + .name = "rtkit_map", + }; + + if (!bfr->iova) { + bfr->buffer = dma_alloc_coherent(helper->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; + return 0; + } + + if (!helper->sram) { + dev_err(helper->dev, + "RTKit buffer request with no SRAM region: %pR", &res); + return -EFAULT; + } + + res.flags = helper->sram->flags; + + if (res.end < res.start || !resource_contains(helper->sram, &res)) { + dev_err(helper->dev, + "RTKit buffer request outside SRAM region: %pR", &res); + return -EFAULT; + } + + bfr->iomem = helper->sram_base + (res.start - helper->sram->start); + bfr->is_mapped = true; + + return 0; +} + +static void apple_rtkit_helper_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +{ + // no-op +} + +static const struct apple_rtkit_ops apple_rtkit_helper_ops = { + .shmem_setup = apple_rtkit_helper_shmem_setup, + .shmem_destroy = apple_rtkit_helper_shmem_destroy, +}; + +static int apple_rtkit_helper_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_rtkit_helper *helper; + int ret; + + /* 44 bits for addresses in standard RTKit requests */ + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) + return ret; + + helper = devm_kzalloc(dev, sizeof(*helper), GFP_KERNEL); + if (!helper) + return -ENOMEM; + + helper->dev = dev; + platform_set_drvdata(pdev, helper); + + helper->asc_base = devm_platform_ioremap_resource_byname(pdev, "asc"); + if (IS_ERR(helper->asc_base)) + return PTR_ERR(helper->asc_base); + + helper->sram = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (helper->sram) { + helper->sram_base = devm_ioremap_resource(dev, helper->sram); + if (IS_ERR(helper->sram_base)) + return dev_err_probe(dev, PTR_ERR(helper->sram_base), + "Failed to map SRAM region"); + } + + helper->rtk = + devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to intialize RTKit"); + + writel_relaxed(APPLE_ASC_CPU_CONTROL_RUN, + helper->asc_base + APPLE_ASC_CPU_CONTROL); + + /* Works for both wake and boot */ + ret = apple_rtkit_wake(helper->rtk); + if (ret != 0) + return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); + + return 0; +} + +static void apple_rtkit_helper_remove(struct platform_device *pdev) +{ + struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + + if (apple_rtkit_is_running(helper->rtk)) + apple_rtkit_quiesce(helper->rtk); + + writel_relaxed(0, helper->asc_base + APPLE_ASC_CPU_CONTROL); +} + +static const struct of_device_id apple_rtkit_helper_of_match[] = { + { .compatible = "apple,rtk-helper-asc4" }, + {}, +}; +MODULE_DEVICE_TABLE(of, apple_rtkit_helper_of_match); + +static struct platform_driver apple_rtkit_helper_driver = { + .driver = { + .name = "rtkit-helper", + .of_match_table = apple_rtkit_helper_of_match, + }, + .probe = apple_rtkit_helper_probe, + .remove = apple_rtkit_helper_remove, +}; +module_platform_driver(apple_rtkit_helper_driver); + +MODULE_AUTHOR("Hector Martin "); +MODULE_LICENSE("Dual MIT/GPL"); +MODULE_DESCRIPTION("Apple RTKit helper driver"); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index b8d5244678f010..c82065a8bf7b03 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -15,9 +15,9 @@ #include #include #include +#include #include #include -#include "mailbox.h" #define APPLE_RTKIT_APP_ENDPOINT_START 0x20 #define APPLE_RTKIT_MAX_ENDPOINTS 0x100 diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index a3fdac8f6f069e..ef0c399ea85230 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -22,6 +22,7 @@ enum { APPLE_RTKIT_EP_DEBUG = 3, APPLE_RTKIT_EP_IOREPORT = 4, APPLE_RTKIT_EP_OSLOG = 8, + APPLE_RTKIT_EP_TRACEKIT = 0xa, }; #define APPLE_RTKIT_MGMT_TYPE GENMASK_ULL(59, 52) @@ -191,6 +192,7 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) case APPLE_RTKIT_EP_DEBUG: case APPLE_RTKIT_EP_IOREPORT: case APPLE_RTKIT_EP_OSLOG: + case APPLE_RTKIT_EP_TRACEKIT: dev_dbg(rtk->dev, "RTKit: Starting system endpoint 0x%02x\n", ep); apple_rtkit_start_ep(rtk, ep); @@ -361,7 +363,7 @@ static void apple_rtkit_memcpy(struct apple_rtkit *rtk, void *dst, static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) { u8 type = FIELD_GET(APPLE_RTKIT_SYSLOG_TYPE, msg); - u8 *bfr; + u8 *bfr __free(kfree) = NULL; if (type != APPLE_RTKIT_CRASHLOG_CRASH) { dev_warn(rtk->dev, "RTKit: Unknown crashlog message: %llx\n", @@ -394,9 +396,7 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) rtk->crashed = true; if (rtk->ops->crashed) - rtk->ops->crashed(rtk->cookie, bfr, rtk->crashlog_buffer.size); - - kfree(bfr); + rtk->ops->crashed(rtk->cookie, bfr, bfr ? rtk->crashlog_buffer.size : 0); } static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) @@ -639,6 +639,12 @@ int apple_rtkit_poll(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_poll); +bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) +{ + return test_bit(ep, rtk->endpoints); +} +EXPORT_SYMBOL_GPL(apple_rtkit_has_endpoint); + int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint) { u64 msg; @@ -974,6 +980,12 @@ struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, } EXPORT_SYMBOL_GPL(devm_apple_rtkit_init); +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk) +{ + devm_release_action(dev, apple_rtkit_free_wrapper, rtk); +} +EXPORT_SYMBOL_GPL(devm_apple_rtkit_free); + MODULE_LICENSE("Dual MIT/GPL"); MODULE_AUTHOR("Sven Peter "); MODULE_DESCRIPTION("Apple RTKit driver"); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs new file mode 100644 index 00000000000000..24d16dd80958ad --- /dev/null +++ b/drivers/soc/apple/sep.rs @@ -0,0 +1,353 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple SEP driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::sync::atomic::{AtomicBool, Ordering}; + +use kernel::{ + bindings, c_str, device, dma, module_platform_driver, new_mutex, of, platform, + prelude::*, + soc::apple::mailbox::{MailCallback, Mailbox, Message}, + sync::{Arc, Mutex}, + types::{ARef, ForeignOwnable}, + workqueue::{self, impl_has_work, new_work, Work, WorkItem}, +}; + +const SHMEM_SIZE: usize = 0x30000; +const MSG_BOOT_TZ0: u64 = 0x5; +const MSG_BOOT_IMG4: u64 = 0x6; +const MSG_SET_SHMEM: u64 = 0x18; +const MSG_BOOT_TZ0_ACK1: u64 = 0x69; +const MSG_BOOT_TZ0_ACK2: u64 = 0xD2; +const MSG_BOOT_IMG4_ACK: u64 = 0x6A; +const MSG_ADVERTISE_EP: u64 = 0; +const EP_DISCOVER: u64 = 0xFD; +const EP_SHMEM: u64 = 0xFE; +const EP_BOOT: u64 = 0xFF; + +const MSG_TYPE_SHIFT: u32 = 16; +const MSG_TYPE_MASK: u64 = 0xFF; +//const MSG_PARAM_SHIFT: u32 = 24; +//const MSG_PARAM_MASK: u64 = 0xFF; + +const MSG_EP_MASK: u64 = 0xFF; +const MSG_DATA_SHIFT: u32 = 32; + +const IOVA_SHIFT: u32 = 0xC; + +type ShMem = dma::CoherentAllocation; + +fn align_up(v: usize, a: usize) -> usize { + (v + a - 1) & !(a - 1) +} + +fn memcpy_to_iomem(iomem: &mut ShMem, off: usize, src: &[u8]) -> Result<()> { + // SAFETY: + // as_slice_mut() checks that off and src.len() are whithin iomem's limits. + // memcpy_to_iomem is only called from within probe() ansuring there are no + // concurrent read and write accesses to the same region while the slice is + // alive per as_slice_mut()'s requiremnts. + unsafe { + let target = iomem.as_slice_mut(off, src.len())?; + target.copy_from_slice(src); + } + Ok(()) +} + +fn build_shmem(dev: &platform::Device) -> Result { + let fwnode = dev.as_ref().fwnode().ok_or(EIO)?; + let mut iomem = + dma::CoherentAllocation::::alloc_coherent(dev.as_ref(), SHMEM_SIZE, GFP_KERNEL)?; + + let panic_offset = 0x4000; + let panic_size = 0x8000; + memcpy_to_iomem(&mut iomem, panic_offset, &1u32.to_le_bytes())?; + + let lpol_offset = panic_offset + panic_size; + let lpol_prop_name = c_str!("local-policy-manifest"); + let lpol_prop_size = fwnode.property_count_elem::(lpol_prop_name)?; + let lpol = fwnode + .property_read_array_vec(lpol_prop_name, lpol_prop_size)? + .required_by(dev.as_ref())?; + memcpy_to_iomem( + &mut iomem, + lpol_offset, + &(lpol_prop_size as u32).to_le_bytes(), + )?; + memcpy_to_iomem(&mut iomem, lpol_offset + 4, &lpol)?; + let lpol_size = align_up(lpol_prop_size + 4, 0x4000); + + let ibot_offset = lpol_offset + lpol_size; + let ibot_prop_name = c_str!("iboot-manifest"); + let ibot_prop_size = fwnode.property_count_elem::(ibot_prop_name)?; + let ibot = fwnode + .property_read_array_vec(ibot_prop_name, ibot_prop_size)? + .required_by(dev.as_ref())?; + memcpy_to_iomem( + &mut iomem, + ibot_offset, + &(ibot_prop_size as u32).to_le_bytes(), + )?; + memcpy_to_iomem(&mut iomem, ibot_offset + 4, &ibot)?; + let ibot_size = align_up(ibot_prop_size + 4, 0x4000); + + memcpy_to_iomem(&mut iomem, 0, b"CNIP")?; + memcpy_to_iomem(&mut iomem, 4, &(panic_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 8, &(panic_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 16, b"OPLA")?; + memcpy_to_iomem(&mut iomem, 16 + 4, &(lpol_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 16 + 8, &(lpol_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 32, b"IPIS")?; + memcpy_to_iomem(&mut iomem, 32 + 4, &(ibot_size as u32).to_le_bytes())?; + memcpy_to_iomem(&mut iomem, 32 + 8, &(ibot_offset as u32).to_le_bytes())?; + + memcpy_to_iomem(&mut iomem, 48, b"llun")?; + Ok(iomem) +} + +#[pin_data] +struct SepReceiveWork { + data: Arc, + msg: Message, + #[pin] + work: Work, +} + +impl_has_work! { + impl HasWork for SepReceiveWork { self.work } +} + +impl SepReceiveWork { + fn new(data: Arc, msg: Message) -> Result> { + Arc::pin_init( + pin_init!(SepReceiveWork { + data, + msg, + work <- new_work!("SepReceiveWork::work"), + }), + GFP_ATOMIC, + ) + } +} + +impl WorkItem for SepReceiveWork { + type Pointer = Arc; + + fn run(this: Arc) { + this.data.process_message(this.msg); + } +} + +struct FwRegionParams { + addr: u64, + size: usize, +} + +#[pin_data] +struct SepData { + dev: ARef, + #[pin] + mbox: Mutex>>, + shmem: ShMem, + region_params: FwRegionParams, + fw_mapped: AtomicBool, +} + +impl SepData { + fn new( + dev: &platform::Device, + region_params: FwRegionParams, + ) -> Result> { + Arc::pin_init( + try_pin_init!(SepData { + shmem: build_shmem(dev)?, + dev: ARef::::from(dev.as_ref()), + mbox <- new_mutex!(None), + region_params, + fw_mapped: AtomicBool::new(false), + }), + GFP_KERNEL, + ) + } + fn start(&self) -> Result<()> { + self.mbox.lock().as_ref().unwrap().send( + Message { + msg0: EP_BOOT | (MSG_BOOT_TZ0 << MSG_TYPE_SHIFT), + msg1: 0, + }, + false, + ) + } + fn load_fw_and_shmem(&self) -> Result<()> { + let fw_addr = unsafe { + let res = bindings::dma_map_resource( + self.dev.as_raw(), + self.region_params.addr, + self.region_params.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + if bindings::dma_mapping_error(self.dev.as_raw(), res) != 0 { + dev_err!(self.dev, "Failed to map firmware"); + return Err(ENOMEM); + } + self.fw_mapped.store(true, Ordering::Relaxed); + res >> IOVA_SHIFT + }; + let guard = self.mbox.lock(); + let mbox = guard.as_ref().unwrap(); + mbox.send( + Message { + msg0: EP_BOOT | (MSG_BOOT_IMG4 << MSG_TYPE_SHIFT) | (fw_addr << MSG_DATA_SHIFT), + msg1: 0, + }, + false, + )?; + let shm_addr = self.shmem.dma_handle() >> IOVA_SHIFT; + mbox.send( + Message { + msg0: EP_SHMEM | (MSG_SET_SHMEM << MSG_TYPE_SHIFT) | (shm_addr << MSG_DATA_SHIFT), + msg1: 0, + }, + false, + )?; + Ok(()) + } + fn process_boot_msg(&self, msg: Message) { + let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; + match ty { + MSG_BOOT_TZ0_ACK1 => {} + MSG_BOOT_TZ0_ACK2 => { + let res = self.load_fw_and_shmem(); + if let Err(e) = res { + dev_err!(self.dev, "Unable to load firmware: {:?}", e); + } + } + MSG_BOOT_IMG4_ACK => {} + _ => { + dev_err!(self.dev, "Unknown boot message type: {}", ty); + } + } + } + fn process_discover_msg(&self, msg: Message) { + let ty = (msg.msg0 >> MSG_TYPE_SHIFT) & MSG_TYPE_MASK; + //let data = (msg.msg0 >> MSG_DATA_SHIFT) as u32; + //let param = (msg.msg0 >> MSG_PARAM_SHIFT) & MSG_PARAM_MASK; + match ty { + MSG_ADVERTISE_EP => { + /*dev_info!( + self.dev, + "Got endpoint {:?} at {}", + core::str::from_utf8(&data.to_be_bytes()), + param + );*/ + } + _ => { + //dev_warn!(self.dev, "Unknown discovery message type: {}", ty); + } + } + } + fn process_message(&self, msg: Message) { + let ep = msg.msg0 & MSG_EP_MASK; + match ep { + EP_BOOT => self.process_boot_msg(msg), + EP_DISCOVER => self.process_discover_msg(msg), + _ => {} // dev_warn!(self.dev, "Message from unknown endpoint: {}", ep), + } + } + fn remove(&self) { + *self.mbox.lock() = None; + if self.fw_mapped.load(Ordering::Relaxed) { + unsafe { + bindings::dma_unmap_resource( + self.dev.as_raw(), + self.region_params.addr, + self.region_params.size, + bindings::dma_data_direction_DMA_TO_DEVICE, + 0, + ); + } + } + } +} + +impl MailCallback for SepData { + type Data = Arc; + fn recv_message(data: ::Borrowed<'_>, msg: Message) { + let work = SepReceiveWork::new(data.into(), msg); + if let Ok(work) = work { + let res = workqueue::system().enqueue(work); + if res.is_err() { + dev_err!( + data.dev, + "Unable to schedule work item for message {}", + msg.msg0 + ); + } + } else { + dev_err!( + data.dev, + "Unable to allocate work item for message {}", + msg.msg0 + ); + } + } +} + +unsafe impl Send for SepData {} +unsafe impl Sync for SepData {} + +struct SepDriver(Arc); + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,sep")), ())] +); + +impl platform::Driver for SepDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe( + pdev: &platform::Device, + _info: Option<&()>, + ) -> impl PinInit { + let of = pdev.as_ref().of_node().ok_or(EIO)?; + let res = of.reserved_mem_region_to_resource_byname(c_str!("sepfw"))?; + let data = SepData::new( + pdev, + FwRegionParams { + addr: res.start(), + size: res.size().try_into()?, + }, + )?; + *data.mbox.lock() = Some(Mailbox::new_byname( + pdev.as_ref(), + c_str!("mbox"), + data.clone(), + )?); + data.start()?; + Ok(Self(data)) + } +} + +impl Drop for SepDriver { + fn drop(&mut self) { + self.0.remove(); + } +} + +module_platform_driver! { + type: SepDriver, + name: "apple_sep", + description: "Secure enclave processor stub driver", + license: "Dual MIT/GPL", +} diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c index ad5899d083f3f0..b80d3f9cff6415 100644 --- a/drivers/soc/qcom/llcc-qcom.c +++ b/drivers/soc/qcom/llcc-qcom.c @@ -3943,7 +3943,7 @@ static const struct llcc_slice_config x1e80100_data[] = { static const struct llcc_edac_reg_offset llcc_v1_edac_reg_offset = { .trp_ecc_error_status0 = 0x20344, .trp_ecc_error_status1 = 0x20348, - .trp_ecc_sb_err_syn0 = 0x2304c, + .trp_ecc_sb_err_syn0 = 0x2034c, .trp_ecc_db_err_syn0 = 0x20370, .trp_ecc_error_cntr_clear = 0x20440, .trp_interrupt_0_status = 0x20480, diff --git a/drivers/soc/qcom/ocmem.c b/drivers/soc/qcom/ocmem.c index 6a23f18b028126..96ca0b87bfc485 100644 --- a/drivers/soc/qcom/ocmem.c +++ b/drivers/soc/qcom/ocmem.c @@ -196,17 +196,16 @@ struct ocmem *of_get_ocmem(struct device *dev) } pdev = of_find_device_by_node(devnode->parent); - if (!pdev) { - dev_err(dev, "Cannot find device node %s\n", devnode->name); - return ERR_PTR(-EPROBE_DEFER); - } + if (!pdev) + return dev_err_ptr_probe(dev, -EPROBE_DEFER, + "Cannot find device node %s\n", + devnode->name); ocmem = platform_get_drvdata(pdev); put_device(&pdev->dev); - if (!ocmem) { - dev_err(dev, "Cannot get ocmem\n"); - return ERR_PTR(-ENODEV); - } + if (!ocmem) + return dev_err_ptr_probe(dev, -EPROBE_DEFER, "Cannot get ocmem\n"); + return ocmem; } EXPORT_SYMBOL_GPL(of_get_ocmem); @@ -308,7 +307,7 @@ static int ocmem_dev_probe(struct platform_device *pdev) ocmem->dev = dev; ocmem->config = device_get_match_data(dev); - ocmem->core_clk = devm_clk_get(dev, "core"); + ocmem->core_clk = devm_clk_get_optional(dev, "core"); if (IS_ERR(ocmem->core_clk)) return dev_err_probe(dev, PTR_ERR(ocmem->core_clk), "Unable to get core clock\n"); diff --git a/drivers/soc/qcom/qcom_aoss.c b/drivers/soc/qcom/qcom_aoss.c index a543ab9bee6c4e..c255662b8fc3d1 100644 --- a/drivers/soc/qcom/qcom_aoss.c +++ b/drivers/soc/qcom/qcom_aoss.c @@ -355,7 +355,7 @@ static int qmp_cdev_set_cur_state(struct thermal_cooling_device *cdev, /* Normalize state */ cdev_state = !!state; - if (qmp_cdev->state == state) + if (qmp_cdev->state == cdev_state) return 0; ret = qmp_send(qmp_cdev->qmp, "{class: volt_flr, event:zero_temp, res:%s, value:%s}", diff --git a/drivers/soc/qcom/ubwc_config.c b/drivers/soc/qcom/ubwc_config.c index 1c25aaf55e523a..8304463f238a6e 100644 --- a/drivers/soc/qcom/ubwc_config.c +++ b/drivers/soc/qcom/ubwc_config.c @@ -231,8 +231,7 @@ static const struct qcom_ubwc_cfg_data x1e80100_data = { static const struct qcom_ubwc_cfg_data glymur_data = { .ubwc_enc_version = UBWC_5_0, .ubwc_dec_version = UBWC_5_0, - .ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 | - UBWC_SWIZZLE_ENABLE_LVL3, + .ubwc_swizzle = 0, .ubwc_bank_spread = true, /* TODO: highest_bank_bit = 15 for LP_DDR4 */ .highest_bank_bit = 16, diff --git a/drivers/soc/tegra/cbb/tegra234-cbb.c b/drivers/soc/tegra/cbb/tegra234-cbb.c index a9adbcecd47cc7..7e387fc54c6b13 100644 --- a/drivers/soc/tegra/cbb/tegra234-cbb.c +++ b/drivers/soc/tegra/cbb/tegra234-cbb.c @@ -313,12 +313,37 @@ static void tegra234_cbb_lookup_apbslv(struct seq_file *file, const char *target } } +static struct tegra234_cbb *tegra234_cbb_get_fabric(u8 fab_id) +{ + struct tegra_cbb *entry; + + list_for_each_entry(entry, &cbb_list, node) { + struct tegra234_cbb *priv = to_tegra234_cbb(entry); + + if (priv->fabric->fab_id == fab_id) + return priv; + } + + return NULL; +} + static void tegra234_sw_lookup_target_timeout(struct seq_file *file, struct tegra234_cbb *cbb, u8 target_id, u8 fab_id) { const struct tegra234_target_lookup *map = cbb->fabric->fab_list[fab_id].target_map; + struct tegra234_cbb *target_cbb = NULL; void __iomem *addr; + if (fab_id == cbb->fabric->fab_id) + target_cbb = cbb; + else + target_cbb = tegra234_cbb_get_fabric(fab_id); + + if (!target_cbb) { + dev_err(cbb->base.dev, "could not find fabric for fab_id:%d\n", fab_id); + return; + } + if (target_id >= cbb->fabric->fab_list[fab_id].max_targets) { tegra_cbb_print_err(file, "\t Invalid target_id:%d\n", target_id); return; @@ -341,7 +366,7 @@ static void tegra234_sw_lookup_target_timeout(struct seq_file *file, struct tegr * e) Goto step-a till all bits are set. */ - addr = cbb->regs + map[target_id].offset; + addr = target_cbb->regs + map[target_id].offset; if (strstr(map[target_id].name, "AXI2APB")) { addr += APB_BLOCK_TMO_STATUS_0; @@ -881,7 +906,7 @@ static const struct tegra234_fabric_lookup tegra234_cbb_fab_list[] = { ARRAY_SIZE(tegra234_common_target_map) }, [T234_AON_FABRIC_ID] = { "aon-fabric", true, tegra234_aon_target_map, - ARRAY_SIZE(tegra234_bpmp_target_map) }, + ARRAY_SIZE(tegra234_aon_target_map) }, [T234_PSC_FABRIC_ID] = { "psc-fabric" }, [T234_BPMP_FABRIC_ID] = { "bpmp-fabric", true, tegra234_bpmp_target_map, @@ -1160,7 +1185,7 @@ static const struct tegra234_fabric_lookup tegra241_cbb_fab_list[] = { [T234_CBB_FABRIC_ID] = { "cbb-fabric", true, tegra241_cbb_target_map, ARRAY_SIZE(tegra241_cbb_target_map) }, [T234_BPMP_FABRIC_ID] = { "bpmp-fabric", true, - tegra241_bpmp_target_map, ARRAY_SIZE(tegra241_cbb_target_map) }, + tegra241_bpmp_target_map, ARRAY_SIZE(tegra241_bpmp_target_map) }, }; static const struct tegra234_cbb_fabric tegra241_cbb_fabric = { .fab_id = T234_CBB_FABRIC_ID, @@ -1586,6 +1611,10 @@ static int __maybe_unused tegra234_cbb_resume_noirq(struct device *dev) { struct tegra234_cbb *cbb = dev_get_drvdata(dev); + /* set ERD bit to mask SError and generate interrupt to report error */ + if (cbb->fabric->off_mask_erd) + tegra234_cbb_mask_serror(cbb); + tegra234_cbb_error_enable(&cbb->base); dev_dbg(dev, "%s resumed\n", cbb->fabric->fab_list[cbb->fabric->fab_id].name); diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index a1a2966512d1a6..6debaabdaa36ad 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -437,7 +437,10 @@ struct tegra_pmc_soc { * @wake_sw_status_map: Bitmap to hold raw status of wakes without mask * @wake_cntrl_level_map: Bitmap to hold wake levels to be programmed in * cntrl register associated with each wake during system suspend. + * @reboot_notifier: PMC reboot notifier handler * @syscore: syscore suspend/resume callbacks + * @wake_work: IRQ work handler for processing wake-up events. + * @wake_status: Status of wake-up events. */ struct tegra_pmc { struct device *dev; @@ -1004,7 +1007,7 @@ static struct tegra_pmc *tegra_pmc_get(struct device *dev) } /** - * tegra_pmc_get() - find the PMC for a given device + * devm_tegra_pmc_get() - find the PMC for a given device * @dev: device for which to find the PMC * * Returns a pointer to the PMC on success or an ERR_PTR()-encoded error code @@ -1746,7 +1749,7 @@ static void tegra_io_pad_unprepare(struct tegra_pmc *pmc) } /** - * tegra_io_pad_power_enable() - enable power to I/O pad + * tegra_pmc_io_pad_power_enable() - enable power to I/O pad * @pmc: power management controller * @id: Tegra I/O pad ID for which to enable power * diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index fb68738dfb9b84..fe5316d93fefe7 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -1899,8 +1899,8 @@ int sdw_handle_slave_status(struct sdw_bus *bus, if (status[i] == SDW_SLAVE_UNATTACHED && slave->status != SDW_SLAVE_UNATTACHED) { - dev_warn(&slave->dev, "Slave %d state check1: UNATTACHED, status was %d\n", - i, slave->status); + dev_dbg(&slave->dev, "Slave %d state check1: UNATTACHED, status was %d\n", + i, slave->status); sdw_modify_slave_status(slave, SDW_SLAVE_UNATTACHED); /* Ensure driver knows that peripheral unattached */ @@ -1951,8 +1951,8 @@ int sdw_handle_slave_status(struct sdw_bus *bus, if (slave->status == SDW_SLAVE_UNATTACHED) break; - dev_warn(&slave->dev, "Slave %d state check2: UNATTACHED, status was %d\n", - i, slave->status); + dev_dbg(&slave->dev, "Slave %d state check2: UNATTACHED, status was %d\n", + i, slave->status); sdw_modify_slave_status(slave, SDW_SLAVE_UNATTACHED); break; diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c index f245c3ffb9e972..b8b62735c8938e 100644 --- a/drivers/soundwire/cadence_master.c +++ b/drivers/soundwire/cadence_master.c @@ -933,6 +933,14 @@ irqreturn_t sdw_cdns_irq(int irq, void *dev_id) cdns_read_response(cdns); + /* + * Clear interrupt before signalling the completion to avoid + * a race between this thread and the main thread starting + * another TX. + */ + cdns_writel(cdns, CDNS_MCP_INTSTAT, CDNS_MCP_INT_RX_WL); + int_status &= ~CDNS_MCP_INT_RX_WL; + if (defer && defer->msg) { cdns_fill_msg_resp(cdns, defer->msg, defer->length, 0); diff --git a/drivers/soundwire/debugfs.c b/drivers/soundwire/debugfs.c index ccc9670ef77ce1..2905ec19b83844 100644 --- a/drivers/soundwire/debugfs.c +++ b/drivers/soundwire/debugfs.c @@ -358,8 +358,8 @@ void sdw_slave_debugfs_init(struct sdw_slave *slave) debugfs_create_file("go", 0200, d, slave, &cmd_go_fops); debugfs_create_file("read_buffer", 0400, d, slave, &read_buffer_fops); - firmware_file = NULL; - debugfs_create_str("firmware_file", 0200, d, &firmware_file); + if (firmware_file) + debugfs_create_str("firmware_file", 0200, d, &firmware_file); slave->debugfs = d; } @@ -371,10 +371,15 @@ void sdw_slave_debugfs_exit(struct sdw_slave *slave) void sdw_debugfs_init(void) { + if (!firmware_file) + firmware_file = kstrdup("", GFP_KERNEL); + sdw_debugfs_root = debugfs_create_dir("soundwire", NULL); } void sdw_debugfs_exit(void) { debugfs_remove_recursive(sdw_debugfs_root); + kfree(firmware_file); + firmware_file = NULL; } diff --git a/drivers/soundwire/intel_ace2x.c b/drivers/soundwire/intel_ace2x.c index 7f01e43ae978a2..20422534baf19f 100644 --- a/drivers/soundwire/intel_ace2x.c +++ b/drivers/soundwire/intel_ace2x.c @@ -82,6 +82,11 @@ static int intel_ace2x_bpt_open_stream(struct sdw_intel *sdw, struct sdw_slave * int len; int i; + if (cdns->bus.bpt_stream) { + dev_err(cdns->dev, "%s: BPT stream already exists\n", __func__); + return -EAGAIN; + } + stream = sdw_alloc_stream("BPT", SDW_STREAM_BPT); if (!stream) return -ENOMEM; diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c index 9d568e385f052a..0280868f7edf5a 100644 --- a/drivers/spi/spi-amlogic-spisg.c +++ b/drivers/spi/spi-amlogic-spisg.c @@ -794,20 +794,19 @@ static int aml_spisg_probe(struct platform_device *pdev) dma_set_max_seg_size(&pdev->dev, SPISG_BLOCK_MAX); + init_completion(&spisg->completion); ret = devm_request_irq(&pdev->dev, irq, aml_spisg_irq, 0, NULL, spisg); if (ret) { dev_err(&pdev->dev, "irq request failed\n"); goto out_clk; } - ret = devm_spi_register_controller(dev, ctlr); + ret = spi_register_controller(ctlr); if (ret) { dev_err(&pdev->dev, "spi controller registration failed\n"); goto out_clk; } - init_completion(&spisg->completion); - pm_runtime_put(&spisg->pdev->dev); return 0; @@ -823,6 +822,8 @@ static void aml_spisg_remove(struct platform_device *pdev) { struct spisg_device *spisg = platform_get_drvdata(pdev); + spi_unregister_controller(spisg->controller); + if (!pm_runtime_suspended(&pdev->dev)) { pinctrl_pm_select_sleep_state(&spisg->pdev->dev); clk_disable_unprepare(spisg->core); diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c index 9c286e534bf0e3..c21323e07d3cad 100644 --- a/drivers/spi/spi-aspeed-smc.c +++ b/drivers/spi/spi-aspeed-smc.c @@ -972,7 +972,7 @@ static int aspeed_spi_probe(struct platform_device *pdev) return -ENOMEM; aspi = spi_controller_get_devdata(ctlr); - platform_set_drvdata(pdev, aspi); + platform_set_drvdata(pdev, ctlr); aspi->data = data; aspi->dev = dev; @@ -1021,7 +1021,7 @@ static int aspeed_spi_probe(struct platform_device *pdev) return ret; } - ret = devm_spi_register_controller(dev, ctlr); + ret = spi_register_controller(ctlr); if (ret) dev_err(&pdev->dev, "spi_register_controller failed\n"); @@ -1030,7 +1030,10 @@ static int aspeed_spi_probe(struct platform_device *pdev) static void aspeed_spi_remove(struct platform_device *pdev) { - struct aspeed_spi *aspi = platform_get_drvdata(pdev); + struct spi_controller *ctlr = platform_get_drvdata(pdev); + struct aspeed_spi *aspi = spi_controller_get_devdata(ctlr); + + spi_unregister_controller(ctlr); aspeed_spi_enable(aspi, false); } diff --git a/drivers/spi/spi-at91-usart.c b/drivers/spi/spi-at91-usart.c index 76eb3ba75ab18e..79edc1cd13c0f4 100644 --- a/drivers/spi/spi-at91-usart.c +++ b/drivers/spi/spi-at91-usart.c @@ -556,7 +556,7 @@ static int at91_usart_spi_probe(struct platform_device *pdev) spin_lock_init(&aus->lock); init_completion(&aus->xfer_completion); - ret = devm_spi_register_controller(&pdev->dev, controller); + ret = spi_register_controller(controller); if (ret) goto at91_usart_fail_register_controller; @@ -634,8 +634,14 @@ static void at91_usart_spi_remove(struct platform_device *pdev) struct spi_controller *ctlr = platform_get_drvdata(pdev); struct at91_usart_spi *aus = spi_controller_get_devdata(ctlr); + spi_controller_get(ctlr); + + spi_unregister_controller(ctlr); + at91_usart_spi_release_dma(ctlr); clk_disable_unprepare(aus->clk); + + spi_controller_put(ctlr); } static const struct dev_pm_ops at91_usart_spi_pm_ops = { diff --git a/drivers/spi/spi-atcspi200.c b/drivers/spi/spi-atcspi200.c index 2665f31a49ceb3..02517af9e39876 100644 --- a/drivers/spi/spi-atcspi200.c +++ b/drivers/spi/spi-atcspi200.c @@ -567,6 +567,8 @@ static int atcspi_probe(struct platform_device *pdev) spi->dev = &pdev->dev; dev_set_drvdata(&pdev->dev, host); + mutex_init(&spi->mutex_lock); + ret = atcspi_init_resources(pdev, spi, &mem_res); if (ret) goto free_controller; @@ -597,7 +599,6 @@ static int atcspi_probe(struct platform_device *pdev) else spi->use_dma = true; } - mutex_init(&spi->mutex_lock); return 0; @@ -605,6 +606,7 @@ static int atcspi_probe(struct platform_device *pdev) clk_disable_unprepare(spi->clk); free_controller: + mutex_destroy(&spi->mutex_lock); spi_controller_put(host); return ret; } diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c index 445d645585bf35..42db85d7ff8e13 100644 --- a/drivers/spi/spi-atmel.c +++ b/drivers/spi/spi-atmel.c @@ -1654,7 +1654,7 @@ static int atmel_spi_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) goto out_free_dma; @@ -1688,8 +1688,12 @@ static void atmel_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct atmel_spi *as = spi_controller_get_devdata(host); + spi_controller_get(host); + pm_runtime_get_sync(&pdev->dev); + spi_unregister_controller(host); + /* reset the hardware and block queue progress */ if (as->use_dma) { atmel_spi_stop_dma(host); @@ -1716,6 +1720,8 @@ static void atmel_spi_remove(struct platform_device *pdev) pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); + + spi_controller_put(host); } static int atmel_spi_runtime_suspend(struct device *dev) diff --git a/drivers/spi/spi-axiado.c b/drivers/spi/spi-axiado.c index 8ddcd27def22b8..2cefc8917b710b 100644 --- a/drivers/spi/spi-axiado.c +++ b/drivers/spi/spi-axiado.c @@ -201,7 +201,7 @@ static void ax_spi_fill_tx_fifo(struct ax_spi *xspi) * then spi control did't work thoroughly, add one byte delay */ if (ax_spi_read(xspi, AX_SPI_IVR) & AX_SPI_IVR_TFOV) - usleep_range(10, 10); + udelay(10); if (xspi->tx_buf) ax_spi_write_b(xspi, AX_SPI_TXFIFO, *xspi->tx_buf++); else @@ -842,7 +842,6 @@ static int ax_spi_probe(struct platform_device *pdev) ctlr->bits_per_word_mask = SPI_BPW_MASK(8); - pm_runtime_mark_last_busy(&pdev->dev); pm_runtime_put_autosuspend(&pdev->dev); ctlr->mem_ops = &ax_spi_mem_ops; diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c index 47266bb23a33cf..40cd7efc4b54ed 100644 --- a/drivers/spi/spi-bcm63xx.c +++ b/drivers/spi/spi-bcm63xx.c @@ -602,7 +602,7 @@ static int bcm63xx_spi_probe(struct platform_device *pdev) goto out_clk_disable; /* register and we are done */ - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(dev, "spi register failed\n"); goto out_clk_disable; @@ -625,11 +625,17 @@ static void bcm63xx_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct bcm63xx_spi *bs = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + /* reset spi block */ bcm_spi_writeb(bs, 0, SPI_INT_MASK); /* HW shutdown */ clk_disable_unprepare(bs->clk); + + spi_controller_put(host); } static int bcm63xx_spi_suspend(struct device *dev) diff --git a/drivers/spi/spi-bcmbca-hsspi.c b/drivers/spi/spi-bcmbca-hsspi.c index ece22260f57001..4ea8cc784e9eba 100644 --- a/drivers/spi/spi-bcmbca-hsspi.c +++ b/drivers/spi/spi-bcmbca-hsspi.c @@ -549,7 +549,7 @@ static int bcmbca_hsspi_probe(struct platform_device *pdev) } /* register and we are done */ - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) goto out_sysgroup_disable; @@ -571,6 +571,8 @@ static void bcmbca_hsspi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct bcmbca_hsspi *bs = spi_controller_get_devdata(host); + spi_unregister_controller(host); + /* reset the hardware and block queue progress */ __raw_writel(0, bs->regs + HSSPI_INT_MASK_REG); clk_disable_unprepare(bs->pll_clk); diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 2ead419e896e2d..057381e56a7fd5 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1544,10 +1544,6 @@ static bool cqspi_supports_mem_op(struct spi_mem *mem, if (op->data.nbytes && op->data.buswidth != 8) return false; - /* A single opcode is supported, it will be repeated */ - if ((op->cmd.opcode >> 8) != (op->cmd.opcode & 0xFF)) - return false; - if (cqspi->is_rzn1) return false; } else if (!all_false) { @@ -1864,14 +1860,10 @@ static int cqspi_probe(struct platform_device *pdev) if (irq < 0) return -ENXIO; - ret = pm_runtime_set_active(dev); - if (ret) - return ret; - ret = clk_bulk_prepare_enable(CLK_QSPI_NUM, cqspi->clks); if (ret) { dev_err(dev, "Cannot enable QSPI clocks.\n"); - goto disable_rpm; + return ret; } /* Obtain QSPI reset control */ @@ -1966,10 +1958,11 @@ static int cqspi_probe(struct platform_device *pdev) cqspi->sclk = 0; if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - pm_runtime_enable(dev); pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT); pm_runtime_use_autosuspend(dev); pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); } host->num_chipselect = cqspi->num_chipselect; @@ -1981,7 +1974,7 @@ static int cqspi_probe(struct platform_device *pdev) ret = cqspi_request_mmap_dma(cqspi); if (ret == -EPROBE_DEFER) { dev_err_probe(&pdev->dev, ret, "Failed to request mmap DMA\n"); - goto disable_controller; + goto disable_rpm; } } @@ -1999,14 +1992,16 @@ static int cqspi_probe(struct platform_device *pdev) release_dma_chan: if (cqspi->rx_chan) dma_release_channel(cqspi->rx_chan); -disable_controller: - cqspi_controller_enable(cqspi, 0); -disable_clks: - if (pm_runtime_get_sync(&pdev->dev) >= 0) - clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks); disable_rpm: - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + pm_runtime_put_noidle(dev); + pm_runtime_dont_use_autosuspend(dev); + } + cqspi_controller_enable(cqspi, 0); +disable_clks: + clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks); return ret; } @@ -2020,28 +2015,29 @@ static void cqspi_remove(struct platform_device *pdev) ddata = of_device_get_match_data(dev); + spi_unregister_controller(cqspi->host); + refcount_set(&cqspi->refcount, 0); if (!refcount_dec_and_test(&cqspi->inflight_ops)) cqspi_wait_idle(cqspi); - spi_unregister_controller(cqspi->host); - if (cqspi->rx_chan) dma_release_channel(cqspi->rx_chan); - cqspi_controller_enable(cqspi, 0); - - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) ret = pm_runtime_get_sync(&pdev->dev); - if (ret >= 0) + if (ret >= 0) { + cqspi_controller_enable(cqspi, 0); clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks); + } if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); } } diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c index caa7a57e6d27b7..891e2ba3695849 100644 --- a/drivers/spi/spi-cadence.c +++ b/drivers/spi/spi-cadence.c @@ -741,7 +741,6 @@ static int cdns_spi_probe(struct platform_device *pdev) /* Set to default valid value */ ctlr->max_speed_hz = xspi->clk_rate / 4; xspi->speed_hz = ctlr->max_speed_hz; - pm_runtime_put_autosuspend(&pdev->dev); } else { ctlr->mode_bits |= SPI_NO_CS; ctlr->target_abort = cdns_target_abort; @@ -752,12 +751,17 @@ static int cdns_spi_probe(struct platform_device *pdev) goto clk_dis_all; } + if (!spi_controller_is_target(ctlr)) + pm_runtime_put_autosuspend(&pdev->dev); + return ret; clk_dis_all: if (!spi_controller_is_target(ctlr)) { pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); } remove_ctlr: spi_controller_put(ctlr); @@ -776,15 +780,26 @@ static void cdns_spi_remove(struct platform_device *pdev) { struct spi_controller *ctlr = platform_get_drvdata(pdev); struct cdns_spi *xspi = spi_controller_get_devdata(ctlr); + int ret = 0; - cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_DISABLE); + if (!spi_controller_is_target(ctlr)) + ret = pm_runtime_get_sync(&pdev->dev); + + spi_controller_get(ctlr); + + spi_unregister_controller(ctlr); + + if (ret >= 0) + cdns_spi_write(xspi, CDNS_SPI_ER, CDNS_SPI_ER_DISABLE); if (!spi_controller_is_target(ctlr)) { pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); } - spi_unregister_controller(ctlr); + spi_controller_put(ctlr); } /** diff --git a/drivers/spi/spi-cavium-octeon.c b/drivers/spi/spi-cavium-octeon.c index 155085a053a109..b95bfa6a301327 100644 --- a/drivers/spi/spi-cavium-octeon.c +++ b/drivers/spi/spi-cavium-octeon.c @@ -54,7 +54,7 @@ static int octeon_spi_probe(struct platform_device *pdev) host->bits_per_word_mask = SPI_BPW_MASK(8); host->max_speed_hz = OCTEON_SPI_MAX_CLOCK_HZ; - err = devm_spi_register_controller(&pdev->dev, host); + err = spi_register_controller(host); if (err) { dev_err(&pdev->dev, "register host failed: %d\n", err); goto fail; @@ -73,8 +73,14 @@ static void octeon_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct octeon_spi *p = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + /* Clear the CSENA* and put everything in a known state. */ writeq(0, p->register_base + OCTEON_SPI_CFG(p)); + + spi_controller_put(host); } static const struct of_device_id octeon_spi_match[] = { diff --git a/drivers/spi/spi-cavium-thunderx.c b/drivers/spi/spi-cavium-thunderx.c index 99aac40a1bbac0..f1a9aa696c8741 100644 --- a/drivers/spi/spi-cavium-thunderx.c +++ b/drivers/spi/spi-cavium-thunderx.c @@ -70,7 +70,7 @@ static int thunderx_spi_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, host); - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) goto error; @@ -90,8 +90,14 @@ static void thunderx_spi_remove(struct pci_dev *pdev) if (!p) return; + spi_controller_get(host); + + spi_unregister_controller(host); + /* Put everything in a known state. */ writeq(0, p->register_base + OCTEON_SPI_CFG(p)); + + spi_controller_put(host); } static const struct pci_device_id thunderx_spi_pci_id_table[] = { diff --git a/drivers/spi/spi-ch341.c b/drivers/spi/spi-ch341.c index 79d2f9ab4ef037..3eaa8f176f63aa 100644 --- a/drivers/spi/spi-ch341.c +++ b/drivers/spi/spi-ch341.c @@ -152,7 +152,7 @@ static int ch341_probe(struct usb_interface *intf, if (ret) return ret; - ctrl = devm_spi_alloc_host(&udev->dev, sizeof(struct ch341_spi_dev)); + ctrl = devm_spi_alloc_host(&intf->dev, sizeof(struct ch341_spi_dev)); if (!ctrl) return -ENOMEM; @@ -163,7 +163,7 @@ static int ch341_probe(struct usb_interface *intf, ch341->read_pipe = usb_rcvbulkpipe(udev, usb_endpoint_num(in)); ch341->rx_len = usb_endpoint_maxp(in); - ch341->rx_buf = devm_kzalloc(&udev->dev, ch341->rx_len, GFP_KERNEL); + ch341->rx_buf = devm_kzalloc(&intf->dev, ch341->rx_len, GFP_KERNEL); if (!ch341->rx_buf) return -ENOMEM; @@ -171,19 +171,18 @@ static int ch341_probe(struct usb_interface *intf, if (!ch341->rx_urb) return -ENOMEM; - ch341->tx_buf = - devm_kzalloc(&udev->dev, CH341_PACKET_LENGTH, GFP_KERNEL); - if (!ch341->tx_buf) - return -ENOMEM; + ch341->tx_buf = devm_kzalloc(&intf->dev, CH341_PACKET_LENGTH, GFP_KERNEL); + if (!ch341->tx_buf) { + ret = -ENOMEM; + goto err_free_urb; + } usb_fill_bulk_urb(ch341->rx_urb, udev, ch341->read_pipe, ch341->rx_buf, ch341->rx_len, ch341_recv, ch341); ret = usb_submit_urb(ch341->rx_urb, GFP_KERNEL); - if (ret) { - usb_free_urb(ch341->rx_urb); - return -ENOMEM; - } + if (ret) + goto err_free_urb; ctrl->bus_num = -1; ctrl->mode_bits = SPI_CPHA; @@ -195,21 +194,34 @@ static int ch341_probe(struct usb_interface *intf, ret = ch341_config_stream(ch341); if (ret) - return ret; + goto err_kill_urb; ret = ch341_enable_pins(ch341, true); if (ret) - return ret; + goto err_kill_urb; ret = spi_register_controller(ctrl); if (ret) - return ret; + goto err_disable_pins; ch341->spidev = spi_new_device(ctrl, &chip); - if (!ch341->spidev) - return -ENOMEM; + if (!ch341->spidev) { + ret = -ENOMEM; + goto err_unregister; + } return 0; + +err_unregister: + spi_unregister_controller(ctrl); +err_disable_pins: + ch341_enable_pins(ch341, false); +err_kill_urb: + usb_kill_urb(ch341->rx_urb); +err_free_urb: + usb_free_urb(ch341->rx_urb); + + return ret; } static void ch341_disconnect(struct usb_interface *intf) @@ -219,6 +231,7 @@ static void ch341_disconnect(struct usb_interface *intf) spi_unregister_device(ch341->spidev); spi_unregister_controller(ch341->ctrl); ch341_enable_pins(ch341, false); + usb_kill_urb(ch341->rx_urb); usb_free_urb(ch341->rx_urb); } diff --git a/drivers/spi/spi-coldfire-qspi.c b/drivers/spi/spi-coldfire-qspi.c index fdf37636cb9f08..b45f44de85dcda 100644 --- a/drivers/spi/spi-coldfire-qspi.c +++ b/drivers/spi/spi-coldfire-qspi.c @@ -410,9 +410,9 @@ static int mcfqspi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, host); pm_runtime_enable(&pdev->dev); - status = devm_spi_register_controller(&pdev->dev, host); + status = spi_register_controller(host); if (status) { - dev_dbg(&pdev->dev, "devm_spi_register_controller failed\n"); + dev_dbg(&pdev->dev, "failed to register controller\n"); goto fail1; } @@ -436,11 +436,17 @@ static void mcfqspi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct mcfqspi *mcfqspi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_disable(&pdev->dev); /* disable the hardware (set the baud rate to 0) */ mcfqspi_wr_qmr(mcfqspi, MCFQSPI_QMR_MSTR); mcfqspi_cs_teardown(mcfqspi); + + spi_controller_put(host); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-dln2.c b/drivers/spi/spi-dln2.c index d90282960ab6c0..392f0d05f50820 100644 --- a/drivers/spi/spi-dln2.c +++ b/drivers/spi/spi-dln2.c @@ -758,7 +758,7 @@ static int dln2_spi_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret < 0) { dev_err(&pdev->dev, "Failed to register host\n"); goto exit_register; @@ -783,10 +783,16 @@ static void dln2_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct dln2_spi *dln2 = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_disable(&pdev->dev); if (dln2_spi_enable(dln2, false) < 0) dev_err(&pdev->dev, "Failed to disable SPI module\n"); + + spi_controller_put(host); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c index 90d5f3ea6508a5..db50018050e549 100644 --- a/drivers/spi/spi-ep93xx.c +++ b/drivers/spi/spi-ep93xx.c @@ -689,7 +689,7 @@ static int ep93xx_spi_probe(struct platform_device *pdev) /* make sure that the hardware is disabled */ writel(0, espi->mmio + SSPCR1); - error = devm_spi_register_controller(&pdev->dev, host); + error = spi_register_controller(host); if (error) { dev_err(&pdev->dev, "failed to register SPI host\n"); goto fail_free_dma; @@ -713,7 +713,13 @@ static void ep93xx_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct ep93xx_spi *espi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + ep93xx_spi_release_dma(espi); + + spi_controller_put(host); } static const struct of_device_id ep93xx_spi_of_ids[] = { diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c index 56270f8fdc1742..45b9974ae91113 100644 --- a/drivers/spi/spi-fsl-espi.c +++ b/drivers/spi/spi-fsl-espi.c @@ -718,7 +718,7 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem, pm_runtime_enable(dev); pm_runtime_get_sync(dev); - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret < 0) goto err_pm; @@ -782,7 +782,15 @@ static int of_fsl_espi_probe(struct platform_device *ofdev) static void of_fsl_espi_remove(struct platform_device *dev) { + struct spi_controller *host = platform_get_drvdata(dev); + + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_disable(&dev->dev); + + spi_controller_put(host); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c index a223b4bc6e6378..57358851029ba5 100644 --- a/drivers/spi/spi-fsl-qspi.c +++ b/drivers/spi/spi-fsl-qspi.c @@ -633,7 +633,7 @@ static int fsl_qspi_do_op(struct fsl_qspi *q, const struct spi_mem_op *op) void __iomem *base = q->iobase; int err = 0; - init_completion(&q->c); + reinit_completion(&q->c); /* * Always start the sequence at the same index since we update @@ -965,6 +965,7 @@ static int fsl_qspi_probe(struct platform_device *pdev) if (ret < 0) return ret; + init_completion(&q->c); ret = devm_request_irq(dev, ret, fsl_qspi_irq_handler, 0, pdev->name, q); if (ret) { diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c index bf3fc3ce0cc2f7..1252c41c206f87 100644 --- a/drivers/spi/spi-fsl-spi.c +++ b/drivers/spi/spi-fsl-spi.c @@ -614,7 +614,7 @@ static struct spi_controller *fsl_spi_probe(struct device *dev, mpc8xxx_spi_write_reg(®_base->mode, regval); - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret < 0) goto err_probe; @@ -705,7 +705,13 @@ static void of_fsl_spi_remove(struct platform_device *ofdev) struct spi_controller *host = platform_get_drvdata(ofdev); struct mpc8xxx_spi *mpc8xxx_spi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + fsl_spi_cpm_free(mpc8xxx_spi); + + spi_controller_put(host); } static struct platform_driver of_fsl_spi_driver = { @@ -751,7 +757,13 @@ static void plat_mpc8xxx_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct mpc8xxx_spi *mpc8xxx_spi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + fsl_spi_cpm_free(mpc8xxx_spi); + + spi_controller_put(host); } MODULE_ALIAS("platform:mpc8xxx_spi"); diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c index 216a0a91fc47db..c42d2a2cdf1e46 100644 --- a/drivers/spi/spi-hisi-kunpeng.c +++ b/drivers/spi/spi-hisi-kunpeng.c @@ -196,8 +196,18 @@ static void hisi_spi_flush_fifo(struct hisi_spi *hs) unsigned long limit = loops_per_jiffy << 1; do { - while (hisi_spi_rx_not_empty(hs)) + unsigned long inner_limit = loops_per_jiffy; + + while (hisi_spi_rx_not_empty(hs) && --inner_limit) { readl(hs->regs + HISI_SPI_DOUT); + cpu_relax(); + } + + if (!inner_limit) { + dev_warn_ratelimited(hs->dev, "RX FIFO flush timeout\n"); + break; + } + } while (hisi_spi_busy(hs) && limit--); } diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c index 902fb64815c989..57625a3ce2f2d6 100644 --- a/drivers/spi/spi-img-spfi.c +++ b/drivers/spi/spi-img-spfi.c @@ -643,7 +643,7 @@ static int img_spfi_probe(struct platform_device *pdev) pm_runtime_set_active(spfi->dev); pm_runtime_enable(spfi->dev); - ret = devm_spi_register_controller(spfi->dev, host); + ret = spi_register_controller(host); if (ret) goto disable_pm; @@ -669,6 +669,10 @@ static void img_spfi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct img_spfi *spfi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + if (spfi->tx_ch) dma_release_channel(spfi->tx_ch); if (spfi->rx_ch) @@ -679,6 +683,8 @@ static void img_spfi_remove(struct platform_device *pdev) clk_disable_unprepare(spfi->spfi_clk); clk_disable_unprepare(spfi->sys_clk); } + + spi_controller_put(host); } #ifdef CONFIG_PM diff --git a/drivers/spi/spi-imx.c b/drivers/spi/spi-imx.c index 64c6c09e1e7bff..164167b593ef42 100644 --- a/drivers/spi/spi-imx.c +++ b/drivers/spi/spi-imx.c @@ -2384,6 +2384,7 @@ static int spi_imx_probe(struct platform_device *pdev) out_runtime_pm_put: pm_runtime_dont_use_autosuspend(spi_imx->dev); pm_runtime_disable(spi_imx->dev); + pm_runtime_put_noidle(spi_imx->dev); pm_runtime_set_suspended(&pdev->dev); clk_disable_unprepare(spi_imx->clk_ipg); @@ -2401,6 +2402,8 @@ static void spi_imx_remove(struct platform_device *pdev) struct spi_imx_data *spi_imx = spi_controller_get_devdata(controller); int ret; + spi_controller_get(controller); + spi_unregister_controller(controller); ret = pm_runtime_get_sync(spi_imx->dev); @@ -2414,6 +2417,8 @@ static void spi_imx_remove(struct platform_device *pdev) pm_runtime_disable(spi_imx->dev); spi_imx_sdma_exit(spi_imx); + + spi_controller_put(controller); } static int spi_imx_runtime_resume(struct device *dev) diff --git a/drivers/spi/spi-lantiq-ssc.c b/drivers/spi/spi-lantiq-ssc.c index f83cb63c9d0c43..75b9af8cb5db71 100644 --- a/drivers/spi/spi-lantiq-ssc.c +++ b/drivers/spi/spi-lantiq-ssc.c @@ -994,7 +994,7 @@ static int lantiq_ssc_probe(struct platform_device *pdev) "Lantiq SSC SPI controller (Rev %i, TXFS %u, RXFS %u, DMA %u)\n", revision, spi->tx_fifo_size, spi->rx_fifo_size, supports_dma); - err = devm_spi_register_controller(dev, host); + err = spi_register_controller(host); if (err) { dev_err(dev, "failed to register spi host\n"); goto err_wq_destroy; @@ -1016,6 +1016,10 @@ static void lantiq_ssc_remove(struct platform_device *pdev) { struct lantiq_ssc_spi *spi = platform_get_drvdata(pdev); + spi_controller_get(spi->host); + + spi_unregister_controller(spi->host); + lantiq_ssc_writel(spi, 0, LTQ_SPI_IRNEN); lantiq_ssc_writel(spi, 0, LTQ_SPI_CLC); rx_fifo_flush(spi); @@ -1024,6 +1028,8 @@ static void lantiq_ssc_remove(struct platform_device *pdev) destroy_workqueue(spi->wq); clk_put(spi->fpi_clk); + + spi_controller_put(spi->host); } static struct platform_driver lantiq_ssc_driver = { diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c index 57768da3205d25..b80f9f457b6674 100644 --- a/drivers/spi/spi-meson-spicc.c +++ b/drivers/spi/spi-meson-spicc.c @@ -1081,7 +1081,7 @@ static int meson_spicc_probe(struct platform_device *pdev) } } - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&pdev->dev, "spi registration failed\n"); goto out_host; @@ -1099,8 +1099,14 @@ static void meson_spicc_remove(struct platform_device *pdev) { struct meson_spicc_device *spicc = platform_get_drvdata(pdev); + spi_controller_get(spicc->host); + + spi_unregister_controller(spicc->host); + /* Disable SPI */ writel(0, spicc->base + SPICC_CONREG); + + spi_controller_put(spicc->host); } static const struct meson_spicc_data meson_spicc_gx_data = { diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c index aafe6cbf2aea71..70215a407b5a32 100644 --- a/drivers/spi/spi-microchip-core-qspi.c +++ b/drivers/spi/spi-microchip-core-qspi.c @@ -74,6 +74,13 @@ #define STATUS_FLAGSX4 BIT(8) #define STATUS_MASK GENMASK(8, 0) +/* + * QSPI Direct Access register defines + */ +#define DIRECT_ACCESS_EN_SSEL BIT(0) +#define DIRECT_ACCESS_OP_SSEL BIT(1) +#define DIRECT_ACCESS_OP_SSEL_SHIFT 1 + #define BYTESUPPER_MASK GENMASK(31, 16) #define BYTESLOWER_MASK GENMASK(15, 0) @@ -158,6 +165,38 @@ static int mchp_coreqspi_set_mode(struct mchp_coreqspi *qspi, const struct spi_m return 0; } +static void mchp_coreqspi_set_cs(struct spi_device *spi, bool enable) +{ + struct mchp_coreqspi *qspi = spi_controller_get_devdata(spi->controller); + u32 val; + + val = readl(qspi->regs + REG_DIRECT_ACCESS); + + val &= ~DIRECT_ACCESS_OP_SSEL; + val |= !enable << DIRECT_ACCESS_OP_SSEL_SHIFT; + + writel(val, qspi->regs + REG_DIRECT_ACCESS); +} + +static int mchp_coreqspi_setup(struct spi_device *spi) +{ + struct mchp_coreqspi *qspi = spi_controller_get_devdata(spi->controller); + u32 val; + + /* + * Active low devices need to be specifically set to their inactive + * states during probe. + */ + if (spi->mode & SPI_CS_HIGH) + return 0; + + val = readl(qspi->regs + REG_DIRECT_ACCESS); + val |= DIRECT_ACCESS_OP_SSEL; + writel(val, qspi->regs + REG_DIRECT_ACCESS); + + return 0; +} + static inline void mchp_coreqspi_read_op(struct mchp_coreqspi *qspi) { u32 control, data; @@ -380,19 +419,6 @@ static int mchp_coreqspi_setup_clock(struct mchp_coreqspi *qspi, struct spi_devi return 0; } -static int mchp_coreqspi_setup_op(struct spi_device *spi_dev) -{ - struct spi_controller *ctlr = spi_dev->controller; - struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr); - u32 control = readl_relaxed(qspi->regs + REG_CONTROL); - - control |= (CONTROL_MASTER | CONTROL_ENABLE); - control &= ~CONTROL_CLKIDLE; - writel_relaxed(control, qspi->regs + REG_CONTROL); - - return 0; -} - static inline void mchp_coreqspi_config_op(struct mchp_coreqspi *qspi, const struct spi_mem_op *op) { u32 idle_cycles = 0; @@ -483,6 +509,7 @@ static int mchp_coreqspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *o reinit_completion(&qspi->data_completion); mchp_coreqspi_config_op(qspi, op); + mchp_coreqspi_set_cs(mem->spi, true); if (op->cmd.opcode) { qspi->txbuf = &opcode; qspi->rxbuf = NULL; @@ -523,6 +550,7 @@ static int mchp_coreqspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *o err = -ETIMEDOUT; error: + mchp_coreqspi_set_cs(mem->spi, false); mutex_unlock(&qspi->op_lock); mchp_coreqspi_disable_ints(qspi); @@ -662,18 +690,28 @@ static int mchp_coreqspi_transfer_one(struct spi_controller *ctlr, struct spi_de struct spi_transfer *t) { struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr); + bool dual_quad = false; qspi->tx_len = t->len; + if (t->tx_nbits == SPI_NBITS_QUAD || t->rx_nbits == SPI_NBITS_QUAD || + t->tx_nbits == SPI_NBITS_DUAL || + t->rx_nbits == SPI_NBITS_DUAL) + dual_quad = true; + if (t->tx_buf) qspi->txbuf = (u8 *)t->tx_buf; if (!t->rx_buf) { mchp_coreqspi_write_op(qspi); - } else { + } else if (!dual_quad) { qspi->rxbuf = (u8 *)t->rx_buf; qspi->rx_len = t->len; mchp_coreqspi_write_read_op(qspi); + } else { + qspi->rxbuf = (u8 *)t->rx_buf; + qspi->rx_len = t->len; + mchp_coreqspi_read_op(qspi); } return 0; @@ -686,13 +724,14 @@ static int mchp_coreqspi_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; int ret; + u32 num_cs, val; ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*qspi)); if (!ctlr) return -ENOMEM; qspi = spi_controller_get_devdata(ctlr); - platform_set_drvdata(pdev, qspi); + platform_set_drvdata(pdev, ctlr); qspi->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qspi->regs)) @@ -718,10 +757,18 @@ static int mchp_coreqspi_probe(struct platform_device *pdev) return ret; } + /* + * The IP core only has a single CS, any more have to be provided via + * gpios + */ + if (of_property_read_u32(pdev->dev.of_node, "num-cs", &num_cs)) + num_cs = 1; + + ctlr->num_chipselect = num_cs; + ctlr->bits_per_word_mask = SPI_BPW_MASK(8); ctlr->mem_ops = &mchp_coreqspi_mem_ops; ctlr->mem_caps = &mchp_coreqspi_mem_caps; - ctlr->setup = mchp_coreqspi_setup_op; ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_DUAL | SPI_TX_QUAD; ctlr->dev.of_node = np; @@ -729,10 +776,22 @@ static int mchp_coreqspi_probe(struct platform_device *pdev) ctlr->prepare_message = mchp_coreqspi_prepare_message; ctlr->unprepare_message = mchp_coreqspi_unprepare_message; ctlr->transfer_one = mchp_coreqspi_transfer_one; - ctlr->num_chipselect = 2; + ctlr->setup = mchp_coreqspi_setup; + ctlr->set_cs = mchp_coreqspi_set_cs; ctlr->use_gpio_descriptors = true; - ret = devm_spi_register_controller(&pdev->dev, ctlr); + val = readl_relaxed(qspi->regs + REG_CONTROL); + val |= (CONTROL_MASTER | CONTROL_ENABLE); + writel_relaxed(val, qspi->regs + REG_CONTROL); + + /* + * Put cs into software controlled mode + */ + val = readl_relaxed(qspi->regs + REG_DIRECT_ACCESS); + val |= DIRECT_ACCESS_EN_SSEL; + writel(val, qspi->regs + REG_DIRECT_ACCESS); + + ret = spi_register_controller(ctlr); if (ret) return dev_err_probe(&pdev->dev, ret, "spi_register_controller failed\n"); @@ -742,9 +801,13 @@ static int mchp_coreqspi_probe(struct platform_device *pdev) static void mchp_coreqspi_remove(struct platform_device *pdev) { - struct mchp_coreqspi *qspi = platform_get_drvdata(pdev); - u32 control = readl_relaxed(qspi->regs + REG_CONTROL); + struct spi_controller *ctlr = platform_get_drvdata(pdev); + struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr); + u32 control; + spi_unregister_controller(ctlr); + + control = readl_relaxed(qspi->regs + REG_CONTROL); mchp_coreqspi_disable_ints(qspi); control &= ~CONTROL_ENABLE; writel_relaxed(control, qspi->regs + REG_CONTROL); diff --git a/drivers/spi/spi-microchip-core-spi.c b/drivers/spi/spi-microchip-core-spi.c index a4c128ae391b5d..be01c178e2b08f 100644 --- a/drivers/spi/spi-microchip-core-spi.c +++ b/drivers/spi/spi-microchip-core-spi.c @@ -384,7 +384,7 @@ static int mchp_corespi_probe(struct platform_device *pdev) mchp_corespi_init(host, spi); - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) { mchp_corespi_disable_ints(spi); mchp_corespi_disable(spi); @@ -399,6 +399,8 @@ static void mchp_corespi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct mchp_corespi *spi = spi_controller_get_devdata(host); + spi_unregister_controller(host); + mchp_corespi_disable_ints(spi); mchp_corespi_disable(spi); } diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c index 05bbd3795e7d80..924d820448fb4a 100644 --- a/drivers/spi/spi-mpc52xx.c +++ b/drivers/spi/spi-mpc52xx.c @@ -498,6 +498,9 @@ static int mpc52xx_spi_probe(struct platform_device *op) err_register: dev_err(&ms->host->dev, "initialization failed\n"); + free_irq(ms->irq0, ms); + free_irq(ms->irq1, ms); + cancel_work_sync(&ms->work); err_gpio: while (i-- > 0) gpiod_put(ms->gpio_cs[i]); @@ -517,15 +520,17 @@ static void mpc52xx_spi_remove(struct platform_device *op) struct mpc52xx_spi *ms = spi_controller_get_devdata(host); int i; - cancel_work_sync(&ms->work); + spi_unregister_controller(host); + free_irq(ms->irq0, ms); free_irq(ms->irq1, ms); + cancel_work_sync(&ms->work); + for (i = 0; i < ms->gpio_cs_count; i++) gpiod_put(ms->gpio_cs[i]); kfree(ms->gpio_cs); - spi_unregister_controller(host); iounmap(ms->regs); spi_controller_put(host); } diff --git a/drivers/spi/spi-mpfs.c b/drivers/spi/spi-mpfs.c index 64d15a6188ac7f..989a379b07008e 100644 --- a/drivers/spi/spi-mpfs.c +++ b/drivers/spi/spi-mpfs.c @@ -574,7 +574,7 @@ static int mpfs_spi_probe(struct platform_device *pdev) mpfs_spi_init(host, spi); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { mpfs_spi_disable_ints(spi); mpfs_spi_disable(spi); @@ -592,6 +592,8 @@ static void mpfs_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct mpfs_spi *spi = spi_controller_get_devdata(host); + spi_unregister_controller(host); + mpfs_spi_disable_ints(spi); mpfs_spi_disable(spi); } diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 0368a26bca9ab7..96f8555be98346 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -1325,7 +1325,7 @@ static int mtk_spi_probe(struct platform_device *pdev) pm_runtime_enable(dev); - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) { pm_runtime_disable(dev); return dev_err_probe(dev, ret, "failed to register host\n"); @@ -1340,6 +1340,8 @@ static void mtk_spi_remove(struct platform_device *pdev) struct mtk_spi *mdata = spi_controller_get_devdata(host); int ret; + spi_unregister_controller(host); + cpu_latency_qos_remove_request(&mdata->qos_request); if (mdata->use_spimem && !completion_done(&mdata->spimem_done)) complete(&mdata->spimem_done); diff --git a/drivers/spi/spi-mtk-nor.c b/drivers/spi/spi-mtk-nor.c index 1e5ec084017499..74f34537b02c9e 100644 --- a/drivers/spi/spi-mtk-nor.c +++ b/drivers/spi/spi-mtk-nor.c @@ -913,7 +913,7 @@ static int mtk_nor_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); pm_runtime_get_noresume(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); if (ret < 0) goto err_probe; @@ -938,6 +938,8 @@ static void mtk_nor_remove(struct platform_device *pdev) struct spi_controller *ctlr = dev_get_drvdata(&pdev->dev); struct mtk_nor *sp = spi_controller_get_devdata(ctlr); + spi_unregister_controller(ctlr); + pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); diff --git a/drivers/spi/spi-mtk-snfi.c b/drivers/spi/spi-mtk-snfi.c index 437edbd658aa2f..73fa84475f0e44 100644 --- a/drivers/spi/spi-mtk-snfi.c +++ b/drivers/spi/spi-mtk-snfi.c @@ -1303,6 +1303,13 @@ static const struct spi_controller_mem_caps mtk_snand_mem_caps = { .ecc = true, }; +static void mtk_unregister_ecc_engine(void *data) +{ + struct nand_ecc_engine *eng = data; + + nand_ecc_unregister_on_host_hw_engine(eng); +} + static irqreturn_t mtk_snand_irq(int irq, void *id) { struct mtk_snand *snf = id; @@ -1443,6 +1450,13 @@ static int mtk_snand_probe(struct platform_device *pdev) goto release_ecc; } + ret = devm_add_action_or_reset(&pdev->dev, mtk_unregister_ecc_engine, + &ms->ecc_eng); + if (ret) { + dev_err_probe(&pdev->dev, ret, "failed to add ECC unregister action\n"); + goto release_ecc; + } + ctlr->num_chipselect = 1; ctlr->mem_ops = &mtk_snand_mem_ops; ctlr->mem_caps = &mtk_snand_mem_caps; diff --git a/drivers/spi/spi-mxic.c b/drivers/spi/spi-mxic.c index f9369c69911c22..b0e7fc828a5098 100644 --- a/drivers/spi/spi-mxic.c +++ b/drivers/spi/spi-mxic.c @@ -832,9 +832,10 @@ static void mxic_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct mxic_spi *mxic = spi_controller_get_devdata(host); + spi_unregister_controller(host); + pm_runtime_disable(&pdev->dev); mxic_spi_mem_ecc_remove(mxic); - spi_unregister_controller(host); } static const struct of_device_id mxic_spi_of_ids[] = { diff --git a/drivers/spi/spi-mxs.c b/drivers/spi/spi-mxs.c index b3301c69e2deb2..0164e04d59a102 100644 --- a/drivers/spi/spi-mxs.c +++ b/drivers/spi/spi-mxs.c @@ -619,7 +619,7 @@ static int mxs_spi_probe(struct platform_device *pdev) if (ret) goto out_pm_runtime_put; - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&pdev->dev, "Cannot register SPI host, %d\n", ret); goto out_pm_runtime_put; @@ -650,11 +650,17 @@ static void mxs_spi_remove(struct platform_device *pdev) spi = spi_controller_get_devdata(host); ssp = &spi->ssp; + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) mxs_spi_runtime_suspend(&pdev->dev); dma_release_channel(ssp->dmach); + + spi_controller_put(host); } static struct platform_driver mxs_spi_driver = { diff --git a/drivers/spi/spi-npcm-pspi.c b/drivers/spi/spi-npcm-pspi.c index e60b3cc398ec3c..cffef0a5977d07 100644 --- a/drivers/spi/spi-npcm-pspi.c +++ b/drivers/spi/spi-npcm-pspi.c @@ -413,7 +413,7 @@ static int npcm_pspi_probe(struct platform_device *pdev) /* set to default clock rate */ npcm_pspi_set_baudrate(priv, NPCM_PSPI_DEFAULT_CLK); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) goto out_disable_clk; @@ -434,8 +434,14 @@ static void npcm_pspi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct npcm_pspi *priv = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + npcm_pspi_reset_hw(priv); clk_disable_unprepare(priv->clk); + + spi_controller_put(host); } static const struct of_device_id npcm_pspi_match[] = { diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index 320b3d93df5711..1e36ae084dd864 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -996,7 +996,7 @@ static int nxp_fspi_do_op(struct nxp_fspi *f, const struct spi_mem_op *op) reg = reg | FSPI_IPRXFCR_CLR; fspi_writel(f, reg, base + FSPI_IPRXFCR); - init_completion(&f->c); + reinit_completion(&f->c); fspi_writel(f, op->addr.val, base + FSPI_IPCR0); /* @@ -1365,6 +1365,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) if (ret < 0) return dev_err_probe(dev, ret, "Failed to disable clock"); + init_completion(&f->c); ret = devm_request_irq(dev, irq, nxp_fspi_irq_handler, 0, pdev->name, f); if (ret) diff --git a/drivers/spi/spi-nxp-xspi.c b/drivers/spi/spi-nxp-xspi.c index 06fcdf22990b86..385302a6e62f2f 100644 --- a/drivers/spi/spi-nxp-xspi.c +++ b/drivers/spi/spi-nxp-xspi.c @@ -958,7 +958,7 @@ static int nxp_xspi_do_op(struct nxp_xspi *xspi, const struct spi_mem_op *op) writel(reg, base + XSPI_RBCT); } - init_completion(&xspi->c); + reinit_completion(&xspi->c); /* Config the data address */ writel(op->addr.val + xspi->memmap_phy, base + XSPI_SFP_TG_SFAR); @@ -1273,6 +1273,7 @@ static int nxp_xspi_probe(struct platform_device *pdev) nxp_xspi_default_setup(xspi); + init_completion(&xspi->c); ret = devm_request_irq(dev, irq, nxp_xspi_irq_handler, 0, pdev->name, xspi); if (ret) diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c index 2207e05c9d062b..cd6d9bf9eaa4ce 100644 --- a/drivers/spi/spi-omap2-mcspi.c +++ b/drivers/spi/spi-omap2-mcspi.c @@ -1585,7 +1585,7 @@ static int omap2_mcspi_probe(struct platform_device *pdev) if (status < 0) goto disable_pm; - status = devm_spi_register_controller(&pdev->dev, ctlr); + status = spi_register_controller(ctlr); if (status < 0) goto disable_pm; @@ -1606,11 +1606,17 @@ static void omap2_mcspi_remove(struct platform_device *pdev) struct spi_controller *ctlr = platform_get_drvdata(pdev); struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr); + spi_controller_get(ctlr); + + spi_unregister_controller(ctlr); + omap2_mcspi_release_dma(ctlr); pm_runtime_dont_use_autosuspend(mcspi->dev); pm_runtime_put_sync(mcspi->dev); pm_runtime_disable(&pdev->dev); + + spi_controller_put(ctlr); } /* work with hotplug and coldplug */ diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c index 7a2186b51b4ccf..a5ce970ff5a8c3 100644 --- a/drivers/spi/spi-orion.c +++ b/drivers/spi/spi-orion.c @@ -774,6 +774,7 @@ static int orion_spi_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_use_autosuspend(&pdev->dev); pm_runtime_set_autosuspend_delay(&pdev->dev, SPI_AUTOSUSPEND_TIMEOUT); + pm_runtime_get_noresume(&pdev->dev); pm_runtime_enable(&pdev->dev); status = orion_spi_reset(spi); @@ -784,10 +785,15 @@ static int orion_spi_probe(struct platform_device *pdev) if (status < 0) goto out_rel_pm; + pm_runtime_put_autosuspend(&pdev->dev); + return status; out_rel_pm: pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); out_rel_axi_clk: clk_disable_unprepare(spi->axi_clk); out: @@ -801,11 +807,19 @@ static void orion_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct orion_spi *spi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_get_sync(&pdev->dev); clk_disable_unprepare(spi->axi_clk); - spi_unregister_controller(host); + spi_controller_put(host); + pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); } MODULE_ALIAS("platform:" DRIVER_NAME); diff --git a/drivers/spi/spi-pic32-sqi.c b/drivers/spi/spi-pic32-sqi.c index 0515900388952e..41662992dbe50c 100644 --- a/drivers/spi/spi-pic32-sqi.c +++ b/drivers/spi/spi-pic32-sqi.c @@ -642,7 +642,7 @@ static int pic32_sqi_probe(struct platform_device *pdev) host->prepare_transfer_hardware = pic32_sqi_prepare_hardware; host->unprepare_transfer_hardware = pic32_sqi_unprepare_hardware; - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&host->dev, "failed registering spi host\n"); free_irq(sqi->irq, sqi); @@ -665,9 +665,15 @@ static void pic32_sqi_remove(struct platform_device *pdev) { struct pic32_sqi *sqi = platform_get_drvdata(pdev); + spi_controller_get(sqi->host); + + spi_unregister_controller(sqi->host); + /* release resources */ free_irq(sqi->irq, sqi); ring_desc_ring_free(sqi); + + spi_controller_put(sqi->host); } static const struct of_device_id pic32_sqi_of_ids[] = { diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c index 369850d1431367..70427e5299456d 100644 --- a/drivers/spi/spi-pic32.c +++ b/drivers/spi/spi-pic32.c @@ -821,7 +821,7 @@ static int pic32_spi_probe(struct platform_device *pdev) } /* register host */ - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&host->dev, "failed registering spi host\n"); goto err_bailout; @@ -840,11 +840,16 @@ static int pic32_spi_probe(struct platform_device *pdev) static void pic32_spi_remove(struct platform_device *pdev) { - struct pic32_spi *pic32s; + struct pic32_spi *pic32s = platform_get_drvdata(pdev); + + spi_controller_get(pic32s->host); + + spi_unregister_controller(pic32s->host); - pic32s = platform_get_drvdata(pdev); pic32_spi_disable(pic32s); pic32_spi_dma_unprep(pic32s); + + spi_controller_put(pic32s->host); } static const struct of_device_id pic32_spi_of_match[] = { diff --git a/drivers/spi/spi-pl022.c b/drivers/spi/spi-pl022.c index c82cc522776ded..e97ffcd13c62eb 100644 --- a/drivers/spi/spi-pl022.c +++ b/drivers/spi/spi-pl022.c @@ -1956,7 +1956,7 @@ static int pl022_probe(struct amba_device *adev, const struct amba_id *id) /* Register with the SPI framework */ amba_set_drvdata(adev, pl022); - status = devm_spi_register_controller(&adev->dev, host); + status = spi_register_controller(host); if (status != 0) { dev_err_probe(&adev->dev, status, "problem registering spi host\n"); @@ -1997,6 +1997,10 @@ pl022_remove(struct amba_device *adev) if (!pl022) return; + spi_controller_get(pl022->host); + + spi_unregister_controller(pl022->host); + /* * undo pm_runtime_put() in probe. I assume that we're not * accessing the primecell here. @@ -2008,6 +2012,8 @@ pl022_remove(struct amba_device *adev) pl022_dma_remove(pl022); amba_release_regions(adev); + + spi_controller_put(pl022->host); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c index 6cbdcd060e8cab..45d9b4cb75e4aa 100644 --- a/drivers/spi/spi-qup.c +++ b/drivers/spi/spi-qup.c @@ -1193,7 +1193,7 @@ static int spi_qup_probe(struct platform_device *pdev) pm_runtime_set_active(dev); pm_runtime_enable(dev); - ret = devm_spi_register_controller(dev, host); + ret = spi_register_controller(host); if (ret) goto disable_pm; @@ -1320,6 +1320,10 @@ static void spi_qup_remove(struct platform_device *pdev) struct spi_qup *controller = spi_controller_get_devdata(host); int ret; + spi_controller_get(host); + + spi_unregister_controller(host); + ret = pm_runtime_get_sync(&pdev->dev); if (ret >= 0) { @@ -1339,6 +1343,8 @@ static void spi_qup_remove(struct platform_device *pdev) pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); + + spi_controller_put(host); } static const struct of_device_id spi_qup_dt_match[] = { diff --git a/drivers/spi/spi-rockchip.c b/drivers/spi/spi-rockchip.c index fd2ebef4903f23..2eb9ede8cc9095 100644 --- a/drivers/spi/spi-rockchip.c +++ b/drivers/spi/spi-rockchip.c @@ -357,7 +357,8 @@ static irqreturn_t rockchip_spi_isr(int irq, void *dev_id) struct rockchip_spi *rs = spi_controller_get_devdata(ctlr); /* When int_cs_inactive comes, spi target abort */ - if (rs->cs_inactive && readl_relaxed(rs->regs + ROCKCHIP_SPI_IMR) & INT_CS_INACTIVE) { + if (rs->cs_inactive && + (readl_relaxed(rs->regs + ROCKCHIP_SPI_ISR) & INT_CS_INACTIVE)) { ctlr->target_abort(ctlr); writel_relaxed(0, rs->regs + ROCKCHIP_SPI_IMR); writel_relaxed(0xffffffff, rs->regs + ROCKCHIP_SPI_ICR); @@ -908,7 +909,7 @@ static int rockchip_spi_probe(struct platform_device *pdev) break; } - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); if (ret < 0) { dev_err(&pdev->dev, "Failed to register controller\n"); goto err_free_dma_rx; @@ -936,6 +937,8 @@ static void rockchip_spi_remove(struct platform_device *pdev) pm_runtime_get_sync(&pdev->dev); + spi_unregister_controller(ctlr); + pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); pm_runtime_set_suspended(&pdev->dev); diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c index c739c1998b4ce6..a8180dece716f3 100644 --- a/drivers/spi/spi-rspi.c +++ b/drivers/spi/spi-rspi.c @@ -1171,8 +1171,14 @@ static void rspi_remove(struct platform_device *pdev) { struct rspi_data *rspi = platform_get_drvdata(pdev); + spi_controller_get(rspi->ctlr); + + spi_unregister_controller(rspi->ctlr); + rspi_release_dma(rspi->ctlr); pm_runtime_disable(&pdev->dev); + + spi_controller_put(rspi->ctlr); } static const struct spi_ops rspi_ops = { @@ -1376,9 +1382,9 @@ static int rspi_probe(struct platform_device *pdev) if (ret < 0) dev_warn(&pdev->dev, "DMA not available, using PIO\n"); - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); if (ret < 0) { - dev_err(&pdev->dev, "devm_spi_register_controller error.\n"); + dev_err(&pdev->dev, "failed to register controller\n"); goto error3; } diff --git a/drivers/spi/spi-rzv2h-rspi.c b/drivers/spi/spi-rzv2h-rspi.c index 23f0e92ae208e2..53c44799fab71d 100644 --- a/drivers/spi/spi-rzv2h-rspi.c +++ b/drivers/spi/spi-rzv2h-rspi.c @@ -50,7 +50,6 @@ /* Register SPBR */ #define RSPI_SPBR_SPR_MIN 0 -#define RSPI_SPBR_SPR_PCLK_MIN 1 #define RSPI_SPBR_SPR_MAX 255 /* Register SPCMD */ @@ -533,6 +532,17 @@ static void rzv2h_rspi_find_rate_fixed(struct clk *clk, u32 hz, for (brdv = RSPI_SPCMD_BRDV_MIN; brdv <= RSPI_SPCMD_BRDV_MAX; brdv++) { spr = DIV_ROUND_UP(clk_rate, hz * (1 << (brdv + 1))); spr--; + /* + * Skip SPR=0 and BRDV=0 as it is not a valid combination: + * - On RZ/G3E, RZ/G3L, RZ/V2H(P) and RZ/V2N, RSPI_n_TCLK is + * fixed at 200MHz and SPR=0 and BRDV=0 results in the maximum + * bit rate of 100Mbps which is prohibited. + * - On RZ/T2H and RZ/N2H, when PCLK (125MHz) is used as + * the clock source, SPR=0 and BRDV=0 is explicitly listed + * as unsupported in the hardware manual (Table 36.7). + */ + if (!spr && !brdv) + continue; if (spr >= spr_min && spr <= spr_max) goto clock_found; } @@ -566,16 +576,12 @@ static u32 rzv2h_rspi_setup_clock(struct rzv2h_rspi_priv *rspi, u32 hz) rspi->info->find_tclk_rate(rspi->tclk, hz, RSPI_SPBR_SPR_MIN, RSPI_SPBR_SPR_MAX, &best_clock); - /* - * T2H and N2H can also use PCLK as a source, which is 125MHz, but not - * when both SPR and BRDV are 0. - */ if (best_clock.error && rspi->info->find_pclk_rate) - rspi->info->find_pclk_rate(rspi->pclk, hz, RSPI_SPBR_SPR_PCLK_MIN, + rspi->info->find_pclk_rate(rspi->pclk, hz, RSPI_SPBR_SPR_MIN, RSPI_SPBR_SPR_MAX, &best_clock); if (!best_clock.clk_rate) - return -EINVAL; + return 0; ret = clk_set_rate(best_clock.clk, best_clock.clk_rate); if (ret) diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c index ba85243d6d89c5..37176e557099d1 100644 --- a/drivers/spi/spi-s3c64xx.c +++ b/drivers/spi/spi-s3c64xx.c @@ -1369,7 +1369,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev) S3C64XX_SPI_INT_TX_OVERRUN_EN | S3C64XX_SPI_INT_TX_UNDERRUN_EN, sdd->regs + S3C64XX_SPI_INT_EN); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret != 0) { dev_err(&pdev->dev, "cannot register SPI host: %d\n", ret); goto err_pm_put; @@ -1399,12 +1399,9 @@ static void s3c64xx_spi_remove(struct platform_device *pdev) pm_runtime_get_sync(&pdev->dev); - writel(0, sdd->regs + S3C64XX_SPI_INT_EN); + spi_unregister_controller(host); - if (!is_polling(sdd)) { - dma_release_channel(sdd->rx_dma.ch); - dma_release_channel(sdd->tx_dma.ch); - } + writel(0, sdd->regs + S3C64XX_SPI_INT_EN); pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/spi/spi-sh-hspi.c b/drivers/spi/spi-sh-hspi.c index e03eaca1b1a7c4..1e3ca718ca736c 100644 --- a/drivers/spi/spi-sh-hspi.c +++ b/drivers/spi/spi-sh-hspi.c @@ -257,9 +257,9 @@ static int hspi_probe(struct platform_device *pdev) ctlr->transfer_one_message = hspi_transfer_one_message; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); if (ret < 0) { - dev_err(&pdev->dev, "devm_spi_register_controller error.\n"); + dev_err(&pdev->dev, "failed to register controller\n"); goto error2; } @@ -279,9 +279,15 @@ static void hspi_remove(struct platform_device *pdev) { struct hspi_priv *hspi = platform_get_drvdata(pdev); + spi_controller_get(hspi->ctlr); + + spi_unregister_controller(hspi->ctlr); + pm_runtime_disable(&pdev->dev); clk_put(hspi->clk); + + spi_controller_put(hspi->ctlr); } static const struct of_device_id hspi_of_match[] = { diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c index 7f3e0881056090..f114b6313f4f65 100644 --- a/drivers/spi/spi-sh-msiof.c +++ b/drivers/spi/spi-sh-msiof.c @@ -1289,9 +1289,9 @@ static int sh_msiof_spi_probe(struct platform_device *pdev) if (ret < 0) dev_warn(dev, "DMA not available, using PIO\n"); - ret = devm_spi_register_controller(dev, ctlr); + ret = spi_register_controller(ctlr); if (ret < 0) { - dev_err(dev, "devm_spi_register_controller error.\n"); + dev_err(dev, "failed to register controller\n"); goto err2; } @@ -1309,8 +1309,14 @@ static void sh_msiof_spi_remove(struct platform_device *pdev) { struct sh_msiof_spi_priv *p = platform_get_drvdata(pdev); + spi_controller_get(p->ctlr); + + spi_unregister_controller(p->ctlr); + sh_msiof_release_dma(p); pm_runtime_disable(&pdev->dev); + + spi_controller_put(p->ctlr); } static const struct platform_device_id spi_driver_ids[] = { diff --git a/drivers/spi/spi-sifive.c b/drivers/spi/spi-sifive.c index 6c7aba8befa07d..74a3e32fd2b5d8 100644 --- a/drivers/spi/spi-sifive.c +++ b/drivers/spi/spi-sifive.c @@ -312,7 +312,8 @@ static int sifive_spi_probe(struct platform_device *pdev) goto put_host; } - spi->clk = devm_clk_get(&pdev->dev, NULL); + /* Spin up the bus clock before hitting registers */ + spi->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(spi->clk)) { dev_err(&pdev->dev, "Unable to find bus clock\n"); ret = PTR_ERR(spi->clk); @@ -342,13 +343,6 @@ static int sifive_spi_probe(struct platform_device *pdev) goto put_host; } - /* Spin up the bus clock before hitting registers */ - ret = clk_prepare_enable(spi->clk); - if (ret) { - dev_err(&pdev->dev, "Unable to enable bus clock\n"); - goto put_host; - } - /* probe the number of CS lines */ spi->cs_inactive = sifive_spi_read(spi, SIFIVE_SPI_REG_CSDEF); sifive_spi_write(spi, SIFIVE_SPI_REG_CSDEF, 0xffffffffU); @@ -357,14 +351,14 @@ static int sifive_spi_probe(struct platform_device *pdev) if (!cs_bits) { dev_err(&pdev->dev, "Could not auto probe CS lines\n"); ret = -EINVAL; - goto disable_clk; + goto put_host; } num_cs = ilog2(cs_bits) + 1; if (num_cs > SIFIVE_SPI_MAX_CS) { dev_err(&pdev->dev, "Invalid number of spi targets\n"); ret = -EINVAL; - goto disable_clk; + goto put_host; } /* Define our host */ @@ -392,22 +386,20 @@ static int sifive_spi_probe(struct platform_device *pdev) dev_name(&pdev->dev), spi); if (ret) { dev_err(&pdev->dev, "Unable to bind to interrupt\n"); - goto disable_clk; + goto put_host; } dev_info(&pdev->dev, "mapped; irq=%d, cs=%d\n", irq, host->num_chipselect); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret < 0) { dev_err(&pdev->dev, "spi_register_host failed\n"); - goto disable_clk; + goto put_host; } return 0; -disable_clk: - clk_disable_unprepare(spi->clk); put_host: spi_controller_put(host); @@ -419,9 +411,14 @@ static void sifive_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct sifive_spi *spi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + /* Disable all the interrupts just in case */ sifive_spi_write(spi, SIFIVE_SPI_REG_IE, 0); - clk_disable_unprepare(spi->clk); + + spi_controller_put(host); } static int sifive_spi_suspend(struct device *dev) diff --git a/drivers/spi/spi-slave-mt27xx.c b/drivers/spi/spi-slave-mt27xx.c index ce889cb33228f3..7aedeaa5889db2 100644 --- a/drivers/spi/spi-slave-mt27xx.c +++ b/drivers/spi/spi-slave-mt27xx.c @@ -453,7 +453,7 @@ static int mtk_spi_slave_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); clk_disable_unprepare(mdata->spi_clk); if (ret) { dev_err(&pdev->dev, @@ -473,7 +473,15 @@ static int mtk_spi_slave_probe(struct platform_device *pdev) static void mtk_spi_slave_remove(struct platform_device *pdev) { + struct spi_controller *ctlr = platform_get_drvdata(pdev); + + spi_controller_get(ctlr); + + spi_unregister_controller(ctlr); + pm_runtime_disable(&pdev->dev); + + spi_controller_put(ctlr); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-sprd.c b/drivers/spi/spi-sprd.c index 0f9fc320363cd2..fd3fd0ce122c93 100644 --- a/drivers/spi/spi-sprd.c +++ b/drivers/spi/spi-sprd.c @@ -977,7 +977,7 @@ static int sprd_spi_probe(struct platform_device *pdev) goto err_rpm_put; } - ret = devm_spi_register_controller(&pdev->dev, sctlr); + ret = spi_register_controller(sctlr); if (ret) goto err_rpm_put; @@ -1008,7 +1008,9 @@ static void sprd_spi_remove(struct platform_device *pdev) if (ret < 0) dev_err(ss->dev, "failed to resume SPI controller\n"); - spi_controller_suspend(sctlr); + spi_controller_get(sctlr); + + spi_unregister_controller(sctlr); if (ret >= 0) { if (ss->dma.enable) @@ -1017,6 +1019,8 @@ static void sprd_spi_remove(struct platform_device *pdev) } pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); + + spi_controller_put(sctlr); } static int __maybe_unused sprd_spi_runtime_suspend(struct device *dev) diff --git a/drivers/spi/spi-st-ssc4.c b/drivers/spi/spi-st-ssc4.c index b173ef70d77ebd..9c8099fe6e19ff 100644 --- a/drivers/spi/spi-st-ssc4.c +++ b/drivers/spi/spi-st-ssc4.c @@ -349,7 +349,7 @@ static int spi_st_probe(struct platform_device *pdev) platform_set_drvdata(pdev, host); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&pdev->dev, "Failed to register host\n"); goto rpm_disable; @@ -371,10 +371,16 @@ static void spi_st_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct spi_st *spi_st = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_disable(&pdev->dev); clk_disable_unprepare(spi_st->clk); + spi_controller_put(host); + pinctrl_pm_select_sleep_state(&pdev->dev); } diff --git a/drivers/spi/spi-sun4i.c b/drivers/spi/spi-sun4i.c index bfdf419a583c17..b7fbb5270edbc7 100644 --- a/drivers/spi/spi-sun4i.c +++ b/drivers/spi/spi-sun4i.c @@ -504,7 +504,7 @@ static int sun4i_spi_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); pm_runtime_idle(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&pdev->dev, "cannot register SPI host\n"); goto err_pm_disable; @@ -522,7 +522,15 @@ static int sun4i_spi_probe(struct platform_device *pdev) static void sun4i_spi_remove(struct platform_device *pdev) { + struct spi_controller *host = platform_get_drvdata(pdev); + + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_force_suspend(&pdev->dev); + + spi_controller_put(host); } static const struct of_device_id sun4i_spi_match[] = { diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c index 240e46f84f7b34..5ac73d324d0630 100644 --- a/drivers/spi/spi-sun6i.c +++ b/drivers/spi/spi-sun6i.c @@ -742,7 +742,7 @@ static int sun6i_spi_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) { dev_err(&pdev->dev, "cannot register SPI host\n"); goto err_pm_disable; @@ -768,12 +768,18 @@ static void sun6i_spi_remove(struct platform_device *pdev) { struct spi_controller *host = platform_get_drvdata(pdev); + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_force_suspend(&pdev->dev); if (host->dma_tx) dma_release_channel(host->dma_tx); if (host->dma_rx) dma_release_channel(host->dma_rx); + + spi_controller_put(host); } static const struct sun6i_spi_cfg sun6i_a31_spi_cfg = { diff --git a/drivers/spi/spi-synquacer.c b/drivers/spi/spi-synquacer.c index d0a875249910da..290c439897c4ff 100644 --- a/drivers/spi/spi-synquacer.c +++ b/drivers/spi/spi-synquacer.c @@ -716,7 +716,7 @@ static int synquacer_spi_probe(struct platform_device *pdev) pm_runtime_set_active(sspi->dev); pm_runtime_enable(sspi->dev); - ret = devm_spi_register_controller(sspi->dev, host); + ret = spi_register_controller(host); if (ret) goto disable_pm; @@ -737,9 +737,15 @@ static void synquacer_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct synquacer_spi *sspi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + pm_runtime_disable(sspi->dev); clk_disable_unprepare(sspi->clk); + + spi_controller_put(host); } static int __maybe_unused synquacer_spi_suspend(struct device *dev) diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c index 848cb6836bd5e2..b8b0ebe0fe931d 100644 --- a/drivers/spi/spi-tegra114.c +++ b/drivers/spi/spi-tegra114.c @@ -1415,7 +1415,7 @@ static int tegra_spi_probe(struct platform_device *pdev) goto exit_pm_disable; } - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret < 0) { dev_err(&pdev->dev, "can not register to host err %d\n", ret); goto exit_free_irq; @@ -1441,6 +1441,10 @@ static void tegra_spi_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct tegra_spi_data *tspi = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + free_irq(tspi->irq, tspi); if (tspi->tx_dma_chan) @@ -1452,6 +1456,8 @@ static void tegra_spi_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) tegra_spi_runtime_suspend(&pdev->dev); + + spi_controller_put(host); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-tegra20-sflash.c b/drivers/spi/spi-tegra20-sflash.c index d9d536d7f7b689..9256729f2d4910 100644 --- a/drivers/spi/spi-tegra20-sflash.c +++ b/drivers/spi/spi-tegra20-sflash.c @@ -505,7 +505,7 @@ static int tegra_sflash_probe(struct platform_device *pdev) tegra_sflash_writel(tsd, tsd->def_command_reg, SPI_COMMAND); pm_runtime_put(&pdev->dev); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret < 0) { dev_err(&pdev->dev, "can not register to host err %d\n", ret); goto exit_pm_disable; @@ -528,11 +528,17 @@ static void tegra_sflash_remove(struct platform_device *pdev) struct spi_controller *host = platform_get_drvdata(pdev); struct tegra_sflash_data *tsd = spi_controller_get_devdata(host); + spi_controller_get(host); + + spi_unregister_controller(host); + free_irq(tsd->irq, tsd); pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) tegra_sflash_runtime_suspend(&pdev->dev); + + spi_controller_put(host); } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c index d1d880a8ed7d8a..1fbd710d616f69 100644 --- a/drivers/spi/spi-ti-qspi.c +++ b/drivers/spi/spi-ti-qspi.c @@ -888,7 +888,7 @@ static int ti_qspi_probe(struct platform_device *pdev) qspi->mmap_enabled = false; qspi->current_cs = -1; - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (!ret) return 0; @@ -903,19 +903,17 @@ static int ti_qspi_probe(struct platform_device *pdev) static void ti_qspi_remove(struct platform_device *pdev) { struct ti_qspi *qspi = platform_get_drvdata(pdev); - int rc; - rc = spi_controller_suspend(qspi->host); - if (rc) { - dev_alert(&pdev->dev, "spi_controller_suspend() failed (%pe)\n", - ERR_PTR(rc)); - return; - } + spi_controller_get(qspi->host); + + spi_unregister_controller(qspi->host); pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); ti_qspi_dma_cleanup(qspi); + + spi_controller_put(qspi->host); } static const struct dev_pm_ops ti_qspi_pm_ops = { diff --git a/drivers/spi/spi-topcliff-pch.c b/drivers/spi/spi-topcliff-pch.c index cae2dcefabea72..14d11450e86dc6 100644 --- a/drivers/spi/spi-topcliff-pch.c +++ b/drivers/spi/spi-topcliff-pch.c @@ -1406,8 +1406,9 @@ static void pch_spi_pd_remove(struct platform_device *plat_dev) dev_dbg(&plat_dev->dev, "%s:[ch%d] irq=%d\n", __func__, plat_dev->id, board_dat->pdev->irq); - if (use_dma) - pch_free_dma_buf(board_dat, data); + spi_controller_get(data->host); + + spi_unregister_controller(data->host); /* check for any pending messages; no action is taken if the queue * is still full; but at least we tried. Unload anyway */ @@ -1432,8 +1433,12 @@ static void pch_spi_pd_remove(struct platform_device *plat_dev) free_irq(board_dat->pdev->irq, data); } + if (use_dma) + pch_free_dma_buf(board_dat, data); + pci_iounmap(board_dat->pdev, data->io_remap_addr); - spi_unregister_controller(data->host); + + spi_controller_put(data->host); } #ifdef CONFIG_PM static int pch_spi_pd_suspend(struct platform_device *pd_dev, diff --git a/drivers/spi/spi-uniphier.c b/drivers/spi/spi-uniphier.c index 9e1d364a619815..eac6c3e8908bd9 100644 --- a/drivers/spi/spi-uniphier.c +++ b/drivers/spi/spi-uniphier.c @@ -666,28 +666,24 @@ static int uniphier_spi_probe(struct platform_device *pdev) } priv->base_dma_addr = res->start; - priv->clk = devm_clk_get(&pdev->dev, NULL); + priv->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(priv->clk)) { dev_err(&pdev->dev, "failed to get clock\n"); ret = PTR_ERR(priv->clk); goto out_host_put; } - ret = clk_prepare_enable(priv->clk); - if (ret) - goto out_host_put; - irq = platform_get_irq(pdev, 0); if (irq < 0) { ret = irq; - goto out_disable_clk; + goto out_host_put; } ret = devm_request_irq(&pdev->dev, irq, uniphier_spi_handler, 0, "uniphier-spi", priv); if (ret) { dev_err(&pdev->dev, "failed to request IRQ\n"); - goto out_disable_clk; + goto out_host_put; } init_completion(&priv->xfer_done); @@ -716,7 +712,7 @@ static int uniphier_spi_probe(struct platform_device *pdev) if (IS_ERR_OR_NULL(host->dma_tx)) { if (PTR_ERR(host->dma_tx) == -EPROBE_DEFER) { ret = -EPROBE_DEFER; - goto out_disable_clk; + goto out_host_put; } host->dma_tx = NULL; dma_tx_burst = INT_MAX; @@ -750,7 +746,7 @@ static int uniphier_spi_probe(struct platform_device *pdev) host->max_dma_len = min(dma_tx_burst, dma_rx_burst); - ret = devm_spi_register_controller(&pdev->dev, host); + ret = spi_register_controller(host); if (ret) goto out_release_dma; @@ -766,9 +762,6 @@ static int uniphier_spi_probe(struct platform_device *pdev) host->dma_tx = NULL; } -out_disable_clk: - clk_disable_unprepare(priv->clk); - out_host_put: spi_controller_put(host); return ret; @@ -777,14 +770,17 @@ static int uniphier_spi_probe(struct platform_device *pdev) static void uniphier_spi_remove(struct platform_device *pdev) { struct spi_controller *host = platform_get_drvdata(pdev); - struct uniphier_spi_priv *priv = spi_controller_get_devdata(host); + + spi_controller_get(host); + + spi_unregister_controller(host); if (host->dma_tx) dma_release_channel(host->dma_tx); if (host->dma_rx) dma_release_channel(host->dma_rx); - clk_disable_unprepare(priv->clk); + spi_controller_put(host); } static const struct of_device_id uniphier_spi_match[] = { diff --git a/drivers/spi/spi-zynq-qspi.c b/drivers/spi/spi-zynq-qspi.c index af252500195ce2..406fd9d5337eec 100644 --- a/drivers/spi/spi-zynq-qspi.c +++ b/drivers/spi/spi-zynq-qspi.c @@ -643,7 +643,7 @@ static int zynq_qspi_probe(struct platform_device *pdev) xqspi = spi_controller_get_devdata(ctlr); xqspi->dev = dev; - platform_set_drvdata(pdev, xqspi); + platform_set_drvdata(pdev, ctlr); xqspi->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(xqspi->regs)) { ret = PTR_ERR(xqspi->regs); @@ -702,9 +702,9 @@ static int zynq_qspi_probe(struct platform_device *pdev) /* QSPI controller initializations */ zynq_qspi_init_hw(xqspi, ctlr->num_chipselect); - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); if (ret) { - dev_err(&pdev->dev, "devm_spi_register_controller failed\n"); + dev_err(&pdev->dev, "failed to register controller\n"); goto remove_ctlr; } @@ -728,9 +728,16 @@ static int zynq_qspi_probe(struct platform_device *pdev) */ static void zynq_qspi_remove(struct platform_device *pdev) { - struct zynq_qspi *xqspi = platform_get_drvdata(pdev); + struct spi_controller *ctlr = platform_get_drvdata(pdev); + struct zynq_qspi *xqspi = spi_controller_get_devdata(ctlr); + + spi_controller_get(ctlr); + + spi_unregister_controller(ctlr); zynq_qspi_write(xqspi, ZYNQ_QSPI_ENABLE_OFFSET, 0); + + spi_controller_put(ctlr); } static const struct of_device_id zynq_qspi_of_match[] = { diff --git a/drivers/spi/spi-zynqmp-gqspi.c b/drivers/spi/spi-zynqmp-gqspi.c index 502fd5eccc8346..f9a1427dabadec 100644 --- a/drivers/spi/spi-zynqmp-gqspi.c +++ b/drivers/spi/spi-zynqmp-gqspi.c @@ -1324,7 +1324,7 @@ static int zynqmp_qspi_probe(struct platform_device *pdev) ctlr->dev.of_node = np; ctlr->auto_runtime_pm = true; - ret = devm_spi_register_controller(&pdev->dev, ctlr); + ret = spi_register_controller(ctlr); if (ret) { dev_err(&pdev->dev, "spi_register_controller failed\n"); goto clk_dis_all; @@ -1362,6 +1362,8 @@ static void zynqmp_qspi_remove(struct platform_device *pdev) pm_runtime_get_sync(&pdev->dev); + spi_unregister_controller(xqspi->ctlr); + zynqmp_gqspi_write(xqspi, GQSPI_EN_OFST, 0x0); pm_runtime_disable(&pdev->dev); diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 9b1125556d2957..2001757fe31c02 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -43,6 +43,8 @@ EXPORT_TRACEPOINT_SYMBOL(spi_transfer_stop); #include "internals.h" +static int __spi_setup(struct spi_device *spi, bool initial_setup); + static DEFINE_IDR(spi_controller_idr); static void spidev_release(struct device *dev) @@ -729,7 +731,7 @@ static int __spi_add_device(struct spi_device *spi) * normally rely on the device being setup. Devices * using SPI_CS_HIGH can't coexist well otherwise... */ - status = spi_setup(spi); + status = __spi_setup(spi, true); if (status < 0) { dev_err(dev, "can't setup %s, status %d\n", dev_name(&spi->dev), status); @@ -3993,27 +3995,7 @@ static int spi_set_cs_timing(struct spi_device *spi) return status; } -/** - * spi_setup - setup SPI mode and clock rate - * @spi: the device whose settings are being modified - * Context: can sleep, and no requests are queued to the device - * - * SPI protocol drivers may need to update the transfer mode if the - * device doesn't work with its default. They may likewise need - * to update clock rates or word sizes from initial values. This function - * changes those settings, and must be called from a context that can sleep. - * Except for SPI_CS_HIGH, which takes effect immediately, the changes take - * effect the next time the device is selected and data is transferred to - * or from it. When this function returns, the SPI device is deselected. - * - * Note that this call will fail if the protocol driver specifies an option - * that the underlying controller or its driver does not support. For - * example, not all hardware supports wire transfers using nine bit words, - * LSB-first wire encoding, or active-high chipselects. - * - * Return: zero on success, else a negative error code. - */ -int spi_setup(struct spi_device *spi) +static int __spi_setup(struct spi_device *spi, bool initial_setup) { unsigned bad_bits, ugly_bits; int status; @@ -4098,7 +4080,7 @@ int spi_setup(struct spi_device *spi) status = spi_set_cs_timing(spi); if (status) { mutex_unlock(&spi->controller->io_mutex); - return status; + goto err_cleanup; } if (spi->controller->auto_runtime_pm && spi->controller->set_cs) { @@ -4107,7 +4089,7 @@ int spi_setup(struct spi_device *spi) mutex_unlock(&spi->controller->io_mutex); dev_err(&spi->controller->dev, "Failed to power device: %d\n", status); - return status; + goto err_cleanup; } /* @@ -4143,6 +4125,37 @@ int spi_setup(struct spi_device *spi) status); return status; + +err_cleanup: + if (initial_setup) + spi_cleanup(spi); + + return status; +} + +/** + * spi_setup - setup SPI mode and clock rate + * @spi: the device whose settings are being modified + * Context: can sleep, and no requests are queued to the device + * + * SPI protocol drivers may need to update the transfer mode if the + * device doesn't work with its default. They may likewise need + * to update clock rates or word sizes from initial values. This function + * changes those settings, and must be called from a context that can sleep. + * Except for SPI_CS_HIGH, which takes effect immediately, the changes take + * effect the next time the device is selected and data is transferred to + * or from it. When this function returns, the SPI device is deselected. + * + * Note that this call will fail if the protocol driver specifies an option + * that the underlying controller or its driver does not support. For + * example, not all hardware supports wire transfers using nine bit words, + * LSB-first wire encoding, or active-high chipselects. + * + * Return: zero on success, else a negative error code. + */ +int spi_setup(struct spi_device *spi) +{ + return __spi_setup(spi, false); } EXPORT_SYMBOL_GPL(spi_setup); @@ -4943,7 +4956,7 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action, * Clear the flag before adding the device so that fw_devlink * doesn't skip adding consumers to this device. */ - rd->dn->fwnode.flags &= ~FWNODE_FLAG_NOT_DEVICE; + fwnode_clear_flag(&rd->dn->fwnode, FWNODE_FLAG_NOT_DEVICE); spi = of_register_spi_device(ctlr, rd->dn); put_device(&ctlr->dev); diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c index 1f58c907c03683..f1f9f6fbc00e52 100644 --- a/drivers/staging/greybus/hid.c +++ b/drivers/staging/greybus/hid.c @@ -201,7 +201,7 @@ static void gb_hid_init_report(struct gb_hid *ghid, struct hid_report *report) * we just need to setup the input fields, so using * hid_report_raw_event is safe. */ - hid_report_raw_event(ghid->hid, report->type, ghid->inbuf, size, 1); + hid_report_raw_event(ghid->hid, report->type, ghid->inbuf, ghid->bufsize, size, 1); } static void gb_hid_init_reports(struct gb_hid *ghid) diff --git a/drivers/staging/greybus/raw.c b/drivers/staging/greybus/raw.c index 3027a2c25bcdee..459aed0f12401d 100644 --- a/drivers/staging/greybus/raw.c +++ b/drivers/staging/greybus/raw.c @@ -21,9 +21,10 @@ struct gb_raw { struct list_head list; int list_data; struct mutex list_lock; - dev_t dev; + struct rw_semaphore disconnect_lock; + bool disconnected; struct cdev cdev; - struct device *device; + struct device dev; }; struct raw_data { @@ -148,6 +149,15 @@ static int gb_raw_send(struct gb_raw *raw, u32 len, const char __user *data) return retval; } +static void raw_dev_release(struct device *dev) +{ + struct gb_raw *raw = container_of(dev, struct gb_raw, dev); + + ida_free(&minors, MINOR(raw->dev.devt)); + + kfree(raw); +} + static int gb_raw_probe(struct gb_bundle *bundle, const struct greybus_bundle_id *id) { @@ -164,63 +174,59 @@ static int gb_raw_probe(struct gb_bundle *bundle, if (cport_desc->protocol_id != GREYBUS_PROTOCOL_RAW) return -ENODEV; - raw = kzalloc_obj(*raw); - if (!raw) + minor = ida_alloc(&minors, GFP_KERNEL); + if (minor < 0) + return minor; + + raw = kzalloc_obj(*raw, GFP_KERNEL); + if (!raw) { + ida_free(&minors, minor); return -ENOMEM; + } + + device_initialize(&raw->dev); + raw->dev.devt = MKDEV(raw_major, minor); + raw->dev.class = &raw_class; + raw->dev.parent = &bundle->dev; + raw->dev.release = raw_dev_release; + retval = dev_set_name(&raw->dev, "gb!raw%d", minor); + if (retval) + goto error_put_device; connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id), gb_raw_request_handler); if (IS_ERR(connection)) { retval = PTR_ERR(connection); - goto error_free; + goto error_put_device; } INIT_LIST_HEAD(&raw->list); mutex_init(&raw->list_lock); + init_rwsem(&raw->disconnect_lock); raw->connection = connection; greybus_set_drvdata(bundle, raw); - minor = ida_alloc(&minors, GFP_KERNEL); - if (minor < 0) { - retval = minor; - goto error_connection_destroy; - } - - raw->dev = MKDEV(raw_major, minor); cdev_init(&raw->cdev, &raw_fops); retval = gb_connection_enable(connection); if (retval) - goto error_remove_ida; + goto error_connection_destroy; - retval = cdev_add(&raw->cdev, raw->dev, 1); + retval = cdev_device_add(&raw->cdev, &raw->dev); if (retval) goto error_connection_disable; - raw->device = device_create(&raw_class, &connection->bundle->dev, - raw->dev, raw, "gb!raw%d", minor); - if (IS_ERR(raw->device)) { - retval = PTR_ERR(raw->device); - goto error_del_cdev; - } - return 0; -error_del_cdev: - cdev_del(&raw->cdev); - error_connection_disable: gb_connection_disable(connection); -error_remove_ida: - ida_free(&minors, minor); - error_connection_destroy: gb_connection_destroy(connection); -error_free: - kfree(raw); +error_put_device: + put_device(&raw->dev); return retval; } @@ -231,11 +237,13 @@ static void gb_raw_disconnect(struct gb_bundle *bundle) struct raw_data *raw_data; struct raw_data *temp; - // FIXME - handle removing a connection when the char device node is open. - device_destroy(&raw_class, raw->dev); - cdev_del(&raw->cdev); + cdev_device_del(&raw->cdev, &raw->dev); + + down_write(&raw->disconnect_lock); + raw->disconnected = true; + up_write(&raw->disconnect_lock); + gb_connection_disable(connection); - ida_free(&minors, MINOR(raw->dev)); gb_connection_destroy(connection); mutex_lock(&raw->list_lock); @@ -244,8 +252,7 @@ static void gb_raw_disconnect(struct gb_bundle *bundle) kfree(raw_data); } mutex_unlock(&raw->list_lock); - - kfree(raw); + put_device(&raw->dev); } /* @@ -278,11 +285,22 @@ static ssize_t raw_write(struct file *file, const char __user *buf, if (count > MAX_PACKET_SIZE) return -E2BIG; + down_read(&raw->disconnect_lock); + + if (raw->disconnected) { + retval = -ENODEV; + goto exit; + } + retval = gb_raw_send(raw, count, buf); if (retval) - return retval; + goto exit; + + retval = count; +exit: + up_read(&raw->disconnect_lock); - return count; + return retval; } static ssize_t raw_read(struct file *file, char __user *buf, size_t count, diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c index a7f22de1c8899f..50366bf10f327f 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c +++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c @@ -1356,6 +1356,10 @@ static int atomisp_s_parm(struct file *file, void *fh, static long atomisp_vidioc_default(struct file *file, void *fh, bool valid_prio, unsigned int cmd, void *arg) { + /* Disable all private IOCTLs for now! */ + if (cmd) + return -EINVAL; + struct video_device *vdev = video_devdata(file); struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd; int err; diff --git a/drivers/staging/media/imx/imx-media-csi.c b/drivers/staging/media/imx/imx-media-csi.c index fd7e37d803e7b0..1bc644f73a9d1c 100644 --- a/drivers/staging/media/imx/imx-media-csi.c +++ b/drivers/staging/media/imx/imx-media-csi.c @@ -97,9 +97,6 @@ struct csi_priv { /* the mipi virtual channel number at link validate */ int vc_num; - /* media bus config of the upstream subdevice CSI is receiving from */ - struct v4l2_mbus_config mbus_cfg; - spinlock_t irqlock; /* protect eof_irq handler */ struct timer_list eof_timeout_timer; int eof_irq; @@ -403,7 +400,8 @@ static void csi_idmac_unsetup_vb2_buf(struct csi_priv *priv, } /* init the SMFC IDMAC channel */ -static int csi_idmac_setup_channel(struct csi_priv *priv) +static int csi_idmac_setup_channel(struct csi_priv *priv, + struct v4l2_mbus_config *mbus_cfg) { struct imx_media_video_dev *vdev = priv->vdev; const struct imx_media_pixfmt *incc; @@ -432,7 +430,7 @@ static int csi_idmac_setup_channel(struct csi_priv *priv) image.phys0 = phys[0]; image.phys1 = phys[1]; - passthrough = requires_passthrough(&priv->mbus_cfg, infmt, incc); + passthrough = requires_passthrough(mbus_cfg, infmt, incc); passthrough_cycles = 1; /* @@ -572,11 +570,12 @@ static void csi_idmac_unsetup(struct csi_priv *priv, csi_idmac_unsetup_vb2_buf(priv, state); } -static int csi_idmac_setup(struct csi_priv *priv) +static int csi_idmac_setup(struct csi_priv *priv, + struct v4l2_mbus_config *mbus_cfg) { int ret; - ret = csi_idmac_setup_channel(priv); + ret = csi_idmac_setup_channel(priv, mbus_cfg); if (ret) return ret; @@ -595,7 +594,8 @@ static int csi_idmac_setup(struct csi_priv *priv) return 0; } -static int csi_idmac_start(struct csi_priv *priv) +static int csi_idmac_start(struct csi_priv *priv, + struct v4l2_mbus_config *mbus_cfg) { struct imx_media_video_dev *vdev = priv->vdev; int ret; @@ -619,7 +619,7 @@ static int csi_idmac_start(struct csi_priv *priv) priv->last_eof = false; priv->nfb4eof = false; - ret = csi_idmac_setup(priv); + ret = csi_idmac_setup(priv, mbus_cfg); if (ret) { v4l2_err(&priv->sd, "csi_idmac_setup failed: %d\n", ret); goto out_free_dma_buf; @@ -701,7 +701,8 @@ static void csi_idmac_stop(struct csi_priv *priv) } /* Update the CSI whole sensor and active windows */ -static int csi_setup(struct csi_priv *priv) +static int csi_setup(struct csi_priv *priv, + struct v4l2_mbus_config *mbus_cfg) { struct v4l2_mbus_framefmt *infmt, *outfmt; const struct imx_media_pixfmt *incc; @@ -719,7 +720,7 @@ static int csi_setup(struct csi_priv *priv) * if cycles is set, we need to handle this over multiple cycles as * generic/bayer data */ - if (is_parallel_bus(&priv->mbus_cfg) && incc->cycles) { + if (is_parallel_bus(mbus_cfg) && incc->cycles) { if_fmt.width *= incc->cycles; crop.width *= incc->cycles; } @@ -730,7 +731,7 @@ static int csi_setup(struct csi_priv *priv) priv->crop.width == 2 * priv->compose.width, priv->crop.height == 2 * priv->compose.height); - ipu_csi_init_interface(priv->csi, &priv->mbus_cfg, &if_fmt, outfmt); + ipu_csi_init_interface(priv->csi, mbus_cfg, &if_fmt, outfmt); ipu_csi_set_dest(priv->csi, priv->dest); @@ -743,14 +744,46 @@ static int csi_setup(struct csi_priv *priv) return 0; } +static void csi_set_src(struct csi_priv *priv, + struct v4l2_mbus_config *mbus_cfg) +{ + bool is_csi2; + + is_csi2 = !is_parallel_bus(mbus_cfg); + if (is_csi2) { + /* + * NOTE! It seems the virtual channels from the mipi csi-2 + * receiver are used only for routing by the video mux's, + * or for hard-wired routing to the CSI's. Once the stream + * enters the CSI's however, they are treated internally + * in the IPU as virtual channel 0. + */ + ipu_csi_set_mipi_datatype(priv->csi, 0, + &priv->format_mbus[CSI_SINK_PAD]); + } + + /* select either parallel or MIPI-CSI2 as input to CSI */ + ipu_set_csi_src_mux(priv->ipu, priv->csi_id, is_csi2); +} + static int csi_start(struct csi_priv *priv) { + struct v4l2_mbus_config mbus_cfg = { .type = 0 }; struct v4l2_fract *input_fi, *output_fi; int ret; + ret = csi_get_upstream_mbus_config(priv, &mbus_cfg); + if (ret) { + v4l2_err(&priv->sd, + "failed to get upstream media bus configuration\n"); + return ret; + } + input_fi = &priv->frame_interval[CSI_SINK_PAD]; output_fi = &priv->frame_interval[priv->active_output_pad]; + csi_set_src(priv, &mbus_cfg); + /* start upstream */ ret = v4l2_subdev_call(priv->src_sd, video, s_stream, 1); ret = (ret && ret != -ENOIOCTLCMD) ? ret : 0; @@ -758,7 +791,7 @@ static int csi_start(struct csi_priv *priv) return ret; /* Skip first few frames from a BT.656 source */ - if (priv->mbus_cfg.type == V4L2_MBUS_BT656) { + if (mbus_cfg.type == V4L2_MBUS_BT656) { u32 delay_usec, bad_frames = 20; delay_usec = DIV_ROUND_UP_ULL((u64)USEC_PER_SEC * @@ -769,12 +802,12 @@ static int csi_start(struct csi_priv *priv) } if (priv->dest == IPU_CSI_DEST_IDMAC) { - ret = csi_idmac_start(priv); + ret = csi_idmac_start(priv, &mbus_cfg); if (ret) goto stop_upstream; } - ret = csi_setup(priv); + ret = csi_setup(priv, &mbus_cfg); if (ret) goto idmac_stop; @@ -1121,7 +1154,6 @@ static int csi_link_validate(struct v4l2_subdev *sd, { struct csi_priv *priv = v4l2_get_subdevdata(sd); struct v4l2_mbus_config mbus_cfg = { .type = 0 }; - bool is_csi2; int ret; ret = v4l2_subdev_link_validate_default(sd, link, @@ -1136,26 +1168,6 @@ static int csi_link_validate(struct v4l2_subdev *sd, return ret; } - mutex_lock(&priv->lock); - - priv->mbus_cfg = mbus_cfg; - is_csi2 = !is_parallel_bus(&mbus_cfg); - if (is_csi2) { - /* - * NOTE! It seems the virtual channels from the mipi csi-2 - * receiver are used only for routing by the video mux's, - * or for hard-wired routing to the CSI's. Once the stream - * enters the CSI's however, they are treated internally - * in the IPU as virtual channel 0. - */ - ipu_csi_set_mipi_datatype(priv->csi, 0, - &priv->format_mbus[CSI_SINK_PAD]); - } - - /* select either parallel or MIPI-CSI2 as input to CSI */ - ipu_set_csi_src_mux(priv->ipu, priv->csi_id, is_csi2); - - mutex_unlock(&priv->lock); return ret; } diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c index b489babe743248..c3f5fc4abd170e 100644 --- a/drivers/staging/rtl8723bs/core/rtw_security.c +++ b/drivers/staging/rtl8723bs/core/rtw_security.c @@ -1291,7 +1291,7 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe) u8 mic[16]; struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; __le16 le_tmp; - __le64 le_tmp64; + __le64 le_tmp64 = 0; ori_len = pattrib->pkt_len - WLAN_HDR_A3_LEN + BIP_AAD_SIZE; BIP_AAD = kzalloc(ori_len, GFP_KERNEL); diff --git a/drivers/staging/rtl8723bs/os_dep/osdep_service.c b/drivers/staging/rtl8723bs/os_dep/osdep_service.c index 7959daeabc6ff0..4cfdf7c623440a 100644 --- a/drivers/staging/rtl8723bs/os_dep/osdep_service.c +++ b/drivers/staging/rtl8723bs/os_dep/osdep_service.c @@ -194,7 +194,8 @@ struct rtw_cbuf *rtw_cbuf_alloc(u32 size) struct rtw_cbuf *cbuf; cbuf = kzalloc_flex(*cbuf, bufs, size); - cbuf->size = size; + if (cbuf) + cbuf->size = size; return cbuf; } diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index 62f6e0cdff4d48..04e45e2bc958fb 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -481,6 +481,9 @@ static int lynxfb_ops_check_var(struct fb_var_screeninfo *var, struct lynxfb_crtc *crtc; resource_size_t request; + if (!var->pixclock) + return -EINVAL; + ret = 0; par = info->par; crtc = &par->crtc; diff --git a/drivers/staging/vme_user/vme_fake.c b/drivers/staging/vme_user/vme_fake.c index be4ad47ed526a9..8abaa3165fbb7b 100644 --- a/drivers/staging/vme_user/vme_fake.c +++ b/drivers/staging/vme_user/vme_fake.c @@ -1230,6 +1230,8 @@ static int __init fake_init(void) err_driver: kfree(fake_bridge); err_struct: + root_device_unregister(vme_root); + return retval; } diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c index a1c91d4515bc56..84124b222a999d 100644 --- a/drivers/target/target_core_configfs.c +++ b/drivers/target/target_core_configfs.c @@ -3227,7 +3227,7 @@ static ssize_t target_tg_pt_gp_members_show(struct config_item *item, config_item_name(&lun->lun_group.cg_item)); cur_len++; /* Extra byte for NULL terminator */ - if ((cur_len + len) > PAGE_SIZE) { + if (cur_len > TG_PT_GROUP_NAME_BUF || (cur_len + len) > PAGE_SIZE) { pr_warn("Ran out of lu_gp_show_attr" "_members buffer\n"); break; diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c index abe91dc8722e4c..21f5cb86d70c00 100644 --- a/drivers/target/target_core_sbc.c +++ b/drivers/target/target_core_sbc.c @@ -1187,7 +1187,8 @@ sbc_execute_unmap(struct se_cmd *cmd) goto err; } - if (lba + range > dev->transport->get_blocks(dev) + 1) { + if (lba + range < lba || + lba + range > dev->transport->get_blocks(dev) + 1) { ret = TCM_ADDRESS_OUT_OF_RANGE; goto err; } diff --git a/drivers/thermal/spear_thermal.c b/drivers/thermal/spear_thermal.c index 603dadcd3df583..5e3e9c1f32f8eb 100644 --- a/drivers/thermal/spear_thermal.c +++ b/drivers/thermal/spear_thermal.c @@ -93,7 +93,7 @@ static int spear_thermal_probe(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; int ret = 0, val; - if (!np || !of_property_read_u32(np, "st,thermal-flags", &val)) { + if (!np || of_property_read_u32(np, "st,thermal-flags", &val)) { dev_err(&pdev->dev, "Failed: DT Pdata not passed\n"); return -EINVAL; } diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c index e546067c962187..44fa45f74da720 100644 --- a/drivers/thermal/sprd_thermal.c +++ b/drivers/thermal/sprd_thermal.c @@ -178,7 +178,7 @@ static int sprd_thm_sensor_calibration(struct device_node *np, static int sprd_thm_rawdata_to_temp(struct sprd_thermal_sensor *sen, u32 rawdata) { - clamp(rawdata, (u32)SPRD_THM_RAW_DATA_LOW, (u32)SPRD_THM_RAW_DATA_HIGH); + rawdata = clamp(rawdata, SPRD_THM_RAW_DATA_LOW, SPRD_THM_RAW_DATA_HIGH); /* * According to the thermal datasheet, the formula of converting @@ -192,7 +192,7 @@ static int sprd_thm_temp_to_rawdata(int temp, struct sprd_thermal_sensor *sen) { u32 val; - clamp(temp, (int)SPRD_THM_TEMP_LOW, (int)SPRD_THM_TEMP_HIGH); + temp = clamp(temp, SPRD_THM_TEMP_LOW, SPRD_THM_TEMP_HIGH); /* * According to the thermal datasheet, the formula of converting diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index d1beee9e15f8ef..cb25e628c24876 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -964,6 +964,8 @@ static void thermal_release(struct device *dev) sizeof("thermal_zone") - 1)) { tz = to_thermal_zone(dev); thermal_zone_destroy_device_groups(tz); + thermal_set_governor(tz, NULL); + ida_destroy(&tz->ida); mutex_destroy(&tz->lock); complete(&tz->removal); } else if (!strncmp(dev_name(dev), "cooling_device", @@ -1611,8 +1613,10 @@ thermal_zone_device_register_with_trips(const char *type, /* sys I/F */ /* Add nodes that are always present via .groups */ result = thermal_zone_create_device_groups(tz); - if (result) + if (result) { + thermal_set_governor(tz, NULL); goto remove_id; + } result = device_register(&tz->device); if (result) @@ -1725,12 +1729,8 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) cancel_delayed_work_sync(&tz->poll_queue); - thermal_set_governor(tz, NULL); - thermal_thresholds_exit(tz); thermal_remove_hwmon_sysfs(tz); - ida_free(&thermal_tz_ida, tz->id); - ida_destroy(&tz->ida); device_del(&tz->device); put_device(&tz->device); @@ -1738,6 +1738,9 @@ void thermal_zone_device_unregister(struct thermal_zone_device *tz) thermal_notify_tz_delete(tz); wait_for_completion(&tz->removal); + + ida_free(&thermal_tz_ida, tz->id); + kfree(tz->tzp); kfree(tz); } diff --git a/drivers/tty/hvc/hvc_iucv.c b/drivers/tty/hvc/hvc_iucv.c index 1dcdb9e99bd8a6..37db8a3e5158e6 100644 --- a/drivers/tty/hvc/hvc_iucv.c +++ b/drivers/tty/hvc/hvc_iucv.c @@ -130,7 +130,7 @@ static struct iucv_handler hvc_iucv_handler = { */ static struct hvc_iucv_private *hvc_iucv_get_private(uint32_t num) { - if (num > hvc_iucv_devices) + if (num >= hvc_iucv_devices) return NULL; return hvc_iucv_table[num]; } diff --git a/drivers/tty/serial/ip22zilog.c b/drivers/tty/serial/ip22zilog.c index 6e19c6713849ae..a12101dc055464 100644 --- a/drivers/tty/serial/ip22zilog.c +++ b/drivers/tty/serial/ip22zilog.c @@ -1025,7 +1025,7 @@ static struct uart_driver ip22zilog_reg = { #endif }; -static void __init ip22zilog_prepare(struct uart_ip22zilog_port *up) +static void ip22zilog_prepare(struct uart_ip22zilog_port *up) { unsigned char sysrq_on = IS_ENABLED(CONFIG_SERIAL_IP22_ZILOG_CONSOLE); int brg; diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index c1fabad6ba1faf..99d83271d8dfbe 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -1298,30 +1299,49 @@ static int apple_s5l_serial_startup(struct uart_port *port) return ret; } +static int __maybe_unused s3c24xx_serial_runtime_suspend(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + int timeout = 10000; + + while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) + udelay(100); + + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + clk_disable_unprepare(ourport->clk); + return 0; +}; + +static int __maybe_unused s3c24xx_serial_runtime_resume(struct device *dev) +{ + struct uart_port *port = dev_get_drvdata(dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); + + clk_prepare_enable(ourport->clk); + + if (!IS_ERR(ourport->baudclk)) + clk_prepare_enable(ourport->baudclk); + return 0; +}; + static void s3c24xx_serial_pm(struct uart_port *port, unsigned int level, unsigned int old) { struct s3c24xx_uart_port *ourport = to_ourport(port); - int timeout = 10000; ourport->pm_level = level; switch (level) { - case 3: - while (--timeout && !s3c24xx_serial_txempty_nofifo(port)) - udelay(100); - - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); - - clk_disable_unprepare(ourport->clk); + case UART_PM_STATE_OFF: + pm_runtime_mark_last_busy(port->dev); + pm_runtime_put_sync(port->dev); break; - case 0: - clk_prepare_enable(ourport->clk); - - if (!IS_ERR(ourport->baudclk)) - clk_prepare_enable(ourport->baudclk); + case UART_PM_STATE_ON: + pm_runtime_get_sync(port->dev); break; default: dev_err(port->dev, "s3c24xx_serial: unknown pm %d\n", level); @@ -2044,18 +2064,15 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) } } + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + dev_dbg(&pdev->dev, "%s: adding port\n", __func__); uart_add_one_port(&s3c24xx_uart_drv, &ourport->port); platform_set_drvdata(pdev, &ourport->port); - /* - * Deactivate the clock enabled in s3c24xx_serial_init_port here, - * so that a potential re-enablement through the pm-callback overlaps - * and keeps the clock enabled in this case. - */ - clk_disable_unprepare(ourport->clk); - if (!IS_ERR(ourport->baudclk)) - clk_disable_unprepare(ourport->baudclk); + pm_runtime_put_sync(&pdev->dev); probe_index++; @@ -2065,26 +2082,40 @@ static int s3c24xx_serial_probe(struct platform_device *pdev) static void s3c24xx_serial_remove(struct platform_device *dev) { struct uart_port *port = s3c24xx_dev_to_port(&dev->dev); + struct s3c24xx_uart_port *ourport = to_ourport(port); - if (port) + if (port) { + pm_runtime_get_sync(&dev->dev); uart_remove_one_port(&s3c24xx_uart_drv, port); + clk_disable_unprepare(ourport->clk); + if (!IS_ERR(ourport->baudclk)) + clk_disable_unprepare(ourport->baudclk); + + pm_runtime_disable(&dev->dev); + pm_runtime_set_suspended(&dev->dev); + pm_runtime_put_noidle(&dev->dev); + } + uart_unregister_driver(&s3c24xx_uart_drv); } /* UART power management code */ -#ifdef CONFIG_PM_SLEEP -static int s3c24xx_serial_suspend(struct device *dev) + +static int __maybe_unused s3c24xx_serial_suspend(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); + if (!console_suspend_enabled && uart_console(port)) + device_set_wakeup_path(dev); + if (port) uart_suspend_port(&s3c24xx_uart_drv, port); return 0; } -static int s3c24xx_serial_resume(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2104,7 +2135,7 @@ static int s3c24xx_serial_resume(struct device *dev) return 0; } -static int s3c24xx_serial_resume_noirq(struct device *dev) +static int __maybe_unused s3c24xx_serial_resume_noirq(struct device *dev) { struct uart_port *port = s3c24xx_dev_to_port(dev); struct s3c24xx_uart_port *ourport = to_ourport(port); @@ -2178,13 +2209,9 @@ static int s3c24xx_serial_resume_noirq(struct device *dev) static const struct dev_pm_ops s3c24xx_serial_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(s3c24xx_serial_suspend, s3c24xx_serial_resume) SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, s3c24xx_serial_resume_noirq) + SET_RUNTIME_PM_OPS(s3c24xx_serial_runtime_suspend, + s3c24xx_serial_runtime_resume, NULL) }; -#define SERIAL_SAMSUNG_PM_OPS (&s3c24xx_serial_pm_ops) - -#else /* !CONFIG_PM_SLEEP */ - -#define SERIAL_SAMSUNG_PM_OPS NULL -#endif /* CONFIG_PM_SLEEP */ /* Console code */ @@ -2672,7 +2699,7 @@ static struct platform_driver samsung_serial_driver = { .id_table = s3c24xx_serial_driver_ids, .driver = { .name = "samsung-uart", - .pm = SERIAL_SAMSUNG_PM_OPS, + .pm = &s3c24xx_serial_pm_ops, .of_match_table = of_match_ptr(s3c24xx_uart_dt_match), }, }; diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c index fac11f20cf0ac2..57d2816cf05f18 100644 --- a/drivers/usb/chipidea/core.c +++ b/drivers/usb/chipidea/core.c @@ -544,30 +544,31 @@ static irqreturn_t ci_irq_handler(int irq, void *data) if (ret == IRQ_HANDLED) return ret; } - } - /* - * Handle id change interrupt, it indicates device/host function - * switch. - */ - if (ci->is_otg && (otgsc & OTGSC_IDIE) && (otgsc & OTGSC_IDIS)) { - ci->id_event = true; - /* Clear ID change irq status */ - hw_write_otgsc(ci, OTGSC_IDIS, OTGSC_IDIS); - ci_otg_queue_work(ci); - return IRQ_HANDLED; - } + /* + * Handle id change interrupt, it indicates device/host function + * switch. + */ + if ((otgsc & OTGSC_IDIE) && (otgsc & OTGSC_IDIS)) { + ci->id_event = true; + /* Clear ID change irq status */ + hw_write_otgsc(ci, OTGSC_IDIS, OTGSC_IDIS); + } - /* - * Handle vbus change interrupt, it indicates device connection - * and disconnection events. - */ - if (ci->is_otg && (otgsc & OTGSC_BSVIE) && (otgsc & OTGSC_BSVIS)) { - ci->b_sess_valid_event = true; - /* Clear BSV irq */ - hw_write_otgsc(ci, OTGSC_BSVIS, OTGSC_BSVIS); - ci_otg_queue_work(ci); - return IRQ_HANDLED; + /* + * Handle vbus change interrupt, it indicates device connection + * and disconnection events. + */ + if ((otgsc & OTGSC_BSVIE) && (otgsc & OTGSC_BSVIS)) { + ci->b_sess_valid_event = true; + /* Clear BSV irq */ + hw_write_otgsc(ci, OTGSC_BSVIS, OTGSC_BSVIS); + } + + if (ci->id_event || ci->b_sess_valid_event) { + ci_otg_queue_work(ci); + return IRQ_HANDLED; + } } /* Handle device/host interrupt */ diff --git a/drivers/usb/chipidea/otg.c b/drivers/usb/chipidea/otg.c index 647e98f4e35110..fecc7d7e2f0d7e 100644 --- a/drivers/usb/chipidea/otg.c +++ b/drivers/usb/chipidea/otg.c @@ -130,6 +130,9 @@ enum ci_role ci_otg_role(struct ci_hdrc *ci) void ci_handle_vbus_change(struct ci_hdrc *ci) { + if (ci->role != CI_ROLE_GADGET) + return; + if (!ci->is_otg) { if (ci->platdata->flags & CI_HDRC_FORCE_VBUS_ACTIVE_ALWAYS) usb_gadget_vbus_connect(&ci->gadget); @@ -187,8 +190,8 @@ void ci_handle_id_switch(struct ci_hdrc *ci) ci_role_stop(ci); - if (role == CI_ROLE_GADGET && - IS_ERR(ci->platdata->vbus_extcon.edev)) + if (role == CI_ROLE_GADGET && !ci->role_switch && + IS_ERR(ci->platdata->vbus_extcon.edev)) /* * Wait vbus lower than OTGSC_BSV before connecting * to host. If connecting status is from an external diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index cf3c3eede1a51f..54059e4fc6ed72 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -114,6 +114,8 @@ static int acm_ctrl_msg(struct acm *acm, int request, int value, int retval; retval = usb_autopm_get_interface(acm->control); +#define VENDOR_CLASS_DATA_IFACE BIT(9) /* data interface uses vendor-specific class */ +#define ALWAYS_POLL_CTRL BIT(10) /* keep ctrl URB active even without an open TTY */ if (retval) return retval; @@ -710,12 +712,14 @@ static int acm_port_activate(struct tty_port *port, struct tty_struct *tty) set_bit(TTY_NO_WRITE_SPLIT, &tty->flags); acm->control->needs_remote_wakeup = 1; - acm->ctrlurb->dev = acm->dev; - retval = usb_submit_urb(acm->ctrlurb, GFP_KERNEL); - if (retval) { - dev_err(&acm->control->dev, - "%s - usb_submit_urb(ctrl irq) failed\n", __func__); - goto error_submit_urb; + if (!(acm->quirks & ALWAYS_POLL_CTRL)) { + acm->ctrlurb->dev = acm->dev; + retval = usb_submit_urb(acm->ctrlurb, GFP_KERNEL); + if (retval) { + dev_err(&acm->control->dev, + "%s - usb_submit_urb(ctrl irq) failed\n", __func__); + goto error_submit_urb; + } } acm_tty_set_termios(tty, NULL); @@ -788,6 +792,14 @@ static void acm_port_shutdown(struct tty_port *port) acm_unpoison_urbs(acm); + if (acm->quirks & ALWAYS_POLL_CTRL) { + acm->ctrlurb->dev = acm->dev; + if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) + dev_dbg(&acm->control->dev, + "ctrl polling restart failed after port close\n"); + /* port_shutdown() cleared DTR/RTS; restore them */ + acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + } } static void acm_tty_cleanup(struct tty_struct *tty) @@ -1328,6 +1340,9 @@ static int acm_probe(struct usb_interface *intf, dev_dbg(&intf->dev, "Your device has switched interfaces.\n"); swap(control_interface, data_interface); + } else if (quirks & VENDOR_CLASS_DATA_IFACE) { + dev_dbg(&intf->dev, + "Vendor-specific data interface class, continuing.\n"); } else { return -EINVAL; } @@ -1522,6 +1537,9 @@ static int acm_probe(struct usb_interface *intf, acm->line.bDataBits = 8; acm_set_line(acm, &acm->line); + if (quirks & ALWAYS_POLL_CTRL) + acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + if (!acm->combined_interfaces) { rv = usb_driver_claim_interface(&acm_driver, data_interface, acm); if (rv) @@ -1543,6 +1561,13 @@ static int acm_probe(struct usb_interface *intf, dev_info(&intf->dev, "ttyACM%d: USB ACM device\n", minor); + if (acm->quirks & ALWAYS_POLL_CTRL) { + acm->ctrlurb->dev = acm->dev; + if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) + dev_warn(&intf->dev, + "failed to start persistent ctrl polling\n"); + } + return 0; err_release_data_interface: @@ -1669,7 +1694,7 @@ static int acm_resume(struct usb_interface *intf) acm_unpoison_urbs(acm); - if (tty_port_initialized(&acm->port)) { + if (tty_port_initialized(&acm->port) || (acm->quirks & ALWAYS_POLL_CTRL)) { rv = usb_submit_urb(acm->ctrlurb, GFP_ATOMIC); for (;;) { @@ -2016,6 +2041,20 @@ static const struct usb_device_id acm_ids[] = { /* CH343 supports CAP_BRK, but doesn't advertise it */ { USB_DEVICE(0x1a86, 0x55d3), .driver_info = MISSING_CAP_BRK, }, + /* + * Lenovo Yoga Book 9 14IAH10 (83KJ) — INGENIC 17EF:6161 touchscreen + * composite device. The CDC ACM control interface (0) uses a standard + * Union descriptor, but the data interface (1) is declared as vendor- + * specific class (0xff) with no CDC data descriptors, so cdc-acm would + * normally reject it. The firmware also requires continuous polling of + * the notification endpoint (EP 0x82) to suppress a 20-second watchdog + * reset; ALWAYS_POLL_CTRL keeps the ctrlurb active even when no TTY is + * open. Match only the control interface by class to avoid probing the + * vendor-specific data interface. + */ + { USB_DEVICE_INTERFACE_CLASS(0x17ef, 0x6161, USB_CLASS_COMM), + .driver_info = VENDOR_CLASS_DATA_IFACE | ALWAYS_POLL_CTRL }, + /* control interfaces without any protocol set */ { USB_INTERFACE_INFO(USB_CLASS_COMM, USB_CDC_SUBCLASS_ACM, USB_CDC_PROTO_NONE) }, diff --git a/drivers/usb/class/usblp.c b/drivers/usb/class/usblp.c index 669b9e6879bfa5..746414763da5d6 100644 --- a/drivers/usb/class/usblp.c +++ b/drivers/usb/class/usblp.c @@ -1178,7 +1178,7 @@ static int usblp_probe(struct usb_interface *intf, } /* Allocate buffer for printer status */ - usblp->statusbuf = kmalloc(STATUS_BUF_SIZE, GFP_KERNEL); + usblp->statusbuf = kzalloc(STATUS_BUF_SIZE, GFP_KERNEL); if (!usblp->statusbuf) { retval = -ENOMEM; goto abort; @@ -1377,6 +1377,7 @@ static int usblp_cache_device_id_string(struct usblp *usblp) { int err, length; + memset(usblp->device_id_string, 0, USBLP_DEVICE_ID_SIZE); err = usblp_get_id(usblp, 0, usblp->device_id_string, USBLP_DEVICE_ID_SIZE - 1); if (err < 0) { dev_dbg(&usblp->intf->dev, diff --git a/drivers/usb/common/ulpi.c b/drivers/usb/common/ulpi.c index b34fb65813c45e..9b69148128e5bf 100644 --- a/drivers/usb/common/ulpi.c +++ b/drivers/usb/common/ulpi.c @@ -286,12 +286,15 @@ static int ulpi_register(struct device *dev, struct ulpi *ulpi) ACPI_COMPANION_SET(&ulpi->dev, ACPI_COMPANION(dev)); ret = ulpi_of_register(ulpi); - if (ret) + if (ret) { + kfree(ulpi); return ret; + } ret = ulpi_read_id(ulpi); if (ret) { of_node_put(ulpi->dev.of_node); + kfree(ulpi); return ret; } diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index 44e38f922bc553..027a5f21c5f791 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -141,6 +141,7 @@ static ssize_t disable_store(struct device *dev, struct device_attribute *attr, usb_disconnect(&port_dev->child); rc = usb_hub_set_port_power(hdev, hub, port1, !disabled); + msleep(2 * hub_power_on_good_delay(hub)); if (disabled) { usb_clear_port_feature(hdev, port1, USB_PORT_FEAT_C_CONNECTION); diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 161a4d58b2cece..0d3c7e7b2262fd 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -1341,12 +1341,6 @@ int dwc3_core_init(struct dwc3 *dwc) hw_mode = DWC3_GHWPARAMS0_MODE(dwc->hwparams.hwparams0); - /* - * Write Linux Version Code to our GUID register so it's easy to figure - * out which kernel version a bug was found. - */ - dwc3_writel(dwc, DWC3_GUID, LINUX_VERSION_CODE); - ret = dwc3_phy_setup(dwc); if (ret) return ret; @@ -1378,6 +1372,12 @@ int dwc3_core_init(struct dwc3 *dwc) if (ret) goto err_exit_phy; + /* + * Write Linux Version Code to our GUID register so it's easy to figure + * out which kernel version a bug was found. + */ + dwc3_writel(dwc, DWC3_GUID, LINUX_VERSION_CODE); + dwc3_core_setup_global_control(dwc); dwc3_core_num_eps(dwc); diff --git a/drivers/usb/gadget/function/f_hid.c b/drivers/usb/gadget/function/f_hid.c index e5ccaec7750cdc..e0c3f39ee95e7b 100644 --- a/drivers/usb/gadget/function/f_hid.c +++ b/drivers/usb/gadget/function/f_hid.c @@ -106,7 +106,7 @@ struct f_hidg { struct list_head report_list; struct device dev; - struct cdev cdev; + struct cdev *cdev; struct usb_function func; struct usb_ep *in_ep; @@ -749,8 +749,9 @@ static int f_hidg_release(struct inode *inode, struct file *fd) static int f_hidg_open(struct inode *inode, struct file *fd) { + struct kobject *parent = inode->i_cdev->kobj.parent; struct f_hidg *hidg = - container_of(inode->i_cdev, struct f_hidg, cdev); + container_of(parent, struct f_hidg, dev.kobj); fd->private_data = hidg; @@ -1276,8 +1277,12 @@ static int hidg_bind(struct usb_configuration *c, struct usb_function *f) } /* create char device */ - cdev_init(&hidg->cdev, &f_hidg_fops); - status = cdev_device_add(&hidg->cdev, &hidg->dev); + hidg->cdev = cdev_alloc(); + if (!hidg->cdev) + goto fail_free_all; + hidg->cdev->ops = &f_hidg_fops; + + status = cdev_device_add(hidg->cdev, &hidg->dev); if (status) goto fail_free_all; @@ -1579,7 +1584,7 @@ static void hidg_unbind(struct usb_configuration *c, struct usb_function *f) { struct f_hidg *hidg = func_to_hidg(f); - cdev_device_del(&hidg->cdev, &hidg->dev); + cdev_device_del(hidg->cdev, &hidg->dev); destroy_workqueue(hidg->workqueue); usb_free_all_descriptors(f); } diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index a6fa5ed3d6cb76..c5bf8a448d6419 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -1210,8 +1210,8 @@ static int ncm_unwrap_ntb(struct gether *port, block_len = get_ncm(&tmp, opts->block_length); /* (d)wBlockLength */ - if (block_len > ntb_max) { - INFO(port->func.config->cdev, "OUT size exceeded\n"); + if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max)) { + INFO(port->func.config->cdev, "Bad block length: %#X\n", block_len); goto err; } diff --git a/drivers/usb/gadget/function/f_phonet.c b/drivers/usb/gadget/function/f_phonet.c index 2c007790ead25a..b1ee9a7c2e9440 100644 --- a/drivers/usb/gadget/function/f_phonet.c +++ b/drivers/usb/gadget/function/f_phonet.c @@ -333,6 +333,15 @@ static void pn_rx_complete(struct usb_ep *ep, struct usb_request *req) if (unlikely(!skb)) break; + if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { + /* Frame count from host exceeds frags[] capacity */ + dev_kfree_skb_any(skb); + if (fp->rx.skb == skb) + fp->rx.skb = NULL; + dev->stats.rx_length_errors++; + break; + } + if (skb->len == 0) { /* First fragment */ skb->protocol = htons(ETH_P_PHONET); skb_reset_mac_header(skb); diff --git a/drivers/usb/gadget/udc/omap_udc.c b/drivers/usb/gadget/udc/omap_udc.c index 91139ae668f480..f3ca79cece1bea 100644 --- a/drivers/usb/gadget/udc/omap_udc.c +++ b/drivers/usb/gadget/udc/omap_udc.c @@ -733,8 +733,6 @@ static void dma_channel_claim(struct omap_ep *ep, unsigned channel) if (status == 0) { omap_writew(reg, UDC_TXDMA_CFG); /* EMIFF or SDRC */ - omap_set_dma_src_burst_mode(ep->lch, - OMAP_DMA_DATA_BURST_4); omap_set_dma_src_data_pack(ep->lch, 1); /* TIPB */ omap_set_dma_dest_params(ep->lch, @@ -756,8 +754,6 @@ static void dma_channel_claim(struct omap_ep *ep, unsigned channel) UDC_DATA_DMA, 0, 0); /* EMIFF or SDRC */ - omap_set_dma_dest_burst_mode(ep->lch, - OMAP_DMA_DATA_BURST_4); omap_set_dma_dest_data_pack(ep->lch, 1); } } diff --git a/drivers/usb/gadget/udc/renesas_usb3.c b/drivers/usb/gadget/udc/renesas_usb3.c index b0b264d34919cd..2c9c3e935a5ec8 100644 --- a/drivers/usb/gadget/udc/renesas_usb3.c +++ b/drivers/usb/gadget/udc/renesas_usb3.c @@ -1669,6 +1669,10 @@ static bool usb3_std_req_get_status(struct renesas_usb3 *usb3, break; case USB_RECIP_ENDPOINT: num = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK; + if (num >= usb3->num_usb3_eps) { + stall = true; + break; + } usb3_ep = usb3_get_ep(usb3, num); if (usb3_ep->halt) status |= 1 << USB_ENDPOINT_HALT; @@ -1781,7 +1785,8 @@ static bool usb3_std_req_feature_endpoint(struct renesas_usb3 *usb3, struct renesas_usb3_ep *usb3_ep; struct renesas_usb3_request *usb3_req; - if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) + if ((le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT) || + (num >= usb3->num_usb3_eps)) return true; /* stall */ usb3_ep = usb3_get_ep(usb3, num); diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index 0a277a07cf7062..b879b438c7e71c 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -51,6 +51,15 @@ config USB_XHCI_PCI_RENESAS installed on your system for this device to work. If unsure, say 'N'. +config USB_XHCI_PCI_ASMEDIA + bool "Support firmware loading for ASMedia xHCI controllers" + default USB_XHCI_PCI if ARCH_APPLE + depends on USB_XHCI_PCI + help + Say 'Y' to enable support for ASMedia xHCI controllers with + host-supplied firmware. These are usually present on Apple devices. + If unsure, say 'N'. + config USB_XHCI_PLATFORM tristate "Generic xHCI driver for a platform device" help diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile index a07e7ba9cd533a..09b4f6c60e2d22 100644 --- a/drivers/usb/host/Makefile +++ b/drivers/usb/host/Makefile @@ -71,6 +71,8 @@ obj-$(CONFIG_USB_UHCI_HCD) += uhci-hcd.o obj-$(CONFIG_USB_FHCI_HCD) += fhci.o obj-$(CONFIG_USB_XHCI_HCD) += xhci-hcd.o obj-$(CONFIG_USB_XHCI_PCI) += xhci-pci.o +xhci-pci-y += xhci-pci-core.o +xhci-pci-$(CONFIG_USB_XHCI_PCI_ASMEDIA) += xhci-pci-asmedia.o obj-$(CONFIG_USB_XHCI_PCI_RENESAS) += xhci-pci-renesas.o obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o obj-$(CONFIG_USB_XHCI_HISTB) += xhci-histb.o diff --git a/drivers/usb/host/xhci-pci-asmedia.c b/drivers/usb/host/xhci-pci-asmedia.c new file mode 100644 index 00000000000000..d6b12f5c540296 --- /dev/null +++ b/drivers/usb/host/xhci-pci-asmedia.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * ASMedia xHCI firmware loader + * Copyright (C) The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include + +#include "xhci.h" +#include "xhci-trace.h" +#include "xhci-pci.h" + +/* Configuration space registers */ +#define ASMT_CFG_CONTROL 0xe0 +#define ASMT_CFG_CONTROL_WRITE BIT(1) +#define ASMT_CFG_CONTROL_READ BIT(0) + +#define ASMT_CFG_SRAM_ADDR 0xe2 + +#define ASMT_CFG_SRAM_ACCESS 0xef +#define ASMT_CFG_SRAM_ACCESS_READ BIT(6) +#define ASMT_CFG_SRAM_ACCESS_ENABLE BIT(7) + +#define ASMT_CFG_DATA_READ0 0xf0 +#define ASMT_CFG_DATA_READ1 0xf4 + +#define ASMT_CFG_DATA_WRITE0 0xf8 +#define ASMT_CFG_DATA_WRITE1 0xfc + +#define ASMT_CMD_GET_FWVER 0x8000060840 +#define ASMT_FWVER_ROM 0x010250090816 + +/* BAR0 registers */ +#define ASMT_REG_ADDR 0x3000 + +#define ASMT_REG_WDATA 0x3004 +#define ASMT_REG_RDATA 0x3008 + +#define ASMT_REG_STATUS 0x3009 +#define ASMT_REG_STATUS_BUSY BIT(7) + +#define ASMT_REG_CODE_WDATA 0x3010 +#define ASMT_REG_CODE_RDATA 0x3018 + +#define ASMT_MMIO_CPU_MISC 0x500e +#define ASMT_MMIO_CPU_MISC_CODE_RAM_WR BIT(0) + +#define ASMT_MMIO_CPU_MODE_NEXT 0x5040 +#define ASMT_MMIO_CPU_MODE_CUR 0x5041 + +#define ASMT_MMIO_CPU_MODE_RAM BIT(0) +#define ASMT_MMIO_CPU_MODE_HALFSPEED BIT(1) + +#define ASMT_MMIO_CPU_EXEC_CTRL 0x5042 +#define ASMT_MMIO_CPU_EXEC_CTRL_RESET BIT(0) +#define ASMT_MMIO_CPU_EXEC_CTRL_HALT BIT(1) + +#define TIMEOUT_USEC 10000 +#define RESET_TIMEOUT_USEC 500000 + +static int asmedia_mbox_tx(struct pci_dev *pdev, u64 data) +{ + u8 op; + int ret, err; + + ret = read_poll_timeout(pci_read_config_byte, err, + err || !(op & ASMT_CFG_CONTROL_WRITE), + 1, TIMEOUT_USEC, false, pdev, ASMT_CFG_CONTROL, + &op); + if (ret) { + dev_err(&pdev->dev, + "Timed out on mailbox tx: 0x%llx\n", + data); + return ret; + } + if (err) + return err; + + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE0, data); + pci_write_config_dword(pdev, ASMT_CFG_DATA_WRITE1, data >> 32); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, ASMT_CFG_CONTROL_WRITE); + + return 0; +} + +static int asmedia_mbox_rx(struct pci_dev *pdev, u64 *data) +{ + u8 op; + u32 low, high; + int ret, err; + + ret = read_poll_timeout(pci_read_config_byte, err, + err || (op & ASMT_CFG_CONTROL_READ), + 1, TIMEOUT_USEC, false, pdev, ASMT_CFG_CONTROL, + &op); + if (ret) { + dev_err(&pdev->dev, "Timed out on mailbox rx\n"); + return ret; + } + if (err) + return err; + + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ0, &low); + pci_read_config_dword(pdev, ASMT_CFG_DATA_READ1, &high); + pci_write_config_byte(pdev, ASMT_CFG_CONTROL, ASMT_CFG_CONTROL_READ); + + *data = ((u64)high << 32) | low; + return 0; +} + +static int asmedia_get_fw_version(struct pci_dev *pdev, u64 *version) +{ + int err = 0; + u64 cmd; + + err = asmedia_mbox_tx(pdev, ASMT_CMD_GET_FWVER); + if (err) + return err; + err = asmedia_mbox_tx(pdev, 0); + if (err) + return err; + + err = asmedia_mbox_rx(pdev, &cmd); + if (err) + return err; + err = asmedia_mbox_rx(pdev, version); + if (err) + return err; + + if (cmd != ASMT_CMD_GET_FWVER) { + dev_err(&pdev->dev, "Unexpected reply command 0x%llx\n", cmd); + return -EIO; + } + + return 0; +} + +static bool asmedia_check_firmware(struct pci_dev *pdev) +{ + u64 fwver; + int ret; + + ret = asmedia_get_fw_version(pdev, &fwver); + if (ret) + return ret; + + dev_info(&pdev->dev, "Firmware version: 0x%llx\n", fwver); + + return fwver != ASMT_FWVER_ROM; +} + +static int asmedia_wait_reset(struct pci_dev *pdev) +{ + struct usb_hcd *hcd = dev_get_drvdata(&pdev->dev); + struct xhci_cap_regs __iomem *cap = hcd->regs; + struct xhci_op_regs __iomem *op; + u32 val; + int ret; + + op = hcd->regs + HC_LENGTH(readl(&cap->hc_capbase)); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (!ret) + return 0; + + dev_err(hcd->self.controller, "Reset timed out, trying to kick it\n"); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = readl_poll_timeout(&op->command, + val, !(val & CMD_RESET), + 1000, RESET_TIMEOUT_USEC); + + if (ret) + dev_err(hcd->self.controller, "Reset timed out, giving up\n"); + + return ret; +} + +static int asmedia_read_reg(struct usb_hcd *hcd, u16 addr, u8 *val) { + void __iomem *regs = hcd->regs; + u8 status; + int ret; + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg wait timed out ([%04x])\n", addr); + return ret; + } + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Read reg addr timed out ([%04x])\n", addr); + return ret; + } + + *val = readb_relaxed(regs + ASMT_REG_RDATA); + return 0; +} + +static int asmedia_write_reg(struct usb_hcd *hcd, u16 addr, u8 data, bool wait) { + void __iomem *regs = hcd->regs; + u8 status, val; + int ret, err; + + writew_relaxed(addr, regs + ASMT_REG_ADDR); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Write reg addr timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + + writeb_relaxed(data, regs + ASMT_REG_WDATA); + + ret = readb_poll_timeout(regs + ASMT_REG_STATUS, status, + !(status & ASMT_REG_STATUS_BUSY), + 1000, TIMEOUT_USEC); + + if (ret) { + dev_err(hcd->self.controller, + "Write reg data timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + + if (!wait) + return 0; + + ret = read_poll_timeout(asmedia_read_reg, err, err || val == data, + 0, TIMEOUT_USEC, false, hcd, addr, &val); + if (ret) { + dev_err(hcd->self.controller, + "Verify register timed out ([%04x] = %02x)\n", + addr, data); + return ret; + } + if (err) { + dev_err(hcd->self.controller, + "Verify register read error ([%04x] = %02x)\n", + addr, data); + return err; + } + + return 0; +} + +static int asmedia_load_fw(struct pci_dev *pdev, const struct firmware *fw) +{ + struct usb_hcd *hcd; + void __iomem *regs; + const u16 *fw_data = (const u16 *)fw->data; + u16 raddr; + u32 data; + size_t index = 0, addr = 0; + size_t words = fw->size >> 1; + int ret, err; + + hcd = dev_get_drvdata(&pdev->dev); + regs = hcd->regs; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_RESET, false); + if (ret) + return ret; + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed pre-upload reset\n"); + return ret; + } + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, + ASMT_MMIO_CPU_EXEC_CTRL_HALT, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, + ASMT_MMIO_CPU_MISC_CODE_RAM_WR, true); + if (ret) + return ret; + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, + ASMT_CFG_SRAM_ACCESS_ENABLE); + + /* The firmware upload is interleaved in 0x4000 word blocks */ + addr = index = 0; + while (index < words) { + data = fw_data[index]; + if ((index | 0x4000) < words) + data |= fw_data[index | 0x4000] << 16; + + pci_write_config_word(pdev, ASMT_CFG_SRAM_ADDR, + addr); + + writel_relaxed(data, regs + ASMT_REG_CODE_WDATA); + + ret = read_poll_timeout(pci_read_config_word, err, + err || (raddr != addr), + 1, TIMEOUT_USEC, false, pdev, + ASMT_CFG_SRAM_ADDR, &raddr); + if (ret) { + dev_err(hcd->self.controller, "Word write timed out\n"); + return ret; + } + if (err) + return err; + + if (++index & 0x4000) + index += 0x4000; + addr += 2; + } + + pci_write_config_byte(pdev, ASMT_CFG_SRAM_ACCESS, 0); + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MISC, 0, true); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_MODE_NEXT, + ASMT_MMIO_CPU_MODE_RAM | + ASMT_MMIO_CPU_MODE_HALFSPEED, false); + if (ret) + return ret; + + ret = asmedia_write_reg(hcd, ASMT_MMIO_CPU_EXEC_CTRL, 0, false); + if (ret) + return ret; + + ret = asmedia_wait_reset(pdev); + if (ret) { + dev_err(hcd->self.controller, "Failed post-upload reset\n"); + return ret; + } + + return 0; +} + +int asmedia_xhci_check_request_fw(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct xhci_driver_data *driver_data = + (struct xhci_driver_data *)id->driver_data; + const char *fw_name = driver_data->firmware; + const struct firmware *fw; + int ret; + + /* Check if device has firmware, if so skip everything */ + ret = asmedia_check_firmware(pdev); + if (ret < 0) + return ret; + else if (ret == 1) + return 0; + + pci_dev_get(pdev); + ret = request_firmware(&fw, fw_name, &pdev->dev); + pci_dev_put(pdev); + if (ret) { + dev_err(&pdev->dev, "Could not load firmware %s: %d\n", + fw_name, ret); + return ret; + } + + ret = asmedia_load_fw(pdev, fw); + if (ret) { + dev_err(&pdev->dev, "Firmware upload failed: %d\n", ret); + goto err; + } + + ret = asmedia_check_firmware(pdev); + if (ret < 0) { + goto err; + } else if (ret != 1) { + dev_err(&pdev->dev, "Firmware version is too old after upload\n"); + ret = -EIO; + } else { + ret = 0; + } + +err: + release_firmware(fw); + return ret; +} diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci-core.c similarity index 97% rename from drivers/usb/host/xhci-pci.c rename to drivers/usb/host/xhci-pci-core.c index 585b2f3117b08a..c58cdcdb3cc862 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci-core.c @@ -569,6 +569,18 @@ static int xhci_pci_setup(struct usb_hcd *hcd) struct pci_dev *pdev = to_pci_dev(hcd->self.controller); int retval; u8 sbrn; + struct xhci_driver_data *driver_data; + const struct pci_device_id *id; + + id = pci_match_id(to_pci_driver(pdev->dev.driver)->id_table, pdev); + if (id && id->driver_data && usb_hcd_is_primary_hcd(hcd)) { + driver_data = (struct xhci_driver_data *)id->driver_data; + if (driver_data->quirks & XHCI_ASMEDIA_FW_QUIRK) { + retval = asmedia_xhci_check_request_fw(pdev, id); + if (retval < 0) + return retval; + } + } xhci = hcd_to_xhci(hcd); @@ -938,10 +950,19 @@ static void xhci_pci_shutdown(struct usb_hcd *hcd) pci_set_power_state(pdev, PCI_D3hot); } +#define ASMEDIA_APPLE_FW_NAME "asmedia/asm2214a-apple.bin" + /*-------------------------------------------------------------------------*/ +static const struct xhci_driver_data asmedia_data = { + .quirks = XHCI_ASMEDIA_FW_QUIRK, + .firmware = ASMEDIA_APPLE_FW_NAME, +}; /* PCI driver selection metadata; PCI hotplugging uses this */ static const struct pci_device_id pci_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_ASMEDIA, 0x2142), + .driver_data = (unsigned long)&asmedia_data, + }, /* handle any USB 3.0 xHCI controller */ { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_XHCI, ~0), }, @@ -949,6 +970,10 @@ static const struct pci_device_id pci_ids[] = { }; MODULE_DEVICE_TABLE(pci, pci_ids); +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +MODULE_FIRMWARE(ASMEDIA_APPLE_FW_NAME); +#endif + /* pci driver glue; this is a "new style" PCI driver module */ static struct pci_driver xhci_pci_driver = { .name = hcd_name, diff --git a/drivers/usb/host/xhci-pci.h b/drivers/usb/host/xhci-pci.h index e87c7d9d76b8e2..452908d1c069ba 100644 --- a/drivers/usb/host/xhci-pci.h +++ b/drivers/usb/host/xhci-pci.h @@ -7,4 +7,22 @@ int xhci_pci_common_probe(struct pci_dev *dev, const struct pci_device_id *id); void xhci_pci_remove(struct pci_dev *dev); +struct xhci_driver_data { + u64 quirks; + const char *firmware; +}; + +#if IS_ENABLED(CONFIG_USB_XHCI_PCI_ASMEDIA) +int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id); + +#else +static inline int asmedia_xhci_check_request_fw(struct pci_dev *dev, + const struct pci_device_id *id) +{ + return 0; +} + +#endif + #endif diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index ef6d8662adecf4..8d8f0865fc1217 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -3285,7 +3285,6 @@ static void xhci_endpoint_disable(struct usb_hcd *hcd, xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n", ep->ep_state); done: - host_ep->hcpriv = NULL; spin_unlock_irqrestore(&xhci->lock, flags); } diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index 2b0796f6d00eae..d262671a3803ee 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1644,6 +1644,7 @@ struct xhci_hcd { #define XHCI_CDNS_SCTX_QUIRK BIT_ULL(48) #define XHCI_ETRON_HOST BIT_ULL(49) #define XHCI_LIMIT_ENDPOINT_INTERVAL_9 BIT_ULL(50) +#define XHCI_ASMEDIA_FW_QUIRK BIT_ULL(51) unsigned int num_active_eps; unsigned int limit_active_eps; diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 313612114db905..42e4cecd28aca6 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -1383,6 +1383,8 @@ static const struct usb_device_id option_ids[] = { .driver_info = NCTRL(2) | RSVD(3) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1073, 0xff), /* Telit FN990A (ECM) */ .driver_info = NCTRL(0) | RSVD(1) }, + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1074, 0xff), /* Telit FN990A (MBIM) */ + .driver_info = NCTRL(5) | RSVD(6) | RSVD(7) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1075, 0xff), /* Telit FN990A (PCIe) */ .driver_info = RSVD(0) }, { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1077, 0xff), /* Telit FN990A (rmnet + audio) */ @@ -1511,7 +1513,11 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1231, 0xff), /* Telit LE910Cx (RNDIS) */ .driver_info = NCTRL(2) | RSVD(3) }, { USB_DEVICE_AND_INTERFACE_INFO(TELIT_VENDOR_ID, 0x1250, 0xff, 0x00, 0x00) }, /* Telit LE910Cx (rmnet) */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1251, 0xff) }, /* Telit LE910Cx (RNDIS) */ { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1252, 0xff) }, /* Telit LE910Cx (MBIM) */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1253, 0xff) }, /* Telit LE910Cx (ECM) */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1254, 0xff) }, /* Telit LE910Cx */ + { USB_DEVICE_INTERFACE_CLASS(TELIT_VENDOR_ID, 0x1255, 0xff) }, /* Telit LE910Cx */ { USB_DEVICE(TELIT_VENDOR_ID, 0x1260), .driver_info = NCTRL(0) | RSVD(1) | RSVD(2) }, { USB_DEVICE(TELIT_VENDOR_ID, 0x1261), diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index 47f50d7a385c8a..255968f9ca42ae 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -2350,10 +2350,11 @@ UNUSUAL_DEV( 0x2027, 0xa001, 0x0000, 0x9999, US_FL_SCM_MULT_TARG ), /* - * Reported by DocMAX - * and Thomas Weißschuh + * Reported by DocMAX , + * Thomas Weißschuh + * and Daniel Brát */ -UNUSUAL_DEV( 0x2109, 0x0715, 0x9999, 0x9999, +UNUSUAL_DEV( 0x2109, 0x0715, 0x0000, 0x9999, "VIA Labs, Inc.", "VL817 SATA Bridge", USB_SC_DEVICE, USB_PR_DEVICE, NULL, diff --git a/drivers/usb/typec/mux/ps883x.c b/drivers/usb/typec/mux/ps883x.c index 5f2879749769e5..1256252eceedcc 100644 --- a/drivers/usb/typec/mux/ps883x.c +++ b/drivers/usb/typec/mux/ps883x.c @@ -444,6 +444,7 @@ static int ps883x_retimer_probe(struct i2c_client *client) goto err_switch_unregister; } + i2c_set_clientdata(client, retimer); return 0; err_switch_unregister: diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c index 19ff8217818e7e..5b1f2750cfc3bb 100644 --- a/drivers/usb/typec/tcpm/fusb302.c +++ b/drivers/usb/typec/tcpm/fusb302.c @@ -1755,8 +1755,9 @@ static int fusb302_probe(struct i2c_client *client) goto destroy_workqueue; } - ret = request_irq(chip->gpio_int_n_irq, fusb302_irq_intn, - IRQF_TRIGGER_LOW, "fsc_interrupt_int_n", chip); + ret = request_threaded_irq(chip->gpio_int_n_irq, NULL, fusb302_irq_intn, + IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "fsc_interrupt_int_n", chip); if (ret < 0) { dev_err(dev, "cannot request IRQ for GPIO Int_N, ret=%d", ret); goto tcpm_unregister_port; diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 8e0e14a2704e02..7612a078bdd188 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -634,9 +634,14 @@ static const char * const pd_rev[] = { (tcpm_cc_is_source((port)->cc2) && \ !tcpm_cc_is_source((port)->cc1))) +#define tcpm_port_is_debug_source(port) \ + (tcpm_cc_is_source((port)->cc1) && tcpm_cc_is_source((port)->cc2)) + +#define tcpm_port_is_debug_sink(port) \ + (tcpm_cc_is_sink((port)->cc1) && tcpm_cc_is_sink((port)->cc2)) + #define tcpm_port_is_debug(port) \ - ((tcpm_cc_is_source((port)->cc1) && tcpm_cc_is_source((port)->cc2)) || \ - (tcpm_cc_is_sink((port)->cc1) && tcpm_cc_is_sink((port)->cc2))) + (tcpm_port_is_debug_source(port) || tcpm_port_is_debug_sink(port)) #define tcpm_port_is_audio(port) \ (tcpm_cc_is_audio((port)->cc1) && tcpm_cc_is_audio((port)->cc2)) @@ -4812,7 +4817,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_state(port, SNK_UNATTACHED, PD_T_DRP_SNK); break; case SRC_ATTACH_WAIT: - if (tcpm_port_is_debug(port)) + if (tcpm_port_is_debug_source(port)) tcpm_set_state(port, DEBUG_ACC_ATTACHED, port->timings.cc_debounce_time); else if (tcpm_port_is_audio(port)) @@ -5070,7 +5075,7 @@ static void run_state_machine(struct tcpm_port *port) tcpm_set_state(port, SRC_UNATTACHED, PD_T_DRP_SRC); break; case SNK_ATTACH_WAIT: - if (tcpm_port_is_debug(port)) + if (tcpm_port_is_debug_sink(port)) tcpm_set_state(port, DEBUG_ACC_ATTACHED, PD_T_CC_DEBOUNCE); else if (tcpm_port_is_audio(port)) @@ -5090,7 +5095,7 @@ static void run_state_machine(struct tcpm_port *port) if (tcpm_port_is_disconnected(port)) tcpm_set_state(port, SNK_UNATTACHED, PD_T_PD_DEBOUNCE); - else if (tcpm_port_is_debug(port)) + else if (tcpm_port_is_debug_sink(port)) tcpm_set_state(port, DEBUG_ACC_ATTACHED, PD_T_CC_DEBOUNCE); else if (tcpm_port_is_audio(port)) @@ -5534,6 +5539,8 @@ static void run_state_machine(struct tcpm_port *port) usb_power_delivery_unregister_capabilities(port->partner_source_caps); port->partner_source_caps = NULL; tcpm_pd_send_control(port, PD_CTRL_ACCEPT, TCPC_TX_SOP); + port->vdm_sm_running = false; + port->explicit_contract = false; tcpm_ams_finish(port); if (port->pwr_role == TYPEC_SOURCE) { port->upcoming_state = SRC_SEND_CAPABILITIES; @@ -5961,10 +5968,10 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, switch (port->state) { case TOGGLING: - if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) || + if (tcpm_port_is_debug_source(port) || tcpm_port_is_audio(port) || tcpm_port_is_source(port)) tcpm_set_state(port, SRC_ATTACH_WAIT, 0); - else if (tcpm_port_is_sink(port)) + else if (tcpm_port_is_debug_sink(port) || tcpm_port_is_sink(port)) tcpm_set_state(port, SNK_ATTACH_WAIT, 0); break; case CHECK_CONTAMINANT: @@ -5972,9 +5979,11 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, break; case SRC_UNATTACHED: case ACC_UNATTACHED: - if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) || + if (tcpm_port_is_debug_source(port) || tcpm_port_is_audio(port) || tcpm_port_is_source(port)) tcpm_set_state(port, SRC_ATTACH_WAIT, 0); + else if (tcpm_port_is_debug_sink(port)) + tcpm_set_state(port, SNK_ATTACH_WAIT, 0); break; case SRC_ATTACH_WAIT: if (tcpm_port_is_disconnected(port) || @@ -5996,7 +6005,7 @@ static void _tcpm_cc_change(struct tcpm_port *port, enum typec_cc_status cc1, } break; case SNK_UNATTACHED: - if (tcpm_port_is_debug(port) || tcpm_port_is_audio(port) || + if (tcpm_port_is_debug_sink(port) || tcpm_port_is_audio(port) || tcpm_port_is_sink(port)) tcpm_set_state(port, SNK_ATTACH_WAIT, 0); break; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index e2b26af2b84a8d..43faec794b95a0 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -820,8 +820,10 @@ static void cd321x_update_work(struct work_struct *work) desc.identity = &st.partner_identity; tps->partner = typec_register_partner(tps->port, &desc); - if (IS_ERR(tps->partner)) - dev_warn(tps->dev, "%s: failed to register partnet\n", __func__); + if (IS_ERR(tps->partner)) { + dev_warn(tps->dev, "%s: failed to register partner\n", __func__); + return; + } if (desc.identity) { typec_partner_set_identity(tps->partner); diff --git a/drivers/usb/usbip/usbip_common.c b/drivers/usb/usbip/usbip_common.c index a2b2da1255dda0..ba9e7c616e1294 100644 --- a/drivers/usb/usbip/usbip_common.c +++ b/drivers/usb/usbip/usbip_common.c @@ -470,6 +470,18 @@ static void usbip_pack_ret_submit(struct usbip_header *pdu, struct urb *urb, urb->status = rpdu->status; urb->actual_length = rpdu->actual_length; urb->start_frame = rpdu->start_frame; + /* + * The number_of_packets field determines the length of + * iso_frame_desc[], which is a flexible array allocated + * at URB creation time. A response must never claim more + * packets than originally submitted; doing so would cause + * an out-of-bounds write in usbip_recv_iso() and + * usbip_pad_iso(). Clamp to zero on violation so both + * functions safely return early. + */ + if (rpdu->number_of_packets < 0 || + rpdu->number_of_packets > urb->number_of_packets) + rpdu->number_of_packets = 0; urb->number_of_packets = rpdu->number_of_packets; urb->error_count = rpdu->error_count; } diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 34874beb0152ee..caf0ee5d6856ca 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -67,57 +67,20 @@ static void vdpa_dev_remove(struct device *d) static int vdpa_dev_match(struct device *dev, const struct device_driver *drv) { - struct vdpa_device *vdev = dev_to_vdpa(dev); + int ret; /* Check override first, and if set, only use the named driver */ - if (vdev->driver_override) - return strcmp(vdev->driver_override, drv->name) == 0; + ret = device_match_driver_override(dev, drv); + if (ret >= 0) + return ret; /* Currently devices must be supported by all vDPA bus drivers */ return 1; } -static ssize_t driver_override_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct vdpa_device *vdev = dev_to_vdpa(dev); - int ret; - - ret = driver_set_override(dev, &vdev->driver_override, buf, count); - if (ret) - return ret; - - return count; -} - -static ssize_t driver_override_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct vdpa_device *vdev = dev_to_vdpa(dev); - ssize_t len; - - device_lock(dev); - len = sysfs_emit(buf, "%s\n", vdev->driver_override); - device_unlock(dev); - - return len; -} -static DEVICE_ATTR_RW(driver_override); - -static struct attribute *vdpa_dev_attrs[] = { - &dev_attr_driver_override.attr, - NULL, -}; - -static const struct attribute_group vdpa_dev_group = { - .attrs = vdpa_dev_attrs, -}; -__ATTRIBUTE_GROUPS(vdpa_dev); - static const struct bus_type vdpa_bus = { .name = "vdpa", - .dev_groups = vdpa_dev_groups, + .driver_override = true, .match = vdpa_dev_match, .probe = vdpa_dev_probe, .remove = vdpa_dev_remove, @@ -132,7 +95,6 @@ static void vdpa_release_dev(struct device *d) ops->free(vdev); ida_free(&vdpa_index_ida, vdev->index); - kfree(vdev->driver_override); kfree(vdev); } diff --git a/drivers/vfio/cdx/intr.c b/drivers/vfio/cdx/intr.c index 8f4402cec9c50f..6dfe0ced3bdd8a 100644 --- a/drivers/vfio/cdx/intr.c +++ b/drivers/vfio/cdx/intr.c @@ -152,6 +152,8 @@ static int vfio_cdx_set_msi_trigger(struct vfio_cdx_device *vdev, if (start + count > cdx_dev->num_msi) return -EINVAL; + guard(mutex)(&vdev->cdx_irqs_lock); + if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) { vfio_cdx_msi_disable(vdev); return 0; @@ -175,6 +177,10 @@ static int vfio_cdx_set_msi_trigger(struct vfio_cdx_device *vdev, return ret; } + /* Ensure MSI is configured before accessing cdx_irqs */ + if (!vdev->config_msi) + return -EINVAL; + for (i = start; i < start + count; i++) { if (!vdev->cdx_irqs[i].trigger) continue; @@ -206,12 +212,5 @@ int vfio_cdx_set_irqs_ioctl(struct vfio_cdx_device *vdev, /* Free All IRQs for the given device */ void vfio_cdx_irqs_cleanup(struct vfio_cdx_device *vdev) { - /* - * Device does not support any interrupt or the interrupts - * were not configured - */ - if (!vdev->cdx_irqs) - return; - vfio_cdx_set_msi_trigger(vdev, 0, 0, 0, VFIO_IRQ_SET_DATA_NONE, NULL); } diff --git a/drivers/vfio/cdx/main.c b/drivers/vfio/cdx/main.c index 8ab97405b2bdda..b31ed4be7bdc19 100644 --- a/drivers/vfio/cdx/main.c +++ b/drivers/vfio/cdx/main.c @@ -8,6 +8,23 @@ #include "private.h" +static int vfio_cdx_init_dev(struct vfio_device *core_vdev) +{ + struct vfio_cdx_device *vdev = + container_of(core_vdev, struct vfio_cdx_device, vdev); + + mutex_init(&vdev->cdx_irqs_lock); + return 0; +} + +static void vfio_cdx_release_dev(struct vfio_device *core_vdev) +{ + struct vfio_cdx_device *vdev = + container_of(core_vdev, struct vfio_cdx_device, vdev); + + mutex_destroy(&vdev->cdx_irqs_lock); +} + static int vfio_cdx_open_device(struct vfio_device *core_vdev) { struct vfio_cdx_device *vdev = @@ -273,6 +290,8 @@ static int vfio_cdx_mmap(struct vfio_device *core_vdev, static const struct vfio_device_ops vfio_cdx_ops = { .name = "vfio-cdx", + .init = vfio_cdx_init_dev, + .release = vfio_cdx_release_dev, .open_device = vfio_cdx_open_device, .close_device = vfio_cdx_close_device, .ioctl = vfio_cdx_ioctl, diff --git a/drivers/vfio/cdx/private.h b/drivers/vfio/cdx/private.h index 172e48caa3a062..94374b5fc9899c 100644 --- a/drivers/vfio/cdx/private.h +++ b/drivers/vfio/cdx/private.h @@ -6,6 +6,8 @@ #ifndef VFIO_CDX_PRIVATE_H #define VFIO_CDX_PRIVATE_H +#include + #define VFIO_CDX_OFFSET_SHIFT 40 static inline u64 vfio_cdx_index_to_offset(u32 index) @@ -31,6 +33,7 @@ struct vfio_cdx_region { struct vfio_cdx_device { struct vfio_device vdev; struct vfio_cdx_region *regions; + struct mutex cdx_irqs_lock; struct vfio_cdx_irq *cdx_irqs; u32 flags; #define BME_SUPPORT BIT(0) diff --git a/drivers/vfio/fsl-mc/vfio_fsl_mc.c b/drivers/vfio/fsl-mc/vfio_fsl_mc.c index 462fae1aa53851..b4c3958201b258 100644 --- a/drivers/vfio/fsl-mc/vfio_fsl_mc.c +++ b/drivers/vfio/fsl-mc/vfio_fsl_mc.c @@ -424,9 +424,7 @@ static int vfio_fsl_mc_bus_notifier(struct notifier_block *nb, if (action == BUS_NOTIFY_ADD_DEVICE && vdev->mc_dev == mc_cont) { - mc_dev->driver_override = kasprintf(GFP_KERNEL, "%s", - vfio_fsl_mc_ops.name); - if (!mc_dev->driver_override) + if (device_set_driver_override(dev, vfio_fsl_mc_ops.name)) dev_warn(dev, "VFIO_FSL_MC: Setting driver override for device in dprc %s failed\n", dev_name(&mc_cont->dev)); else diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index d43745fe4c8433..e34c7e1ba1c81f 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -734,10 +734,10 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev) #if IS_ENABLED(CONFIG_EEH) eeh_dev_release(vdev->pdev); #endif - vfio_pci_core_disable(vdev); - vfio_pci_dma_buf_cleanup(vdev); + vfio_pci_core_disable(vdev); + mutex_lock(&vdev->igate); vfio_pci_eventfd_replace_locked(vdev, &vdev->err_trigger, NULL); vfio_pci_eventfd_replace_locked(vdev, &vdev->req_trigger, NULL); @@ -1670,21 +1670,16 @@ vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev, if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev)) return VM_FAULT_SIGBUS; - switch (order) { - case 0: + if (!order) return vmf_insert_pfn(vmf->vma, vmf->address, pfn); -#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP - case PMD_ORDER: + + if (IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP) && order == PMD_ORDER) return vmf_insert_pfn_pmd(vmf, pfn, false); -#endif -#ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP - case PUD_ORDER: + + if (IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP) && order == PUD_ORDER) return vmf_insert_pfn_pud(vmf, pfn, false); - break; -#endif - default: - return VM_FAULT_FALLBACK; - } + + return VM_FAULT_FALLBACK; } EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn); @@ -1987,9 +1982,8 @@ static int vfio_pci_bus_notifier(struct notifier_block *nb, pdev->is_virtfn && physfn == vdev->pdev) { pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", pci_name(pdev)); - pdev->driver_override = kasprintf(GFP_KERNEL, "%s", - vdev->vdev.ops->name); - WARN_ON(!pdev->driver_override); + WARN_ON(device_set_driver_override(&pdev->dev, + vdev->vdev.ops->name)); } else if (action == BUS_NOTIFY_BOUND_DRIVER && pdev->is_virtfn && physfn == vdev->pdev) { struct pci_driver *drv = pci_dev_driver(pdev); diff --git a/drivers/vfio/pci/virtio/common.h b/drivers/vfio/pci/virtio/common.h index cb3d5e57d3a3e9..3ccbd49e6abe90 100644 --- a/drivers/vfio/pci/virtio/common.h +++ b/drivers/vfio/pci/virtio/common.h @@ -68,7 +68,7 @@ struct virtiovf_migration_file { enum virtiovf_migf_state state; enum virtiovf_load_state load_state; /* synchronize access to the lists */ - spinlock_t list_lock; + struct mutex list_lock; struct list_head buf_list; struct list_head avail_list; struct virtiovf_data_buffer *buf; diff --git a/drivers/vfio/pci/virtio/migrate.c b/drivers/vfio/pci/virtio/migrate.c index 35fa2d6ed61173..15fcd936528b7f 100644 --- a/drivers/vfio/pci/virtio/migrate.c +++ b/drivers/vfio/pci/virtio/migrate.c @@ -142,9 +142,9 @@ virtiovf_alloc_data_buffer(struct virtiovf_migration_file *migf, size_t length) static void virtiovf_put_data_buffer(struct virtiovf_data_buffer *buf) { - spin_lock_irq(&buf->migf->list_lock); + mutex_lock(&buf->migf->list_lock); list_add_tail(&buf->buf_elm, &buf->migf->avail_list); - spin_unlock_irq(&buf->migf->list_lock); + mutex_unlock(&buf->migf->list_lock); } static int @@ -170,21 +170,21 @@ virtiovf_get_data_buffer(struct virtiovf_migration_file *migf, size_t length) INIT_LIST_HEAD(&free_list); - spin_lock_irq(&migf->list_lock); + mutex_lock(&migf->list_lock); list_for_each_entry_safe(buf, temp_buf, &migf->avail_list, buf_elm) { list_del_init(&buf->buf_elm); if (buf->allocated_length >= length) { - spin_unlock_irq(&migf->list_lock); + mutex_unlock(&migf->list_lock); goto found; } /* * Prevent holding redundant buffers. Put in a free - * list and call at the end not under the spin lock + * list and call at the end not under the mutex * (&migf->list_lock) to minimize its scope usage. */ list_add(&buf->buf_elm, &free_list); } - spin_unlock_irq(&migf->list_lock); + mutex_unlock(&migf->list_lock); buf = virtiovf_alloc_data_buffer(migf, length); found: @@ -295,6 +295,7 @@ static int virtiovf_release_file(struct inode *inode, struct file *filp) struct virtiovf_migration_file *migf = filp->private_data; virtiovf_disable_fd(migf); + mutex_destroy(&migf->list_lock); mutex_destroy(&migf->lock); kfree(migf); return 0; @@ -308,7 +309,7 @@ virtiovf_get_data_buff_from_pos(struct virtiovf_migration_file *migf, bool found = false; *end_of_data = false; - spin_lock_irq(&migf->list_lock); + mutex_lock(&migf->list_lock); if (list_empty(&migf->buf_list)) { *end_of_data = true; goto end; @@ -329,7 +330,7 @@ virtiovf_get_data_buff_from_pos(struct virtiovf_migration_file *migf, migf->state = VIRTIOVF_MIGF_STATE_ERROR; end: - spin_unlock_irq(&migf->list_lock); + mutex_unlock(&migf->list_lock); return found ? buf : NULL; } @@ -369,10 +370,10 @@ static ssize_t virtiovf_buf_read(struct virtiovf_data_buffer *vhca_buf, } if (*pos >= vhca_buf->start_pos + vhca_buf->length) { - spin_lock_irq(&vhca_buf->migf->list_lock); + mutex_lock(&vhca_buf->migf->list_lock); list_del_init(&vhca_buf->buf_elm); list_add_tail(&vhca_buf->buf_elm, &vhca_buf->migf->avail_list); - spin_unlock_irq(&vhca_buf->migf->list_lock); + mutex_unlock(&vhca_buf->migf->list_lock); } return done; @@ -554,9 +555,9 @@ virtiovf_add_buf_header(struct virtiovf_data_buffer *header_buf, header_buf->length = sizeof(header); header_buf->start_pos = header_buf->migf->max_pos; migf->max_pos += header_buf->length; - spin_lock_irq(&migf->list_lock); + mutex_lock(&migf->list_lock); list_add_tail(&header_buf->buf_elm, &migf->buf_list); - spin_unlock_irq(&migf->list_lock); + mutex_unlock(&migf->list_lock); return 0; } @@ -621,9 +622,9 @@ virtiovf_read_device_context_chunk(struct virtiovf_migration_file *migf, buf->start_pos = buf->migf->max_pos; migf->max_pos += buf->length; - spin_lock(&migf->list_lock); + mutex_lock(&migf->list_lock); list_add_tail(&buf->buf_elm, &migf->buf_list); - spin_unlock_irq(&migf->list_lock); + mutex_unlock(&migf->list_lock); return 0; out_header: @@ -692,7 +693,7 @@ virtiovf_pci_save_device_data(struct virtiovf_pci_core_device *virtvdev, mutex_init(&migf->lock); INIT_LIST_HEAD(&migf->buf_list); INIT_LIST_HEAD(&migf->avail_list); - spin_lock_init(&migf->list_lock); + mutex_init(&migf->list_lock); migf->virtvdev = virtvdev; lockdep_assert_held(&virtvdev->state_mutex); @@ -1082,7 +1083,7 @@ virtiovf_pci_resume_device_data(struct virtiovf_pci_core_device *virtvdev) mutex_init(&migf->lock); INIT_LIST_HEAD(&migf->buf_list); INIT_LIST_HEAD(&migf->avail_list); - spin_lock_init(&migf->list_lock); + mutex_init(&migf->list_lock); buf = virtiovf_alloc_data_buffer(migf, VIRTIOVF_TARGET_INITIAL_BUF_SIZE); if (IS_ERR(buf)) { diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c index fff95b2d5dde1c..15513825090434 100644 --- a/drivers/vfio/pci/xe/main.c +++ b/drivers/vfio/pci/xe/main.c @@ -454,39 +454,46 @@ static const struct vfio_migration_ops xe_vfio_pci_migration_ops = { static void xe_vfio_pci_migration_init(struct xe_vfio_pci_core_device *xe_vdev) { struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; - struct pci_dev *pdev = to_pci_dev(core_vdev->dev); - struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); - if (!xe) + if (!xe_sriov_vfio_migration_supported(xe_vdev->xe)) return; - if (!xe_sriov_vfio_migration_supported(xe)) - return; - - mutex_init(&xe_vdev->state_mutex); - spin_lock_init(&xe_vdev->reset_lock); - - /* PF internal control uses vfid index starting from 1 */ - xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; - xe_vdev->xe = xe; core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; core_vdev->mig_ops = &xe_vfio_pci_migration_ops; } -static void xe_vfio_pci_migration_fini(struct xe_vfio_pci_core_device *xe_vdev) +static int xe_vfio_pci_vf_init(struct xe_vfio_pci_core_device *xe_vdev) { - if (!xe_vdev->vfid) - return; + struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; + struct pci_dev *pdev = to_pci_dev(core_vdev->dev); + struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); - mutex_destroy(&xe_vdev->state_mutex); + if (!pdev->is_virtfn) + return 0; + if (!xe) + return -ENODEV; + xe_vdev->xe = xe; + + /* PF internal control uses vfid index starting from 1 */ + xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; + + xe_vfio_pci_migration_init(xe_vdev); + + return 0; } static int xe_vfio_pci_init_dev(struct vfio_device *core_vdev) { struct xe_vfio_pci_core_device *xe_vdev = container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + int ret; - xe_vfio_pci_migration_init(xe_vdev); + mutex_init(&xe_vdev->state_mutex); + spin_lock_init(&xe_vdev->reset_lock); + + ret = xe_vfio_pci_vf_init(xe_vdev); + if (ret) + return ret; return vfio_pci_core_init_dev(core_vdev); } @@ -496,7 +503,8 @@ static void xe_vfio_pci_release_dev(struct vfio_device *core_vdev) struct xe_vfio_pci_core_device *xe_vdev = container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); - xe_vfio_pci_migration_fini(xe_vdev); + mutex_destroy(&xe_vdev->state_mutex); + vfio_pci_core_release_dev(core_vdev); } static const struct vfio_device_ops xe_vfio_pci_ops = { diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 80965181920c36..c6536cad9c4f94 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -560,7 +560,7 @@ static void vhost_net_busy_poll(struct vhost_net *net, busyloop_timeout = poll_rx ? rvq->busyloop_timeout: tvq->busyloop_timeout; - preempt_disable(); + migrate_disable(); endtime = busy_clock() + busyloop_timeout; while (vhost_can_busy_poll(endtime)) { @@ -577,7 +577,7 @@ static void vhost_net_busy_poll(struct vhost_net *net, cpu_relax(); } - preempt_enable(); + migrate_enable(); if (poll_rx || sock_has_rx_data(sock)) vhost_net_busy_poll_try_queue(net, vq); diff --git a/drivers/video/backlight/sky81452-backlight.c b/drivers/video/backlight/sky81452-backlight.c index 2749231f038544..b2679b24de14bf 100644 --- a/drivers/video/backlight/sky81452-backlight.c +++ b/drivers/video/backlight/sky81452-backlight.c @@ -202,6 +202,9 @@ static struct sky81452_bl_platform_data *sky81452_bl_parse_dt( pdata->dpwm_mode = of_property_read_bool(np, "skyworks,dpwm-mode"); pdata->phase_shift = of_property_read_bool(np, "skyworks,phase-shift"); pdata->gpiod_enable = devm_gpiod_get_optional(dev, NULL, GPIOD_OUT_HIGH); + if (IS_ERR(pdata->gpiod_enable)) + return dev_err_cast_probe(dev, pdata->gpiod_enable, + "failed to get gpio\n"); ret = of_property_count_u32_elems(np, "led-sources"); if (ret < 0) { diff --git a/drivers/video/fbdev/core/fb_defio.c b/drivers/video/fbdev/core/fb_defio.c index ca48b89a323d35..93bd2f696fa475 100644 --- a/drivers/video/fbdev/core/fb_defio.c +++ b/drivers/video/fbdev/core/fb_defio.c @@ -24,6 +24,75 @@ #include #include +/* + * struct fb_deferred_io_state + */ + +struct fb_deferred_io_state { + struct kref ref; + + struct mutex lock; /* mutex that protects the pageref list */ + /* fields protected by lock */ + struct fb_info *info; +}; + +static struct fb_deferred_io_state *fb_deferred_io_state_alloc(void) +{ + struct fb_deferred_io_state *fbdefio_state; + + fbdefio_state = kzalloc_obj(*fbdefio_state); + if (!fbdefio_state) + return NULL; + + kref_init(&fbdefio_state->ref); + mutex_init(&fbdefio_state->lock); + + return fbdefio_state; +} + +static void fb_deferred_io_state_release(struct fb_deferred_io_state *fbdefio_state) +{ + mutex_destroy(&fbdefio_state->lock); + + kfree(fbdefio_state); +} + +static void fb_deferred_io_state_get(struct fb_deferred_io_state *fbdefio_state) +{ + kref_get(&fbdefio_state->ref); +} + +static void __fb_deferred_io_state_release(struct kref *ref) +{ + struct fb_deferred_io_state *fbdefio_state = + container_of(ref, struct fb_deferred_io_state, ref); + + fb_deferred_io_state_release(fbdefio_state); +} + +static void fb_deferred_io_state_put(struct fb_deferred_io_state *fbdefio_state) +{ + kref_put(&fbdefio_state->ref, __fb_deferred_io_state_release); +} + +/* + * struct vm_operations_struct + */ + +static void fb_deferred_io_vm_open(struct vm_area_struct *vma) +{ + struct fb_deferred_io_state *fbdefio_state = vma->vm_private_data; + + fb_deferred_io_state_get(fbdefio_state); +} + +static void fb_deferred_io_vm_close(struct vm_area_struct *vma) +{ + struct fb_deferred_io_state *fbdefio_state = vma->vm_private_data; + + fb_deferred_io_state_put(fbdefio_state); +} + static struct page *fb_deferred_io_get_page(struct fb_info *info, unsigned long offs) { struct fb_deferred_io *fbdefio = info->fbdefio; @@ -121,25 +190,46 @@ static void fb_deferred_io_pageref_put(struct fb_deferred_io_pageref *pageref, /* this is to find and return the vmalloc-ed fb pages */ static vm_fault_t fb_deferred_io_fault(struct vm_fault *vmf) { + struct fb_info *info; unsigned long offset; struct page *page; - struct fb_info *info = vmf->vma->vm_private_data; + vm_fault_t ret; + struct fb_deferred_io_state *fbdefio_state = vmf->vma->vm_private_data; + + mutex_lock(&fbdefio_state->lock); + + info = fbdefio_state->info; + if (!info) { + ret = VM_FAULT_SIGBUS; /* our device is gone */ + goto err_mutex_unlock; + } offset = vmf->pgoff << PAGE_SHIFT; - if (offset >= info->fix.smem_len) - return VM_FAULT_SIGBUS; + if (offset >= info->fix.smem_len) { + ret = VM_FAULT_SIGBUS; + goto err_mutex_unlock; + } page = fb_deferred_io_get_page(info, offset); - if (!page) - return VM_FAULT_SIGBUS; + if (!page) { + ret = VM_FAULT_SIGBUS; + goto err_mutex_unlock; + } if (!vmf->vma->vm_file) fb_err(info, "no mapping available\n"); BUG_ON(!info->fbdefio->mapping); + mutex_unlock(&fbdefio_state->lock); + vmf->page = page; + return 0; + +err_mutex_unlock: + mutex_unlock(&fbdefio_state->lock); + return ret; } int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasync) @@ -166,15 +256,24 @@ EXPORT_SYMBOL_GPL(fb_deferred_io_fsync); * Adds a page to the dirty list. Call this from struct * vm_operations_struct.page_mkwrite. */ -static vm_fault_t fb_deferred_io_track_page(struct fb_info *info, unsigned long offset, - struct page *page) +static vm_fault_t fb_deferred_io_track_page(struct fb_deferred_io_state *fbdefio_state, + unsigned long offset, struct page *page) { - struct fb_deferred_io *fbdefio = info->fbdefio; + struct fb_info *info; + struct fb_deferred_io *fbdefio; struct fb_deferred_io_pageref *pageref; vm_fault_t ret; /* protect against the workqueue changing the page list */ - mutex_lock(&fbdefio->lock); + mutex_lock(&fbdefio_state->lock); + + info = fbdefio_state->info; + if (!info) { + ret = VM_FAULT_SIGBUS; /* our device is gone */ + goto err_mutex_unlock; + } + + fbdefio = info->fbdefio; pageref = fb_deferred_io_pageref_get(info, offset, page); if (WARN_ON_ONCE(!pageref)) { @@ -192,50 +291,38 @@ static vm_fault_t fb_deferred_io_track_page(struct fb_info *info, unsigned long */ lock_page(pageref->page); - mutex_unlock(&fbdefio->lock); + mutex_unlock(&fbdefio_state->lock); /* come back after delay to process the deferred IO */ schedule_delayed_work(&info->deferred_work, fbdefio->delay); return VM_FAULT_LOCKED; err_mutex_unlock: - mutex_unlock(&fbdefio->lock); + mutex_unlock(&fbdefio_state->lock); return ret; } -/* - * fb_deferred_io_page_mkwrite - Mark a page as written for deferred I/O - * @fb_info: The fbdev info structure - * @vmf: The VM fault - * - * This is a callback we get when userspace first tries to - * write to the page. We schedule a workqueue. That workqueue - * will eventually mkclean the touched pages and execute the - * deferred framebuffer IO. Then if userspace touches a page - * again, we repeat the same scheme. - * - * Returns: - * VM_FAULT_LOCKED on success, or a VM_FAULT error otherwise. - */ -static vm_fault_t fb_deferred_io_page_mkwrite(struct fb_info *info, struct vm_fault *vmf) +static vm_fault_t fb_deferred_io_page_mkwrite(struct fb_deferred_io_state *fbdefio_state, + struct vm_fault *vmf) { unsigned long offset = vmf->pgoff << PAGE_SHIFT; struct page *page = vmf->page; file_update_time(vmf->vma->vm_file); - return fb_deferred_io_track_page(info, offset, page); + return fb_deferred_io_track_page(fbdefio_state, offset, page); } -/* vm_ops->page_mkwrite handler */ static vm_fault_t fb_deferred_io_mkwrite(struct vm_fault *vmf) { - struct fb_info *info = vmf->vma->vm_private_data; + struct fb_deferred_io_state *fbdefio_state = vmf->vma->vm_private_data; - return fb_deferred_io_page_mkwrite(info, vmf); + return fb_deferred_io_page_mkwrite(fbdefio_state, vmf); } static const struct vm_operations_struct fb_deferred_io_vm_ops = { + .open = fb_deferred_io_vm_open, + .close = fb_deferred_io_vm_close, .fault = fb_deferred_io_fault, .page_mkwrite = fb_deferred_io_mkwrite, }; @@ -252,7 +339,10 @@ int fb_deferred_io_mmap(struct fb_info *info, struct vm_area_struct *vma) vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); if (!(info->flags & FBINFO_VIRTFB)) vm_flags_set(vma, VM_IO); - vma->vm_private_data = info; + vma->vm_private_data = info->fbdefio_state; + + fb_deferred_io_state_get(info->fbdefio_state); /* released in vma->vm_ops->close() */ + return 0; } EXPORT_SYMBOL_GPL(fb_deferred_io_mmap); @@ -263,9 +353,10 @@ static void fb_deferred_io_work(struct work_struct *work) struct fb_info *info = container_of(work, struct fb_info, deferred_work.work); struct fb_deferred_io_pageref *pageref, *next; struct fb_deferred_io *fbdefio = info->fbdefio; + struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; /* here we wrprotect the page's mappings, then do all deferred IO. */ - mutex_lock(&fbdefio->lock); + mutex_lock(&fbdefio_state->lock); #ifdef CONFIG_MMU list_for_each_entry(pageref, &fbdefio->pagereflist, list) { struct page *page = pageref->page; @@ -283,12 +374,13 @@ static void fb_deferred_io_work(struct work_struct *work) list_for_each_entry_safe(pageref, next, &fbdefio->pagereflist, list) fb_deferred_io_pageref_put(pageref, info); - mutex_unlock(&fbdefio->lock); + mutex_unlock(&fbdefio_state->lock); } int fb_deferred_io_init(struct fb_info *info) { struct fb_deferred_io *fbdefio = info->fbdefio; + struct fb_deferred_io_state *fbdefio_state; struct fb_deferred_io_pageref *pagerefs; unsigned long npagerefs; int ret; @@ -298,7 +390,11 @@ int fb_deferred_io_init(struct fb_info *info) if (WARN_ON(!info->fix.smem_len)) return -EINVAL; - mutex_init(&fbdefio->lock); + fbdefio_state = fb_deferred_io_state_alloc(); + if (!fbdefio_state) + return -ENOMEM; + fbdefio_state->info = info; + INIT_DELAYED_WORK(&info->deferred_work, fb_deferred_io_work); INIT_LIST_HEAD(&fbdefio->pagereflist); if (fbdefio->delay == 0) /* set a default of 1 s */ @@ -315,10 +411,12 @@ int fb_deferred_io_init(struct fb_info *info) info->npagerefs = npagerefs; info->pagerefs = pagerefs; + info->fbdefio_state = fbdefio_state; + return 0; err: - mutex_destroy(&fbdefio->lock); + fb_deferred_io_state_release(fbdefio_state); return ret; } EXPORT_SYMBOL_GPL(fb_deferred_io_init); @@ -352,11 +450,19 @@ EXPORT_SYMBOL_GPL(fb_deferred_io_release); void fb_deferred_io_cleanup(struct fb_info *info) { struct fb_deferred_io *fbdefio = info->fbdefio; + struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; fb_deferred_io_lastclose(info); + info->fbdefio_state = NULL; + + mutex_lock(&fbdefio_state->lock); + fbdefio_state->info = NULL; + mutex_unlock(&fbdefio_state->lock); + + fb_deferred_io_state_put(fbdefio_state); + kvfree(info->pagerefs); - mutex_destroy(&fbdefio->lock); fbdefio->mapping = NULL; } EXPORT_SYMBOL_GPL(fb_deferred_io_cleanup); diff --git a/drivers/video/fbdev/core/fbcon_rotate.c b/drivers/video/fbdev/core/fbcon_rotate.c index 1562a8f20b4f59..5348f6c6f57c7c 100644 --- a/drivers/video/fbdev/core/fbcon_rotate.c +++ b/drivers/video/fbdev/core/fbcon_rotate.c @@ -46,6 +46,10 @@ int fbcon_rotate_font(struct fb_info *info, struct vc_data *vc) info->fbops->fb_sync(info); if (par->fd_size < d_cellsize * len) { + kfree(par->fontbuffer); + par->fontbuffer = NULL; + par->fd_size = 0; + dst = kmalloc_array(len, d_cellsize, GFP_KERNEL); if (dst == NULL) { @@ -54,7 +58,6 @@ int fbcon_rotate_font(struct fb_info *info, struct vc_data *vc) } par->fd_size = d_cellsize * len; - kfree(par->fontbuffer); par->fontbuffer = dst; } diff --git a/drivers/video/fbdev/matrox/g450_pll.c b/drivers/video/fbdev/matrox/g450_pll.c index e2c1478aa47f95..6a08f78cd1acb7 100644 --- a/drivers/video/fbdev/matrox/g450_pll.c +++ b/drivers/video/fbdev/matrox/g450_pll.c @@ -409,7 +409,7 @@ static int __g450_setclk(struct matrox_fb_info *minfo, unsigned int fout, case M_VIDEO_PLL: { u_int8_t tmp; - unsigned int mnp; + unsigned int mnp __maybe_unused; unsigned long flags; matroxfb_DAC_lock_irqsave(flags); diff --git a/drivers/video/fbdev/offb.c b/drivers/video/fbdev/offb.c index f85428e13996ba..166b2dff36f59e 100644 --- a/drivers/video/fbdev/offb.c +++ b/drivers/video/fbdev/offb.c @@ -640,8 +640,13 @@ static void offb_init_nodriver(struct platform_device *parent, struct device_nod vid = be32_to_cpup(vidp); did = be32_to_cpup(didp); pdev = pci_get_device(vid, did, NULL); - if (!pdev || pci_enable_device(pdev)) + if (!pdev) return; + + if (pci_enable_device(pdev)) { + pci_dev_put(pdev); + return; + } } #endif /* kludge for valkyrie */ diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index 51ebe78359ec34..531fb8478e2071 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -496,6 +496,9 @@ static int tdfxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info) } } + if (!var->pixclock) + return -EINVAL; + if (PICOS2KHZ(var->pixclock) > par->max_pixclock) { DPRINTK("pixclock too high (%ldKHz)\n", PICOS2KHZ(var->pixclock)); diff --git a/drivers/video/fbdev/udlfb.c b/drivers/video/fbdev/udlfb.c index 3c6a9b5758d955..fdbb8671a810c7 100644 --- a/drivers/video/fbdev/udlfb.c +++ b/drivers/video/fbdev/udlfb.c @@ -321,12 +321,32 @@ static int dlfb_set_video_mode(struct dlfb_data *dlfb, return retval; } +static void dlfb_vm_open(struct vm_area_struct *vma) +{ + struct dlfb_data *dlfb = vma->vm_private_data; + + atomic_inc(&dlfb->mmap_count); +} + +static void dlfb_vm_close(struct vm_area_struct *vma) +{ + struct dlfb_data *dlfb = vma->vm_private_data; + + atomic_dec(&dlfb->mmap_count); +} + +static const struct vm_operations_struct dlfb_vm_ops = { + .open = dlfb_vm_open, + .close = dlfb_vm_close, +}; + static int dlfb_ops_mmap(struct fb_info *info, struct vm_area_struct *vma) { unsigned long start = vma->vm_start; unsigned long size = vma->vm_end - vma->vm_start; unsigned long offset = vma->vm_pgoff << PAGE_SHIFT; unsigned long page, pos; + struct dlfb_data *dlfb = info->par; if (info->fbdefio) return fb_deferred_io_mmap(info, vma); @@ -358,6 +378,9 @@ static int dlfb_ops_mmap(struct fb_info *info, struct vm_area_struct *vma) size = 0; } + vma->vm_ops = &dlfb_vm_ops; + vma->vm_private_data = dlfb; + atomic_inc(&dlfb->mmap_count); return 0; } @@ -1018,6 +1041,9 @@ static int dlfb_ops_check_var(struct fb_var_screeninfo *var, struct fb_videomode mode; struct dlfb_data *dlfb = info->par; + if (!var->pixclock) + return -EINVAL; + /* set device-specific elements of var unrelated to mode */ dlfb_var_color_format(var); @@ -1173,7 +1199,6 @@ static void dlfb_deferred_vfree(struct dlfb_data *dlfb, void *mem) /* * Assumes &info->lock held by caller - * Assumes no active clients have framebuffer open */ static int dlfb_realloc_framebuffer(struct dlfb_data *dlfb, struct fb_info *info, u32 new_len) { @@ -1185,6 +1210,13 @@ static int dlfb_realloc_framebuffer(struct dlfb_data *dlfb, struct fb_info *info new_len = PAGE_ALIGN(new_len); if (new_len > old_len) { + if (atomic_read(&dlfb->mmap_count) > 0) { + dev_warn(info->dev, + "refusing realloc: %d active mmaps\n", + atomic_read(&dlfb->mmap_count)); + return -EBUSY; + } + /* * Alloc system memory for virtual framebuffer */ diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 0c9ea24a200c98..66d00b6ceb789b 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -157,7 +157,8 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) } while (info.result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); - if (info.result != RSI_SUCCESS) { + /* Break out in case of failure */ + if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page; diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c index e001e6769a43fc..910a1de0d5a72f 100644 --- a/drivers/virt/coco/sev-guest/sev-guest.c +++ b/drivers/virt/coco/sev-guest/sev-guest.c @@ -176,7 +176,6 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques struct snp_guest_req req = {}; int ret, npages = 0, resp_len; sockptr_t certs_address; - struct page *page; if (sockptr_is_null(io->req_data) || sockptr_is_null(io->resp_data)) return -EINVAL; @@ -211,16 +210,15 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques * zeros to indicate that certificate data was not provided. */ npages = report_req->certs_len >> PAGE_SHIFT; - page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, - get_order(report_req->certs_len)); - if (!page) + req.certs_data = alloc_pages_exact(npages << PAGE_SHIFT, + GFP_KERNEL_ACCOUNT | __GFP_ZERO); + if (!req.certs_data) return -ENOMEM; - req.certs_data = page_address(page); ret = set_memory_decrypted((unsigned long)req.certs_data, npages); if (ret) { pr_err("failed to mark page shared, ret=%d\n", ret); - __free_pages(page, get_order(report_req->certs_len)); + free_pages_exact(req.certs_data, npages << PAGE_SHIFT); return -EFAULT; } @@ -277,7 +275,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques if (set_memory_encrypted((unsigned long)req.certs_data, npages)) WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n"); else - __free_pages(page, get_order(report_req->certs_len)); + free_pages_exact(req.certs_data, npages << PAGE_SHIFT); } return ret; } diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c index 66a158f67a712b..6b9b0f9b05cedf 100644 --- a/drivers/watchdog/apple_wdt.c +++ b/drivers/watchdog/apple_wdt.c @@ -218,6 +218,7 @@ static int apple_wdt_suspend(struct device *dev) static DEFINE_SIMPLE_DEV_PM_OPS(apple_wdt_pm_ops, apple_wdt_suspend, apple_wdt_resume); static const struct of_device_id apple_wdt_of_match[] = { + { .compatible = "apple,t8103-wdt" }, { .compatible = "apple,wdt" }, {}, }; diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index 15ba592236e845..725a49a0eee72e 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -1620,6 +1620,12 @@ static void privcmd_close(struct vm_area_struct *vma) kvfree(pages); } +static int privcmd_may_split(struct vm_area_struct *area, unsigned long addr) +{ + /* Forbid splitting, avoids double free via privcmd_close(). */ + return -EINVAL; +} + static vm_fault_t privcmd_fault(struct vm_fault *vmf) { printk(KERN_DEBUG "privcmd_fault: vma=%p %lx-%lx, pgoff=%lx, uv=%p\n", @@ -1631,6 +1637,7 @@ static vm_fault_t privcmd_fault(struct vm_fault *vmf) static const struct vm_operations_struct privcmd_vm_ops = { .close = privcmd_close, + .may_split = privcmd_may_split, .fault = privcmd_fault }; diff --git a/drivers/xen/sys-hypervisor.c b/drivers/xen/sys-hypervisor.c index b1bb01ba82f880..91923242a5ae71 100644 --- a/drivers/xen/sys-hypervisor.c +++ b/drivers/xen/sys-hypervisor.c @@ -366,6 +366,8 @@ static ssize_t buildid_show(struct hyp_sysfs_attr *attr, char *buffer) ret = sprintf(buffer, ""); return ret; } + if (ret > PAGE_SIZE) + return -ENOSPC; buildid = kmalloc(sizeof(*buildid) + ret, GFP_KERNEL); if (!buildid) @@ -373,8 +375,10 @@ static ssize_t buildid_show(struct hyp_sysfs_attr *attr, char *buffer) buildid->len = ret; ret = HYPERVISOR_xen_version(XENVER_build_id, buildid); - if (ret > 0) - ret = sprintf(buffer, "%s", buildid->buf); + if (ret > 0) { + /* Build id is binary, not a string. */ + memcpy(buffer, buildid->buf, ret); + } kfree(buildid); return ret; diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index e4b27aecbf0591..79a2b5dfd6941f 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -598,6 +598,8 @@ static int pcistub_seize(struct pci_dev *dev, return err; } +static struct pci_driver xen_pcibk_pci_driver; + /* Called when 'bind'. This means we must _NOT_ call pci_reset_function or * other functions that take the sysfs lock. */ static int pcistub_probe(struct pci_dev *dev, const struct pci_device_id *id) @@ -609,8 +611,8 @@ static int pcistub_probe(struct pci_dev *dev, const struct pci_device_id *id) match = pcistub_match(dev); - if ((dev->driver_override && - !strcmp(dev->driver_override, PCISTUB_DRIVER_NAME)) || + if (device_match_driver_override(&dev->dev, + &xen_pcibk_pci_driver.driver) > 0 || match) { if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c index 057487efaaeb2d..acda42499ca9ef 100644 --- a/fs/9p/v9fs.c +++ b/fs/9p/v9fs.c @@ -413,7 +413,11 @@ static void v9fs_apply_options(struct v9fs_session_info *v9ses, /* * Note that we must |= flags here as session_init already * set basic flags. This adds in flags from parsed options. + * Default access flags must be cleared if session options + * changes them to avoid mangling the setting. */ + if (ctx->session_opts.flags & V9FS_ACCESS_MASK) + v9ses->flags &= ~V9FS_ACCESS_MASK; v9ses->flags |= ctx->session_opts.flags; #ifdef CONFIG_9P_FSCACHE v9ses->cachetag = ctx->session_opts.cachetag; diff --git a/fs/adfs/super.c b/fs/adfs/super.c index 2c5b2076acf9e8..a4cd0a5159dd17 100644 --- a/fs/adfs/super.c +++ b/fs/adfs/super.c @@ -317,6 +317,9 @@ static int adfs_validate_bblk(struct super_block *sb, struct buffer_head *bh, if (adfs_checkdiscrecord(dr)) return -EILSEQ; + if ((dr->nzones | dr->nzones_high << 8) == 0) + return -EILSEQ; + *drp = dr; return 0; } diff --git a/fs/afs/file.c b/fs/afs/file.c index f609366fd2ac07..74d04af51ff4ae 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -19,7 +19,7 @@ #include #include "internal.h" -static int afs_file_mmap_prepare(struct vm_area_desc *desc); +static int afs_file_mmap(struct file *file, struct vm_area_struct *vma); static ssize_t afs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter); static ssize_t afs_file_splice_read(struct file *in, loff_t *ppos, @@ -35,7 +35,7 @@ const struct file_operations afs_file_operations = { .llseek = generic_file_llseek, .read_iter = afs_file_read_iter, .write_iter = netfs_file_write_iter, - .mmap_prepare = afs_file_mmap_prepare, + .mmap = afs_file_mmap, .splice_read = afs_file_splice_read, .splice_write = iter_file_splice_write, .fsync = afs_fsync, @@ -492,16 +492,16 @@ static void afs_drop_open_mmap(struct afs_vnode *vnode) /* * Handle setting up a memory mapping on an AFS file. */ -static int afs_file_mmap_prepare(struct vm_area_desc *desc) +static int afs_file_mmap(struct file *file, struct vm_area_struct *vma) { - struct afs_vnode *vnode = AFS_FS_I(file_inode(desc->file)); + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); int ret; afs_add_open_mmap(vnode); - ret = generic_file_mmap_prepare(desc); + ret = generic_file_mmap(file, vma); if (ret == 0) - desc->vm_ops = &afs_vm_ops; + vma->vm_ops = &afs_vm_ops; else afs_drop_open_mmap(vnode); return ret; diff --git a/fs/backing-file.c b/fs/backing-file.c index 45da8600d56440..1f3bbfc758823f 100644 --- a/fs/backing-file.c +++ b/fs/backing-file.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "internal.h" @@ -29,14 +30,15 @@ * returned file into a container structure that also stores the stacked * file's path, which can be retrieved using backing_file_user_path(). */ -struct file *backing_file_open(const struct path *user_path, int flags, +struct file *backing_file_open(const struct file *user_file, int flags, const struct path *real_path, const struct cred *cred) { + const struct path *user_path = &user_file->f_path; struct file *f; int error; - f = alloc_empty_backing_file(flags, cred); + f = alloc_empty_backing_file(flags, cred, user_file); if (IS_ERR(f)) return f; @@ -52,15 +54,16 @@ struct file *backing_file_open(const struct path *user_path, int flags, } EXPORT_SYMBOL_GPL(backing_file_open); -struct file *backing_tmpfile_open(const struct path *user_path, int flags, +struct file *backing_tmpfile_open(const struct file *user_file, int flags, const struct path *real_parentpath, umode_t mode, const struct cred *cred) { struct mnt_idmap *real_idmap = mnt_idmap(real_parentpath->mnt); + const struct path *user_path = &user_file->f_path; struct file *f; int error; - f = alloc_empty_backing_file(flags, cred); + f = alloc_empty_backing_file(flags, cred, user_file); if (IS_ERR(f)) return f; @@ -336,8 +339,13 @@ int backing_file_mmap(struct file *file, struct vm_area_struct *vma, vma_set_file(vma, file); - scoped_with_creds(ctx->cred) + scoped_with_creds(ctx->cred) { + ret = security_mmap_backing_file(vma, file, user_file); + if (ret) + return ret; + ret = vfs_mmap(vma->vm_file, vma); + } if (ctx->accessed) ctx->accessed(user_file); diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 1b0eb246b7147a..37ca4ef879029d 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -4709,6 +4709,7 @@ static void btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info, free_extent_buffer_stale(eb); } } + btrfs_extent_io_tree_release(dirty_pages); } static void btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index f643a05208720d..dc2b22e65bad5b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -811,7 +811,8 @@ static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, * do not even bother try compression, as there will be no space saving * and will always fallback to regular write later. */ - if (start != 0 && end + 1 - start <= fs_info->sectorsize) + if (end + 1 - start <= fs_info->sectorsize && + (start > 0 || end + 1 < inode->disk_i_size)) return 0; /* Defrag ioctl takes precedence over mount options and properties. */ if (inode->defrag_compress == BTRFS_DEFRAG_DONT_COMPRESS) @@ -1060,6 +1061,12 @@ static void compress_file_range(struct btrfs_work *work) mapping_set_error(mapping, -EIO); return; } + /* + * If a single block at file offset 0 cannot be inlined, fall back to + * regular writes without marking the file incompressible. + */ + if (start == 0 && end <= blocksize) + goto cleanup_and_bail_uncompressed; /* * We aren't doing an inline extent. Round the compressed size up to a @@ -1238,7 +1245,7 @@ static void submit_one_async_extent(struct async_chunk *async_chunk, NULL, &cached, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | - EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, + EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV, PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); if (async_extent->cb) @@ -4961,6 +4968,8 @@ static int btrfs_rmdir(struct inode *vfs_dir, struct dentry *dentry) if (ret) goto out; + btrfs_record_unlink_dir(trans, dir, inode, false); + /* now the directory is empty */ ret = btrfs_unlink_inode(trans, dir, inode, &fname.disk_name); if (!ret) diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index d75d31b606e499..4a1d27e4884df0 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -2897,7 +2897,7 @@ static long btrfs_ioctl_space_info(struct btrfs_fs_info *fs_info, return -ENOMEM; space_args.total_spaces = 0; - dest = kmalloc(alloc_size, GFP_KERNEL); + dest = kzalloc(alloc_size, GFP_KERNEL); if (!dest) return -ENOMEM; dest_orig = dest; @@ -2953,7 +2953,8 @@ static long btrfs_ioctl_space_info(struct btrfs_fs_info *fs_info, user_dest = (struct btrfs_ioctl_space_info __user *) (arg + sizeof(struct btrfs_ioctl_space_args)); - if (copy_to_user(user_dest, dest_orig, alloc_size)) + if (copy_to_user(user_dest, dest_orig, + space_args.total_spaces * sizeof(*dest_orig))) return -EFAULT; out: diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 314cb95ba846d2..55d8d238e58626 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -322,6 +322,51 @@ static int clone_copy_inline_extent(struct btrfs_inode *inode, ret = copy_inline_to_page(inode, new_key->offset, inline_data, size, datal, comp_type); + + /* + * If we copied the inline extent data to a page/folio beyond the i_size + * of the destination inode, then we need to increase the i_size before + * we start a transaction to update the inode item. This is to prevent a + * deadlock when the flushoncommit mount option is used, which happens + * like this: + * + * 1) Task A clones an inline extent from inode X to an offset of inode + * Y that is beyond Y's current i_size. This means we copied the + * inline extent's data to a folio of inode Y that is beyond its EOF, + * using the call above to copy_inline_to_page(); + * + * 2) Task B starts a transaction commit and calls + * btrfs_start_delalloc_flush() to flush delalloc; + * + * 3) The delalloc flushing sees the new dirty folio of inode Y and when + * it attempts to flush it, it ends up at extent_writepage() and sees + * that the offset of the folio is beyond the i_size of inode Y, so + * it attempts to invalidate the folio by calling folio_invalidate(), + * which ends up at btrfs' folio invalidate callback - + * btrfs_invalidate_folio(). There it tries to lock the folio's range + * in inode Y's extent io tree, but it blocks since it's currently + * locked by task A - during reflink we lock the inodes and the + * source and destination ranges after flushing all delalloc and + * waiting for ordered extent completion - after that we don't expect + * to have dirty folios in the ranges, the exception is if we have to + * copy an inline extent's data (because the destination offset is + * not zero); + * + * 4) Task A then does the 'goto out' below and attempts to start a + * transaction to update the inode item, and then it's blocked since + * the current transaction is in the TRANS_STATE_COMMIT_START state. + * Therefore task A has to wait for the current transaction to become + * unblocked (its state >= TRANS_STATE_UNBLOCKED). + * + * This leads to a deadlock - the task committing the transaction + * waiting for the delalloc flushing which is blocked during folio + * invalidation on the inode's extent lock and the reflink task waiting + * for the current transaction to be unblocked so that it can start a + * a new one to update the inode item (while holding the extent lock). + */ + if (ret == 0 && new_key->offset + datal > i_size_read(&inode->vfs_inode)) + i_size_write(&inode->vfs_inode, new_key->offset + datal); + goto out; } diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 033f74fd6225cd..d8cd04fe9a4cd5 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -3884,7 +3884,7 @@ static int add_remap_tree_entries(struct btrfs_trans_handle *trans, struct btrfs ret = btrfs_insert_empty_items(trans, fs_info->remap_root, path, &batch); btrfs_release_path(path); - if (num_entries <= max_items) + if (ret || num_entries <= max_items) break; num_entries -= max_items; @@ -4182,6 +4182,12 @@ static int move_existing_remap(struct btrfs_fs_info *fs_info, return ret; } + if (ins.offset < length) { + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, ins.offset - length); + spin_unlock(&sinfo->lock); + } + dest_addr = ins.objectid; dest_length = ins.offset; @@ -5008,6 +5014,12 @@ static int do_remap_reloc_trans(struct btrfs_fs_info *fs_info, return ret; } + if (ins.offset < remap_length) { + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, ins.offset - remap_length); + spin_unlock(&sinfo->lock); + } + made_reservation = true; new_addr = ins.objectid; diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 87cbc051cb12ff..4e5196cf7b352f 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -276,10 +276,8 @@ static int create_space_info_sub_group(struct btrfs_space_info *parent, u64 flag sub_group->subgroup_id = id; ret = btrfs_sysfs_add_space_info_type(sub_group); - if (ret) { - kfree(sub_group); + if (ret) parent->sub_group[index] = NULL; - } return ret; } @@ -311,7 +309,7 @@ static int create_space_info(struct btrfs_fs_info *info, u64 flags) ret = btrfs_sysfs_add_space_info_type(space_info); if (ret) - goto out_free; + return ret; list_add(&space_info->list, &info->space_info); if (flags & BTRFS_BLOCK_GROUP_DATA) diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 8dd77c431974d1..ec3c15fc7ae398 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1275,14 +1275,13 @@ static int btrfs_write_and_wait_transaction(struct btrfs_trans_handle *trans) blk_finish_plug(&plug); ret2 = btrfs_wait_extents(fs_info, dirty_pages); - btrfs_extent_io_tree_release(&trans->transaction->dirty_pages); - if (ret) return ret; - else if (ret2) + if (ret2) return ret2; - else - return 0; + + btrfs_extent_io_tree_release(&trans->transaction->dirty_pages); + return 0; } /* diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 6b8e810a35ce1a..fd400fbc654d9c 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -8457,7 +8457,12 @@ bool btrfs_verify_dev_items(const struct btrfs_fs_info *fs_info) mutex_lock(&uuid_mutex); list_for_each_entry(dev, &fs_info->fs_devices->devices, dev_list) { - if (!test_bit(BTRFS_DEV_STATE_ITEM_FOUND, &dev->dev_state)) { + /* + * Replace target dev item (devid 0) is not inserted into chunk tree. + * So skip the DEV_STATE_ITEM check. + */ + if (dev->devid != BTRFS_DEV_REPLACE_DEVID && + !test_bit(BTRFS_DEV_STATE_ITEM_FOUND, &dev->dev_state)) { btrfs_err(fs_info, "devid %llu path %s is registered but not found in chunk tree", dev->devid, btrfs_dev_name(dev)); diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 2090fc78529cb4..0c01fcb73ddde8 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -1322,6 +1322,7 @@ void ceph_process_folio_batch(struct address_space *mapping, ceph_wbc, folio); if (rc == -ENODATA) { folio_unlock(folio); + folio_put(folio); ceph_wbc->fbatch.folios[i] = NULL; continue; } else if (rc == -E2BIG) { @@ -1332,6 +1333,7 @@ void ceph_process_folio_batch(struct address_space *mapping, if (!folio_clear_dirty_for_io(folio)) { doutc(cl, "%p !folio_clear_dirty_for_io\n", folio); folio_unlock(folio); + folio_put(folio); ceph_wbc->fbatch.folios[i] = NULL; continue; } @@ -1365,6 +1367,10 @@ void ceph_process_folio_batch(struct address_space *mapping, rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, folio); if (rc) { + /* Did we just begin a new contiguous op? Nevermind! */ + if (ceph_wbc->len == 0) + ceph_wbc->num_ops--; + folio_redirty_for_writepage(wbc, folio); folio_unlock(folio); break; diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index bac9cfb6b982f2..27ce9e55e94768 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -769,7 +769,8 @@ struct dentry *ceph_finish_lookup(struct ceph_mds_request *req, d_drop(dentry); err = -ENOENT; } else { - d_add(dentry, NULL); + if (d_unhashed(dentry)) + d_add(dentry, NULL); } } } @@ -840,7 +841,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, spin_unlock(&ci->i_ceph_lock); doutc(cl, " dir %llx.%llx complete, -ENOENT\n", ceph_vinop(dir)); - d_add(dentry, NULL); + if (d_unhashed(dentry)) + d_add(dentry, NULL); di->lease_shared_gen = atomic_read(&ci->i_shared_gen); return NULL; } diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index 5f87f62091a144..e773be07f7674a 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -1254,6 +1254,22 @@ int __ceph_setxattr(struct inode *inode, const char *name, ceph_vinop(inode), name, ceph_cap_string(issued)); __build_xattrs(inode); + /* + * __build_xattrs() may have released and reacquired i_ceph_lock, + * during which handle_cap_grant() could have replaced i_xattrs.blob + * with a newer MDS-provided blob and bumped i_xattrs.version. If that + * caused __build_xattrs() to rebuild the rb-tree from the new blob, + * count/names_size/vals_size may now be larger than when + * required_blob_size was computed above. Recompute it here so the + * prealloc_blob size check below reflects the current tree state. + */ + required_blob_size = __get_required_blob_size(ci, name_len, val_len); + if (required_blob_size > mdsc->mdsmap->m_max_xattr_size) { + doutc(cl, "sync (size too large): %d > %llu\n", + required_blob_size, mdsc->mdsmap->m_max_xattr_size); + goto do_sync; + } + if (!ci->i_xattrs.prealloc_blob || required_blob_size > ci->i_xattrs.prealloc_blob->alloc_len) { struct ceph_buffer *blob; @@ -1294,6 +1310,7 @@ int __ceph_setxattr(struct inode *inode, const char *name, do_sync: spin_unlock(&ci->i_ceph_lock); + ceph_buffer_put(old_blob); do_sync_unlocked: if (lock_snap_rwsem) up_read(&mdsc->snap_rwsem); diff --git a/fs/d_path.c b/fs/d_path.c index bb365511066b21..a48957c0971ef6 100644 --- a/fs/d_path.c +++ b/fs/d_path.c @@ -301,18 +301,19 @@ EXPORT_SYMBOL(d_path); char *dynamic_dname(char *buffer, int buflen, const char *fmt, ...) { va_list args; - char temp[64]; + char *start; int sz; va_start(args, fmt); - sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1; + sz = vsnprintf(buffer, buflen, fmt, args) + 1; va_end(args); - if (sz > sizeof(temp) || sz > buflen) + if (sz > NAME_MAX || sz > buflen) return ERR_PTR(-ENAMETOOLONG); - buffer += buflen - sz; - return memcpy(buffer, temp, sz); + /* Move the formatted d_name to the end of the buffer. */ + start = buffer + (buflen - sz); + return memmove(start, buffer, sz); } char *simple_dname(struct dentry *dentry, char *buffer, int buflen) diff --git a/fs/dcache.c b/fs/dcache.c index 7ba1801d813299..24f4f3acaa8cff 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -3257,7 +3257,7 @@ static void __init dcache_init_early(void) HASH_EARLY | HASH_ZERO, &d_hash_shift, NULL, - 0, + 2, 0); d_hash_shift = 32 - d_hash_shift; @@ -3289,7 +3289,7 @@ static void __init dcache_init(void) HASH_ZERO, &d_hash_shift, NULL, - 0, + 2, 0); d_hash_shift = 32 - d_hash_shift; diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c index 3376ab6a519d17..edd6aafbfbaaf3 100644 --- a/fs/debugfs/file.c +++ b/fs/debugfs/file.c @@ -1047,7 +1047,6 @@ ssize_t debugfs_read_file_str(struct file *file, char __user *user_buf, return ret; } -EXPORT_SYMBOL_GPL(debugfs_create_str); static ssize_t debugfs_write_file_str(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) @@ -1127,7 +1126,7 @@ static const struct file_operations fops_str_wo = { * directory dentry if set. If this parameter is %NULL, then the * file will be created in the root of the debugfs filesystem. * @value: a pointer to the variable that the file should read to and write - * from. + * from. This pointer and the string it points to must not be %NULL. * * This function creates a file in debugfs with the given name that * contains the value of the variable @value. If the @mode variable is so @@ -1136,9 +1135,13 @@ static const struct file_operations fops_str_wo = { void debugfs_create_str(const char *name, umode_t mode, struct dentry *parent, char **value) { + if (WARN_ON(!value || !*value)) + return; + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_str, &fops_str_ro, &fops_str_wo); } +EXPORT_SYMBOL_GPL(debugfs_create_str); static ssize_t read_file_blob(struct file *file, char __user *user_buf, size_t count, loff_t *ppos) diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 3c54e95964c9fb..2b065f8c3f711e 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -145,6 +145,7 @@ static void *z_erofs_lz4_handle_overlap(const struct z_erofs_decompress_req *rq, oend = rq->pageofs_out + rq->outputsize; omargin = PAGE_ALIGN(oend) - oend; if (!rq->partial_decoding && may_inplace && + rq->outpages >= rq->inpages && omargin >= LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize)) { for (i = 0; i < rq->inpages; ++i) if (rq->out[rq->outpages - rq->inpages + i] != diff --git a/fs/erofs/dir.c b/fs/erofs/dir.c index e5132575b9d3ef..4aa52a5f204a0f 100644 --- a/fs/erofs/dir.c +++ b/fs/erofs/dir.c @@ -19,20 +19,18 @@ static int erofs_fill_dentries(struct inode *dir, struct dir_context *ctx, const char *de_name = (char *)dentry_blk + nameoff; unsigned int de_namelen; - /* the last dirent in the block? */ - if (de + 1 >= end) - de_namelen = strnlen(de_name, maxsize - nameoff); - else + /* non-trailing dirent in the directory block? */ + if (de + 1 < end) de_namelen = le16_to_cpu(de[1].nameoff) - nameoff; + else if (maxsize <= nameoff) + goto err_bogus; + else + de_namelen = strnlen(de_name, maxsize - nameoff); - /* a corrupted entry is found */ - if (nameoff + de_namelen > maxsize || - de_namelen > EROFS_NAME_LEN) { - erofs_err(dir->i_sb, "bogus dirent @ nid %llu", - EROFS_I(dir)->nid); - DBG_BUGON(1); - return -EFSCORRUPTED; - } + /* a corrupted entry is found (including negative namelen) */ + if (!in_range32(de_namelen, 1, EROFS_NAME_LEN) || + nameoff + de_namelen > maxsize) + goto err_bogus; if (!dir_emit(ctx, de_name, de_namelen, erofs_nid_to_ino64(EROFS_SB(dir->i_sb), @@ -42,6 +40,10 @@ static int erofs_fill_dentries(struct inode *dir, struct dir_context *ctx, ctx->pos += sizeof(struct erofs_dirent); } return 0; +err_bogus: + erofs_err(dir->i_sb, "bogus dirent @ nid %llu", EROFS_I(dir)->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; } static int erofs_readdir(struct file *f, struct dir_context *ctx) @@ -88,7 +90,7 @@ static int erofs_readdir(struct file *f, struct dir_context *ctx) } nameoff = le16_to_cpu(de->nameoff); - if (nameoff < sizeof(struct erofs_dirent) || nameoff >= bsz) { + if (!nameoff || nameoff >= bsz || (nameoff % sizeof(*de))) { erofs_err(sb, "invalid de[0].nameoff %u @ nid %llu", nameoff, EROFS_I(dir)->nid); err = -EFSCORRUPTED; diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index b80c6bb33a58c2..7871b16c1d3336 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -44,9 +44,9 @@ struct erofs_deviceslot { u8 tag[64]; /* digest(sha256), etc. */ __le32 blocks_lo; /* total blocks count of this device */ __le32 uniaddr_lo; /* unified starting block of this device */ - __le32 blocks_hi; /* total blocks count MSB */ + __le16 blocks_hi; /* total blocks count MSB */ __le16 uniaddr_hi; /* unified starting block MSB */ - u8 reserved[50]; + u8 reserved[52]; }; #define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot) diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 4b3d21402e101f..a188c570087aec 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -351,7 +351,7 @@ static int erofs_ioctl_get_volume_label(struct inode *inode, void __user *arg) ret = clear_user(arg, 1); else ret = copy_to_user(arg, sbi->volume_name, - strlen(sbi->volume_name)); + strlen(sbi->volume_name) + 1); return ret ? -EFAULT : 0; } diff --git a/fs/erofs/ishare.c b/fs/erofs/ishare.c index ec433bacc59252..6ed66b17359ba1 100644 --- a/fs/erofs/ishare.c +++ b/fs/erofs/ishare.c @@ -4,6 +4,7 @@ */ #include #include +#include #include "internal.h" #include "xattr.h" @@ -106,7 +107,8 @@ static int erofs_ishare_file_open(struct inode *inode, struct file *file) if (file->f_flags & O_DIRECT) return -EINVAL; - realfile = alloc_empty_backing_file(O_RDONLY|O_NOATIME, current_cred()); + realfile = alloc_empty_backing_file(O_RDONLY|O_NOATIME, current_cred(), + file); if (IS_ERR(realfile)) return PTR_ERR(realfile); ihold(sharedinode); @@ -150,8 +152,14 @@ static ssize_t erofs_ishare_file_read_iter(struct kiocb *iocb, static int erofs_ishare_mmap(struct file *file, struct vm_area_struct *vma) { struct file *realfile = file->private_data; + int err; vma_set_file(vma, realfile); + + err = security_mmap_backing_file(vma, realfile, file); + if (err) + return err; + return generic_file_readonly_mmap(file, vma); } diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 972a0c82198d7e..802add6652fda6 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -129,6 +129,7 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, struct erofs_fscache *fscache; struct erofs_deviceslot *dis; struct file *file; + bool _48bit; dis = erofs_read_metabuf(buf, sb, *pos, false); if (IS_ERR(dis)) @@ -175,8 +176,11 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, dif->file = file; } - dif->blocks = le32_to_cpu(dis->blocks_lo); - dif->uniaddr = le32_to_cpu(dis->uniaddr_lo); + _48bit = erofs_sb_has_48bit(sbi); + dif->blocks = le32_to_cpu(dis->blocks_lo) | + (_48bit ? (u64)le16_to_cpu(dis->blocks_hi) << 32 : 0); + dif->uniaddr = le32_to_cpu(dis->uniaddr_lo) | + (_48bit ? (u64)le16_to_cpu(dis->uniaddr_hi) << 32 : 0); sbi->total_blocks += dif->blocks; *pos += EROFS_DEVT_SLOT_SIZE; return 0; diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index 30775502b56da2..abf7ddc64c63b8 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -10,7 +10,7 @@ struct z_erofs_maprecorder { struct inode *inode; struct erofs_map_blocks *map; - unsigned long lcn; + u64 lcn; /* compression extent information gathered */ u8 type, headtype; u16 clusterofs; @@ -20,8 +20,7 @@ struct z_erofs_maprecorder { bool partialref, in_mbox; }; -static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, - unsigned long lcn) +static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, u64 lcn) { struct inode *const inode = m->inode; struct erofs_inode *const vi = EROFS_I(inode); @@ -94,7 +93,7 @@ static int get_compacted_la_distance(unsigned int lobits, } static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, - unsigned long lcn, bool lookahead) + u64 lcn, bool lookahead) { struct inode *const inode = m->inode; struct erofs_inode *const vi = EROFS_I(inode); @@ -234,7 +233,7 @@ static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, } static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, - unsigned int lcn, bool lookahead) + u64 lcn, bool lookahead) { struct erofs_inode *vi = EROFS_I(m->inode); int err; @@ -249,7 +248,7 @@ static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, return err; if (m->type >= Z_EROFS_LCLUSTER_TYPE_MAX) { - erofs_err(m->inode->i_sb, "unknown type %u @ lcn %u of nid %llu", + erofs_err(m->inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", m->type, lcn, EROFS_I(m->inode)->nid); DBG_BUGON(1); return -EOPNOTSUPP; @@ -269,7 +268,7 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, const unsigned int lclusterbits = vi->z_lclusterbits; while (m->lcn >= lookback_distance) { - unsigned long lcn = m->lcn - lookback_distance; + u64 lcn = m->lcn - lookback_distance; int err; if (!lookback_distance) @@ -286,7 +285,7 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, m->map->m_la = (lcn << lclusterbits) | m->clusterofs; return 0; } - erofs_err(sb, "bogus lookback distance %u @ lcn %lu of nid %llu", + erofs_err(sb, "bogus lookback distance %u @ lcn %llu of nid %llu", lookback_distance, m->lcn, vi->nid); DBG_BUGON(1); return -EFSCORRUPTED; @@ -300,7 +299,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, struct erofs_inode *vi = EROFS_I(inode); bool bigpcl1 = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; bool bigpcl2 = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2; - unsigned long lcn = m->lcn + 1; + u64 lcn = m->lcn + 1; int err; DBG_BUGON(m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); @@ -331,7 +330,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD && m->delta[0] != 1) { - erofs_err(sb, "bogus CBLKCNT @ lcn %lu of nid %llu", lcn, vi->nid); + erofs_err(sb, "bogus CBLKCNT @ lcn %llu of nid %llu", lcn, vi->nid); DBG_BUGON(1); return -EFSCORRUPTED; } diff --git a/fs/eventpoll.c b/fs/eventpoll.c index 4b43bf41296d4b..8c03de028c4824 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -827,36 +827,47 @@ static void ep_free(struct eventpoll *ep) } /* - * Removes a "struct epitem" from the eventpoll RB tree and deallocates - * all the associated resources. Must be called with "mtx" held. - * If the dying flag is set, do the removal only if force is true. - * This prevents ep_clear_and_put() from dropping all the ep references - * while running concurrently with eventpoll_release_file(). - * Returns true if the eventpoll can be disposed. + * The ffd.file pointer may be in the process of being torn down due to + * being closed, but we may not have finished eventpoll_release() yet. + * + * Normally, even with the atomic_long_inc_not_zero, the file may have + * been free'd and then gotten re-allocated to something else (since + * files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU). + * + * But for epoll, users hold the ep->mtx mutex, and as such any file in + * the process of being free'd will block in eventpoll_release_file() + * and thus the underlying file allocation will not be free'd, and the + * file re-use cannot happen. + * + * For the same reason we can avoid a rcu_read_lock() around the + * operation - 'ffd.file' cannot go away even if the refcount has + * reached zero (but we must still not call out to ->poll() functions + * etc). */ -static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) +static struct file *epi_fget(const struct epitem *epi) { - struct file *file = epi->ffd.file; - struct epitems_head *to_free; - struct hlist_head *head; + struct file *file; - lockdep_assert_irqs_enabled(); + file = epi->ffd.file; + if (!file_ref_get(&file->f_ref)) + file = NULL; + return file; +} - /* - * Removes poll wait queue hooks. - */ - ep_unregister_pollwait(ep, epi); +/* + * Called with &file->f_lock held, + * returns with it released + */ +static void ep_remove_file(struct eventpoll *ep, struct epitem *epi, + struct file *file) +{ + struct epitems_head *to_free = NULL; + struct hlist_head *head = file->f_ep; - /* Remove the current item from the list of epoll hooks */ - spin_lock(&file->f_lock); - if (epi->dying && !force) { - spin_unlock(&file->f_lock); - return false; - } + lockdep_assert_held(&ep->mtx); + lockdep_assert_held(&file->f_lock); - to_free = NULL; - head = file->f_ep; - if (head->first == &epi->fllink && !epi->fllink.next) { + if (hlist_is_singular_node(&epi->fllink, head)) { /* See eventpoll_release() for details. */ WRITE_ONCE(file->f_ep, NULL); if (!is_file_epoll(file)) { @@ -869,6 +880,11 @@ static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) hlist_del_rcu(&epi->fllink); spin_unlock(&file->f_lock); free_ephead(to_free); +} + +static bool ep_remove_epi(struct eventpoll *ep, struct epitem *epi) +{ + lockdep_assert_held(&ep->mtx); rb_erase_cached(&epi->rbn, &ep->rbr); @@ -896,7 +912,29 @@ static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force) */ static void ep_remove_safe(struct eventpoll *ep, struct epitem *epi) { - if (__ep_remove(ep, epi, false)) + struct file *file __free(fput) = NULL; + + lockdep_assert_irqs_enabled(); + lockdep_assert_held(&ep->mtx); + + ep_unregister_pollwait(ep, epi); + + /* cheap sync with eventpoll_release_file() */ + if (unlikely(READ_ONCE(epi->dying))) + return; + + /* + * If we manage to grab a reference it means we're not in + * eventpoll_release_file() and aren't going to be. + */ + file = epi_fget(epi); + if (!file) + return; + + spin_lock(&file->f_lock); + ep_remove_file(ep, epi, file); + + if (ep_remove_epi(ep, epi)) WARN_ON_ONCE(ep_refcount_dec_and_test(ep)); } @@ -1012,34 +1050,6 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep return res; } -/* - * The ffd.file pointer may be in the process of being torn down due to - * being closed, but we may not have finished eventpoll_release() yet. - * - * Normally, even with the atomic_long_inc_not_zero, the file may have - * been free'd and then gotten re-allocated to something else (since - * files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU). - * - * But for epoll, users hold the ep->mtx mutex, and as such any file in - * the process of being free'd will block in eventpoll_release_file() - * and thus the underlying file allocation will not be free'd, and the - * file re-use cannot happen. - * - * For the same reason we can avoid a rcu_read_lock() around the - * operation - 'ffd.file' cannot go away even if the refcount has - * reached zero (but we must still not call out to ->poll() functions - * etc). - */ -static struct file *epi_fget(const struct epitem *epi) -{ - struct file *file; - - file = epi->ffd.file; - if (!file_ref_get(&file->f_ref)) - file = NULL; - return file; -} - /* * Differs from ep_eventpoll_poll() in that internal callers already have * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested() @@ -1128,7 +1138,7 @@ void eventpoll_release_file(struct file *file) spin_lock(&file->f_lock); if (file->f_ep && file->f_ep->first) { epi = hlist_entry(file->f_ep->first, struct epitem, fllink); - epi->dying = true; + WRITE_ONCE(epi->dying, true); spin_unlock(&file->f_lock); /* @@ -1137,7 +1147,13 @@ void eventpoll_release_file(struct file *file) */ ep = epi->ep; mutex_lock(&ep->mtx); - dispose = __ep_remove(ep, epi, true); + + ep_unregister_pollwait(ep, epi); + + spin_lock(&file->f_lock); + ep_remove_file(ep, epi, file); + dispose = ep_remove_epi(ep, epi); + mutex_unlock(&ep->mtx); if (dispose && ep_refcount_dec_and_test(ep)) diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index dbfe9098a1245d..39d972722f5f84 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -1430,9 +1430,17 @@ struct inode *ext2_iget (struct super_block *sb, unsigned long ino) * the test is that same one that e2fsck uses * NeilBrown 1999oct15 */ - if (inode->i_nlink == 0 && (inode->i_mode == 0 || ei->i_dtime)) { - /* this inode is deleted */ - ret = -ESTALE; + if (inode->i_nlink == 0) { + if (inode->i_mode == 0 || ei->i_dtime) { + /* this inode is deleted */ + ret = -ESTALE; + } else { + ext2_error(sb, __func__, + "inode %lu has zero i_nlink with mode 0%o and no dtime, " + "filesystem may be corrupt", + ino, inode->i_mode); + ret = -EFSCORRUPTED; + } goto bad_inode; } inode->i_blocks = le32_to_cpu(raw_inode->i_blocks); diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c index 5496b2c8e2cd3a..6b53a3f39fcd69 100644 --- a/fs/ext4/extents-test.c +++ b/fs/ext4/extents-test.c @@ -142,10 +142,14 @@ static struct file_system_type ext_fs_type = { static void extents_kunit_exit(struct kunit *test) { - struct super_block *sb = k_ctx.k_ei->vfs_inode.i_sb; - struct ext4_sb_info *sbi = sb->s_fs_info; + struct ext4_sb_info *sbi; + if (!k_ctx.k_ei) + return; + + sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info; ext4_es_unregister_shrinker(sbi); + deactivate_super(sbi->s_sb); kfree(sbi); kfree(k_ctx.k_ei); kfree(k_ctx.k_data); @@ -224,34 +228,38 @@ static int extents_kunit_init(struct kunit *test) (struct kunit_ext_test_param *)(test->param_value); int err; - sb = sget(&ext_fs_type, NULL, ext_set, 0, NULL); - if (IS_ERR(sb)) - return PTR_ERR(sb); - - sb->s_blocksize = 4096; - sb->s_blocksize_bits = 12; - sbi = kzalloc_obj(struct ext4_sb_info); if (sbi == NULL) return -ENOMEM; + sb = sget(&ext_fs_type, NULL, ext_set, 0, NULL); + if (IS_ERR(sb)) { + kfree(sbi); + return PTR_ERR(sb); + } + sbi->s_sb = sb; sb->s_fs_info = sbi; + sb->s_blocksize = 4096; + sb->s_blocksize_bits = 12; + if (!param || !param->disable_zeroout) sbi->s_extent_max_zeroout_kb = 32; + err = ext4_es_register_shrinker(sbi); + if (err) + goto out_deactivate; + /* setup the mock inode */ k_ctx.k_ei = kzalloc_obj(struct ext4_inode_info); - if (k_ctx.k_ei == NULL) - return -ENOMEM; + if (k_ctx.k_ei == NULL) { + err = -ENOMEM; + goto out; + } ei = k_ctx.k_ei; inode = &ei->vfs_inode; - err = ext4_es_register_shrinker(sbi); - if (err) - return err; - ext4_es_init_tree(&ei->i_es_tree); rwlock_init(&ei->i_es_lock); INIT_LIST_HEAD(&ei->i_es_list); @@ -266,8 +274,10 @@ static int extents_kunit_init(struct kunit *test) inode->i_sb = sb; k_ctx.k_data = kzalloc(EXT_DATA_LEN * 4096, GFP_KERNEL); - if (k_ctx.k_data == NULL) - return -ENOMEM; + if (k_ctx.k_data == NULL) { + err = -ENOMEM; + goto out; + } /* * set the data area to a junk value @@ -309,7 +319,23 @@ static int extents_kunit_init(struct kunit *test) kunit_activate_static_stub(test, ext4_ext_zeroout, ext4_ext_zeroout_stub); kunit_activate_static_stub(test, ext4_issue_zeroout, ext4_issue_zeroout_stub); + up_write(&sb->s_umount); + return 0; + +out: + kfree(k_ctx.k_ei); + k_ctx.k_ei = NULL; + + kfree(k_ctx.k_data); + k_ctx.k_data = NULL; + + ext4_es_unregister_shrinker(sbi); +out_deactivate: + deactivate_locked_super(sb); + kfree(sbi); + + return err; } /* diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c index 6f5bfbb0e8a42c..95cb644cd32fa8 100644 --- a/fs/ext4/mballoc-test.c +++ b/fs/ext4/mballoc-test.c @@ -362,7 +362,6 @@ static int mbt_kunit_init(struct kunit *test) return ret; } - test->priv = sb; kunit_activate_static_stub(test, ext4_read_block_bitmap_nowait, ext4_read_block_bitmap_nowait_stub); @@ -383,6 +382,8 @@ static int mbt_kunit_init(struct kunit *test) return -ENOMEM; } + test->priv = sb; + return 0; } @@ -390,6 +391,9 @@ static void mbt_kunit_exit(struct kunit *test) { struct super_block *sb = (struct super_block *)test->priv; + if (!sb) + return; + mbt_mb_release(sb); mbt_ctx_release(sb); mbt_ext4_free_super_block(sb); diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index 7bf9ba19a89db2..a4eaee58e54503 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -226,7 +226,7 @@ check_xattrs(struct inode *inode, struct buffer_head *bh, /* Find the end of the names list */ while (!IS_LAST_ENTRY(e)) { struct ext4_xattr_entry *next = EXT4_XATTR_NEXT(e); - if ((void *)next >= end) { + if ((void *)next + sizeof(u32) > end) { err_str = "e_name out of bounds"; goto errout; } @@ -1165,7 +1165,7 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, { struct inode *ea_inode; struct ext4_xattr_entry *entry; - struct ext4_iloc iloc; + struct ext4_iloc iloc = { .bh = NULL }; bool dirty = false; unsigned int ea_ino; int err; @@ -1260,6 +1260,8 @@ ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, ext4_warning_inode(parent, "handle dirty metadata err=%d", err); } + + brelse(iloc.bh); } /* diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c index 8c76400ba6317d..aa8ba4cdfe34e1 100644 --- a/fs/f2fs/compress.c +++ b/fs/f2fs/compress.c @@ -1491,10 +1491,10 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) f2fs_compress_free_page(page); - dec_page_count(sbi, type); - - if (atomic_dec_return(&cic->pending_pages)) + if (atomic_dec_return(&cic->pending_pages)) { + dec_page_count(sbi, type); return; + } for (i = 0; i < cic->nr_rpages; i++) { WARN_ON(!cic->rpages[i]); @@ -1504,6 +1504,14 @@ void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) page_array_free(sbi, cic->rpages, cic->nr_rpages); kmem_cache_free(cic_entry_slab, cic); + + /* + * Make sure dec_page_count() is the last access to sbi. + * Once it drops the F2FS_WB_CP_DATA counter to zero, the + * unmount thread can proceed to destroy sbi and + * sbi->page_array_slab. + */ + dec_page_count(sbi, type); } static int f2fs_write_raw_pages(struct compress_ctx *cc, diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 338df7a2aea6be..e35e2f03cfbc17 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -173,7 +173,8 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task) while (nr_pages--) dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); - if (F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && + if (bio->bi_status == BLK_STS_OK && + F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && f2fs_sanity_check_node_footer(F2FS_F_SB(folio), folio, folio->index, NODE_TYPE_REGULAR, true)) bio->bi_status = BLK_STS_IOERR; @@ -386,6 +387,8 @@ static void f2fs_write_end_io(struct bio *bio) folio->index, NODE_TYPE_REGULAR, true); f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); } + if (f2fs_in_warm_node_list(sbi, folio)) + f2fs_del_fsync_node_entry(sbi, folio); dec_page_count(sbi, type); @@ -397,8 +400,6 @@ static void f2fs_write_end_io(struct bio *bio) wq_has_sleeper(&sbi->cp_wait)) wake_up(&sbi->cp_wait); - if (f2fs_in_warm_node_list(sbi, folio)) - f2fs_del_fsync_node_entry(sbi, folio); folio_clear_f2fs_gcing(folio); folio_end_writeback(folio); } @@ -1575,7 +1576,8 @@ static bool f2fs_map_blocks_cached(struct inode *inode, f2fs_wait_on_block_writeback_range(inode, map->m_pblk, map->m_len); - if (f2fs_allow_multi_device_dio(sbi, flag)) { + map->m_multidev_dio = f2fs_allow_multi_device_dio(sbi, flag); + if (map->m_multidev_dio) { int bidx = f2fs_target_device_index(sbi, map->m_pblk); struct f2fs_dev_info *dev = &sbi->devs[bidx]; @@ -1635,8 +1637,26 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) lfs_dio_write = (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && map->m_may_create); - if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) - goto out; + if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) { + struct extent_info ei; + + /* + * 1. If map->m_multidev_dio is true, map->m_pblk cannot be + * waitted by f2fs_wait_on_block_writeback_range() and are not + * mergeable. + * 2. If pgofs hits the read extent cache, it means the mapping + * is already cached in the extent cache, but it is not + * mergeable, and there is no need to query the mapping again + * via f2fs_get_dnode_of_data(). + */ + pgofs = (pgoff_t)map->m_lblk + map->m_len; + if (map->m_len == maxblocks || + map->m_multidev_dio || + f2fs_lookup_read_extent_cache(inode, pgofs, &ei)) + goto out; + ofs = map->m_len; + goto map_more; + } map->m_bdev = inode->i_sb->s_bdev; map->m_multidev_dio = @@ -1647,7 +1667,8 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) /* it only supports block size == page size */ pgofs = (pgoff_t)map->m_lblk; - end = pgofs + maxblocks; +map_more: + end = (pgoff_t)map->m_lblk + maxblocks; if (flag == F2FS_GET_BLOCK_PRECACHE) mode = LOOKUP_NODE_RA; diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c index 0ed84cc065a7ed..87169fd29d8972 100644 --- a/fs/f2fs/extent_cache.c +++ b/fs/f2fs/extent_cache.c @@ -119,9 +119,10 @@ static bool __may_extent_tree(struct inode *inode, enum extent_type type) if (!__init_may_extent_tree(inode, type)) return false; + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return false; + if (type == EX_READ) { - if (is_inode_flag_set(inode, FI_NO_EXTENT)) - return false; if (is_inode_flag_set(inode, FI_COMPRESSED_FILE) && !f2fs_sb_has_readonly(F2FS_I_SB(inode))) return false; @@ -644,6 +645,8 @@ static unsigned int __destroy_extent_node(struct inode *inode, while (atomic_read(&et->node_cnt)) { write_lock(&et->lock); + if (!is_inode_flag_set(inode, FI_NO_EXTENT)) + set_inode_flag(inode, FI_NO_EXTENT); node_cnt += __free_extent_tree(sbi, et, nr_shrink); write_unlock(&et->lock); } @@ -688,12 +691,12 @@ static void __update_extent_tree_range(struct inode *inode, write_lock(&et->lock); - if (type == EX_READ) { - if (is_inode_flag_set(inode, FI_NO_EXTENT)) { - write_unlock(&et->lock); - return; - } + if (is_inode_flag_set(inode, FI_NO_EXTENT)) { + write_unlock(&et->lock); + return; + } + if (type == EX_READ) { prev = et->largest; dei.len = 0; diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index bb34e864d0efc7..ef3961c6d8cf16 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -2042,6 +2042,9 @@ struct f2fs_sb_info { spinlock_t iostat_lat_lock; struct iostat_lat_info *iostat_io_lat; #endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lock_class_key cp_global_sem_key; +#endif }; /* Definitions to access f2fs_sb_info */ @@ -3947,6 +3950,8 @@ int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, enum node_type ntype, bool in_irq); struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); +int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, + bool mark_dirty, enum iostat_type io_type); int f2fs_move_node_folio(struct folio *node_folio, int gc_type); void f2fs_flush_inline_data(struct f2fs_sb_info *sbi); int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, @@ -3989,7 +3994,7 @@ bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); -bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi); +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check); void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc); void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi); diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index f46b2673d31f5a..ec0680187c0dbe 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -1230,7 +1230,7 @@ static int ra_data_block(struct inode *inode, pgoff_t index) .encrypted_page = NULL, .in_list = 0, }; - int err; + int err = 0; folio = f2fs_grab_cache_folio(mapping, index, true); if (IS_ERR(folio)) @@ -1283,6 +1283,9 @@ static int ra_data_block(struct inode *inode, pgoff_t index) fio.encrypted_page = &efolio->page; + if (folio_test_uptodate(efolio)) + goto put_encrypted_page; + err = f2fs_submit_page_bio(&fio); if (err) goto put_encrypted_page; diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 0a1052d5ee624e..62a8a1192a41a4 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -792,7 +792,7 @@ int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, int f2fs_inline_data_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, __u64 start, __u64 len) { - __u64 byteaddr, ilen; + __u64 byteaddr = 0, ilen; __u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED | FIEMAP_EXTENT_LAST; struct node_info ni; @@ -814,6 +814,15 @@ int f2fs_inline_data_fiemap(struct inode *inode, goto out; } + if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { + err = f2fs_write_single_node_folio(ifolio, true, false, FS_NODE_IO); + if (err) + return err; + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + } ilen = min_t(size_t, MAX_INLINE_DATA(inode), i_size_read(inode)); if (start >= ilen) goto out; @@ -825,9 +834,14 @@ int f2fs_inline_data_fiemap(struct inode *inode, if (err) goto out; - byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits; - byteaddr += (char *)inline_data_addr(inode, ifolio) - - (char *)F2FS_INODE(ifolio); + if (__is_valid_data_blkaddr(ni.blk_addr)) { + byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits; + byteaddr += (char *)inline_data_addr(inode, ifolio) - + (char *)F2FS_INODE(ifolio); + } else { + f2fs_bug_on(F2FS_I_SB(inode), ni.blk_addr != NEW_ADDR); + flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN; + } err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags); trace_f2fs_fiemap(inode, start, byteaddr, ilen, flags, err); out: diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c index e0f850b3f0c387..89240be8cc59b0 100644 --- a/fs/f2fs/inode.c +++ b/fs/f2fs/inode.c @@ -687,7 +687,7 @@ void f2fs_update_inode(struct inode *inode, struct folio *node_folio) ri->i_uid = cpu_to_le32(i_uid_read(inode)); ri->i_gid = cpu_to_le32(i_gid_read(inode)); ri->i_links = cpu_to_le32(inode->i_nlink); - ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(inode->i_blocks) + 1); + ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(READ_ONCE(inode->i_blocks)) + 1); if (!f2fs_is_atomic_file(inode) || is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index e360f08a9586ab..6ef21deeef1c8a 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -964,6 +964,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, return err; err = f2fs_create_whiteout(idmap, old_dir, &whiteout, &fname); + f2fs_free_filename(&fname); if (err) return err; } diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 74992fd9c9b660..a2ead811c3161f 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -427,7 +427,9 @@ bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino) struct f2fs_nm_info *nm_i = NM_I(sbi); struct nat_entry *e; bool need_update = true; + struct f2fs_lock_context lc; + f2fs_down_read_trace(&sbi->node_write, &lc); f2fs_down_read(&nm_i->nat_tree_lock); e = __lookup_nat_cache(nm_i, ino, false); if (e && get_nat_flag(e, HAS_LAST_FSYNC) && @@ -435,6 +437,7 @@ bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino) get_nat_flag(e, HAS_FSYNCED_INODE))) need_update = false; f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&sbi->node_write, &lc); return need_update; } @@ -1729,9 +1732,10 @@ static struct folio *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) return last_folio; } -static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted, - struct writeback_control *wbc, bool do_balance, - enum iostat_type io_type, unsigned int *seq_id) +static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, + bool *submitted, struct writeback_control *wbc, + bool do_balance, enum iostat_type io_type, + unsigned int *seq_id) { struct f2fs_sb_info *sbi = F2FS_F_SB(folio); nid_t nid; @@ -1801,13 +1805,14 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted goto redirty_out; } - if (atomic) { - if (!test_opt(sbi, NOBARRIER)) - fio.op_flags |= REQ_PREFLUSH | REQ_FUA; - if (IS_INODE(folio)) - set_dentry_mark(folio, + if (atomic && !test_opt(sbi, NOBARRIER)) + fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + + set_dentry_mark(folio, false); + set_fsync_mark(folio, do_fsync); + if (IS_INODE(folio) && (atomic || is_fsync_dnode(folio))) + set_dentry_mark(folio, f2fs_need_dentry_mark(sbi, ino_of_node(folio))); - } /* should add to global list before clearing PAGECACHE status */ if (f2fs_in_warm_node_list(sbi, folio)) { @@ -1843,41 +1848,51 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool *submitted return false; } -int f2fs_move_node_folio(struct folio *node_folio, int gc_type) +int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, + bool mark_dirty, enum iostat_type io_type) { int err = 0; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 1, + }; - if (gc_type == FG_GC) { - struct writeback_control wbc = { - .sync_mode = WB_SYNC_ALL, - .nr_to_write = 1, - }; + if (!sync_mode) { + /* set page dirty and write it */ + if (!folio_test_writeback(node_folio)) + folio_mark_dirty(node_folio); + goto out_folio; + } - f2fs_folio_wait_writeback(node_folio, NODE, true, true); + f2fs_folio_wait_writeback(node_folio, NODE, true, true); + if (mark_dirty) folio_mark_dirty(node_folio); + else if (!folio_test_dirty(node_folio)) + goto out_folio; - if (!folio_clear_dirty_for_io(node_folio)) { - err = -EAGAIN; - goto out_page; - } - - if (!__write_node_folio(node_folio, false, NULL, - &wbc, false, FS_GC_NODE_IO, NULL)) - err = -EAGAIN; - goto release_page; - } else { - /* set page dirty and write it */ - if (!folio_test_writeback(node_folio)) - folio_mark_dirty(node_folio); + if (!folio_clear_dirty_for_io(node_folio)) { + err = -EAGAIN; + goto out_folio; } -out_page: + + if (!__write_node_folio(node_folio, false, false, NULL, + &wbc, false, FS_GC_NODE_IO, NULL)) + err = -EAGAIN; + goto release_folio; +out_folio: folio_unlock(node_folio); -release_page: +release_folio: f2fs_folio_put(node_folio, false); return err; } +int f2fs_move_node_folio(struct folio *node_folio, int gc_type) +{ + return f2fs_write_single_node_folio(node_folio, gc_type == FG_GC, + true, FS_GC_NODE_IO); +} + int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, struct writeback_control *wbc, bool atomic, unsigned int *seq_id) @@ -1908,6 +1923,7 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, for (i = 0; i < nr_folios; i++) { struct folio *folio = fbatch.folios[i]; bool submitted = false; + bool do_fsync = false; if (unlikely(f2fs_cp_error(sbi))) { f2fs_folio_put(last_folio, false); @@ -1938,19 +1954,13 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, f2fs_folio_wait_writeback(folio, NODE, true, true); - set_fsync_mark(folio, 0); - set_dentry_mark(folio, 0); - if (!atomic || folio == last_folio) { - set_fsync_mark(folio, 1); + do_fsync = true; percpu_counter_inc(&sbi->rf_node_block_count); if (IS_INODE(folio)) { if (is_inode_flag_set(inode, FI_DIRTY_INODE)) f2fs_update_inode(inode, folio); - if (!atomic) - set_dentry_mark(folio, - f2fs_need_dentry_mark(sbi, ino)); } /* may be written by other thread */ if (!folio_test_dirty(folio)) @@ -1962,8 +1972,9 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, if (!__write_node_folio(folio, atomic && folio == last_folio, - &submitted, wbc, true, - FS_NODE_IO, seq_id)) { + do_fsync, &submitted, + wbc, true, FS_NODE_IO, + seq_id)) { f2fs_folio_put(last_folio, false); folio_batch_release(&fbatch); ret = -EIO; @@ -2163,10 +2174,7 @@ int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, if (!folio_clear_dirty_for_io(folio)) goto continue_unlock; - set_fsync_mark(folio, 0); - set_dentry_mark(folio, 0); - - if (!__write_node_folio(folio, false, &submitted, + if (!__write_node_folio(folio, false, false, &submitted, wbc, do_balance, io_type, NULL)) { folio_batch_release(&fbatch); ret = -EIO; diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 6a97fe76712b57..8390994a8826cd 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -1880,7 +1880,7 @@ void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) * * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. */ -bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check) { struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; struct discard_policy dpolicy; @@ -1897,7 +1897,7 @@ bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi) /* just to make sure there is no pending discard commands */ __wait_all_discard_cmd(sbi, NULL); - f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt)); + f2fs_bug_on(sbi, need_check && atomic_read(&dcc->discard_cmd_cnt)); return !dropped; } @@ -2367,7 +2367,7 @@ static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) * Recovery can cache discard commands, so in error path of * fill_super(), it needs to give a chance to handle them. */ - f2fs_issue_discard_timeout(sbi); + f2fs_issue_discard_timeout(sbi, true); kfree(dcc); SM_I(sbi)->dcc_info = NULL; diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index 8774c60b4be44e..79cc7b39802f8a 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -336,9 +336,12 @@ static const struct fs_parameter_spec f2fs_param_specs[] = { fsparam_flag("usrquota", Opt_usrquota), fsparam_flag("grpquota", Opt_grpquota), fsparam_flag("prjquota", Opt_prjquota), - fsparam_string_empty("usrjquota", Opt_usrjquota), - fsparam_string_empty("grpjquota", Opt_grpjquota), - fsparam_string_empty("prjjquota", Opt_prjjquota), + fsparam_string("usrjquota", Opt_usrjquota), + fsparam_flag("usrjquota", Opt_usrjquota), + fsparam_string("grpjquota", Opt_grpjquota), + fsparam_flag("grpjquota", Opt_grpjquota), + fsparam_string("prjjquota", Opt_prjjquota), + fsparam_flag("prjjquota", Opt_prjjquota), fsparam_flag("nat_bits", Opt_nat_bits), fsparam_enum("jqfmt", Opt_jqfmt, f2fs_param_jqfmt), fsparam_enum("alloc_mode", Opt_alloc, f2fs_param_alloc_mode), @@ -979,26 +982,26 @@ static int f2fs_parse_param(struct fs_context *fc, struct fs_parameter *param) ctx_set_opt(ctx, F2FS_MOUNT_PRJQUOTA); break; case Opt_usrjquota: - if (!*param->string) - ret = f2fs_unnote_qf_name(fc, USRQUOTA); - else + if (param->type == fs_value_is_string && *param->string) ret = f2fs_note_qf_name(fc, USRQUOTA, param); + else + ret = f2fs_unnote_qf_name(fc, USRQUOTA); if (ret) return ret; break; case Opt_grpjquota: - if (!*param->string) - ret = f2fs_unnote_qf_name(fc, GRPQUOTA); - else + if (param->type == fs_value_is_string && *param->string) ret = f2fs_note_qf_name(fc, GRPQUOTA, param); + else + ret = f2fs_unnote_qf_name(fc, GRPQUOTA); if (ret) return ret; break; case Opt_prjjquota: - if (!*param->string) - ret = f2fs_unnote_qf_name(fc, PRJQUOTA); - else + if (param->type == fs_value_is_string && *param->string) ret = f2fs_note_qf_name(fc, PRJQUOTA, param); + else + ret = f2fs_unnote_qf_name(fc, PRJQUOTA); if (ret) return ret; break; @@ -1515,6 +1518,7 @@ static int f2fs_check_opt_consistency(struct fs_context *fc, F2FS_OPTION(sbi).root_reserved_blocks); ctx_clear_opt(ctx, F2FS_MOUNT_RESERVE_ROOT); ctx->opt_mask &= ~BIT(F2FS_MOUNT_RESERVE_ROOT); + ctx->spec_mask &= ~F2FS_SPEC_reserve_root; } if (test_opt(sbi, RESERVE_NODE) && (ctx->opt_mask & BIT(F2FS_MOUNT_RESERVE_NODE)) && @@ -1523,6 +1527,7 @@ static int f2fs_check_opt_consistency(struct fs_context *fc, F2FS_OPTION(sbi).root_reserved_nodes); ctx_clear_opt(ctx, F2FS_MOUNT_RESERVE_NODE); ctx->opt_mask &= ~BIT(F2FS_MOUNT_RESERVE_NODE); + ctx->spec_mask &= ~F2FS_SPEC_reserve_node; } err = f2fs_check_test_dummy_encryption(fc, sb); @@ -2009,7 +2014,7 @@ static void f2fs_put_super(struct super_block *sb) } /* be sure to wait for any on-going discard commands */ - done = f2fs_issue_discard_timeout(sbi); + done = f2fs_issue_discard_timeout(sbi, true); if (f2fs_realtime_discard_enable(sbi) && !sbi->discard_blks && done) { struct cp_control cpc = { .reason = CP_UMOUNT | CP_TRIMMED, @@ -2152,7 +2157,7 @@ static int f2fs_unfreeze(struct super_block *sb) * will recover after removal of snapshot. */ if (test_opt(sbi, DISCARD) && !f2fs_hw_support_discard(sbi)) - f2fs_issue_discard_timeout(sbi); + f2fs_issue_discard_timeout(sbi, true); clear_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING); return 0; @@ -2957,7 +2962,12 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) need_stop_discard = true; } else { f2fs_stop_discard_thread(sbi); - f2fs_issue_discard_timeout(sbi); + /* + * f2fs_ioc_fitrim() won't race w/ "remount ro" + * so it's safe to check discard_cmd_cnt in + * f2fs_issue_discard_timeout(). + */ + f2fs_issue_discard_timeout(sbi, flags & SB_RDONLY); need_restart_discard = true; } } @@ -4948,6 +4958,11 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) init_f2fs_rwsem_trace(&sbi->gc_lock, sbi, LOCK_NAME_GC_LOCK); mutex_init(&sbi->writepages); init_f2fs_rwsem_trace(&sbi->cp_global_sem, sbi, LOCK_NAME_CP_GLOBAL); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + lockdep_register_key(&sbi->cp_global_sem_key); + lockdep_set_class(&sbi->cp_global_sem.internal_rwsem, + &sbi->cp_global_sem_key); +#endif init_f2fs_rwsem_trace(&sbi->node_write, sbi, LOCK_NAME_NODE_WRITE); init_f2fs_rwsem_trace(&sbi->node_change, sbi, LOCK_NAME_NODE_CHANGE); spin_lock_init(&sbi->stat_lock); @@ -5419,6 +5434,9 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) free_sb_buf: kfree(raw_super); free_sbi: +#ifdef CONFIG_DEBUG_LOCK_ALLOC + lockdep_unregister_key(&sbi->cp_global_sem_key); +#endif kfree(sbi); sb->s_fs_info = NULL; @@ -5500,6 +5518,9 @@ static void kill_f2fs_super(struct super_block *sb) /* Release block devices last, after fscrypt_destroy_keyring(). */ if (sbi) { destroy_device_list(sbi); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + lockdep_unregister_key(&sbi->cp_global_sem_key); +#endif kfree(sbi); sb->s_fs_info = NULL; } diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 5fbfdc96e502d6..0d05ecd64ca037 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -379,10 +379,12 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, if (!strcmp(a->attr.name, "extension_list")) { __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; - int cold_count = le32_to_cpu(sbi->raw_super->extension_count); - int hot_count = sbi->raw_super->hot_ext_count; + int cold_count, hot_count; int len = 0, i; + f2fs_down_read(&sbi->sb_lock); + cold_count = le32_to_cpu(sbi->raw_super->extension_count); + hot_count = sbi->raw_super->hot_ext_count; len += sysfs_emit_at(buf, len, "cold file extension:\n"); for (i = 0; i < cold_count; i++) len += sysfs_emit_at(buf, len, "%s\n", extlist[i]); @@ -390,6 +392,7 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, len += sysfs_emit_at(buf, len, "hot file extension:\n"); for (i = cold_count; i < cold_count + hot_count; i++) len += sysfs_emit_at(buf, len, "%s\n", extlist[i]); + f2fs_up_read(&sbi->sb_lock); return len; } @@ -1984,24 +1987,26 @@ int __init f2fs_init_sysfs(void) ret = kobject_init_and_add(&f2fs_feat, &f2fs_feat_ktype, NULL, "features"); if (ret) - goto put_kobject; + goto unregister_kset; ret = kobject_init_and_add(&f2fs_tune, &f2fs_tune_ktype, NULL, "tuning"); if (ret) - goto put_kobject; + goto put_feat; f2fs_proc_root = proc_mkdir("fs/f2fs", NULL); if (!f2fs_proc_root) { ret = -ENOMEM; - goto put_kobject; + goto put_tune; } return 0; -put_kobject: +put_tune: kobject_put(&f2fs_tune); +put_feat: kobject_put(&f2fs_feat); +unregister_kset: kset_unregister(&f2fs_kset); return ret; } diff --git a/fs/file_table.c b/fs/file_table.c index aaa5faaace1e96..d19d879b6efc1d 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -50,6 +50,9 @@ struct backing_file { struct path user_path; freeptr_t bf_freeptr; }; +#ifdef CONFIG_SECURITY + void *security; +#endif }; #define backing_file(f) container_of(f, struct backing_file, file) @@ -66,6 +69,25 @@ void backing_file_set_user_path(struct file *f, const struct path *path) } EXPORT_SYMBOL_GPL(backing_file_set_user_path); +#ifdef CONFIG_SECURITY +void *backing_file_security(const struct file *f) +{ + return backing_file(f)->security; +} + +void backing_file_set_security(struct file *f, void *security) +{ + backing_file(f)->security = security; +} +#endif /* CONFIG_SECURITY */ + +static inline void backing_file_free(struct backing_file *ff) +{ + security_backing_file_free(&ff->file); + path_put(&ff->user_path); + kmem_cache_free(bfilp_cachep, ff); +} + static inline void file_free(struct file *f) { security_file_free(f); @@ -73,8 +95,7 @@ static inline void file_free(struct file *f) percpu_counter_dec(&nr_files); put_cred(f->f_cred); if (unlikely(f->f_mode & FMODE_BACKING)) { - path_put(backing_file_user_path(f)); - kmem_cache_free(bfilp_cachep, backing_file(f)); + backing_file_free(backing_file(f)); } else { kmem_cache_free(filp_cachep, f); } @@ -283,6 +304,14 @@ struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred) return f; } +static int init_backing_file(struct backing_file *ff, + const struct file *user_file) +{ + memset(&ff->user_path, 0, sizeof(ff->user_path)); + backing_file_set_security(&ff->file, NULL); + return security_backing_file_alloc(&ff->file, user_file); +} + /* * Variant of alloc_empty_file() that allocates a backing_file container * and doesn't check and modify nr_files. @@ -290,7 +319,8 @@ struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred) * This is only for kernel internal use, and the allocate file must not be * installed into file tables or such. */ -struct file *alloc_empty_backing_file(int flags, const struct cred *cred) +struct file *alloc_empty_backing_file(int flags, const struct cred *cred, + const struct file *user_file) { struct backing_file *ff; int error; @@ -305,7 +335,14 @@ struct file *alloc_empty_backing_file(int flags, const struct cred *cred) return ERR_PTR(error); } + /* The f_mode flags must be set before fput(). */ ff->file.f_mode |= FMODE_BACKING | FMODE_NOACCOUNT; + error = init_backing_file(ff, user_file); + if (unlikely(error)) { + fput(&ff->file); + return ERR_PTR(error); + } + return &ff->file; } EXPORT_SYMBOL_GPL(alloc_empty_backing_file); diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 3c75ee025bdaf9..d63baa1b6fecf7 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -570,28 +570,30 @@ void inode_switch_wbs_work_fn(struct work_struct *work) struct inode_switch_wbs_context *isw, *next_isw; struct llist_node *list; + list = llist_del_all(&new_wb->switch_wbs_ctxs); /* - * Grab out reference to wb so that it cannot get freed under us + * Nothing to do? That would be a problem as references held by isw + * items protect wb from freeing... + */ + if (WARN_ON_ONCE(!list)) + return; + + /* + * Grab our reference to wb so that it cannot get freed under us * after we process all the isw items. */ wb_get(new_wb); - while (1) { - list = llist_del_all(&new_wb->switch_wbs_ctxs); - /* Nothing to do? */ - if (!list) - break; - /* - * In addition to synchronizing among switchers, I_WB_SWITCH - * tells the RCU protected stat update paths to grab the i_page - * lock so that stat transfer can synchronize against them. - * Let's continue after I_WB_SWITCH is guaranteed to be - * visible. - */ - synchronize_rcu(); + /* + * In addition to synchronizing among switchers, I_WB_SWITCH + * tells the RCU protected stat update paths to grab the i_page + * lock so that stat transfer can synchronize against them. + * Let's continue after I_WB_SWITCH is guaranteed to be + * visible. + */ + synchronize_rcu(); - llist_for_each_entry_safe(isw, next_isw, list, list) - process_inode_switch_wbs(new_wb, isw); - } + llist_for_each_entry_safe(isw, next_isw, list, list) + process_inode_switch_wbs(new_wb, isw); wb_put(new_wb); } diff --git a/fs/fuse/control.c b/fs/fuse/control.c index 140bd5730d9984..f902a7fb4630cc 100644 --- a/fs/fuse/control.c +++ b/fs/fuse/control.c @@ -121,7 +121,7 @@ static ssize_t fuse_conn_max_background_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - unsigned val; + unsigned int val = 0; ssize_t ret; ret = fuse_conn_limit_write(file, buf, count, ppos, &val, @@ -163,7 +163,7 @@ static ssize_t fuse_conn_congestion_threshold_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - unsigned val; + unsigned int val = 0; struct fuse_conn *fc; ssize_t ret; diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 0b0241f47170d4..24ee9e87026379 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -570,6 +570,11 @@ static void request_wait_answer(struct fuse_req *req) if (!err) return; + if (req->args->abort_on_kill) { + fuse_abort_conn(fc); + return; + } + if (test_bit(FR_URING, &req->flags)) removed = fuse_uring_remove_pending_req(req); else @@ -676,7 +681,8 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap, fuse_force_creds(req); __set_bit(FR_WAITING, &req->flags); - __set_bit(FR_FORCE, &req->flags); + if (!args->abort_on_kill) + __set_bit(FR_FORCE, &req->flags); } else { WARN_ON(args->nocreds); req = fuse_get_req(idmap, fm, false); @@ -1011,6 +1017,9 @@ static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop folio_clear_uptodate(newfolio); folio_clear_mappedtodisk(newfolio); + if (folio_test_large(newfolio)) + goto out_fallback_unlock; + if (fuse_check_folio(newfolio) != 0) goto out_fallback_unlock; @@ -2590,9 +2599,8 @@ static int fuse_device_clone(struct fuse_conn *fc, struct file *new) static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) { - int res; int oldfd; - struct fuse_dev *fud = NULL; + struct fuse_dev *fud; if (get_user(oldfd, argp)) return -EFAULT; @@ -2605,17 +2613,15 @@ static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) * Check against file->f_op because CUSE * uses the same ioctl handler. */ - if (fd_file(f)->f_op == file->f_op) - fud = __fuse_get_dev(fd_file(f)); + if (fd_file(f)->f_op != file->f_op) + return -EINVAL; - res = -EINVAL; - if (fud) { - mutex_lock(&fuse_mutex); - res = fuse_device_clone(fud->fc, file); - mutex_unlock(&fuse_mutex); - } + fud = fuse_get_dev(fd_file(f)); + if (IS_ERR(fud)) + return PTR_ERR(fud); - return res; + guard(mutex)(&fuse_mutex); + return fuse_device_clone(fud->fc, file); } static long fuse_dev_ioctl_backing_open(struct file *file, diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 7ac6b232ef1232..d3acfd346ab527 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -481,6 +481,11 @@ static int fuse_dentry_init(struct dentry *dentry) fd->dentry = dentry; RB_CLEAR_NODE(&fd->node); dentry->d_fsdata = fd; + /* + * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * if compared to fc->epoch, which is initialized to '1'. + */ + dentry->d_time = 0; return 0; } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 7f16049387d15e..23a241f18623a4 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -345,6 +345,7 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool invalidate_vmap:1; + bool abort_on_kill:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index c795abe47a4f4a..bc05c9479f5716 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1551,6 +1551,7 @@ int fuse_send_init(struct fuse_mount *fm) int err; if (fm->fc->sync_init) { + ia->args.abort_on_kill = true; err = fuse_simple_request(fm, &ia->args); /* Ignore size of init reply */ if (err > 0) diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c index 72de97c03d0eeb..f2d08ac2459b7e 100644 --- a/fs/fuse/passthrough.c +++ b/fs/fuse/passthrough.c @@ -167,7 +167,7 @@ struct fuse_backing *fuse_passthrough_open(struct file *file, int backing_id) goto out; /* Allocate backing file per fuse file to store fuse path */ - backing_file = backing_file_open(&file->f_path, file->f_flags, + backing_file = backing_file_open(file, file->f_flags, &fb->file->f_path, fb->cred); err = PTR_ERR(backing_file); if (IS_ERR(backing_file)) { diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index c2aae2eef0868b..aae657fd56c0ed 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -41,6 +41,10 @@ static void fuse_add_dirent_to_cache(struct file *file, unsigned int offset; void *addr; + /* Dirent doesn't fit in readdir cache page? Skip caching. */ + if (reclen > PAGE_SIZE) + return; + spin_lock(&fi->rdc.lock); /* * Is cache already completed? Or this entry does not go at the end of diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c index e79ad087512a05..6a6ded7a61d208 100644 --- a/fs/gfs2/aops.c +++ b/fs/gfs2/aops.c @@ -158,6 +158,7 @@ static int gfs2_writepages(struct address_space *mapping, struct writeback_control *wbc) { struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping); + long initial_nr_to_write = wbc->nr_to_write; struct iomap_writepage_ctx wpc = { .inode = mapping->host, .wbc = wbc, @@ -166,13 +167,13 @@ static int gfs2_writepages(struct address_space *mapping, int ret; /* - * Even if we didn't write enough pages here, we might still be holding + * Even if we didn't write any pages here, we might still be holding * dirty pages in the ail. We forcibly flush the ail because we don't * want balance_dirty_pages() to loop indefinitely trying to write out * pages held in the ail that it can't find. */ ret = iomap_writepages(&wpc); - if (ret == 0 && wbc->nr_to_write > 0) + if (ret == 0 && wbc->nr_to_write == initial_nr_to_write) set_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags); return ret; } diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c index 8344040ecaf79e..e9bf4879c07f73 100644 --- a/fs/gfs2/inode.c +++ b/fs/gfs2/inode.c @@ -892,7 +892,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, goto fail_gunlock4; mark_inode_dirty(inode); - d_instantiate(dentry, inode); + d_instantiate_new(dentry, inode); /* After instantiate, errors should result in evict which will destroy * both inode and iopen glocks properly. */ if (file) { @@ -904,7 +904,6 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, gfs2_glock_dq_uninit(&gh); gfs2_glock_put(io_gl); gfs2_qa_put(dip); - unlock_new_inode(inode); return error; fail_gunlock4: diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c index 347df29d610e67..c483d3c7be6989 100644 --- a/fs/gfs2/log.c +++ b/fs/gfs2/log.c @@ -467,8 +467,9 @@ void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) { atomic_add(blks, &sdp->sd_log_blks_free); trace_gfs2_log_blocks(sdp, blks); - gfs2_assert_withdraw(sdp, atomic_read(&sdp->sd_log_blks_free) <= - sdp->sd_jdesc->jd_blocks); + gfs2_assert_withdraw(sdp, !sdp->sd_jdesc || + atomic_read(&sdp->sd_log_blks_free) <= + sdp->sd_jdesc->jd_blocks); if (atomic_read(&sdp->sd_log_blks_needed)) wake_up(&sdp->sd_log_waitq); } @@ -1018,14 +1019,15 @@ static void trans_drain(struct gfs2_trans *tr) } /** - * gfs2_log_flush - flush incore transaction(s) + * __gfs2_log_flush - flush incore transaction(s) * @sdp: The filesystem * @gl: The glock structure to flush. If NULL, flush the whole incore log * @flags: The log header flags: GFS2_LOG_HEAD_FLUSH_* and debug flags * */ -void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) +static void __gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, + u32 flags) { struct gfs2_trans *tr = NULL; unsigned int reserved_blocks = 0, used_blocks = 0; @@ -1033,7 +1035,6 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) unsigned int first_log_head; unsigned int reserved_revokes = 0; - down_write(&sdp->sd_log_flush_lock); trace_gfs2_log_flush(sdp, 1, flags); repeat: @@ -1145,7 +1146,6 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) gfs2_assert_withdraw(sdp, used_blocks < reserved_blocks); gfs2_log_release(sdp, reserved_blocks - used_blocks); } - up_write(&sdp->sd_log_flush_lock); gfs2_trans_free(sdp, tr); trace_gfs2_log_flush(sdp, 0, flags); return; @@ -1166,6 +1166,13 @@ void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) goto out_end; } +void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) +{ + down_write(&sdp->sd_log_flush_lock); + __gfs2_log_flush(sdp, gl, flags); + up_write(&sdp->sd_log_flush_lock); +} + /** * gfs2_merge_trans - Merge a new transaction into a cached transaction * @sdp: the filesystem @@ -1297,19 +1304,25 @@ int gfs2_logd(void *data) break; if (gfs2_jrnl_flush_reqd(sdp) || t == 0) { + down_write(&sdp->sd_log_flush_lock); gfs2_ail1_empty(sdp, 0); - gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | - GFS2_LFC_LOGD_JFLUSH_REQD); + __gfs2_log_flush(sdp, NULL, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_LOGD_JFLUSH_REQD); + up_write(&sdp->sd_log_flush_lock); } if (test_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags) || gfs2_ail_flush_reqd(sdp)) { clear_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags); + down_write(&sdp->sd_log_flush_lock); gfs2_ail1_start(sdp); gfs2_ail1_wait(sdp); gfs2_ail1_empty(sdp, 0); - gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | - GFS2_LFC_LOGD_AIL_FLUSH_REQD); + __gfs2_log_flush(sdp, NULL, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_LOGD_AIL_FLUSH_REQD); + up_write(&sdp->sd_log_flush_lock); } t = gfs2_tune_get(sdp, gt_logd_secs) * HZ; diff --git a/fs/hfsplus/bfind.c b/fs/hfsplus/bfind.c index 336d654861c597..9a55fa6d529429 100644 --- a/fs/hfsplus/bfind.c +++ b/fs/hfsplus/bfind.c @@ -287,3 +287,54 @@ int hfs_brec_goto(struct hfs_find_data *fd, int cnt) fd->bnode = bnode; return res; } + +/** + * hfsplus_brec_read_cat - read and validate a catalog record + * @fd: find data structure + * @entry: pointer to catalog entry to read into + * + * Reads a catalog record and validates its size matches the expected + * size based on the record type. + * + * Returns 0 on success, or negative error code on failure. + */ +int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry) +{ + int res; + u32 expected_size; + + res = hfs_brec_read(fd, entry, sizeof(hfsplus_cat_entry)); + if (res) + return res; + + /* Validate catalog record size based on type */ + switch (be16_to_cpu(entry->type)) { + case HFSPLUS_FOLDER: + expected_size = sizeof(struct hfsplus_cat_folder); + break; + case HFSPLUS_FILE: + expected_size = sizeof(struct hfsplus_cat_file); + break; + case HFSPLUS_FOLDER_THREAD: + case HFSPLUS_FILE_THREAD: + /* Ensure we have at least the fixed fields before reading nodeName.length */ + if (fd->entrylength < HFSPLUS_MIN_THREAD_SZ) { + pr_err("thread record too short (got %u)\n", fd->entrylength); + return -EIO; + } + expected_size = hfsplus_cat_thread_size(&entry->thread); + break; + default: + pr_err("unknown catalog record type %d\n", + be16_to_cpu(entry->type)); + return -EIO; + } + + if (fd->entrylength != expected_size) { + pr_err("catalog record size mismatch (type %d, got %u, expected %u)\n", + be16_to_cpu(entry->type), fd->entrylength, expected_size); + return -EIO; + } + + return 0; +} diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c index 02c1eee4a4b860..6c8380f7208df4 100644 --- a/fs/hfsplus/catalog.c +++ b/fs/hfsplus/catalog.c @@ -194,12 +194,12 @@ static int hfsplus_fill_cat_thread(struct super_block *sb, int hfsplus_find_cat(struct super_block *sb, u32 cnid, struct hfs_find_data *fd) { - hfsplus_cat_entry tmp; + hfsplus_cat_entry tmp = {0}; int err; u16 type; hfsplus_cat_build_key_with_cnid(sb, fd->search_key, cnid); - err = hfs_brec_read(fd, &tmp, sizeof(hfsplus_cat_entry)); + err = hfsplus_brec_read_cat(fd, &tmp); if (err) return err; diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index d559bf8625f853..25535592234c9b 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -49,7 +49,7 @@ static struct dentry *hfsplus_lookup(struct inode *dir, struct dentry *dentry, if (unlikely(err < 0)) goto fail; again: - err = hfs_brec_read(&fd, &entry, sizeof(entry)); + err = hfsplus_brec_read_cat(&fd, &entry); if (err) { if (err == -ENOENT) { hfs_find_exit(&fd); diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 5f891b73a64677..61d52091dd28d7 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -509,6 +509,15 @@ int hfsplus_submit_bio(struct super_block *sb, sector_t sector, void *buf, void **data, blk_opf_t opf); int hfsplus_read_wrapper(struct super_block *sb); +static inline u32 hfsplus_cat_thread_size(const struct hfsplus_cat_thread *thread) +{ + return offsetof(struct hfsplus_cat_thread, nodeName) + + offsetof(struct hfsplus_unistr, unicode) + + be16_to_cpu(thread->nodeName.length) * sizeof(hfsplus_unichr); +} + +int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry); + /* * time helpers: convert between 1904-base and 1970-base timestamps * diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index 7229a8ae89f946..67df3af9cf1559 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -569,9 +569,11 @@ static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) if (err) goto out_put_root; err = hfsplus_cat_build_key(sb, fd.search_key, HFSPLUS_ROOT_CNID, &str); - if (unlikely(err < 0)) + if (unlikely(err < 0)) { + hfs_find_exit(&fd); goto out_put_root; - if (!hfs_brec_read(&fd, &entry, sizeof(entry))) { + } + if (!hfsplus_brec_read_cat(&fd, &entry)) { hfs_find_exit(&fd); if (entry.type != cpu_to_be16(HFSPLUS_FOLDER)) { err = -EIO; diff --git a/fs/internal.h b/fs/internal.h index cbc384a1aa096a..8c1f6c548dbfd7 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -106,7 +106,8 @@ extern void chroot_fs_refs(const struct path *, const struct path *); */ struct file *alloc_empty_file(int flags, const struct cred *cred); struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred); -struct file *alloc_empty_backing_file(int flags, const struct cred *cred); +struct file *alloc_empty_backing_file(int flags, const struct cred *cred, + const struct file *user_file); void backing_file_set_user_path(struct file *f, const struct path *path); static inline void file_put_write_access(struct file *file) @@ -199,7 +200,6 @@ extern int build_open_flags(const struct open_how *how, struct open_flags *op); struct file *file_close_fd_locked(struct files_struct *files, unsigned fd); int do_ftruncate(struct file *file, loff_t length, int small); -int do_sys_ftruncate(unsigned int fd, loff_t length, int small); int chmod_common(const struct path *path, umode_t mode); int do_fchownat(int dfd, const char __user *filename, uid_t user, gid_t group, int flag); diff --git a/fs/isofs/export.c b/fs/isofs/export.c index 421d247fae5230..78f80c1a5c54a0 100644 --- a/fs/isofs/export.c +++ b/fs/isofs/export.c @@ -24,7 +24,7 @@ isofs_export_iget(struct super_block *sb, { struct inode *inode; - if (block == 0) + if (block == 0 || block >= ISOFS_SB(sb)->s_nzones) return ERR_PTR(-ESTALE); inode = isofs_iget(sb, block, offset); if (IS_ERR(inode)) diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c index 6fe6dbd0c740f7..1232fab59a4e68 100644 --- a/fs/isofs/rock.c +++ b/fs/isofs/rock.c @@ -101,6 +101,15 @@ static int rock_continue(struct rock_state *rs) goto out; } + if ((unsigned)rs->cont_extent >= ISOFS_SB(rs->inode->i_sb)->s_nzones) { + printk(KERN_NOTICE "rock: corrupted directory entry. " + "extent=%u out of volume (nzones=%lu)\n", + (unsigned)rs->cont_extent, + ISOFS_SB(rs->inode->i_sb)->s_nzones); + ret = -EIO; + goto out; + } + if (rs->cont_extent) { struct buffer_head *bh; diff --git a/fs/jbd2/revoke.c b/fs/jbd2/revoke.c index 9016ddb824474d..e4c2fbd381f123 100644 --- a/fs/jbd2/revoke.c +++ b/fs/jbd2/revoke.c @@ -428,6 +428,7 @@ void jbd2_journal_cancel_revoke(handle_t *handle, struct journal_head *jh) journal_t *journal = handle->h_transaction->t_journal; int need_cancel; struct buffer_head *bh = jh2bh(jh); + struct address_space *bh_mapping = bh->b_folio->mapping; jbd2_debug(4, "journal_head %p, cancelling revoke\n", jh); @@ -464,13 +465,14 @@ void jbd2_journal_cancel_revoke(handle_t *handle, struct journal_head *jh) * buffer_head? If so, we'd better make sure we clear the * revoked status on any hashed alias too, otherwise the revoke * state machine will get very upset later on. */ - if (need_cancel) { + if (need_cancel && !sb_is_blkdev_sb(bh_mapping->host->i_sb)) { struct buffer_head *bh2; + bh2 = __find_get_block_nonatomic(bh->b_bdev, bh->b_blocknr, bh->b_size); if (bh2) { - if (bh2 != bh) - clear_buffer_revoked(bh2); + WARN_ON_ONCE(bh2 == bh); + clear_buffer_revoked(bh2); __brelse(bh2); } } diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index 255a847ca0b6b9..abc65dc79f8543 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -80,6 +80,11 @@ static const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) return buf; } +#else +static inline const char *nlmdbg_cookie2a(const struct nlm_cookie *cookie) +{ + return "???"; +} #endif /* diff --git a/fs/mbcache.c b/fs/mbcache.c index 480d02d6ebf03e..2a6319b4072cca 100644 --- a/fs/mbcache.c +++ b/fs/mbcache.c @@ -406,6 +406,7 @@ void mb_cache_destroy(struct mb_cache *cache) { struct mb_cache_entry *entry, *next; + cancel_work_sync(&cache->c_shrink_work); shrinker_free(cache->c_shrink); /* diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c index 154a14bb2d7f78..429e4396e1b000 100644 --- a/fs/netfs/iterator.c +++ b/fs/netfs/iterator.c @@ -22,7 +22,7 @@ * * Extract the page fragments from the given amount of the source iterator and * build up a second iterator that refers to all of those bits. This allows - * the original iterator to disposed of. + * the original iterator to be disposed of. * * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be * allowed on the pages extracted. @@ -43,7 +43,7 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, unsigned int max_pages; unsigned int npages = 0; unsigned int i; - ssize_t ret; + ssize_t ret = 0; size_t count = orig_len, offset, len; size_t bv_size, pg_size; @@ -67,8 +67,8 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, ret = iov_iter_extract_pages(orig, &pages, count, max_pages - npages, extraction_flags, &offset); - if (ret < 0) { - pr_err("Couldn't get user pages (rc=%zd)\n", ret); + if (unlikely(ret <= 0)) { + ret = ret ?: -EIO; break; } @@ -97,6 +97,13 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, npages += cur_npages; } + if (ret < 0 && (ret == -ENOMEM || npages == 0)) { + for (i = 0; i < npages; i++) + unpin_user_page(bv[i].bv_page); + kvfree(bv); + return ret; + } + iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count); return npages; } diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index cb0a645aeb5016..94e85ad9067e5e 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -381,14 +381,13 @@ bl_write_pagelist(struct nfs_pgio_header *header, int sync) sector_t isect, extent_length = 0; struct parallel_io *par = NULL; loff_t offset = header->args.offset; - size_t count = header->args.count; struct page **pages = header->args.pages; int pg_index = header->args.pgbase >> PAGE_SHIFT; unsigned int pg_len; struct blk_plug plug; int i; - dprintk("%s enter, %zu@%lld\n", __func__, count, offset); + dprintk("%s enter, %u@%lld\n", __func__, header->args.count, offset); /* At this point, header->page_aray is a (sequential) list of nfs_pages. * We want to write each, and if there is an error set pnfs_error @@ -429,7 +428,6 @@ bl_write_pagelist(struct nfs_pgio_header *header, int sync) } offset += pg_len; - count -= pg_len; isect += (pg_len >> SECTOR_SHIFT); extent_length -= (pg_len >> SECTOR_SHIFT); } diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 63e09dfc27a8f7..0338603e9674c8 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -253,6 +253,8 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, u32 minor_version); extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *, struct inode *); +extern void nfs4_session_limit_rwsize(struct nfs_server *server); +extern void nfs4_session_limit_xasize(struct nfs_server *server); extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index c211639949c27e..71c271a1700a92 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -855,7 +855,7 @@ EXPORT_SYMBOL_GPL(nfs4_set_ds_client); * Limit the mount rsize, wsize and dtsize using negotiated fore * channel attributes. */ -static void nfs4_session_limit_rwsize(struct nfs_server *server) +void nfs4_session_limit_rwsize(struct nfs_server *server) { struct nfs4_session *sess; u32 server_resp_sz; @@ -878,7 +878,7 @@ static void nfs4_session_limit_rwsize(struct nfs_server *server) /* * Limit xattr sizes using the channel attributes. */ -static void nfs4_session_limit_xasize(struct nfs_server *server) +void nfs4_session_limit_xasize(struct nfs_server *server) { #ifdef CONFIG_NFS_V4_2 struct nfs4_session *sess; diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 91bcf67bd743f7..655617ffca8db5 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -10618,6 +10618,9 @@ static struct nfs_server *nfs4_clone_server(struct nfs_server *source, if (IS_ERR(server)) return server; + nfs4_session_limit_rwsize(server); + nfs4_session_limit_xasize(server); + error = nfs4_delegation_hash_alloc(server); if (error) { nfs_free_server(server); diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 6880c5c520e75d..6b8f01129ecc1a 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1413,6 +1413,9 @@ nfsd4_clone(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, dst, clone->cl_dst_pos, clone->cl_count, EX_ISSYNC(cstate->current_fh.fh_export)); + if (!status && (READ_ONCE(dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0) + nfsd_update_cmtime_attr(dst->nf_file, 0); + nfsd_file_put(dst); nfsd_file_put(src); out: @@ -2118,8 +2121,10 @@ static int nfsd4_do_async_copy(void *data) set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); trace_nfsd_copy_async_done(copy); - nfsd4_send_cb_offload(copy); atomic_dec(©->cp_nn->pending_async_copies); + if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update) + nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); + nfsd4_send_cb_offload(copy); return 0; } @@ -2179,6 +2184,9 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid, sizeof(result->cb_stateid)); dup_copy_fields(copy, async_copy); + if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & + FMODE_NOCMTIME) != 0) + async_copy->attr_update = true; memcpy(async_copy->cp_cb_offload.co_referring_sessionid.data, cstate->session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN); @@ -2197,6 +2205,10 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } else { status = nfsd4_do_copy(copy, copy->nf_src->nf_file, copy->nf_dst->nf_file, true); + if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & + FMODE_NOCMTIME) != 0 && + copy->cp_res.wr_bytes_written > 0) + nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); } out: trace_nfsd_copy_done(copy, status); @@ -2535,10 +2547,6 @@ nfsd4_get_dir_delegation(struct svc_rqst *rqstp, dd = nfsd_get_dir_deleg(cstate, gdd, nf); nfsd_file_put(nf); if (IS_ERR(dd)) { - int err = PTR_ERR(dd); - - if (err != -EAGAIN) - return nfserrno(err); gdd->gddrnf_status = GDD4_UNAVAIL; return nfs_ok; } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 6b9c399b89dfb7..44b1a93f219af5 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -1226,10 +1226,6 @@ static void put_deleg_file(struct nfs4_file *fp) static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct file *f) { - struct iattr ia = { .ia_valid = ATTR_ATIME | ATTR_CTIME | ATTR_MTIME | ATTR_DELEG }; - struct inode *inode = file_inode(f); - int ret; - /* don't do anything if FMODE_NOCMTIME isn't set */ if ((READ_ONCE(f->f_mode) & FMODE_NOCMTIME) == 0) return; @@ -1247,17 +1243,7 @@ static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct f return; /* Stamp everything to "now" */ - inode_lock(inode); - ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &ia, NULL); - inode_unlock(inode); - if (ret) { - struct inode *inode = file_inode(f); - - pr_notice_ratelimited("nfsd: Unable to update timestamps on inode %02x:%02x:%lu: %d\n", - MAJOR(inode->i_sb->s_dev), - MINOR(inode->i_sb->s_dev), - inode->i_ino, ret); - } + nfsd_update_cmtime_attr(f, ATTR_ATIME); } static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp) @@ -1495,8 +1481,24 @@ release_all_access(struct nfs4_ol_stateid *stp) } } +/** + * nfs4_replay_free_cache - release dynamically allocated replay buffer + * @rp: replay cache to reset + * + * If @rp->rp_buf points to a kmalloc'd buffer, free it and reset + * rp_buf to the inline rp_ibuf. Always zeroes rp_buflen. + */ +void nfs4_replay_free_cache(struct nfs4_replay *rp) +{ + if (rp->rp_buf != rp->rp_ibuf) + kfree(rp->rp_buf); + rp->rp_buf = rp->rp_ibuf; + rp->rp_buflen = 0; +} + static inline void nfs4_free_stateowner(struct nfs4_stateowner *sop) { + nfs4_replay_free_cache(&sop->so_replay); kfree(sop->so_owner.data); sop->so_ops->so_free(sop); } @@ -6257,12 +6259,12 @@ nfsd4_add_rdaccess_to_wrdeleg(struct svc_rqst *rqstp, struct nfsd4_open *open, return (false); fp = stp->st_stid.sc_file; spin_lock(&fp->fi_lock); - __nfs4_file_get_access(fp, NFS4_SHARE_ACCESS_READ); if (!fp->fi_fds[O_RDONLY]) { + __nfs4_file_get_access(fp, NFS4_SHARE_ACCESS_READ); fp->fi_fds[O_RDONLY] = nf; + fp->fi_rdeleg_file = nfsd_file_get(fp->fi_fds[O_RDONLY]); nf = NULL; } - fp->fi_rdeleg_file = nfsd_file_get(fp->fi_fds[O_RDONLY]); spin_unlock(&fp->fi_lock); if (nf) nfsd_file_put(nf); @@ -6352,7 +6354,6 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, } open->op_delegate_type = deleg_ts ? OPEN_DELEGATE_WRITE_ATTRS_DELEG : OPEN_DELEGATE_WRITE; - dp->dl_cb_fattr.ncf_cur_fsize = stat.size; dp->dl_cb_fattr.ncf_initial_cinfo = nfsd4_change_attribute(&stat); dp->dl_atime = stat.atime; dp->dl_ctime = stat.ctime; @@ -9401,11 +9402,15 @@ nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, if (status != nfserr_jukebox || !nfsd_wait_for_delegreturn(rqstp, inode)) goto out_status; + status = nfs_ok; + goto out_status; + } + if (!ncf->ncf_file_modified) { + if (ncf->ncf_initial_cinfo != ncf->ncf_cb_change) + ncf->ncf_file_modified = true; + else if (i_size_read(inode) != ncf->ncf_cb_fsize) + ncf->ncf_file_modified = true; } - if (!ncf->ncf_file_modified && - (ncf->ncf_initial_cinfo != ncf->ncf_cb_change || - ncf->ncf_cur_fsize != ncf->ncf_cb_fsize)) - ncf->ncf_file_modified = true; if (ncf->ncf_file_modified) { int err; @@ -9531,3 +9536,31 @@ nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, put_nfs4_file(fp); return ERR_PTR(status); } + +/** + * nfsd_update_cmtime_attr - update file's delegated ctime/mtime, + * and optionally other attributes (ie ATTR_ATIME). + * @f: pointer to an opened file + * @flags: any additional flags that should be updated + * + * Given upon opening a file delegated attributes were issues, update + * @f attributes to current times. + */ +void nfsd_update_cmtime_attr(struct file *f, unsigned int flags) +{ + int ret; + struct inode *inode = file_inode(f); + struct iattr attr = { + .ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_DELEG | flags, + }; + + inode_lock(inode); + ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &attr, NULL); + inode_unlock(inode); + if (ret) + pr_notice_ratelimited("nfsd: Unable to update timestamps on " + "inode %02x:%02x:%lu: %d\n", + MAJOR(inode->i_sb->s_dev), + MINOR(inode->i_sb->s_dev), + inode->i_ino, ret); +} diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 9d234913100b9b..ef663331063b15 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -6281,14 +6281,23 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) int len = xdr->buf->len - (op_status_offset + XDR_UNIT); so->so_replay.rp_status = op->status; - if (len <= NFSD4_REPLAY_ISIZE) { - so->so_replay.rp_buflen = len; - read_bytes_from_xdr_buf(xdr->buf, - op_status_offset + XDR_UNIT, - so->so_replay.rp_buf, len); - } else { - so->so_replay.rp_buflen = 0; + if (len > NFSD4_REPLAY_ISIZE) { + char *buf = kmalloc(len, GFP_KERNEL); + + nfs4_replay_free_cache(&so->so_replay); + if (buf) { + so->so_replay.rp_buf = buf; + } else { + /* rp_buflen already zeroed; skip caching */ + goto status; + } + } else if (so->so_replay.rp_buf != so->so_replay.rp_ibuf) { + nfs4_replay_free_cache(&so->so_replay); } + so->so_replay.rp_buflen = len; + read_bytes_from_xdr_buf(xdr->buf, + op_status_offset + XDR_UNIT, + so->so_replay.rp_buf, len); } status: op->status = nfsd4_map_status(op->status, diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index ed85dd43da18e6..68b629fbaaeb9b 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -105,9 +105,12 @@ static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, { /* Check if the request originated from a secure port. */ if (rqstp && !nfsd_originating_port_ok(rqstp, cred, exp)) { - RPC_IFDEBUG(char buf[RPC_MAX_ADDRBUFLEN]); - dprintk("nfsd: request from insecure port %s!\n", - svc_print_addr(rqstp, buf, sizeof(buf))); + if (IS_ENABLED(CONFIG_SUNRPC_DEBUG)) { + char buf[RPC_MAX_ADDRBUFLEN]; + + dprintk("nfsd: request from insecure port %s!\n", + svc_print_addr(rqstp, buf, sizeof(buf))); + } return nfserr_perm; } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index c0ca115c3b74bb..5afee1342aa9a0 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -549,10 +549,10 @@ struct nfs4_client_reclaim { * ~32(deleg. ace) = 112 bytes * * Some responses can exceed this. A LOCK denial includes the conflicting - * lock owner, which can be up to 1024 bytes (NFS4_OPAQUE_LIMIT). Responses - * larger than REPLAY_ISIZE are not cached in rp_ibuf; only rp_status is - * saved. Enlarging this constant increases the size of every - * nfs4_stateowner. + * lock owner, which can be up to 1024 bytes (NFS4_OPAQUE_LIMIT). When a + * response exceeds REPLAY_ISIZE, a buffer is dynamically allocated. If + * that allocation fails, only rp_status is saved. Enlarging this constant + * increases the size of every nfs4_stateowner. */ #define NFSD4_REPLAY_ISIZE 112 @@ -564,12 +564,14 @@ struct nfs4_client_reclaim { struct nfs4_replay { __be32 rp_status; unsigned int rp_buflen; - char *rp_buf; + char *rp_buf; /* rp_ibuf or kmalloc'd */ struct knfsd_fh rp_openfh; int rp_locked; char rp_ibuf[NFSD4_REPLAY_ISIZE]; }; +extern void nfs4_replay_free_cache(struct nfs4_replay *rp); + struct nfs4_stateowner; struct nfs4_stateowner_operations { @@ -832,6 +834,7 @@ extern void nfsd4_shutdown_copy(struct nfs4_client *clp); void nfsd4_put_client(struct nfs4_client *clp); void nfsd4_async_copy_reaper(struct nfsd_net *nn); bool nfsd4_has_active_async_copies(struct nfs4_client *clp); +void nfsd_update_cmtime_attr(struct file *f, unsigned int flags); extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, struct nfsd_net *nn); extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn); diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 417e9ad9fbb397..9a4124c77e049d 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -752,6 +752,7 @@ struct nfsd4_copy { struct nfsd_file *nf_src; struct nfsd_file *nf_dst; + bool attr_update; copy_stateid_t cp_stateid; diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c index 674380837ab98b..888dc1831c86ee 100644 --- a/fs/nilfs2/dat.c +++ b/fs/nilfs2/dat.c @@ -524,6 +524,9 @@ int nilfs_dat_read(struct super_block *sb, size_t entry_size, if (err) goto failed; + err = nilfs_attach_btree_node_cache(dat); + if (err) + goto failed; err = nilfs_read_inode_common(dat, raw_inode); if (err) goto failed; diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index e17b8da6649133..e0a606643e8792 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -736,6 +736,12 @@ static int nilfs_ioctl_mark_blocks_dirty(struct the_nilfs *nilfs, int ret, i; for (i = 0; i < nmembs; i++) { + /* + * bd_oblocknr must never be 0 as block 0 + * is never a valid GC target block + */ + if (unlikely(!bdescs[i].bd_oblocknr)) + return -EINVAL; /* XXX: use macro or inline func to check liveness */ ret = nilfs_bmap_lookup_at_level(bmap, bdescs[i].bd_offset, diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index c2dcb25151deb2..ae904451dfc095 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -1210,6 +1210,7 @@ static int fanotify_find_path(int dfd, const char __user *filename, *path = fd_file(f)->f_path; path_get(path); + ret = 0; } else { unsigned int lookup_flags = 0; @@ -1219,22 +1220,7 @@ static int fanotify_find_path(int dfd, const char __user *filename, lookup_flags |= LOOKUP_DIRECTORY; ret = user_path_at(dfd, filename, lookup_flags, path); - if (ret) - goto out; - } - - /* you can only watch an inode if you have read permissions on it */ - ret = path_permission(path, MAY_READ); - if (ret) { - path_put(path); - goto out; } - - ret = security_path_notify(path, mask, obj_type); - if (ret) - path_put(path); - -out: return ret; } @@ -1615,17 +1601,18 @@ SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) pr_debug("%s: flags=%x event_f_flags=%x\n", __func__, flags, event_f_flags); - if (!capable(CAP_SYS_ADMIN)) { - /* - * An unprivileged user can setup an fanotify group with - * limited functionality - an unprivileged group is limited to - * notification events with file handles or mount ids and it - * cannot use unlimited queue/marks. - */ - if ((flags & FANOTIFY_ADMIN_INIT_FLAGS) || - !(flags & (FANOTIFY_FID_BITS | FAN_REPORT_MNT))) - return -EPERM; + /* + * An unprivileged user can setup an fanotify group with limited + * functionality - an unprivileged group is limited to notification + * events with file handles or mount ids and it cannot use unlimited + * queue/marks. + */ + if (((flags & FANOTIFY_ADMIN_INIT_FLAGS) || + !(flags & (FANOTIFY_FID_BITS | FAN_REPORT_MNT))) && + !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) { /* * Setting the internal flag FANOTIFY_UNPRIV on the group * prevents setting mount/filesystem marks on this group and @@ -1990,8 +1977,8 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, * A user is allowed to setup sb/mount/mntns marks only if it is * capable in the user ns where the group was created. */ - if (!ns_capable(group->user_ns, CAP_SYS_ADMIN) && - mark_type != FAN_MARK_INODE) + if (mark_type != FAN_MARK_INODE && + !ns_capable(group->user_ns, CAP_SYS_ADMIN)) return -EPERM; /* @@ -2057,6 +2044,15 @@ static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, goto path_put_and_out; } + /* you can only watch an inode if you have read permissions on it */ + ret = path_permission(&path, MAY_READ); + if (ret) + goto path_put_and_out; + + ret = security_path_notify(&path, mask, obj_type); + if (ret) + goto path_put_and_out; + if (fid_mode) { ret = fanotify_test_fsid(path.dentry, flags, &__fsid); if (ret) diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c index 9995de1710e596..b646a861a84c64 100644 --- a/fs/notify/fsnotify.c +++ b/fs/notify/fsnotify.c @@ -388,7 +388,7 @@ static struct fsnotify_mark *fsnotify_first_mark(struct fsnotify_mark_connector return hlist_entry_safe(node, struct fsnotify_mark, obj_list); } -static struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark) +struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark) { struct hlist_node *node = NULL; diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 5e1845f2c25dd2..2edac3b391787c 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -621,6 +621,7 @@ static int inotify_new_watch(struct fsnotify_group *group, if (ret) { /* we failed to get on the inode, get off the idr */ inotify_remove_from_idr(group, tmp_i_mark); + dec_inotify_watches(group->inotify_data.ucounts); goto out_err; } diff --git a/fs/notify/mark.c b/fs/notify/mark.c index c2ed5b11b0fe63..e256b420100dc8 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -238,7 +238,12 @@ static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn, return inode; } -static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) +/* + * Calculate mask of events for a list of marks. + * + * Return true if any of the attached marks want to hold an inode reference. + */ +static bool __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) { u32 new_mask = 0; bool want_iref = false; @@ -262,6 +267,34 @@ static void *__fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) */ WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask); + return want_iref; +} + +/* + * Calculate mask of events for a list of marks after attach/modify mark + * and get an inode reference for the connector if needed. + * + * A concurrent add of evictable mark and detach of non-evictable mark can + * lead to __fsnotify_recalc_mask() returning false want_iref, but in this + * case we defer clearing iref to fsnotify_recalc_mask_clear_iref() called + * from fsnotify_put_mark(). + */ +static void fsnotify_recalc_mask_set_iref(struct fsnotify_mark_connector *conn) +{ + bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF; + bool want_iref = __fsnotify_recalc_mask(conn) || has_iref; + + (void) fsnotify_update_iref(conn, want_iref); +} + +/* + * Calculate mask of events for a list of marks after detach mark + * and return the inode object if its reference is no longer needed. + */ +static void *fsnotify_recalc_mask_clear_iref(struct fsnotify_mark_connector *conn) +{ + bool want_iref = __fsnotify_recalc_mask(conn); + return fsnotify_update_iref(conn, want_iref); } @@ -298,7 +331,7 @@ void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) spin_lock(&conn->lock); update_children = !fsnotify_conn_watches_children(conn); - __fsnotify_recalc_mask(conn); + fsnotify_recalc_mask_set_iref(conn); update_children &= fsnotify_conn_watches_children(conn); spin_unlock(&conn->lock); /* @@ -419,7 +452,7 @@ void fsnotify_put_mark(struct fsnotify_mark *mark) /* Update watched objects after detaching mark */ if (sb) fsnotify_update_sb_watchers(sb, conn); - objp = __fsnotify_recalc_mask(conn); + objp = fsnotify_recalc_mask_clear_iref(conn); type = conn->type; } WRITE_ONCE(mark->connector, NULL); @@ -457,9 +490,6 @@ EXPORT_SYMBOL_GPL(fsnotify_put_mark); */ static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark) { - if (!mark) - return true; - if (refcount_inc_not_zero(&mark->refcnt)) { spin_lock(&mark->lock); if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) { @@ -500,15 +530,22 @@ bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info) int type; fsnotify_foreach_iter_type(type) { + struct fsnotify_mark *mark = iter_info->marks[type]; + /* This can fail if mark is being removed */ - if (!fsnotify_get_mark_safe(iter_info->marks[type])) { - __release(&fsnotify_mark_srcu); - goto fail; + while (mark && !fsnotify_get_mark_safe(mark)) { + if (mark->group == iter_info->current_group) { + __release(&fsnotify_mark_srcu); + goto fail; + } + /* This is a mark in an unrelated group, skip */ + mark = fsnotify_next_mark(mark); + iter_info->marks[type] = mark; } } /* - * Now that both marks are pinned by refcount in the inode / vfsmount + * Now that all marks are pinned by refcount in the inode / vfsmount / etc * lists, we can drop SRCU lock, and safely resume the list iteration * once userspace returns. */ diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c index 6cb9bc5d605c26..76e581d3961d05 100644 --- a/fs/ntfs3/attrib.c +++ b/fs/ntfs3/attrib.c @@ -1152,6 +1152,21 @@ int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, if (err) goto out; } + + if (vcn0 < svcn || evcn1 <= vcn0) { + struct ATTRIB *attr2; + + attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, NULL, + 0, &vcn0, &mi); + if (!attr2) { + err = -EINVAL; + goto out; + } + err = attr_load_runs(attr2, ni, run, NULL); + if (err) + goto out; + } + da = false; /* no delalloc for compressed file. */ } diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index 272e452761436e..037df47fa9f3af 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -2791,13 +2791,14 @@ static inline bool check_file_record(const struct MFT_REC *rec, u16 fn = le16_to_cpu(rec->rhdr.fix_num); u16 ao = le16_to_cpu(rec->attr_off); u32 rs = sbi->record_size; + u32 used = le32_to_cpu(rec->used); /* Check the file record header for consistency. */ if (rec->rhdr.sign != NTFS_FILE_SIGNATURE || fo > (SECTOR_SIZE - ((rs >> SECTOR_SHIFT) + 1) * sizeof(short)) || (fn - 1) * SECTOR_SIZE != rs || ao < MFTRECORD_FIXUP_OFFSET_1 || ao > sbi->record_size - SIZEOF_RESIDENT || !is_rec_inuse(rec) || - le32_to_cpu(rec->total) != rs) { + le32_to_cpu(rec->total) != rs || used > rs || used < ao) { return false; } @@ -2809,6 +2810,15 @@ static inline bool check_file_record(const struct MFT_REC *rec, return false; } + /* + * The do_action() handlers compute memmove lengths as + * "rec->used - ", which underflows when + * rec->used is smaller than the attribute walk reached. At this + * point attr is the ATTR_END marker; rec->used must cover it. + */ + if (used < PtrOffset(rec, attr) + sizeof(attr->type)) + return false; + return true; } diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c index 6e65066ebcc1ae..eac421cf98a875 100644 --- a/fs/ntfs3/inode.c +++ b/fs/ntfs3/inode.c @@ -822,6 +822,11 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return err; } + if (!clen) { + /* broken file? */ + return -EINVAL; + } + if (lcn == EOF_LCN) { /* request out of file. */ if (flags & IOMAP_REPORT) { @@ -855,11 +860,6 @@ static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, return 0; } - if (!clen) { - /* broken file? */ - return -EINVAL; - } - iomap->bdev = inode->i_sb->s_bdev; iomap->offset = offset; iomap->length = ((loff_t)clen << cluster_bits) - off; diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c index c0324cdc174dd0..1ce7d92fb27482 100644 --- a/fs/ntfs3/run.c +++ b/fs/ntfs3/run.c @@ -1008,6 +1008,9 @@ int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, if (size_size > sizeof(len)) return -EINVAL; + if (run_buf + size_size > run_last) + return -EINVAL; + len = run_unpack_s64(run_buf, size_size, 0); /* Skip size_size. */ run_buf += size_size; @@ -1020,6 +1023,9 @@ int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, else if (offset_size <= sizeof(s64)) { s64 dlcn; + if (run_buf + offset_size > run_last) + return -EINVAL; + /* Initial value of dlcn is -1 or 0. */ dlcn = (run_buf[offset_size - 1] & 0x80) ? (s64)-1 : 0; dlcn = run_unpack_s64(run_buf, offset_size, dlcn); @@ -1059,9 +1065,15 @@ int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, return -EOPNOTSUPP; } #endif - if (lcn != SPARSE_LCN64 && lcn + len > sbi->used.bitmap.nbits) { - /* LCN range is out of volume. */ - return -EINVAL; + if (lcn != SPARSE_LCN64) { + u64 lcn_end; + + if (check_add_overflow(lcn, len, &lcn_end)) + return -EINVAL; + if (lcn_end > sbi->used.bitmap.nbits) { + /* LCN range is out of volume. */ + return -EINVAL; + } } if (!run) diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 174a7cb202a08c..9ed485f9efbae2 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -1339,8 +1339,13 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) le32_to_cpu(attr->res.data_size) >> 1, UTF16_LITTLE_ENDIAN, sbi->volume.label, sizeof(sbi->volume.label)); - if (err < 0) + if (err < 0) { sbi->volume.label[0] = 0; + } else if (err >= sizeof(sbi->volume.label)) { + sbi->volume.label[sizeof(sbi->volume.label) - 1] = 0; + } else { + sbi->volume.label[err] = 0; + } } else { /* Should we break mounting here? */ //err = -EINVAL; diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c index 17ba79f443ee73..c6dbec1693b125 100644 --- a/fs/ocfs2/aops.c +++ b/fs/ocfs2/aops.c @@ -37,6 +37,8 @@ #include "namei.h" #include "sysfile.h" +#define OCFS2_DIO_MARK_EXTENT_BATCH 200 + static int ocfs2_symlink_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create) { @@ -2277,7 +2279,7 @@ static int ocfs2_dio_end_io_write(struct inode *inode, struct ocfs2_alloc_context *meta_ac = NULL; handle_t *handle = NULL; loff_t end = offset + bytes; - int ret = 0, credits = 0; + int ret = 0, credits = 0, batch = 0; ocfs2_init_dealloc_ctxt(&dealloc); @@ -2295,19 +2297,6 @@ static int ocfs2_dio_end_io_write(struct inode *inode, } down_write(&oi->ip_alloc_sem); - - /* Delete orphan before acquire i_rwsem. */ - if (dwc->dw_orphaned) { - BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); - - end = end > i_size_read(inode) ? end : 0; - - ret = ocfs2_del_inode_from_orphan(osb, inode, di_bh, - !!end, end); - if (ret < 0) - mlog_errno(ret); - } - di = (struct ocfs2_dinode *)di_bh->b_data; ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); @@ -2327,24 +2316,25 @@ static int ocfs2_dio_end_io_write(struct inode *inode, credits = ocfs2_calc_extend_credits(inode->i_sb, &di->id2.i_list); - handle = ocfs2_start_trans(osb, credits); - if (IS_ERR(handle)) { - ret = PTR_ERR(handle); - mlog_errno(ret); - goto unlock; - } - ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, - OCFS2_JOURNAL_ACCESS_WRITE); - if (ret) { - mlog_errno(ret); - goto commit; - } - list_for_each_entry(ue, &dwc->dw_zero_list, ue_node) { + if (!handle) { + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto unlock; + } + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto commit; + } + } ret = ocfs2_assure_trans_credits(handle, credits); if (ret < 0) { mlog_errno(ret); - break; + goto commit; } ret = ocfs2_mark_extent_written(inode, &et, handle, ue->ue_cpos, 1, @@ -2352,19 +2342,44 @@ static int ocfs2_dio_end_io_write(struct inode *inode, meta_ac, &dealloc); if (ret < 0) { mlog_errno(ret); - break; + goto commit; + } + + if (++batch == OCFS2_DIO_MARK_EXTENT_BATCH) { + ocfs2_commit_trans(osb, handle); + handle = NULL; + batch = 0; } } if (end > i_size_read(inode)) { + if (!handle) { + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto unlock; + } + } ret = ocfs2_set_inode_size(handle, inode, di_bh, end); if (ret < 0) mlog_errno(ret); } + commit: - ocfs2_commit_trans(osb, handle); + if (handle) + ocfs2_commit_trans(osb, handle); unlock: up_write(&oi->ip_alloc_sem); + + /* everything looks good, let's start the cleanup */ + if (!ret && dwc->dw_orphaned) { + BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); + + ret = ocfs2_del_inode_from_orphan(osb, inode, di_bh, 0, 0); + if (ret < 0) + mlog_errno(ret); + } ocfs2_inode_unlock(inode, 1); brelse(di_bh); out: diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c index 70ca79e4bdc3f9..dc9da9133c8ee6 100644 --- a/fs/ocfs2/dlm/dlmdomain.c +++ b/fs/ocfs2/dlm/dlmdomain.c @@ -980,6 +980,14 @@ static int dlm_match_regions(struct dlm_ctxt *dlm, goto bail; } + if (qr->qr_numregions > O2NM_MAX_REGIONS) { + mlog(ML_ERROR, "Domain %s: Joining node %d has invalid " + "number of heartbeat regions %u\n", + qr->qr_domain, qr->qr_node, qr->qr_numregions); + status = -EINVAL; + goto bail; + } + r = remote; for (i = 0; i < qr->qr_numregions; ++i) { mlog(0, "Region %.*s\n", O2HB_MAX_REGION_NAME_LEN, r); @@ -994,7 +1002,7 @@ static int dlm_match_regions(struct dlm_ctxt *dlm, for (i = 0; i < localnr; ++i) { foundit = 0; r = remote; - for (j = 0; j <= qr->qr_numregions; ++j) { + for (j = 0; j < qr->qr_numregions; ++j) { if (!memcmp(l, r, O2HB_MAX_REGION_NAME_LEN)) { foundit = 1; break; diff --git a/fs/ocfs2/ioctl.c b/fs/ocfs2/ioctl.c index bfed0fb35f9bd8..cbe59d23166636 100644 --- a/fs/ocfs2/ioctl.c +++ b/fs/ocfs2/ioctl.c @@ -441,13 +441,16 @@ static int ocfs2_info_freefrag_scan_chain(struct ocfs2_super *osb, struct buffer_head *bh = NULL; struct ocfs2_group_desc *bg = NULL; - unsigned int max_bits, num_clusters; + unsigned int max_bits, max_bitmap_bits, num_clusters; unsigned int offset = 0, cluster, chunk; unsigned int chunk_free, last_chunksize = 0; if (!le32_to_cpu(rec->c_free)) goto bail; + max_bitmap_bits = 8 * ocfs2_group_bitmap_size(osb->sb, 0, + osb->s_feature_incompat); + do { if (!bg) blkno = le64_to_cpu(rec->c_blkno); @@ -479,6 +482,19 @@ static int ocfs2_info_freefrag_scan_chain(struct ocfs2_super *osb, continue; max_bits = le16_to_cpu(bg->bg_bits); + + /* + * Non-coherent scans read raw blocks and do not get the + * bg_bits validation from + * ocfs2_read_group_descriptor(). + */ + if (max_bits > max_bitmap_bits) { + mlog(ML_ERROR, + "Group desc #%llu has %u bits, max bitmap bits %u\n", + (unsigned long long)blkno, max_bits, max_bitmap_bits); + max_bits = max_bitmap_bits; + } + offset = 0; for (chunk = 0; chunk < chunks_in_group; chunk++) { diff --git a/fs/ocfs2/mmap.c b/fs/ocfs2/mmap.c index 50e2faf64c1976..6c570157caf16e 100644 --- a/fs/ocfs2/mmap.c +++ b/fs/ocfs2/mmap.c @@ -30,7 +30,8 @@ static vm_fault_t ocfs2_fault(struct vm_fault *vmf) { - struct vm_area_struct *vma = vmf->vma; + unsigned long long ip_blkno = + OCFS2_I(file_inode(vmf->vma->vm_file))->ip_blkno; sigset_t oldset; vm_fault_t ret; @@ -38,11 +39,9 @@ static vm_fault_t ocfs2_fault(struct vm_fault *vmf) ret = filemap_fault(vmf); ocfs2_unblock_signals(&oldset); - trace_ocfs2_fault(OCFS2_I(vma->vm_file->f_mapping->host)->ip_blkno, - vma, vmf->page, vmf->pgoff); + trace_ocfs2_fault(ip_blkno, vmf->page, vmf->pgoff); return ret; } - static vm_fault_t __ocfs2_page_mkwrite(struct file *file, struct buffer_head *di_bh, struct folio *folio) { diff --git a/fs/ocfs2/ocfs2_trace.h b/fs/ocfs2/ocfs2_trace.h index 4b32fb5658ad7a..6c2c97a9804fb0 100644 --- a/fs/ocfs2/ocfs2_trace.h +++ b/fs/ocfs2/ocfs2_trace.h @@ -1246,22 +1246,20 @@ TRACE_EVENT(ocfs2_write_end_inline, TRACE_EVENT(ocfs2_fault, TP_PROTO(unsigned long long ino, - void *area, void *page, unsigned long pgoff), - TP_ARGS(ino, area, page, pgoff), + void *page, unsigned long pgoff), + TP_ARGS(ino, page, pgoff), TP_STRUCT__entry( __field(unsigned long long, ino) - __field(void *, area) __field(void *, page) __field(unsigned long, pgoff) ), TP_fast_assign( __entry->ino = ino; - __entry->area = area; __entry->page = page; __entry->pgoff = pgoff; ), - TP_printk("%llu %p %p %lu", - __entry->ino, __entry->area, __entry->page, __entry->pgoff) + TP_printk("%llu %p %lu", + __entry->ino, __entry->page, __entry->pgoff) ); /* End of trace events for fs/ocfs2/mmap.c. */ diff --git a/fs/ocfs2/resize.c b/fs/ocfs2/resize.c index ac3ec2c2111963..6375d503597251 100644 --- a/fs/ocfs2/resize.c +++ b/fs/ocfs2/resize.c @@ -303,9 +303,13 @@ int ocfs2_group_extend(struct inode * inode, int new_clusters) fe = (struct ocfs2_dinode *)main_bm_bh->b_data; - /* main_bm_bh is validated by inode read inside ocfs2_inode_lock(), - * so any corruption is a code bug. */ - BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + /* JBD-managed buffers can bypass validation, so treat this as corruption. */ + if (!OCFS2_IS_VALID_DINODE(fe)) { + ret = ocfs2_error(main_bm_inode->i_sb, + "Invalid dinode #%llu\n", + (unsigned long long)OCFS2_I(main_bm_inode)->ip_blkno); + goto out_unlock; + } if (le16_to_cpu(fe->id2.i_chain.cl_cpg) != ocfs2_group_bitmap_size(osb->sb, 0, @@ -504,14 +508,14 @@ int ocfs2_group_add(struct inode *inode, struct ocfs2_new_group_input *input) goto out_unlock; } - ocfs2_set_new_buffer_uptodate(INODE_CACHE(inode), group_bh); - ret = ocfs2_verify_group_and_input(main_bm_inode, fe, input, group_bh); if (ret) { mlog_errno(ret); goto out_free_group_bh; } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(main_bm_inode), group_bh); + trace_ocfs2_group_add((unsigned long long)input->group, input->chain, input->clusters, input->frees); @@ -519,7 +523,7 @@ int ocfs2_group_add(struct inode *inode, struct ocfs2_new_group_input *input) if (IS_ERR(handle)) { mlog_errno(PTR_ERR(handle)); ret = -EINVAL; - goto out_free_group_bh; + goto out_remove_cache; } cl_bpc = le16_to_cpu(fe->id2.i_chain.cl_bpc); @@ -573,9 +577,11 @@ int ocfs2_group_add(struct inode *inode, struct ocfs2_new_group_input *input) out_commit: ocfs2_commit_trans(osb, handle); -out_free_group_bh: +out_remove_cache: if (ret < 0) - ocfs2_remove_from_cache(INODE_CACHE(inode), group_bh); + ocfs2_remove_from_cache(INODE_CACHE(main_bm_inode), group_bh); + +out_free_group_bh: brelse(group_bh); out_unlock: diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 42ee5db362d3eb..b9a6bdbf596c2b 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -911,8 +911,8 @@ static int ocfs2_xattr_list_entry(struct super_block *sb, total_len = prefix_len + name_len + 1; *result += total_len; - /* we are just looking for how big our buffer needs to be */ - if (!size) + /* No buffer means we are only looking for the required size. */ + if (!buffer) return 0; if (*result > size) diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c index 90ae07c69349e5..834cae1e622331 100644 --- a/fs/omfs/inode.c +++ b/fs/omfs/inode.c @@ -513,6 +513,12 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_brelse_bh; } + if (sbi->s_sys_blocksize < OMFS_DIR_START) { + printk(KERN_ERR "omfs: sysblock size (%d) is too small\n", + sbi->s_sys_blocksize); + goto out_brelse_bh; + } + if (sbi->s_blocksize < sbi->s_sys_blocksize || sbi->s_blocksize > OMFS_MAX_BLOCK_SIZE) { printk(KERN_ERR "omfs: block size (%d) is out of range\n", diff --git a/fs/open.c b/fs/open.c index 91f1139591abed..412d0d6fbaa757 100644 --- a/fs/open.c +++ b/fs/open.c @@ -197,7 +197,7 @@ int do_ftruncate(struct file *file, loff_t length, int small) ATTR_MTIME | ATTR_CTIME, file); } -int do_sys_ftruncate(unsigned int fd, loff_t length, int small) +int ksys_ftruncate(unsigned int fd, loff_t length, unsigned int flags) { if (length < 0) return -EINVAL; @@ -205,18 +205,18 @@ int do_sys_ftruncate(unsigned int fd, loff_t length, int small) if (fd_empty(f)) return -EBADF; - return do_ftruncate(fd_file(f), length, small); + return do_ftruncate(fd_file(f), length, !(flags & FTRUNCATE_LFS)); } SYSCALL_DEFINE2(ftruncate, unsigned int, fd, off_t, length) { - return do_sys_ftruncate(fd, length, 1); + return ksys_ftruncate(fd, length, 0); } #ifdef CONFIG_COMPAT COMPAT_SYSCALL_DEFINE2(ftruncate, unsigned int, fd, compat_off_t, length) { - return do_sys_ftruncate(fd, length, 1); + return ksys_ftruncate(fd, length, 0); } #endif @@ -229,7 +229,7 @@ SYSCALL_DEFINE2(truncate64, const char __user *, path, loff_t, length) SYSCALL_DEFINE2(ftruncate64, unsigned int, fd, loff_t, length) { - return do_sys_ftruncate(fd, length, 0); + return ksys_ftruncate(fd, length, FTRUNCATE_LFS); } #endif /* BITS_PER_LONG == 32 */ @@ -245,7 +245,7 @@ COMPAT_SYSCALL_DEFINE3(truncate64, const char __user *, pathname, COMPAT_SYSCALL_DEFINE3(ftruncate64, unsigned int, fd, compat_arg_u64_dual(length)) { - return ksys_ftruncate(fd, compat_arg_u64_glue(length)); + return ksys_ftruncate(fd, compat_arg_u64_glue(length), FTRUNCATE_LFS); } #endif diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index ff3dbd1ca61f28..f2f20a611af3ad 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -1374,7 +1374,7 @@ static int ovl_create_tmpfile(struct file *file, struct dentry *dentry, return PTR_ERR(cred); ovl_path_upper(dentry->d_parent, &realparentpath); - realfile = backing_tmpfile_open(&file->f_path, flags, &realparentpath, + realfile = backing_tmpfile_open(file, flags, &realparentpath, mode, current_cred()); err = PTR_ERR_OR_ZERO(realfile); pr_debug("tmpfile/open(%pd2, 0%o) = %i\n", realparentpath.dentry, mode, err); diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index 97bed2286030da..27cc07738f33bf 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -48,7 +48,7 @@ static struct file *ovl_open_realfile(const struct file *file, if (!inode_owner_or_capable(real_idmap, realinode)) flags &= ~O_NOATIME; - realfile = backing_file_open(file_user_path(file), + realfile = backing_file_open(file, flags, realpath, current_cred()); } } diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c index 2ea769f311c340..339cb0a3942bfa 100644 --- a/fs/overlayfs/util.c +++ b/fs/overlayfs/util.c @@ -1379,7 +1379,7 @@ int ovl_ensure_verity_loaded(const struct path *datapath) struct inode *inode = d_inode(datapath->dentry); struct file *filp; - if (!fsverity_active(inode) && IS_VERITY(inode)) { + if (IS_VERITY(inode) && fsverity_get_info(inode) == NULL) { /* * If this inode was not yet opened, the verity info hasn't been * loaded yet, so we need to do that here to force it into memory. diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c index ed97494abf60fa..0713ef986c204c 100644 --- a/fs/pstore/ram_core.c +++ b/fs/pstore/ram_core.c @@ -488,6 +488,10 @@ static void *persistent_ram_iomap(phys_addr_t start, size_t size, else va = ioremap_wc(start, size); + /* We must release the mem region if ioremap fails. */ + if (!va) + release_mem_region(start, size); + /* * Since request_mem_region() and ioremap() are byte-granularity * there is no need handle anything special like we do when the diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c index 376739f6420ed5..64cf4272149654 100644 --- a/fs/quota/dquot.c +++ b/fs/quota/dquot.c @@ -363,6 +363,31 @@ static inline int dquot_active(struct dquot *dquot) return test_bit(DQ_ACTIVE_B, &dquot->dq_flags); } +static struct dquot *__dqgrab(struct dquot *dquot) +{ + lockdep_assert_held(&dq_list_lock); + if (!atomic_read(&dquot->dq_count)) + remove_free_dquot(dquot); + atomic_inc(&dquot->dq_count); + return dquot; +} + +/* + * Get reference to dquot when we got pointer to it by some other means. The + * dquot has to be active and the caller has to make sure it cannot get + * deactivated under our hands. + */ +struct dquot *dqgrab(struct dquot *dquot) +{ + spin_lock(&dq_list_lock); + WARN_ON_ONCE(!dquot_active(dquot)); + dquot = __dqgrab(dquot); + spin_unlock(&dq_list_lock); + + return dquot; +} +EXPORT_SYMBOL_GPL(dqgrab); + static inline int dquot_dirty(struct dquot *dquot) { return test_bit(DQ_MOD_B, &dquot->dq_flags); @@ -641,15 +666,14 @@ int dquot_scan_active(struct super_block *sb, continue; if (dquot->dq_sb != sb) continue; - /* Now we have active dquot so we can just increase use count */ - atomic_inc(&dquot->dq_count); + __dqgrab(dquot); spin_unlock(&dq_list_lock); dqput(old_dquot); old_dquot = dquot; /* * ->release_dquot() can be racing with us. Our reference - * protects us from new calls to it so just wait for any - * outstanding call and recheck the DQ_ACTIVE_B after that. + * protects us from dquot_release() proceeding so just wait for + * any outstanding call and recheck the DQ_ACTIVE_B after that. */ wait_on_dquot(dquot); if (dquot_active(dquot)) { @@ -717,7 +741,7 @@ int dquot_writeback_dquots(struct super_block *sb, int type) /* Now we have active dquot from which someone is * holding reference so we can safely just increase * use count */ - dqgrab(dquot); + __dqgrab(dquot); spin_unlock(&dq_list_lock); err = dquot_write_dquot(dquot); if (err && !ret) @@ -963,9 +987,7 @@ struct dquot *dqget(struct super_block *sb, struct kqid qid) spin_unlock(&dq_list_lock); dqstats_inc(DQST_LOOKUPS); } else { - if (!atomic_read(&dquot->dq_count)) - remove_free_dquot(dquot); - atomic_inc(&dquot->dq_count); + __dqgrab(dquot); spin_unlock(&dq_list_lock); dqstats_inc(DQST_CACHE_HITS); dqstats_inc(DQST_LOOKUPS); diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index cc4237c57cbe48..2ef53161ce119e 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -992,6 +992,7 @@ static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, } } + rdt_last_cmd_printf("Invalid domain %lu\n", dom_id); return -EINVAL; } diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index 04bb95091f498a..64e22c064fa0a3 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -286,6 +286,14 @@ int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, &rqst[0], &oplock, &oparms, utf16_path); if (rc) goto oshr_free; + + if (oplock != SMB2_OPLOCK_LEVEL_II) { + rc = -EINVAL; + cifs_dbg(FYI, "%s: Oplock level %d not suitable for cached directory\n", + __func__, oplock); + goto oshr_free; + } + smb2_set_next_command(tcon, &rqst[0]); memset(&qi_iov, 0, sizeof(qi_iov)); diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index c920039d733c3b..e5d0d581c130bf 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -758,6 +758,77 @@ static void dump_ace(struct smb_ace *pace, char *end_of_acl) } #endif +static int validate_dacl(struct smb_acl *pdacl, char *end_of_acl) +{ + int i, ace_hdr_size, ace_size, min_ace_size; + u16 dacl_size, num_aces; + char *acl_base, *end_of_dacl; + struct smb_ace *pace; + + if (!pdacl) + return 0; + + if (end_of_acl < (char *)pdacl + sizeof(struct smb_acl)) { + cifs_dbg(VFS, "ACL too small to parse DACL\n"); + return -EINVAL; + } + + dacl_size = le16_to_cpu(pdacl->size); + if (dacl_size < sizeof(struct smb_acl) || + end_of_acl < (char *)pdacl + dacl_size) { + cifs_dbg(VFS, "ACL too small to parse DACL\n"); + return -EINVAL; + } + + num_aces = le16_to_cpu(pdacl->num_aces); + if (!num_aces) + return 0; + + ace_hdr_size = offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth); + min_ace_size = ace_hdr_size + sizeof(__le32); + if (num_aces > (dacl_size - sizeof(struct smb_acl)) / min_ace_size) { + cifs_dbg(VFS, "ACL too small to parse DACL\n"); + return -EINVAL; + } + + end_of_dacl = (char *)pdacl + dacl_size; + acl_base = (char *)pdacl; + ace_size = sizeof(struct smb_acl); + + for (i = 0; i < num_aces; ++i) { + if (end_of_dacl - acl_base < ace_size) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + + pace = (struct smb_ace *)(acl_base + ace_size); + acl_base = (char *)pace; + + if (end_of_dacl - acl_base < ace_hdr_size || + pace->sid.num_subauth == 0 || + pace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + + ace_size = ace_hdr_size + sizeof(__le32) * pace->sid.num_subauth; + if (end_of_dacl - acl_base < ace_size || + le16_to_cpu(pace->size) < ace_size) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + + ace_size = le16_to_cpu(pace->size); + if (end_of_dacl - acl_base < ace_size) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + } + + return 0; +} + static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, struct smb_sid *pownersid, struct smb_sid *pgrpsid, struct cifs_fattr *fattr, bool mode_from_special_sid) @@ -765,7 +836,7 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, int i; u16 num_aces = 0; int acl_size; - char *acl_base; + char *acl_base, *end_of_dacl; struct smb_ace **ppace; /* BB need to add parm so we can store the SID BB */ @@ -777,12 +848,8 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, return; } - /* validate that we do not go past end of acl */ - if (end_of_acl < (char *)pdacl + sizeof(struct smb_acl) || - end_of_acl < (char *)pdacl + le16_to_cpu(pdacl->size)) { - cifs_dbg(VFS, "ACL too small to parse DACL\n"); + if (validate_dacl(pdacl, end_of_acl)) return; - } cifs_dbg(NOISY, "DACL revision %d size %d num aces %d\n", le16_to_cpu(pdacl->revision), le16_to_cpu(pdacl->size), @@ -793,6 +860,7 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, user/group/other have no permissions */ fattr->cf_mode &= ~(0777); + end_of_dacl = (char *)pdacl + le16_to_cpu(pdacl->size); acl_base = (char *)pdacl; acl_size = sizeof(struct smb_acl); @@ -800,37 +868,18 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, if (num_aces > 0) { umode_t denied_mode = 0; - if (num_aces > (le16_to_cpu(pdacl->size) - sizeof(struct smb_acl)) / - (offsetof(struct smb_ace, sid) + - offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) - return; - ppace = kmalloc_objs(struct smb_ace *, num_aces); if (!ppace) return; for (i = 0; i < num_aces; ++i) { - if (end_of_acl - acl_base < acl_size) - break; - ppace[i] = (struct smb_ace *) (acl_base + acl_size); - acl_base = (char *)ppace[i]; - acl_size = offsetof(struct smb_ace, sid) + - offsetof(struct smb_sid, sub_auth); - - if (end_of_acl - acl_base < acl_size || - ppace[i]->sid.num_subauth == 0 || - ppace[i]->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || - (end_of_acl - acl_base < - acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth) || - (le16_to_cpu(ppace[i]->size) < - acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) - break; #ifdef CONFIG_CIFS_DEBUG2 - dump_ace(ppace[i], end_of_acl); + dump_ace(ppace[i], end_of_dacl); #endif if (mode_from_special_sid && + ppace[i]->sid.num_subauth >= 3 && (compare_sids(&(ppace[i]->sid), &sid_unix_NFS_mode) == 0)) { /* @@ -870,6 +919,7 @@ static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, (void *)ppace[i], sizeof(struct smb_ace)); */ + acl_base = (char *)ppace[i]; acl_size = le16_to_cpu(ppace[i]->size); } @@ -1214,6 +1264,17 @@ static int parse_sid(struct smb_sid *psid, char *end_of_acl) return 0; } +static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset) +{ + if (acl_len < sizeof(struct smb_acl)) + return false; + + if (dacloffset < sizeof(struct smb_ntsd)) + return false; + + return dacloffset <= acl_len - sizeof(struct smb_acl); +} + /* Convert CIFS ACL to POSIX form */ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, @@ -1234,7 +1295,6 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, group_sid_ptr = (struct smb_sid *)((char *)pntsd + le32_to_cpu(pntsd->gsidoffset)); dacloffset = le32_to_cpu(pntsd->dacloffset); - dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset), le32_to_cpu(pntsd->gsidoffset), @@ -1265,11 +1325,18 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb, return rc; } - if (dacloffset) + if (dacloffset) { + if (!dacl_offset_valid(acl_len, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); parse_dacl(dacl_ptr, end_of_acl, owner_sid_ptr, group_sid_ptr, fattr, get_mode_from_special_sid); - else + } else { cifs_dbg(FYI, "no ACL\n"); /* BB grant all or default perms? */ + } return rc; } @@ -1292,11 +1359,15 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { - dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); - if (end_of_acl < (char *)dacl_ptr + le16_to_cpu(dacl_ptr->size)) { - cifs_dbg(VFS, "Server returned illegal ACL size\n"); + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); return -EINVAL; } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); + rc = validate_dacl(dacl_ptr, end_of_acl); + if (rc) + return rc; } owner_sid_ptr = (struct smb_sid *)((char *)pntsd + @@ -1661,7 +1732,19 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, nsecdesclen = sizeof(struct smb_ntsd) + (sizeof(struct smb_sid) * 2); dacloffset = le32_to_cpu(pntsd->dacloffset); if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + rc = -EINVAL; + goto id_mode_to_cifs_acl_exit; + } + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); + rc = validate_dacl(dacl_ptr, (char *)pntsd + secdesclen); + if (rc) { + kfree(pntsd); + cifs_put_tlink(tlink); + return rc; + } if (mode_from_sid) nsecdesclen += le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct smb_ace); @@ -1677,7 +1760,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, * descriptor parameters, and security descriptor itself */ nsecdesclen = max_t(u32, nsecdesclen, DEFAULT_SEC_DESC_LEN); - pnntsd = kmalloc(nsecdesclen, GFP_KERNEL); + pnntsd = kzalloc(nsecdesclen, GFP_KERNEL); if (!pnntsd) { kfree(pntsd); cifs_put_tlink(tlink); @@ -1697,6 +1780,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); } +id_mode_to_cifs_acl_exit: cifs_put_tlink(tlink); kfree(pnntsd); diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index 69b38f0ccf2b26..e9eeb9f8a56192 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -3610,7 +3610,6 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) server = mnt_ctx->server; ctx = mnt_ctx->fs_ctx; cifs_sb = mnt_ctx->cifs_sb; - sbflags = cifs_sb_flags(cifs_sb); /* search for existing tcon to this server share */ tcon = cifs_get_tcon(mnt_ctx->ses, ctx); @@ -3625,9 +3624,10 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) * path (i.e., do not remap / and \ and do not map any special characters) */ if (tcon->posix_extensions) { - sbflags |= CIFS_MOUNT_POSIX_PATHS; - sbflags &= ~(CIFS_MOUNT_MAP_SFM_CHR | - CIFS_MOUNT_MAP_SPECIAL_CHR); + atomic_or(CIFS_MOUNT_POSIX_PATHS, &cifs_sb->mnt_cifs_flags); + atomic_andnot(CIFS_MOUNT_MAP_SFM_CHR | + CIFS_MOUNT_MAP_SPECIAL_CHR, + &cifs_sb->mnt_cifs_flags); } #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY @@ -3651,6 +3651,7 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ tcon->unix_ext = 0; /* server does not support them */ + sbflags = cifs_sb_flags(cifs_sb); /* do not care if a following call succeed - informational */ if (!tcon->pipe && server->ops->qfs_tcon) { server->ops->qfs_tcon(mnt_ctx->xid, tcon, cifs_sb); @@ -3675,7 +3676,6 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) out: mnt_ctx->tcon = tcon; - atomic_set(&cifs_sb->mnt_cifs_flags, sbflags); return rc; } diff --git a/fs/smb/client/ioctl.c b/fs/smb/client/ioctl.c index 9afab3237e54c3..17408bb8ab65bf 100644 --- a/fs/smb/client/ioctl.c +++ b/fs/smb/client/ioctl.c @@ -296,7 +296,7 @@ static int cifs_dump_full_key(struct cifs_tcon *tcon, struct smb3_full_key_debug break; case SMB2_ENCRYPTION_AES256_CCM: case SMB2_ENCRYPTION_AES256_GCM: - out.session_key_length = CIFS_SESS_KEY_SIZE; + out.session_key_length = ses->auth_key.len; out.server_in_key_length = out.server_out_key_length = SMB3_GCM256_CRYPTKEY_SIZE; break; default: diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c index 9694117050a6c7..e198e3dda91772 100644 --- a/fs/smb/client/smb1ops.c +++ b/fs/smb/client/smb1ops.c @@ -49,7 +49,6 @@ void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon, if (!CIFSSMBQFSUnixInfo(xid, tcon)) { __u64 cap = le64_to_cpu(tcon->fsUnixInfo.Capability); - unsigned int sbflags; cifs_dbg(FYI, "unix caps which server supports %lld\n", cap); /* @@ -76,29 +75,27 @@ void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon, if (cap & CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP) cifs_dbg(VFS, "per-share encryption not supported yet\n"); - if (cifs_sb) - sbflags = cifs_sb_flags(cifs_sb); - cap &= CIFS_UNIX_CAP_MASK; if (ctx && ctx->no_psx_acl) cap &= ~CIFS_UNIX_POSIX_ACL_CAP; else if (CIFS_UNIX_POSIX_ACL_CAP & cap) { cifs_dbg(FYI, "negotiated posix acl support\n"); - if (cifs_sb) - sbflags |= CIFS_MOUNT_POSIXACL; + if (cifs_sb) { + atomic_or(CIFS_MOUNT_POSIXACL, + &cifs_sb->mnt_cifs_flags); + } } if (ctx && ctx->posix_paths == 0) cap &= ~CIFS_UNIX_POSIX_PATHNAMES_CAP; else if (cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) { cifs_dbg(FYI, "negotiate posix pathnames\n"); - if (cifs_sb) - sbflags |= CIFS_MOUNT_POSIX_PATHS; + if (cifs_sb) { + atomic_or(CIFS_MOUNT_POSIX_PATHS, + &cifs_sb->mnt_cifs_flags); + } } - if (cifs_sb) - atomic_set(&cifs_sb->mnt_cifs_flags, sbflags); - cifs_dbg(FYI, "Negotiate caps 0x%x\n", (int)cap); #ifdef CONFIG_CIFS_DEBUG2 if (cap & CIFS_UNIX_FCNTL_CAP) diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index ed651c946251d0..6860eff3169329 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -27,10 +27,11 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) { struct smb2_err_rsp *err = iov->iov_base; struct smb2_symlink_err_rsp *sym = ERR_PTR(-EINVAL); + u8 *end = (u8 *)err + iov->iov_len; u32 len; if (err->ErrorContextCount) { - struct smb2_error_context_rsp *p, *end; + struct smb2_error_context_rsp *p; len = (u32)err->ErrorContextCount * (offsetof(struct smb2_error_context_rsp, ErrorContextData) + @@ -39,8 +40,7 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) return ERR_PTR(-EINVAL); p = (struct smb2_error_context_rsp *)err->ErrorData; - end = (struct smb2_error_context_rsp *)((u8 *)err + iov->iov_len); - do { + while ((u8 *)p + sizeof(*p) <= end) { if (le32_to_cpu(p->ErrorId) == SMB2_ERROR_ID_DEFAULT) { sym = (struct smb2_symlink_err_rsp *)p->ErrorContextData; break; @@ -49,15 +49,20 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) __func__, le32_to_cpu(p->ErrorId)); len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); + if (len > end - ((u8 *)p + sizeof(*p))) + return ERR_PTR(-EINVAL); + p = (struct smb2_error_context_rsp *)(p->ErrorContextData + len); - } while (p < end); + } } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) && iov->iov_len >= SMB2_SYMLINK_STRUCT_SIZE) { sym = (struct smb2_symlink_err_rsp *)err->ErrorData; } - if (!IS_ERR(sym) && (le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || - le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) + if (!IS_ERR(sym) && + ((u8 *)sym + sizeof(*sym) > end || + le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || + le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) sym = ERR_PTR(-EINVAL); return sym; @@ -128,8 +133,10 @@ int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec print_len = le16_to_cpu(sym->PrintNameLength); print_offs = le16_to_cpu(sym->PrintNameOffset); - if (iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + sub_offs + sub_len || - iov->iov_len < SMB2_SYMLINK_STRUCT_SIZE + print_offs + print_len) + if ((char *)sym->PathBuffer + sub_offs + sub_len > + (char *)iov->iov_base + iov->iov_len || + (char *)sym->PathBuffer + print_offs + print_len > + (char *)iov->iov_base + iov->iov_len) return -EINVAL; return smb2_parse_native_symlink(path, diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 364bdcff9c9d5b..3b09cf8ab0f276 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -111,7 +111,7 @@ static int check_wsl_eas(struct kvec *rsp_iov) u32 outlen, next; u16 vlen; u8 nlen; - u8 *end; + u8 *ea_end, *iov_end; outlen = le32_to_cpu(rsp->OutputBufferLength); if (outlen < SMB2_WSL_MIN_QUERY_EA_RESP_SIZE || @@ -120,15 +120,19 @@ static int check_wsl_eas(struct kvec *rsp_iov) ea = (void *)((u8 *)rsp_iov->iov_base + le16_to_cpu(rsp->OutputBufferOffset)); - end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + ea_end = (u8 *)ea + outlen; + iov_end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + if (ea_end > iov_end) + return -EINVAL; + for (;;) { - if ((u8 *)ea > end - sizeof(*ea)) + if ((u8 *)ea > ea_end - sizeof(*ea)) return -EINVAL; nlen = ea->ea_name_length; vlen = le16_to_cpu(ea->ea_value_length); if (nlen != SMB2_WSL_XATTR_NAME_LEN || - (u8 *)ea + nlen + 1 + vlen > end) + (u8 *)ea->ea_data + nlen + 1 + vlen > ea_end) return -EINVAL; switch (vlen) { diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index 973fce3c959c4b..2a7355ce1a0783 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -241,7 +241,8 @@ smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, if (len != calc_len) { /* create failed on symlink */ if (command == SMB2_CREATE_HE && - shdr->Status == STATUS_STOPPED_ON_SYMLINK) + shdr->Status == STATUS_STOPPED_ON_SYMLINK && + len > calc_len) return 0; /* Windows 7 server returns 24 bytes more */ if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE) diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 509fcea28a429d..ccc06c83956b5a 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -111,10 +111,21 @@ smb2_add_credits(struct TCP_Server_Info *server, cifs_trace_rw_credits_zero_in_flight); } server->in_flight--; + + /* + * Rebalance credits when an op drains in_flight. For session setup, + * do this only when the total accumulated credits are high enough (>2) + * so that a newly established secondary channel can reserve credits for + * echoes and oplocks. We expect this to happen at the end of the final + * session setup response. + */ if (server->in_flight == 0 && ((optype & CIFS_OP_MASK) != CIFS_NEG_OP) && ((optype & CIFS_OP_MASK) != CIFS_SESS_OP)) rc = change_conf(server); + else if (server->in_flight == 0 && + ((optype & CIFS_OP_MASK) == CIFS_SESS_OP) && *val > 2) + rc = change_conf(server); /* * Sometimes server returns 0 credits on oplock break ack - we need to * rebalance credits in this case. @@ -1783,6 +1794,12 @@ smb2_ioctl_query_info(const unsigned int xid, qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; if (le32_to_cpu(qi_rsp->OutputBufferLength) < qi.input_buffer_length) qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength); + if (qi.input_buffer_length > 0 && + struct_size(qi_rsp, Buffer, qi.input_buffer_length) > + rsp_iov[1].iov_len) { + rc = -EFAULT; + goto out; + } if (copy_to_user(&pqi->input_buffer_length, &qi.input_buffer_length, sizeof(qi.input_buffer_length))) { diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c index 81be2b226e2644..bcd7ec9c952174 100644 --- a/fs/smb/client/smb2transport.c +++ b/fs/smb/client/smb2transport.c @@ -259,7 +259,8 @@ smb2_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, } static int generate_key(struct cifs_ses *ses, struct kvec label, - struct kvec context, __u8 *key, unsigned int key_size) + struct kvec context, __u8 *key, unsigned int key_size, + unsigned int full_key_size) { unsigned char zero = 0x0; __u8 i[4] = {0, 0, 0, 1}; @@ -280,7 +281,7 @@ static int generate_key(struct cifs_ses *ses, struct kvec label, } hmac_sha256_init_usingrawkey(&hmac_ctx, ses->auth_key.response, - SMB2_NTLMV2_SESSKEY_SIZE); + full_key_size); hmac_sha256_update(&hmac_ctx, i, 4); hmac_sha256_update(&hmac_ctx, label.iov_base, label.iov_len); hmac_sha256_update(&hmac_ctx, &zero, 1); @@ -314,6 +315,7 @@ generate_smb3signingkey(struct cifs_ses *ses, struct TCP_Server_Info *server, const struct derivation_triplet *ptriplet) { + unsigned int full_key_size = SMB2_NTLMV2_SESSKEY_SIZE; int rc; bool is_binding = false; int chan_index = 0; @@ -348,17 +350,31 @@ generate_smb3signingkey(struct cifs_ses *ses, rc = generate_key(ses, ptriplet->signing.label, ptriplet->signing.context, ses->chans[chan_index].signkey, - SMB3_SIGN_KEY_SIZE); + SMB3_SIGN_KEY_SIZE, + SMB2_NTLMV2_SESSKEY_SIZE); if (rc) return rc; } else { rc = generate_key(ses, ptriplet->signing.label, ptriplet->signing.context, ses->smb3signingkey, - SMB3_SIGN_KEY_SIZE); + SMB3_SIGN_KEY_SIZE, + SMB2_NTLMV2_SESSKEY_SIZE); if (rc) return rc; + /* + * Per MS-SMB2 3.2.5.3.1, signing key always uses Session.SessionKey + * (first 16 bytes). Encryption/decryption keys use + * Session.FullSessionKey when dialect is 3.1.1 and cipher is + * AES-256-CCM or AES-256-GCM, otherwise Session.SessionKey. + */ + + if (server->dialect == SMB311_PROT_ID && + (server->cipher_type == SMB2_ENCRYPTION_AES256_CCM || + server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + full_key_size = ses->auth_key.len; + /* safe to access primary channel, since it will never go away */ spin_lock(&ses->chan_lock); memcpy(ses->chans[chan_index].signkey, ses->smb3signingkey, @@ -368,13 +384,15 @@ generate_smb3signingkey(struct cifs_ses *ses, rc = generate_key(ses, ptriplet->encryption.label, ptriplet->encryption.context, ses->smb3encryptionkey, - SMB3_ENC_DEC_KEY_SIZE); + SMB3_ENC_DEC_KEY_SIZE, + full_key_size); if (rc) return rc; rc = generate_key(ses, ptriplet->decryption.label, ptriplet->decryption.context, ses->smb3decryptionkey, - SMB3_ENC_DEC_KEY_SIZE); + SMB3_ENC_DEC_KEY_SIZE, + full_key_size); if (rc) return rc; } @@ -389,7 +407,7 @@ generate_smb3signingkey(struct cifs_ses *ses, &ses->Suid); cifs_dbg(VFS, "Cipher type %d\n", server->cipher_type); cifs_dbg(VFS, "Session Key %*ph\n", - SMB2_NTLMV2_SESSKEY_SIZE, ses->auth_key.response); + (int)ses->auth_key.len, ses->auth_key.response); cifs_dbg(VFS, "Signing Key %*ph\n", SMB3_SIGN_KEY_SIZE, ses->smb3signingkey); if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index c79304012b08c3..d0fcc777941567 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -1551,17 +1551,25 @@ static int smbd_post_send_iter(struct smbdirect_socket *sc, rc = smbd_post_send(sc, batch, request); if (!rc) { + /* + * From here request is moved to batch + * and we should not free it explicitly. + */ + if (batch != &_batch) return 0; rc = smbd_send_batch_flush(sc, batch, true); if (!rc) return 0; + + goto err_flush; } err_dma: smbd_free_send_io(request); +err_flush: err_alloc: atomic_inc(&sc->send_io.credits.count); wake_up(&sc->send_io.credits.wait_queue); @@ -2912,7 +2920,7 @@ struct smbdirect_mr_io *smbd_register_mr(struct smbd_connection *info, struct smbdirect_socket *sc = &info->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; struct smbdirect_mr_io *mr; - int rc, num_pages; + int rc, num_pages, num_mapped; struct ib_reg_wr *reg_wr; num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1); @@ -2940,18 +2948,21 @@ struct smbdirect_mr_io *smbd_register_mr(struct smbd_connection *info, num_pages, iov_iter_count(iter), sp->max_frmr_depth); smbd_iter_to_mr(iter, &mr->sgt, sp->max_frmr_depth); - rc = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); - if (!rc) { - log_rdma_mr(ERR, "ib_dma_map_sg num_pages=%x dir=%x rc=%x\n", - num_pages, mr->dir, rc); + num_mapped = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + if (!num_mapped) { + log_rdma_mr(ERR, "ib_dma_map_sg num_pages=%x dir=%x num_mapped=%x\n", + num_pages, mr->dir, num_mapped); + rc = -EIO; goto dma_map_error; } - rc = ib_map_mr_sg(mr->mr, mr->sgt.sgl, mr->sgt.nents, NULL, PAGE_SIZE); - if (rc != mr->sgt.nents) { + rc = ib_map_mr_sg(mr->mr, mr->sgt.sgl, num_mapped, NULL, PAGE_SIZE); + if (rc != num_mapped) { log_rdma_mr(ERR, - "ib_map_mr_sg failed rc = %d nents = %x\n", - rc, mr->sgt.nents); + "ib_map_mr_sg failed rc = %d num_mapped = %x\n", + rc, num_mapped); + if (rc >= 0) + rc = -EIO; goto map_mr_error; } diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index af5f4030433171..7d0691f7263fe2 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -827,6 +827,7 @@ int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, struct smb2_transform_hdr *tr_hdr = smb_get_msg(iov[0].iov_base); unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 20; int rc; + DECLARE_CRYPTO_WAIT(wait); struct scatterlist *sg; u8 sign[SMB2_SIGNATURE_SIZE] = {}; u8 key[SMB3_ENC_DEC_KEY_SIZE]; @@ -913,12 +914,12 @@ int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, aead_request_set_crypt(req, sg, sg, crypt_len, iv); aead_request_set_ad(req, assoc_data_len); - aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); - if (enc) - rc = crypto_aead_encrypt(req); - else - rc = crypto_aead_decrypt(req); + rc = crypto_wait_req(enc ? crypto_aead_encrypt(req) : + crypto_aead_decrypt(req), &wait); if (rc) goto free_iv; diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index 1bb2081c492c38..fb9918e5d9871a 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -96,7 +96,17 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) xa_destroy(&conn->sessions); kvfree(conn->request_buf); kfree(conn->preauth_info); + kfree(conn->mechToken); if (atomic_dec_and_test(&conn->refcnt)) { + /* + * async_ida is embedded in struct ksmbd_conn, so pair + * ida_destroy() with the final kfree() rather than with + * the unconditional field teardown above. This keeps + * the IDA valid for the entire lifetime of the struct, + * even while other refcount holders (oplock / vfs + * durable handles) still reference the connection. + */ + ida_destroy(&conn->async_ida); conn->transport->ops->free_transport(conn->transport); kfree(conn); } @@ -236,7 +246,7 @@ int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) { struct ksmbd_conn *conn; int rc, retry_count = 0, max_timeout = 120; - int rcount = 1, bkt; + int rcount, bkt; retry_idle: if (retry_count >= max_timeout) @@ -245,8 +255,7 @@ int ksmbd_conn_wait_idle_sess_id(struct ksmbd_conn *curr_conn, u64 sess_id) down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { if (conn->binding || xa_load(&conn->sessions, sess_id)) { - if (conn == curr_conn) - rcount = 2; + rcount = (conn == curr_conn) ? 2 : 1; if (atomic_read(&conn->req_running) >= rcount) { rc = wait_event_timeout(conn->req_running_q, atomic_read(&conn->req_running) < rcount, @@ -535,24 +544,54 @@ int ksmbd_conn_transport_init(void) static void stop_sessions(void) { - struct ksmbd_conn *conn; + struct ksmbd_conn *conn, *target; struct ksmbd_transport *t; + bool any; int bkt; + /* + * Serialised via init_lock; no concurrent stop_sessions() can + * touch conn->stop_called, so writing it under the read lock is + * safe. + */ again: + target = NULL; + any = false; down_read(&conn_list_lock); hash_for_each(conn_list, bkt, conn, hlist) { - t = conn->transport; - ksmbd_conn_set_exiting(conn); - if (t->ops->shutdown) { - up_read(&conn_list_lock); + any = true; + if (conn->stop_called) + continue; + atomic_inc(&conn->refcnt); + conn->stop_called = true; + /* + * Mark the connection EXITING while still holding the + * read lock so the selection and the status transition + * happen together. Do not regress a connection that has + * already advanced to RELEASING on its own (e.g. the + * handler exited its receive loop for an unrelated + * reason). + */ + if (READ_ONCE(conn->status) != KSMBD_SESS_RELEASING) + ksmbd_conn_set_exiting(conn); + target = conn; + break; + } + up_read(&conn_list_lock); + + if (target) { + t = target->transport; + if (t->ops->shutdown) t->ops->shutdown(t); - down_read(&conn_list_lock); + if (atomic_dec_and_test(&target->refcnt)) { + ida_destroy(&target->async_ida); + t->ops->free_transport(t); + kfree(target); } + goto again; } - up_read(&conn_list_lock); - if (!hash_empty(conn_list)) { + if (any) { msleep(100); goto again; } diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index 1e2587036bca63..4df31e85a77cd9 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -49,6 +49,7 @@ struct ksmbd_conn { struct mutex srv_mutex; int status; unsigned int cli_cap; + bool stop_called; union { __be32 inet_addr; #if IS_ENABLED(CONFIG_IPV6) diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c index a3183fe5c5365f..cf45841d9d1b99 100644 --- a/fs/smb/server/mgmt/user_config.c +++ b/fs/smb/server/mgmt/user_config.c @@ -56,12 +56,6 @@ struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, goto err_free; if (resp_ext) { - if (resp_ext->ngroups > NGROUPS_MAX) { - pr_err("ngroups(%u) from login response exceeds max groups(%d)\n", - resp_ext->ngroups, NGROUPS_MAX); - goto err_free; - } - user->sgid = kmemdup(resp_ext->____payload, resp_ext->ngroups * sizeof(gid_t), KSMBD_DEFAULT_GFP); diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 39be2d2be86c3a..de58aed76cb426 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -382,17 +382,16 @@ void ksmbd_session_destroy(struct ksmbd_session *sess) return; delete_proc_session(sess); - + ksmbd_tree_conn_session_logoff(sess); + ksmbd_destroy_file_table(sess); if (sess->user) ksmbd_free_user(sess->user); - - ksmbd_tree_conn_session_logoff(sess); - ksmbd_destroy_file_table(&sess->file_table); ksmbd_launch_ksmbd_durable_scavenger(); ksmbd_session_rpc_clear_list(sess); free_channel_list(sess); kfree(sess->Preauth_HashValue); ksmbd_release_id(&session_ida, sess->id); + ida_destroy(&sess->tree_conn_ida); kfree(sess); } @@ -549,8 +548,13 @@ struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, struct ksmbd_session *sess; sess = ksmbd_session_lookup(conn, id); - if (!sess && conn->binding) + if (!sess && conn->binding) { sess = ksmbd_session_lookup_slowpath(id); + if (sess && !xa_load(&sess->ksmbd_chann_list, (long)conn)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } if (sess && sess->state != SMB2_SESSION_VALID) { ksmbd_user_session_put(sess); sess = NULL; @@ -618,7 +622,7 @@ void destroy_previous_session(struct ksmbd_conn *conn, goto out; } - ksmbd_destroy_file_table(&prev_sess->file_table); + ksmbd_destroy_file_table(prev_sess); prev_sess->state = SMB2_SESSION_EXPIRED; ksmbd_all_conn_set_status(id, KSMBD_SESS_NEED_SETUP); ksmbd_launch_ksmbd_durable_scavenger(); @@ -667,6 +671,8 @@ static struct ksmbd_session *__session_create(int protocol) if (!sess) return NULL; + ida_init(&sess->tree_conn_ida); + if (ksmbd_init_file_table(&sess->file_table)) goto error; @@ -686,8 +692,6 @@ static struct ksmbd_session *__session_create(int protocol) if (ret) goto error; - ida_init(&sess->tree_conn_ida); - down_write(&sessions_table_lock); hash_add(sessions_table, &sess->hlist, sess->id); up_write(&sessions_table_lock); diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index 9b2bb8764a805b..cd3f28b0e7cb24 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -1841,6 +1841,7 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, struct ksmbd_share_config *share, struct ksmbd_file *fp, struct lease_ctx_info *lctx, + struct ksmbd_user *user, char *name) { struct oplock_info *opinfo = opinfo_get(fp); @@ -1849,6 +1850,12 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, if (!opinfo) return 0; + if (ksmbd_vfs_compare_durable_owner(fp, user) == false) { + ksmbd_debug(SMB, "Durable handle reconnect failed: owner mismatch\n"); + ret = -EBADF; + goto out; + } + if (opinfo->is_lease == false) { if (lctx) { pr_err("create context include lease\n"); diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index 921e3199e4df43..d91a8266e065ef 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -126,5 +126,6 @@ int smb2_check_durable_oplock(struct ksmbd_conn *conn, struct ksmbd_share_config *share, struct ksmbd_file *fp, struct lease_ctx_info *lctx, + struct ksmbd_user *user, char *name); #endif /* __KSMBD_OPLOCK_H */ diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index 8e4cfdc0ba025a..c3c7688f0fa80b 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -1915,7 +1915,7 @@ int smb2_sess_setup(struct ksmbd_work *work) else if (rc) rsp->hdr.Status = STATUS_LOGON_FAILURE; - if (conn->use_spnego && conn->mechToken) { + if (conn->mechToken) { kfree(conn->mechToken); conn->mechToken = NULL; } @@ -2845,6 +2845,8 @@ static int parse_durable_handle_context(struct ksmbd_work *work, dh_info->reconnected = true; goto out; } + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; } if ((lc && (lc->req_state & SMB2_LEASE_HANDLE_CACHING_LE)) || @@ -3013,30 +3015,25 @@ int smb2_open(struct ksmbd_work *work) } if (dh_info.reconnected == true) { - rc = smb2_check_durable_oplock(conn, share, dh_info.fp, lc, name); - if (rc) { - ksmbd_put_durable_fd(dh_info.fp); + rc = smb2_check_durable_oplock(conn, share, dh_info.fp, + lc, sess->user, name); + if (rc) goto err_out2; - } rc = ksmbd_reopen_durable_fd(work, dh_info.fp); - if (rc) { - ksmbd_put_durable_fd(dh_info.fp); + if (rc) goto err_out2; - } fp = dh_info.fp; if (ksmbd_override_fsids(work)) { rc = -ENOMEM; - ksmbd_put_durable_fd(dh_info.fp); goto err_out2; } file_info = FILE_OPENED; rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); - ksmbd_put_durable_fd(fp); if (rc) goto err_out2; @@ -3806,6 +3803,9 @@ int smb2_open(struct ksmbd_work *work) ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status); } + if (dh_info.reconnected) + ksmbd_put_durable_fd(dh_info.fp); + kfree(name); kfree(lc); @@ -4716,6 +4716,11 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, ea_req = (struct smb2_ea_info_req *)((char *)req + le16_to_cpu(req->InputBufferOffset)); + + if (le32_to_cpu(req->InputBufferLength) < + offsetof(struct smb2_ea_info_req, name) + + ea_req->EaNameLength) + return -EINVAL; } else { /* need to send all EAs, if no specific EA is requested*/ if (le32_to_cpu(req->Flags) & SL_RETURN_SINGLE_ENTRY) @@ -4816,6 +4821,8 @@ static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, /* align next xattr entry at 4 byte bundary */ alignment_bytes = ((next_offset + 3) & ~3) - next_offset; if (alignment_bytes) { + if (buf_free_len < alignment_bytes) + break; memset(ptr, '\0', alignment_bytes); ptr += alignment_bytes; next_offset += alignment_bytes; diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index c30d01877c4185..c1d1f34581d69d 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -451,7 +451,8 @@ static void parse_dacl(struct mnt_idmap *idmap, ppace[i]->access_req = smb_map_generic_desired_access(ppace[i]->access_req); - if (!(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { + if (ppace[i]->sid.num_subauth >= 3 && + !(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { fattr->cf_mode = le32_to_cpu(ppace[i]->sid.sub_auth[2]); break; @@ -595,6 +596,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, struct smb_sid *sid; struct smb_ace *ntace; int i, j; + u16 ace_sz; if (!fattr->cf_acls) goto posix_default_acl; @@ -639,8 +641,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, flags = 0x03; ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -649,8 +653,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, if (S_ISDIR(fattr->cf_mode) && (pace->e_tag == ACL_USER || pace->e_tag == ACL_GROUP)) { ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x03, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -690,8 +696,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, } ntace = (struct smb_ace *)((char *)pndace + *size); - *size += fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) + break; (*num_aces)++; if (pace->e_tag == ACL_USER) ntace->access_req |= @@ -727,7 +735,8 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, break; memcpy((char *)pndace + size, ntace, nt_ace_size); - size += nt_ace_size; + if (check_add_overflow(size, nt_ace_size, &size)) + break; aces_size -= nt_ace_size; ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); num_aces++; @@ -1059,7 +1068,26 @@ static void smb_set_ace(struct smb_ace *ace, const struct smb_sid *sid, u8 type, ace->flags = flags; ace->access_req = access_req; smb_copy_sid(&ace->sid, sid); - ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 + (sid->num_subauth * 4)); + ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 + + (ace->sid.num_subauth * 4)); +} + +static int smb_append_inherited_ace(struct smb_ace **ace, int *nt_size, + u16 *ace_cnt, const struct smb_sid *sid, + u8 type, u8 flags, __le32 access_req) +{ + int ace_size; + + smb_set_ace(*ace, sid, type, flags, access_req); + ace_size = le16_to_cpu((*ace)->size); + /* pdacl->size is __le16 and includes struct smb_acl. */ + if (check_add_overflow(*nt_size, ace_size, nt_size) || + *nt_size > U16_MAX - (int)sizeof(struct smb_acl)) + return -EINVAL; + + (*ace_cnt)++; + *ace = (struct smb_ace *)((char *)*ace + ace_size); + return 0; } int smb_inherit_dacl(struct ksmbd_conn *conn, @@ -1105,8 +1133,24 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, goto free_parent_pntsd; } - aces_base = kmalloc(sizeof(struct smb_ace) * num_aces * 2, - KSMBD_DEFAULT_GFP); + aces_size = pdacl_size - sizeof(struct smb_acl); + + /* + * Validate num_aces against the DACL payload before allocating. + * Each ACE must be at least as large as its fixed-size header + * (up to the SID base), so num_aces cannot exceed the payload + * divided by the minimum ACE size. This mirrors the existing + * check in parse_dacl(). + */ + if (num_aces > aces_size / (offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth) + + sizeof(__le16))) { + rc = -EINVAL; + goto free_parent_pntsd; + } + + aces_base = kmalloc_array(num_aces * 2, sizeof(struct smb_ace), + KSMBD_DEFAULT_GFP); if (!aces_base) { rc = -ENOMEM; goto free_parent_pntsd; @@ -1115,7 +1159,6 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, aces = (struct smb_ace *)aces_base; parent_aces = (struct smb_ace *)((char *)parent_pdacl + sizeof(struct smb_acl)); - aces_size = acl_len - sizeof(struct smb_acl); if (pntsd_type & DACL_AUTO_INHERITED) inherited_flags = INHERITED_ACE; @@ -1123,11 +1166,20 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, for (i = 0; i < num_aces; i++) { int pace_size; - if (offsetof(struct smb_ace, access_req) > aces_size) + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; pace_size = le16_to_cpu(parent_aces->size); - if (pace_size > aces_size) + if (pace_size > aces_size || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + + if (parent_aces->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * parent_aces->sid.num_subauth) break; aces_size -= pace_size; @@ -1159,22 +1211,24 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, } if (is_dir && creator && flags & CONTAINER_INHERIT_ACE) { - smb_set_ace(aces, psid, parent_aces->type, inherited_flags, - parent_aces->access_req); - nt_size += le16_to_cpu(aces->size); - ace_cnt++; - aces = (struct smb_ace *)((char *)aces + le16_to_cpu(aces->size)); + rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, + psid, parent_aces->type, + inherited_flags, + parent_aces->access_req); + if (rc) + goto free_aces_base; flags |= INHERIT_ONLY_ACE; psid = creator; } else if (is_dir && !(parent_aces->flags & NO_PROPAGATE_INHERIT_ACE)) { psid = &parent_aces->sid; } - smb_set_ace(aces, psid, parent_aces->type, flags | inherited_flags, - parent_aces->access_req); - nt_size += le16_to_cpu(aces->size); - aces = (struct smb_ace *)((char *)aces + le16_to_cpu(aces->size)); - ace_cnt++; + rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, psid, + parent_aces->type, + flags | inherited_flags, + parent_aces->access_req); + if (rc) + goto free_aces_base; pass: parent_aces = (struct smb_ace *)((char *)parent_aces + pace_size); } @@ -1184,7 +1238,7 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, struct smb_acl *pdacl; struct smb_sid *powner_sid = NULL, *pgroup_sid = NULL; int powner_sid_size = 0, pgroup_sid_size = 0, pntsd_size; - int pntsd_alloc_size; + size_t pntsd_alloc_size; if (parent_pntsd->osidoffset) { powner_sid = (struct smb_sid *)((char *)parent_pntsd + @@ -1197,8 +1251,19 @@ int smb_inherit_dacl(struct ksmbd_conn *conn, pgroup_sid_size = 1 + 1 + 6 + (pgroup_sid->num_subauth * 4); } - pntsd_alloc_size = sizeof(struct smb_ntsd) + powner_sid_size + - pgroup_sid_size + sizeof(struct smb_acl) + nt_size; + if (check_add_overflow(sizeof(struct smb_ntsd), + (size_t)powner_sid_size, + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, + (size_t)pgroup_sid_size, + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, sizeof(struct smb_acl), + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, (size_t)nt_size, + &pntsd_alloc_size)) { + rc = -EINVAL; + goto free_aces_base; + } pntsd = kzalloc(pntsd_alloc_size, KSMBD_DEFAULT_GFP); if (!pntsd) { @@ -1341,10 +1406,13 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, access_req) > aces_size) + if (offsetof(struct smb_ace, sid) + + aces_size < CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); - if (ace_size > aces_size) + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; granted |= le32_to_cpu(ace->access_req); @@ -1359,13 +1427,19 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); aces_size = acl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { - if (offsetof(struct smb_ace, access_req) > aces_size) + if (offsetof(struct smb_ace, sid) + + aces_size < CIFS_SID_BASE_SIZE) break; ace_size = le16_to_cpu(ace->size); - if (ace_size > aces_size) + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; aces_size -= ace_size; + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) + break; + if (!compare_sids(&sid, &ace->sid) || !compare_sids(&sid_unix_NFS_mode, &ace->sid)) { found = 1; diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index 2dbabe2d800554..1c5645238bd3f6 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "vfs_cache.h" #include "transport_ipc.h" @@ -497,7 +498,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_rpc_command *resp = entry->response; - msg_sz = sizeof(struct ksmbd_rpc_command) + resp->payload_sz; + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), + resp->payload_sz, &msg_sz)) + return -EINVAL; break; } case KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST: @@ -516,8 +519,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) if (resp->payload_sz < resp->veto_list_sz) return -EINVAL; - msg_sz = sizeof(struct ksmbd_share_config_response) + - resp->payload_sz; + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; } break; } @@ -526,6 +530,12 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) struct ksmbd_login_response_ext *resp = entry->response; if (resp->ngroups) { + if (resp->ngroups < 0 || + resp->ngroups > NGROUPS_MAX) { + pr_err("ngroups(%d) from login response exceeds max groups(%d)\n", + resp->ngroups, NGROUPS_MAX); + return -EINVAL; + } msg_sz = sizeof(struct ksmbd_login_response_ext) + resp->ngroups * sizeof(gid_t); } diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index 188572491d53ff..dbc8dedb85dced 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -1588,15 +1588,21 @@ static int smb_direct_post_send_data(struct smbdirect_socket *sc, if (ret) goto err; + /* + * From here msg is moved to send_ctx + * and we should not free it explicitly. + */ + if (send_ctx == &_send_ctx) { ret = smb_direct_flush_send_list(sc, send_ctx, true); if (ret) - goto err; + goto flush_failed; } return 0; err: smb_direct_free_sendmsg(sc, msg); +flush_failed: header_failed: atomic_inc(&sc->send_io.credits.count); credit_failed: diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index 7e29b06820e216..13b711ea575d8e 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -183,6 +183,8 @@ static int ksmbd_tcp_new_connection(struct socket *client_sk) t = alloc_transport(client_sk); if (!t) { sock_release(client_sk); + if (server_conf.max_connections) + atomic_dec(&active_num_conn); return -ENOMEM; } @@ -279,7 +281,7 @@ static int ksmbd_kthread_fn(void *p) skip_max_ip_conns_limit: if (server_conf.max_connections && - atomic_inc_return(&active_num_conn) >= server_conf.max_connections) { + atomic_inc_return(&active_num_conn) > server_conf.max_connections) { pr_info_ratelimited("Limit the maximum number of connections(%u)\n", atomic_read(&active_num_conn)); atomic_dec(&active_num_conn); diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index 168f2dd7e200b7..3551f01a3fa035 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -19,6 +19,7 @@ #include "misc.h" #include "mgmt/tree_connect.h" #include "mgmt/user_session.h" +#include "mgmt/user_config.h" #include "smb_common.h" #include "server.h" #include "smb2pdu.h" @@ -463,9 +464,11 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) * there are not accesses to fp->lock_list. */ list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { - spin_lock(&fp->conn->llist_lock); - list_del(&smb_lock->clist); - spin_unlock(&fp->conn->llist_lock); + if (!list_empty(&smb_lock->clist) && fp->conn) { + spin_lock(&fp->conn->llist_lock); + list_del(&smb_lock->clist); + spin_unlock(&fp->conn->llist_lock); + } list_del(&smb_lock->flist); locks_free_lock(smb_lock->fl); @@ -474,6 +477,8 @@ static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) if (ksmbd_stream_fd(fp)) kfree(fp->stream.name); + kfree(fp->owner.name); + kmem_cache_free(filp_cache, fp); } @@ -785,11 +790,13 @@ void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, } static int -__close_file_table_ids(struct ksmbd_file_table *ft, +__close_file_table_ids(struct ksmbd_session *sess, struct ksmbd_tree_connect *tcon, bool (*skip)(struct ksmbd_tree_connect *tcon, - struct ksmbd_file *fp)) + struct ksmbd_file *fp, + struct ksmbd_user *user)) { + struct ksmbd_file_table *ft = &sess->file_table; struct ksmbd_file *fp; unsigned int id = 0; int num = 0; @@ -802,7 +809,7 @@ __close_file_table_ids(struct ksmbd_file_table *ft, break; } - if (skip(tcon, fp) || + if (skip(tcon, fp, sess->user) || !atomic_dec_and_test(&fp->refcount)) { id++; write_unlock(&ft->lock); @@ -854,7 +861,8 @@ static inline bool is_reconnectable(struct ksmbd_file *fp) } static bool tree_conn_fd_check(struct ksmbd_tree_connect *tcon, - struct ksmbd_file *fp) + struct ksmbd_file *fp, + struct ksmbd_user *user) { return fp->tcon != tcon; } @@ -989,16 +997,74 @@ void ksmbd_stop_durable_scavenger(void) kthread_stop(server_conf.dh_task); } +/* + * ksmbd_vfs_copy_durable_owner - Copy owner info for durable reconnect + * @fp: ksmbd file pointer to store owner info + * @user: user pointer to copy from + * + * This function binds the current user's identity to the file handle + * to satisfy MS-SMB2 Step 8 (SecurityContext matching) during reconnect. + * + * Return: 0 on success, or negative error code on failure + */ +static int ksmbd_vfs_copy_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + if (!user) + return -EINVAL; + + /* Duplicate the user name to ensure identity persistence */ + fp->owner.name = kstrdup(user->name, GFP_KERNEL); + if (!fp->owner.name) + return -ENOMEM; + + fp->owner.uid = user->uid; + fp->owner.gid = user->gid; + + return 0; +} + +/** + * ksmbd_vfs_compare_durable_owner - Verify if the requester is original owner + * @fp: existing ksmbd file pointer + * @user: user pointer of the reconnect requester + * + * Compares the UID, GID, and name of the current requester against the + * original owner stored in the file handle. + * + * Return: true if the user matches, false otherwise + */ +bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + if (!user || !fp->owner.name) + return false; + + /* Check if the UID and GID match first (fast path) */ + if (fp->owner.uid != user->uid || fp->owner.gid != user->gid) + return false; + + /* Validate the account name to ensure the same SecurityContext */ + if (strcmp(fp->owner.name, user->name)) + return false; + + return true; +} + static bool session_fd_check(struct ksmbd_tree_connect *tcon, - struct ksmbd_file *fp) + struct ksmbd_file *fp, struct ksmbd_user *user) { struct ksmbd_inode *ci; struct oplock_info *op; struct ksmbd_conn *conn; + struct ksmbd_lock *smb_lock, *tmp_lock; if (!is_reconnectable(fp)) return false; + if (ksmbd_vfs_copy_durable_owner(fp, user)) + return false; + conn = fp->conn; ci = fp->f_ci; down_write(&ci->m_lock); @@ -1011,6 +1077,12 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, } up_write(&ci->m_lock); + list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { + spin_lock(&fp->conn->llist_lock); + list_del_init(&smb_lock->clist); + spin_unlock(&fp->conn->llist_lock); + } + fp->conn = NULL; fp->tcon = NULL; fp->volatile_id = KSMBD_NO_FID; @@ -1024,7 +1096,7 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) { - int num = __close_file_table_ids(&work->sess->file_table, + int num = __close_file_table_ids(work->sess, work->tcon, tree_conn_fd_check); @@ -1033,7 +1105,7 @@ void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) void ksmbd_close_session_fds(struct ksmbd_work *work) { - int num = __close_file_table_ids(&work->sess->file_table, + int num = __close_file_table_ids(work->sess, work->tcon, session_fd_check); @@ -1090,6 +1162,9 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) { struct ksmbd_inode *ci; struct oplock_info *op; + struct ksmbd_conn *conn = work->conn; + struct ksmbd_lock *smb_lock; + unsigned int old_f_state; if (!fp->is_durable || fp->conn || fp->tcon) { pr_err("Invalid durable fd [%p:%p]\n", fp->conn, fp->tcon); @@ -1101,9 +1176,23 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) return -EBADF; } - fp->conn = work->conn; + old_f_state = fp->f_state; + fp->f_state = FP_NEW; + __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); + if (!has_file_id(fp->volatile_id)) { + fp->f_state = old_f_state; + return -EBADF; + } + + fp->conn = conn; fp->tcon = work->tcon; + list_for_each_entry(smb_lock, &fp->lock_list, flist) { + spin_lock(&conn->llist_lock); + list_add_tail(&smb_lock->clist, &conn->lock_list); + spin_unlock(&conn->llist_lock); + } + ci = fp->f_ci; down_write(&ci->m_lock); list_for_each_entry_rcu(op, &ci->m_op_list, op_entry) { @@ -1114,13 +1203,10 @@ int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) } up_write(&ci->m_lock); - fp->f_state = FP_NEW; - __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); - if (!has_file_id(fp->volatile_id)) { - fp->conn = NULL; - fp->tcon = NULL; - return -EBADF; - } + fp->owner.uid = fp->owner.gid = 0; + kfree(fp->owner.name); + fp->owner.name = NULL; + return 0; } @@ -1135,12 +1221,14 @@ int ksmbd_init_file_table(struct ksmbd_file_table *ft) return 0; } -void ksmbd_destroy_file_table(struct ksmbd_file_table *ft) +void ksmbd_destroy_file_table(struct ksmbd_session *sess) { + struct ksmbd_file_table *ft = &sess->file_table; + if (!ft->idr) return; - __close_file_table_ids(ft, NULL, session_fd_check); + __close_file_table_ids(sess, NULL, session_fd_check); idr_destroy(ft->idr); kfree(ft->idr); ft->idr = NULL; diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index 78b506c5ef03b7..866f32c10d4dda 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -68,6 +68,13 @@ enum { FP_CLOSED }; +/* Owner information for durable handle reconnect */ +struct durable_owner { + unsigned int uid; + unsigned int gid; + char *name; +}; + struct ksmbd_file { struct file *filp; u64 persistent_id; @@ -114,6 +121,7 @@ struct ksmbd_file { bool is_resilient; bool is_posix_ctxt; + struct durable_owner owner; }; static inline void set_ctx_actor(struct dir_context *ctx, @@ -140,7 +148,7 @@ static inline bool ksmbd_stream_fd(struct ksmbd_file *fp) } int ksmbd_init_file_table(struct ksmbd_file_table *ft); -void ksmbd_destroy_file_table(struct ksmbd_file_table *ft); +void ksmbd_destroy_file_table(struct ksmbd_session *sess); int ksmbd_close_fd(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_fd_fast(struct ksmbd_work *work, u64 id); struct ksmbd_file *ksmbd_lookup_foreign_fd(struct ksmbd_work *work, u64 id); @@ -166,6 +174,8 @@ void ksmbd_free_global_file_table(void); void ksmbd_set_fd_limit(unsigned long limit); void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, unsigned int state); +bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user); /* * INODE hash diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c index e1e639f515a0fd..989edd6c6c23ca 100644 --- a/fs/sysfs/group.c +++ b/fs/sysfs/group.c @@ -517,8 +517,11 @@ static int sysfs_group_attrs_change_owner(struct kobject *kobj, struct attribute *const *attr; for (i = 0, attr = grp->attrs; *attr; i++, attr++) { - if (grp->is_visible) { - mode = grp->is_visible(kobj, *attr, i); + if (grp->is_visible || grp->is_visible_const) { + if (grp->is_visible) + mode = grp->is_visible(kobj, *attr, i); + else + mode = grp->is_visible_const(kobj, *attr, i); if (mode & SYSFS_GROUP_INVISIBLE) break; if (!mode) diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 8e5ac464b32849..26b6453de30ef6 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -180,29 +180,25 @@ static int eventfs_set_attr(struct mnt_idmap *idmap, struct dentry *dentry, const char *name; int ret; - mutex_lock(&eventfs_mutex); + guard(mutex)(&eventfs_mutex); ei = dentry->d_fsdata; - if (ei->is_freed) { - /* Do not allow changes if the event is about to be removed. */ - mutex_unlock(&eventfs_mutex); + /* Do not allow changes if the event is about to be removed. */ + if (ei->is_freed) return -ENODEV; - } /* Preallocate the children mode array if necessary */ if (!(dentry->d_inode->i_mode & S_IFDIR)) { if (!ei->entry_attrs) { ei->entry_attrs = kzalloc_objs(*ei->entry_attrs, ei->nr_entries, GFP_NOFS); - if (!ei->entry_attrs) { - ret = -ENOMEM; - goto out; - } + if (!ei->entry_attrs) + return -ENOMEM; } } ret = simple_setattr(idmap, dentry, iattr); if (ret < 0) - goto out; + return ret; /* * If this is a dir, then update the ei cache, only the file @@ -225,8 +221,6 @@ static int eventfs_set_attr(struct mnt_idmap *idmap, struct dentry *dentry, } } } - out: - mutex_unlock(&eventfs_mutex); return ret; } @@ -250,6 +244,8 @@ static void eventfs_set_attrs(struct eventfs_inode *ei, bool update_uid, kuid_t { struct eventfs_inode *ei_child; + lockdep_assert_held(&eventfs_mutex); + /* Update events// */ if (WARN_ON_ONCE(level > 3)) return; @@ -528,26 +524,24 @@ static struct dentry *eventfs_root_lookup(struct inode *dir, struct tracefs_inode *ti; struct eventfs_inode *ei; const char *name = dentry->d_name.name; - struct dentry *result = NULL; ti = get_tracefs(dir); if (WARN_ON_ONCE(!(ti->flags & TRACEFS_EVENT_INODE))) return ERR_PTR(-EIO); - mutex_lock(&eventfs_mutex); + guard(mutex)(&eventfs_mutex); ei = ti->private; if (!ei || ei->is_freed) - goto out; + return NULL; list_for_each_entry(ei_child, &ei->children, list) { if (strcmp(ei_child->name, name) != 0) continue; /* A child is freed and removed from the list at the same time */ if (WARN_ON_ONCE(ei_child->is_freed)) - goto out; - result = lookup_dir_entry(dentry, ei, ei_child); - goto out; + return NULL; + return lookup_dir_entry(dentry, ei, ei_child); } for (int i = 0; i < ei->nr_entries; i++) { @@ -561,14 +555,12 @@ static struct dentry *eventfs_root_lookup(struct inode *dir, data = ei->data; if (entry->callback(name, &mode, &data, &fops) <= 0) - goto out; + return NULL; + + return lookup_file_dentry(dentry, ei, i, mode, data, fops); - result = lookup_file_dentry(dentry, ei, i, mode, data, fops); - goto out; } - out: - mutex_unlock(&eventfs_mutex); - return result; + return NULL; } /* @@ -584,8 +576,6 @@ static int eventfs_iterate(struct file *file, struct dir_context *ctx) struct eventfs_inode *ei; const char *name; umode_t mode; - int idx; - int ret = -EINVAL; int ino; int i, r, c; @@ -598,22 +588,18 @@ static int eventfs_iterate(struct file *file, struct dir_context *ctx) c = ctx->pos - 2; - idx = srcu_read_lock(&eventfs_srcu); - - mutex_lock(&eventfs_mutex); - ei = READ_ONCE(ti->private); - if (ei && ei->is_freed) - ei = NULL; - mutex_unlock(&eventfs_mutex); + guard(srcu)(&eventfs_srcu); - if (!ei) - goto out; + scoped_guard(mutex, &eventfs_mutex) { + ei = READ_ONCE(ti->private); + if (!ei || ei->is_freed) + return -EINVAL; + } /* * Need to create the dentries and inodes to have a consistent * inode number. */ - ret = 0; /* Start at 'c' to jump over already read entries */ for (i = c; i < ei->nr_entries; i++, ctx->pos++) { @@ -622,21 +608,19 @@ static int eventfs_iterate(struct file *file, struct dir_context *ctx) entry = &ei->entries[i]; name = entry->name; - mutex_lock(&eventfs_mutex); /* If ei->is_freed then just bail here, nothing more to do */ - if (ei->is_freed) { - mutex_unlock(&eventfs_mutex); - goto out; + scoped_guard(mutex, &eventfs_mutex) { + if (ei->is_freed) + return -EINVAL; + r = entry->callback(name, &mode, &cdata, &fops); } - r = entry->callback(name, &mode, &cdata, &fops); - mutex_unlock(&eventfs_mutex); if (r <= 0) continue; ino = EVENTFS_FILE_INODE_INO; if (!dir_emit(ctx, name, strlen(name), ino, DT_REG)) - goto out; + return -EINVAL; } /* Subtract the skipped entries above */ @@ -659,19 +643,13 @@ static int eventfs_iterate(struct file *file, struct dir_context *ctx) ino = eventfs_dir_ino(ei_child); - if (!dir_emit(ctx, name, strlen(name), ino, DT_DIR)) - goto out_dec; + if (!dir_emit(ctx, name, strlen(name), ino, DT_DIR)) { + /* Incremented ctx->pos without adding something, reset it */ + ctx->pos--; + return -EINVAL; + } } - ret = 1; - out: - srcu_read_unlock(&eventfs_srcu, idx); - - return ret; - - out_dec: - /* Incremented ctx->pos without adding something, reset it */ - ctx->pos--; - goto out; + return 1; } /** @@ -728,11 +706,10 @@ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode INIT_LIST_HEAD(&ei->children); INIT_LIST_HEAD(&ei->list); - mutex_lock(&eventfs_mutex); - if (!parent->is_freed) - list_add_tail(&ei->list, &parent->children); - mutex_unlock(&eventfs_mutex); - + scoped_guard(mutex, &eventfs_mutex) { + if (!parent->is_freed) + list_add_tail_rcu(&ei->list, &parent->children); + } /* Was the parent freed? */ if (list_empty(&ei->list)) { cleanup_ei(ei); @@ -878,9 +855,8 @@ void eventfs_remove_dir(struct eventfs_inode *ei) if (!ei) return; - mutex_lock(&eventfs_mutex); + guard(mutex)(&eventfs_mutex); eventfs_remove_rec(ei, 0); - mutex_unlock(&eventfs_mutex); } /** @@ -912,3 +888,15 @@ void eventfs_remove_events_dir(struct eventfs_inode *ei) d_invalidate(dentry); d_make_discardable(dentry); } + +int eventfs_remount_lock(void) +{ + mutex_lock(&eventfs_mutex); + return srcu_read_lock(&eventfs_srcu); +} + +void eventfs_remount_unlock(int srcu_idx) +{ + srcu_read_unlock(&eventfs_srcu, srcu_idx); + mutex_unlock(&eventfs_mutex); +} diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c index 51c00c8fa17553..40477513cce1e4 100644 --- a/fs/tracefs/inode.c +++ b/fs/tracefs/inode.c @@ -336,6 +336,7 @@ static int tracefs_apply_options(struct super_block *sb, bool remount) struct inode *inode = d_inode(sb->s_root); struct tracefs_inode *ti; bool update_uid, update_gid; + int srcu_idx; umode_t tmp_mode; /* @@ -360,6 +361,7 @@ static int tracefs_apply_options(struct super_block *sb, bool remount) update_uid = fsi->opts & BIT(Opt_uid); update_gid = fsi->opts & BIT(Opt_gid); + srcu_idx = eventfs_remount_lock(); rcu_read_lock(); list_for_each_entry_rcu(ti, &tracefs_inodes, list) { if (update_uid) { @@ -381,6 +383,7 @@ static int tracefs_apply_options(struct super_block *sb, bool remount) eventfs_remount(ti, update_uid, update_gid); } rcu_read_unlock(); + eventfs_remount_unlock(srcu_idx); } return 0; @@ -426,7 +429,7 @@ static int tracefs_drop_inode(struct inode *inode) * This inode is being freed and cannot be used for * eventfs. Clear the flag so that it doesn't call into * eventfs during the remount flag updates. The eventfs_inode - * gets freed after an RCU cycle, so the content will still + * gets freed after an SRCU cycle, so the content will still * be safe if the iteration is going on now. */ ti->flags &= ~TRACEFS_EVENT_INODE; @@ -491,6 +494,7 @@ static int tracefs_fill_super(struct super_block *sb, struct fs_context *fc) return err; sb->s_op = &tracefs_super_operations; + tracefs_apply_options(sb, false); set_default_d_op(sb, &tracefs_dentry_operations); return 0; diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h index d83c2a25f288e0..a4a7f8431affb0 100644 --- a/fs/tracefs/internal.h +++ b/fs/tracefs/internal.h @@ -76,4 +76,7 @@ struct inode *tracefs_get_inode(struct super_block *sb); void eventfs_remount(struct tracefs_inode *ti, bool update_uid, bool update_gid); void eventfs_d_release(struct dentry *dentry); +int eventfs_remount_lock(void); +void eventfs_remount_unlock(int srcu_idx); + #endif /* _TRACEFS_INTERNAL_H */ diff --git a/fs/udf/misc.c b/fs/udf/misc.c index 0788593b6a1d8b..6928e378fbbdcb 100644 --- a/fs/udf/misc.c +++ b/fs/udf/misc.c @@ -230,8 +230,12 @@ struct buffer_head *udf_read_tagged(struct super_block *sb, uint32_t block, } /* Verify the descriptor CRC */ - if (le16_to_cpu(tag_p->descCRCLength) + sizeof(struct tag) > sb->s_blocksize || - le16_to_cpu(tag_p->descCRC) == crc_itu_t(0, + if (le16_to_cpu(tag_p->descCRCLength) + sizeof(struct tag) > sb->s_blocksize) { + udf_err(sb, "block %u: CRC length %u exceeds block size\n", + block, le16_to_cpu(tag_p->descCRCLength)); + goto error_out; + } + if (le16_to_cpu(tag_p->descCRC) == crc_itu_t(0, bh->b_data + sizeof(struct tag), le16_to_cpu(tag_p->descCRCLength))) return bh; diff --git a/fs/udf/super.c b/fs/udf/super.c index 27f463fd1d89e8..df2b62eddfc0ca 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1694,8 +1694,9 @@ static struct udf_vds_record *handle_partition_descriptor( return &(data->part_descs_loc[i].rec); if (data->num_part_descs >= data->size_part_descs) { struct part_desc_seq_scan_data *new_loc; - unsigned int new_size = ALIGN(partnum, PART_DESC_ALLOC_STEP); + unsigned int new_size; + new_size = data->num_part_descs + PART_DESC_ALLOC_STEP; new_loc = kzalloc_objs(*new_loc, new_size); if (!new_loc) return ERR_PTR(-ENOMEM); @@ -1705,6 +1706,7 @@ static struct udf_vds_record *handle_partition_descriptor( data->part_descs_loc = new_loc; data->size_part_descs = new_size; } + data->part_descs_loc[data->num_part_descs].partnum = partnum; return &(data->part_descs_loc[data->num_part_descs++].rec); } diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index bdc84e5219cde3..4b53dc4a32664c 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1238,8 +1238,6 @@ static __always_inline int validate_unaligned_range( return -EINVAL; if (!len) return -EINVAL; - if (start < mmap_min_addr) - return -EINVAL; if (start >= task_size) return -EINVAL; if (len > task_size - start) diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index d2f3c50d80e70f..faed2ed587dda4 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1831,6 +1831,7 @@ xfs_alloc_buftarg( return btp; error_free: + fs_put_dax(btp->bt_daxdev, mp); kfree(btp); return ERR_PTR(error); } diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c index 6c790983823463..4527119b29619f 100644 --- a/fs/xfs/xfs_sysfs.c +++ b/fs/xfs/xfs_sysfs.c @@ -14,6 +14,7 @@ #include "xfs_log_priv.h" #include "xfs_mount.h" #include "xfs_zones.h" +#include "xfs_zone_alloc.h" struct xfs_sysfs_attr { struct attribute attr; @@ -724,6 +725,7 @@ zonegc_low_space_store( const char *buf, size_t count) { + struct xfs_mount *mp = zoned_to_mp(kobj); int ret; unsigned int val; @@ -734,7 +736,10 @@ zonegc_low_space_store( if (val > 100) return -EINVAL; - zoned_to_mp(kobj)->m_zonegc_low_space = val; + if (mp->m_zonegc_low_space != val) { + mp->m_zonegc_low_space = val; + xfs_zone_gc_wakeup(mp); + } return count; } diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c index e3d19b6dc64a88..6b42572f99840f 100644 --- a/fs/xfs/xfs_zone_alloc.c +++ b/fs/xfs/xfs_zone_alloc.c @@ -1214,7 +1214,7 @@ xfs_alloc_zone_info( return zi; out_free_bitmaps: - while (--i > 0) + while (--i >= 0) kvfree(zi->zi_used_bucket_bitmap[i]); kfree(zi); return NULL; diff --git a/fs/xfs/xfs_zone_alloc.h b/fs/xfs/xfs_zone_alloc.h index 4db02816d0fdad..8b2ef98c81eff8 100644 --- a/fs/xfs/xfs_zone_alloc.h +++ b/fs/xfs/xfs_zone_alloc.h @@ -51,6 +51,7 @@ int xfs_mount_zones(struct xfs_mount *mp); void xfs_unmount_zones(struct xfs_mount *mp); void xfs_zone_gc_start(struct xfs_mount *mp); void xfs_zone_gc_stop(struct xfs_mount *mp); +void xfs_zone_gc_wakeup(struct xfs_mount *mp); #else static inline int xfs_mount_zones(struct xfs_mount *mp) { @@ -65,6 +66,9 @@ static inline void xfs_zone_gc_start(struct xfs_mount *mp) static inline void xfs_zone_gc_stop(struct xfs_mount *mp) { } +static inline void xfs_zone_gc_wakeup(struct xfs_mount *mp) +{ +} #endif /* CONFIG_XFS_RT */ #endif /* _XFS_ZONE_ALLOC_H */ diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c index 309f700985243e..9f19cd456ed84d 100644 --- a/fs/xfs/xfs_zone_gc.c +++ b/fs/xfs/xfs_zone_gc.c @@ -1159,6 +1159,23 @@ xfs_zone_gc_stop( kthread_park(mp->m_zone_info->zi_gc_thread); } +void +xfs_zone_gc_wakeup( + struct xfs_mount *mp) +{ + struct super_block *sb = mp->m_super; + + /* + * If we are unmounting the file system we must not try to + * wake gc as m_zone_info might have been freed already. + */ + if (down_read_trylock(&sb->s_umount)) { + if (!xfs_is_readonly(mp)) + wake_up_process(mp->m_zone_info->zi_gc_thread); + up_read(&sb->s_umount); + } +} + int xfs_zone_gc_mount( struct xfs_mount *mp) diff --git a/include/acpi/actbl1.h b/include/acpi/actbl1.h index 4e15583e0d254c..f72e00517eb3d1 100644 --- a/include/acpi/actbl1.h +++ b/include/acpi/actbl1.h @@ -1386,6 +1386,12 @@ enum acpi_einj_command_status { #define ACPI_EINJ_CXL_MEM_FATAL (1<<17) #define ACPI_EINJ_VENDOR_DEFINED (1<<31) +/* EINJV2 error types from EINJV2_GET_ERROR_TYPE (ACPI 6.6) */ + +#define ACPI_EINJV2_PROCESSOR (1) +#define ACPI_EINJV2_MEMORY (1<<1) +#define ACPI_EINJV2_PCIE (1<<2) + /******************************************************************************* * * ERST - Error Record Serialization Table (ACPI 4.0) diff --git a/include/asm-generic/codetag.lds.h b/include/asm-generic/codetag.lds.h index a14f4bdafddae8..4948e5d4e9d99b 100644 --- a/include/asm-generic/codetag.lds.h +++ b/include/asm-generic/codetag.lds.h @@ -18,7 +18,7 @@ IF_MEM_ALLOC_PROFILING(SECTION_WITH_BOUNDARIES(alloc_tags)) #define MOD_SEPARATE_CODETAG_SECTION(_name) \ - .codetag.##_name : { \ + .codetag.##_name 0 : { \ SECTION_WITH_BOUNDARIES(_name) \ } diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index 655bd9104ffb24..aca218734dcdec 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -57,10 +57,19 @@ enum drm_gpuva_flags { */ DRM_GPUVA_SPARSE = (1 << 1), + /** + * @DRM_GPUVA_REPEAT: + * + * Flag indicating that the &drm_gpuva is a mapping of a GEM + * object with a certain range that is repeated multiple times to + * fill the virtual address range. + */ + DRM_GPUVA_REPEAT = (1 << 2), + /** * @DRM_GPUVA_USERBITS: user defined bits */ - DRM_GPUVA_USERBITS = (1 << 2), + DRM_GPUVA_USERBITS = (1 << 3), }; /** @@ -112,6 +121,18 @@ struct drm_gpuva { */ u64 offset; + /* + * @gem.range: the range of the GEM that is mapped + * + * When dealing with normal mappings, this must be zero. + * When flags has DRM_GPUVA_REPEAT set, this field must be + * smaller than va.range and va.range must be a multiple of + * gem.range. + * This is a u32 not a u64 because we expect repeated mappings + * to be pointing to relatively small portions of a GEM object. + */ + u32 range; + /** * @gem.obj: the mapped &drm_gem_object */ @@ -882,11 +903,27 @@ struct drm_gpuva_op_map { */ u64 offset; + /* + * @gem.range: the range of the GEM that is mapped + * + * When dealing with normal mappings, this must be zero. + * When flags has DRM_GPUVA_REPEAT set, it must be a multiple + * of va.range. This is a u32 not a u64 because we expect + * repeated mappings to be pointing to a relatively small + * portion of a GEM object. + */ + u32 range; + /** * @gem.obj: the &drm_gem_object to map */ struct drm_gem_object *obj; } gem; + + /** + * @flags: requested flags for the &drm_gpuva for this mapping + */ + enum drm_gpuva_flags flags; }; /** @@ -1123,6 +1160,7 @@ void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, const struct drm_gpuva_op_map *op) { + va->flags = op->flags; va->va.addr = op->va.addr; va->va.range = op->va.range; va->gem.obj = op->gem.obj; @@ -1248,6 +1286,16 @@ struct drm_gpuvm_ops { * used. */ int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); + + /** + * @sm_can_merge_flags: called during &drm_gpuvm_sm_map + * + * This callback is called to determine whether two va ranges can be merged, + * based on their flags. + * + * If NULL, va ranges can only be merged if their flags are equal. + */ + bool (*sm_can_merge_flags)(enum drm_gpuva_flags a, enum drm_gpuva_flags b); }; int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, @@ -1255,6 +1303,7 @@ int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, u64 addr, u64 range); +int drm_gpuvm_bo_unmap(struct drm_gpuvm_bo *bo, void *priv); int drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, unsigned int num_fences, diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 78e07c2507c7a4..e29fb50e9f4ccc 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -301,6 +301,11 @@ struct drm_sched_fence { * @lock: the lock used by the scheduled and the finished fences. */ spinlock_t lock; + /** + * @sched_name: the name of the scheduler that owns this fence. We + * keep a copy here since fences can outlive their scheduler. + */ + char sched_name[16]; /** * @owner: job owner for debugging */ diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h index 33e80f30b8b822..a5d386583fb6e0 100644 --- a/include/drm/ttm/ttm_resource.h +++ b/include/drm/ttm/ttm_resource.h @@ -448,6 +448,8 @@ void ttm_resource_add_bulk_move(struct ttm_resource *res, struct ttm_buffer_object *bo); void ttm_resource_del_bulk_move(struct ttm_resource *res, struct ttm_buffer_object *bo); +void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res, + struct ttm_buffer_object *bo); void ttm_resource_move_to_lru_tail(struct ttm_resource *res); void ttm_resource_init(struct ttm_buffer_object *bo, diff --git a/include/dt-bindings/clock/qcom,gcc-sc8180x.h b/include/dt-bindings/clock/qcom,gcc-sc8180x.h index b9d8438a15ffbb..9ed7b794aacc47 100644 --- a/include/dt-bindings/clock/qcom,gcc-sc8180x.h +++ b/include/dt-bindings/clock/qcom,gcc-sc8180x.h @@ -322,5 +322,10 @@ #define USB30_MP_GDSC 8 #define USB30_PRIM_GDSC 9 #define USB30_SEC_GDSC 10 +#define HLOS1_VOTE_MMNOC_MMU_TBU_HF0_GDSC 11 +#define HLOS1_VOTE_MMNOC_MMU_TBU_HF1_GDSC 12 +#define HLOS1_VOTE_MMNOC_MMU_TBU_SF_GDSC 13 +#define HLOS1_VOTE_TURING_MMU_TBU0_GDSC 14 +#define HLOS1_VOTE_TURING_MMU_TBU1_GDSC 15 #endif diff --git a/include/dt-bindings/clock/qcom,glymur-gcc.h b/include/dt-bindings/clock/qcom,glymur-gcc.h index 10c12b8c51c34c..6907653c79927f 100644 --- a/include/dt-bindings/clock/qcom,glymur-gcc.h +++ b/include/dt-bindings/clock/qcom,glymur-gcc.h @@ -574,5 +574,6 @@ #define GCC_VIDEO_AXI0_CLK_ARES 89 #define GCC_VIDEO_AXI1_CLK_ARES 90 #define GCC_VIDEO_BCR 91 +#define GCC_VIDEO_AXI0C_CLK_ARES 92 #endif diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index d40ac39bfbe8d9..02de2ede560f3a 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -163,9 +163,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) { WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } +void alloc_tag_add_early_pfn(unsigned long pfn); #else static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} static inline void alloc_tag_sub_check(union codetag_ref *ref) {} +static inline void alloc_tag_add_early_pfn(unsigned long pfn) {} #endif /* Caller should verify both ref and tag to be valid */ diff --git a/include/linux/backing-file.h b/include/linux/backing-file.h index 1476a6ed1bfd77..c939cd222730c4 100644 --- a/include/linux/backing-file.h +++ b/include/linux/backing-file.h @@ -18,10 +18,10 @@ struct backing_file_ctx { void (*end_write)(struct kiocb *iocb, ssize_t); }; -struct file *backing_file_open(const struct path *user_path, int flags, +struct file *backing_file_open(const struct file *user_file, int flags, const struct path *real_path, const struct cred *cred); -struct file *backing_tmpfile_open(const struct path *user_path, int flags, +struct file *backing_tmpfile_open(const struct file *user_file, int flags, const struct path *real_parentpath, umode_t mode, const struct cred *cred); ssize_t backing_file_read_iter(struct file *file, struct iov_iter *iter, diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index b907e6c2307d85..260d7968cf7202 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -108,6 +108,7 @@ int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, extern unsigned int cdrom_check_events(struct cdrom_device_info *cdi, unsigned int clearing); +extern void cdrom_probe_write_features(struct cdrom_device_info *cdi); extern int register_cdrom(struct gendisk *disk, struct cdrom_device_info *cdi); extern void unregister_cdrom(struct cdrom_device_info *cdi); diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 7f87399938fa2b..8e7afb5a5980a7 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -609,8 +609,8 @@ struct cgroup { /* used to wait for offlining of csses */ wait_queue_head_t offline_waitq; - /* used by cgroup_rmdir() to wait for dying tasks to leave */ - wait_queue_head_t dying_populated_waitq; + /* defers killing csses after removal until cgroup is depopulated */ + struct work_struct finish_destroy_work; /* used to schedule release agent */ struct work_struct release_agent_work; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index cc894fc3897105..4317c5a312bd1a 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -372,7 +372,7 @@ struct cpufreq_driver { * conditions) scale invariance can be disabled, which causes the * schedutil governor to fall back to the latter. */ - void (*adjust_perf)(unsigned int cpu, + void (*adjust_perf)(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity); @@ -617,7 +617,7 @@ struct cpufreq_governor { /* Pass a target to the cpufreq driver */ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq); -void cpufreq_driver_adjust_perf(unsigned int cpu, +void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, unsigned long capacity); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 62cd7b35a29c94..22ba327ec2278c 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -92,7 +92,6 @@ enum cpuhp_state { CPUHP_NET_DEV_DEAD, CPUHP_IOMMU_IOVA_DEAD, CPUHP_AP_ARM_CACHE_B15_RAC_DEAD, - CPUHP_PADATA_DEAD, CPUHP_AP_DTPM_CPU_DEAD, CPUHP_RANDOM_PREPARE, CPUHP_WORKQUEUE_PREP, diff --git a/include/linux/damon.h b/include/linux/damon.h index be3d198043ff9f..d94d965103e1ac 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -805,9 +805,11 @@ struct damon_ctx { /* lists of &struct damon_call_control */ struct list_head call_controls; + bool call_controls_obsolete; struct mutex call_controls_lock; struct damos_walk_control *walk_control; + bool walk_control_obsolete; struct mutex walk_control_lock; /* diff --git a/include/linux/device.h b/include/linux/device.h index e65d564f01cd7c..ac972e7bead464 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -458,6 +458,22 @@ struct device_physical_location { bool lid; }; +/** + * enum struct_device_flags - Flags in struct device + * + * Each flag should have a set of accessor functions created via + * __create_dev_flag_accessors() for each access. + * + * @DEV_FLAG_READY_TO_PROBE: If set then device_add() has finished enough + * initialization that probe could be called. + * @DEV_FLAG_COUNT: Number of defined struct_device_flags. + */ +enum struct_device_flags { + DEV_FLAG_READY_TO_PROBE = 0, + + DEV_FLAG_COUNT +}; + /** * struct device - The basic device structure * @parent: The device's "parent" device, the device to which it is attached. @@ -553,6 +569,7 @@ struct device_physical_location { * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers. * @dma_iommu: Device is using default IOMMU implementation for DMA and * doesn't rely on dma_ops structure. + * @flags: DEV_FLAG_XXX flags. Use atomic bitfield operations to modify. * * At the lowest level, every device in a Linux system is represented by an * instance of struct device. The device structure contains the information @@ -675,8 +692,36 @@ struct device { #ifdef CONFIG_IOMMU_DMA bool dma_iommu:1; #endif + + DECLARE_BITMAP(flags, DEV_FLAG_COUNT); }; +#define __create_dev_flag_accessors(accessor_name, flag_name) \ +static inline bool dev_##accessor_name(const struct device *dev) \ +{ \ + return test_bit(flag_name, dev->flags); \ +} \ +static inline void dev_set_##accessor_name(struct device *dev) \ +{ \ + set_bit(flag_name, dev->flags); \ +} \ +static inline void dev_clear_##accessor_name(struct device *dev) \ +{ \ + clear_bit(flag_name, dev->flags); \ +} \ +static inline void dev_assign_##accessor_name(struct device *dev, bool value) \ +{ \ + assign_bit(flag_name, dev->flags, value); \ +} \ +static inline bool dev_test_and_set_##accessor_name(struct device *dev) \ +{ \ + return test_and_set_bit(flag_name, dev->flags); \ +} + +__create_dev_flag_accessors(ready_to_probe, DEV_FLAG_READY_TO_PROBE); + +#undef __create_dev_flag_accessors + /** * struct device_link - Device link representation. * @supplier: The device on the supplier end of the link. diff --git a/include/linux/dmi.h b/include/linux/dmi.h index 927f8a8b7a1ddd..2eedf44e680126 100644 --- a/include/linux/dmi.h +++ b/include/linux/dmi.h @@ -60,6 +60,7 @@ enum dmi_entry_type { DMI_ENTRY_OOB_REMOTE_ACCESS, DMI_ENTRY_BIS_ENTRY, DMI_ENTRY_SYSTEM_BOOT, + DMI_ENTRY_64_MEM_ERROR, DMI_ENTRY_MGMT_DEV, DMI_ENTRY_MGMT_DEV_COMPONENT, DMI_ENTRY_MGMT_DEV_THRES, @@ -69,6 +70,10 @@ enum dmi_entry_type { DMI_ENTRY_ADDITIONAL, DMI_ENTRY_ONBOARD_DEV_EXT, DMI_ENTRY_MGMT_CONTROLLER_HOST, + DMI_ENTRY_TPM_DEVICE, + DMI_ENTRY_PROCESSOR_ADDITIONAL, + DMI_ENTRY_FIRMWARE_INVENTORY, + DMI_ENTRY_STRING_PROPERTY, DMI_ENTRY_INACTIVE = 126, DMI_ENTRY_END_OF_TABLE = 127, }; diff --git a/include/linux/dpll.h b/include/linux/dpll.h index 2ce295b46b8cdc..8f97120ee7b37d 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -276,6 +276,7 @@ int dpll_pin_ref_sync_pair_add(struct dpll_pin *pin, int dpll_device_change_ntf(struct dpll_device *dpll); +int __dpll_pin_change_ntf(struct dpll_pin *pin); int dpll_pin_change_ntf(struct dpll_pin *pin); int register_dpll_notifier(struct notifier_block *nb); diff --git a/include/linux/fb.h b/include/linux/fb.h index 6d4a58084fd5f3..aed17567fe508d 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -218,13 +218,14 @@ struct fb_deferred_io { unsigned long delay; bool sort_pagereflist; /* sort pagelist by offset */ int open_count; /* number of opened files; protected by fb_info lock */ - struct mutex lock; /* mutex that protects the pageref list */ struct list_head pagereflist; /* list of pagerefs for touched pages */ struct address_space *mapping; /* page cache object for fb device */ /* callback */ struct page *(*get_page)(struct fb_info *info, unsigned long offset); void (*deferred_io)(struct fb_info *info, struct list_head *pagelist); }; + +struct fb_deferred_io_state; #endif /* @@ -487,6 +488,7 @@ struct fb_info { unsigned long npagerefs; struct fb_deferred_io_pageref *pagerefs; struct fb_deferred_io *fbdefio; + struct fb_deferred_io_state *fbdefio_state; #endif const struct fb_ops *fbops; diff --git a/include/linux/firmware/samsung/exynos-acpm-protocol.h b/include/linux/firmware/samsung/exynos-acpm-protocol.h index 2091da965a5ad2..13f17dc4443b8f 100644 --- a/include/linux/firmware/samsung/exynos-acpm-protocol.h +++ b/include/linux/firmware/samsung/exynos-acpm-protocol.h @@ -14,30 +14,24 @@ struct acpm_handle; struct device_node; struct acpm_dvfs_ops { - int (*set_rate)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, unsigned int clk_id, - unsigned long rate); - unsigned long (*get_rate)(const struct acpm_handle *handle, + int (*set_rate)(struct acpm_handle *handle, unsigned int acpm_chan_id, + unsigned int clk_id, unsigned long rate); + unsigned long (*get_rate)(struct acpm_handle *handle, unsigned int acpm_chan_id, unsigned int clk_id); }; struct acpm_pmic_ops { - int (*read_reg)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 *buf); - int (*bulk_read)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 count, u8 *buf); - int (*write_reg)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 value); - int (*bulk_write)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 count, const u8 *buf); - int (*update_reg)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 value, u8 mask); + int (*read_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 *buf); + int (*bulk_read)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 count, u8 *buf); + int (*write_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 value); + int (*bulk_write)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 count, const u8 *buf); + int (*update_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 value, u8 mask); }; struct acpm_ops { @@ -56,12 +50,12 @@ struct acpm_handle { struct device; #if IS_ENABLED(CONFIG_EXYNOS_ACPM_PROTOCOL) -const struct acpm_handle *devm_acpm_get_by_node(struct device *dev, - struct device_node *np); +struct acpm_handle *devm_acpm_get_by_node(struct device *dev, + struct device_node *np); #else -static inline const struct acpm_handle *devm_acpm_get_by_node(struct device *dev, - struct device_node *np) +static inline struct acpm_handle *devm_acpm_get_by_node(struct device *dev, + struct device_node *np) { return NULL; } diff --git a/include/linux/fs.h b/include/linux/fs.h index 8b3dd145b25ec1..ef17f9e211e4fe 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2058,8 +2058,6 @@ static inline bool can_mmap_file(struct file *file) return true; } -int __compat_vma_mmap(const struct file_operations *f_op, - struct file *file, struct vm_area_struct *vma); int compat_vma_mmap(struct file *file, struct vm_area_struct *vma); static inline int vfs_mmap(struct file *file, struct vm_area_struct *vma) @@ -2475,6 +2473,19 @@ struct file *dentry_create(struct path *path, int flags, umode_t mode, const struct cred *cred); const struct path *backing_file_user_path(const struct file *f); +#ifdef CONFIG_SECURITY +void *backing_file_security(const struct file *f); +void backing_file_set_security(struct file *f, void *security); +#else +static inline void *backing_file_security(const struct file *f) +{ + return NULL; +} +static inline void backing_file_set_security(struct file *f, void *security) +{ +} +#endif /* CONFIG_SECURITY */ + /* * When mmapping a file on a stackable filesystem (e.g., overlayfs), the file * stored in ->vm_file is a backing file whose f_inode is on the underlying diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index 897d6211c1635c..1da63f2d704014 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -178,9 +178,6 @@ struct fsl_mc_obj_desc { * @regions: pointer to array of MMIO region entries * @irqs: pointer to array of pointers to interrupts allocated to this device * @resource: generic resource associated with this MC object device, if any. - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * * Generic device object for MC object devices that are "attached" to a * MC bus. @@ -214,7 +211,6 @@ struct fsl_mc_device { struct fsl_mc_device_irq **irqs; struct fsl_mc_resource *resource; struct device_link *consumer_link; - const char *driver_override; }; #define to_fsl_mc_device(_dev) \ diff --git a/include/linux/fsl/ntmp.h b/include/linux/fsl/ntmp.h index 916dc4fe7de3b5..83a449b4d6ec46 100644 --- a/include/linux/fsl/ntmp.h +++ b/include/linux/fsl/ntmp.h @@ -31,6 +31,12 @@ struct netc_tbl_vers { u8 rsst_ver; }; +struct netc_swcbd { + void *buf; + dma_addr_t dma; + size_t size; +}; + struct netc_cbdr { struct device *dev; struct netc_cbdr_regs regs; @@ -44,9 +50,10 @@ struct netc_cbdr { void *addr_base_align; dma_addr_t dma_base; dma_addr_t dma_base_align; + struct netc_swcbd *swcbd; /* Serialize the order of command BD ring */ - spinlock_t ring_lock; + struct mutex ring_lock; }; struct ntmp_user { diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 95985400d3d8e2..e5cde39d6e85d6 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -915,6 +915,7 @@ extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group, unsigned int obj_type); extern void fsnotify_get_mark(struct fsnotify_mark *mark); extern void fsnotify_put_mark(struct fsnotify_mark *mark); +struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark); extern void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info); extern bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info); diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 097be89487bf5c..154910a621ac22 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -15,6 +15,7 @@ #define _LINUX_FWNODE_H_ #include +#include #include #include #include @@ -42,12 +43,12 @@ struct device; * suppliers. Only enforce ordering with suppliers that have * drivers. */ -#define FWNODE_FLAG_LINKS_ADDED BIT(0) -#define FWNODE_FLAG_NOT_DEVICE BIT(1) -#define FWNODE_FLAG_INITIALIZED BIT(2) -#define FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD BIT(3) -#define FWNODE_FLAG_BEST_EFFORT BIT(4) -#define FWNODE_FLAG_VISITED BIT(5) +#define FWNODE_FLAG_LINKS_ADDED 0 +#define FWNODE_FLAG_NOT_DEVICE 1 +#define FWNODE_FLAG_INITIALIZED 2 +#define FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD 3 +#define FWNODE_FLAG_BEST_EFFORT 4 +#define FWNODE_FLAG_VISITED 5 struct fwnode_handle { struct fwnode_handle *secondary; @@ -57,7 +58,7 @@ struct fwnode_handle { struct device *dev; struct list_head suppliers; struct list_head consumers; - u8 flags; + unsigned long flags; }; /* @@ -212,16 +213,37 @@ static inline void fwnode_init(struct fwnode_handle *fwnode, INIT_LIST_HEAD(&fwnode->suppliers); } +static inline void fwnode_set_flag(struct fwnode_handle *fwnode, + unsigned int bit) +{ + set_bit(bit, &fwnode->flags); +} + +static inline void fwnode_clear_flag(struct fwnode_handle *fwnode, + unsigned int bit) +{ + clear_bit(bit, &fwnode->flags); +} + +static inline void fwnode_assign_flag(struct fwnode_handle *fwnode, + unsigned int bit, bool value) +{ + assign_bit(bit, &fwnode->flags, value); +} + +static inline bool fwnode_test_flag(struct fwnode_handle *fwnode, + unsigned int bit) +{ + return test_bit(bit, &fwnode->flags); +} + static inline void fwnode_dev_initialized(struct fwnode_handle *fwnode, bool initialized) { if (IS_ERR_OR_NULL(fwnode)) return; - if (initialized) - fwnode->flags |= FWNODE_FLAG_INITIALIZED; - else - fwnode->flags &= ~FWNODE_FLAG_INITIALIZED; + fwnode_assign_flag(fwnode, FWNODE_FLAG_INITIALIZED, initialized); } int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, @@ -229,5 +251,6 @@ int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); bool fw_devlink_is_strict(void); +int fw_devlink_count_absent_consumers(struct fwnode_handle *fwnode); #endif diff --git a/include/linux/hid.h b/include/linux/hid.h index 31324609af4df1..6a907570ce0e0a 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -625,7 +625,9 @@ struct hid_input { enum hid_type { HID_TYPE_OTHER = 0, HID_TYPE_USBMOUSE, - HID_TYPE_USBNONE + HID_TYPE_USBNONE, + HID_TYPE_SPI_KEYBOARD, + HID_TYPE_SPI_MOUSE, }; enum hid_battery_status { @@ -787,6 +789,8 @@ struct hid_descriptor { .bus = BUS_BLUETOOTH, .vendor = (ven), .product = (prod) #define HID_I2C_DEVICE(ven, prod) \ .bus = BUS_I2C, .vendor = (ven), .product = (prod) +#define HID_SPI_DEVICE(ven, prod) \ + .bus = BUS_SPI, .vendor = (ven), .product = (prod) #define HID_REPORT_ID(rep) \ .report_type = (rep) @@ -998,6 +1002,8 @@ struct hid_field *hid_find_field(struct hid_device *hdev, unsigned int report_ty int hid_set_field(struct hid_field *, unsigned, __s32); int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, int interrupt); +int hid_safe_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, + size_t bufsize, u32 size, int interrupt); struct hid_field *hidinput_get_led_field(struct hid_device *hid); unsigned int hidinput_count_leds(struct hid_device *hid); __s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code); @@ -1266,8 +1272,8 @@ static inline u32 hid_report_len(struct hid_report *report) return DIV_ROUND_UP(report->size, 8) + (report->id > 0); } -int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, - int interrupt); +int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, + size_t bufsize, u32 size, int interrupt); /* HID quirks API */ unsigned long hid_lookup_quirk(const struct hid_device *hdev); diff --git a/include/linux/hid_bpf.h b/include/linux/hid_bpf.h index a2e47dbcf82c8b..19fffa4574a47c 100644 --- a/include/linux/hid_bpf.h +++ b/include/linux/hid_bpf.h @@ -72,8 +72,8 @@ struct hid_ops { int (*hid_hw_output_report)(struct hid_device *hdev, __u8 *buf, size_t len, u64 source, bool from_bpf); int (*hid_input_report)(struct hid_device *hid, enum hid_report_type type, - u8 *data, u32 size, int interrupt, u64 source, bool from_bpf, - bool lock_already_taken); + u8 *data, size_t bufsize, u32 size, int interrupt, u64 source, + bool from_bpf, bool lock_already_taken); struct module *owner; const struct bus_type *bus_type; }; @@ -200,7 +200,8 @@ struct hid_bpf { #ifdef CONFIG_HID_BPF u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type, u8 *data, - u32 *size, int interrupt, u64 source, bool from_bpf); + size_t *buf_size, u32 *size, int interrupt, u64 source, + bool from_bpf); int dispatch_hid_bpf_raw_requests(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, u32 size, enum hid_report_type rtype, @@ -215,8 +216,11 @@ int hid_bpf_device_init(struct hid_device *hid); const u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, const u8 *rdesc, unsigned int *size); #else /* CONFIG_HID_BPF */ static inline u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type, - u8 *data, u32 *size, int interrupt, - u64 source, bool from_bpf) { return data; } + u8 *data, size_t *buf_size, u32 *size, + int interrupt, u64 source, bool from_bpf) +{ + return data; +} static inline int dispatch_hid_bpf_raw_requests(struct hid_device *hdev, unsigned char reportnum, u8 *buf, u32 size, enum hid_report_type rtype, diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 65910437be1ca3..67d4f092464649 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -796,6 +796,23 @@ static inline unsigned huge_page_shift(struct hstate *h) return h->order + PAGE_SHIFT; } +/** + * hugetlb_linear_page_index() - linear_page_index() but in hugetlb + * page size granularity. + * @vma: the hugetlb VMA + * @address: the virtual address within the VMA + * + * Return: the page offset within the mapping in huge page units. + */ +static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, + unsigned long address) +{ + struct hstate *h = hstate_vma(vma); + + return ((address - vma->vm_start) >> huge_page_shift(h)) + + (vma->vm_pgoff >> huge_page_order(h)); +} + static inline bool order_is_gigantic(unsigned int order) { return order > MAX_PAGE_ORDER; diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h index 593f5d4e108b2c..755281fab23de2 100644 --- a/include/linux/hugetlb_inline.h +++ b/include/linux/hugetlb_inline.h @@ -30,7 +30,7 @@ static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) #endif -static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma) +static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma) { return is_vm_hugetlb_flags(vma->vm_flags); } diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h index f8e9f5d36d2a2d..a97b1d01f3acf0 100644 --- a/include/linux/ieee80211-eht.h +++ b/include/linux/ieee80211-eht.h @@ -251,8 +251,8 @@ struct ieee80211_eht_operation_info { #define IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF 0x40 #define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK 0x07 -#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ 0x08 -#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ 0x30 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ 0x10 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ 0x20 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ 0x40 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK 0x78 #define IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP 0x80 diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index e6272f9c5e42cc..20cc16ea4e5abc 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -147,11 +147,13 @@ extern __be16 vlan_dev_vlan_proto(const struct net_device *dev); * @priority: skb priority * @vlan_qos: vlan priority: (skb->priority << 13) & 0xE000 * @next: pointer to next struct + * @rcu: used for deferred freeing of mapping nodes */ struct vlan_priority_tci_mapping { u32 priority; u16 vlan_qos; - struct vlan_priority_tci_mapping *next; + struct vlan_priority_tci_mapping __rcu *next; + struct rcu_head rcu; }; struct proc_dir_entry; @@ -177,7 +179,7 @@ struct vlan_dev_priv { unsigned int nr_ingress_mappings; u32 ingress_priority_map[8]; unsigned int nr_egress_mappings; - struct vlan_priority_tci_mapping *egress_priority_map[16]; + struct vlan_priority_tci_mapping __rcu *egress_priority_map[16]; __be16 vlan_proto; u16 vlan_id; @@ -209,19 +211,24 @@ static inline u16 vlan_dev_get_egress_qos_mask(struct net_device *dev, u32 skprio) { struct vlan_priority_tci_mapping *mp; + u16 vlan_qos = 0; - smp_rmb(); /* coupled with smp_wmb() in vlan_dev_set_egress_priority() */ + rcu_read_lock(); - mp = vlan_dev_priv(dev)->egress_priority_map[(skprio & 0xF)]; + mp = rcu_dereference(vlan_dev_priv(dev)->egress_priority_map[skprio & 0xF]); while (mp) { if (mp->priority == skprio) { - return mp->vlan_qos; /* This should already be shifted - * to mask correctly with the - * VLAN's TCI */ + vlan_qos = READ_ONCE(mp->vlan_qos); + break; } - mp = mp->next; + mp = rcu_dereference(mp->next); } - return 0; + rcu_read_unlock(); + + /* This should already be shifted to mask correctly with + * the VLAN's TCI. + */ + return vlan_qos; } extern bool vlan_do_receive(struct sk_buff **skb); diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index e19872e37e067f..c3b1b92910d94e 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -177,7 +177,7 @@ struct io_pgtable_cfg { } arm_mali_lpae_cfg; struct { - u64 ttbr[4]; + void *ttbr[4]; u32 n_ttbrs; u32 n_levels; } apple_dart_cfg; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 555597b54083cd..5816d8fb9163ab 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -289,12 +289,18 @@ enum iommu_resv_type { IOMMU_RESV_MSI, /* Software-managed MSI translation window */ IOMMU_RESV_SW_MSI, + /* + * Memory regions which must be mapped with the specified mapping + * at all times. + */ + IOMMU_RESV_TRANSLATED, }; /** * struct iommu_resv_region - descriptor for a reserved memory region * @list: Linked list pointers * @start: System physical start address of the region + * @start: Device virtual start address of the region for IOMMU_RESV_TRANSLATED * @length: Length of the region in bytes * @prot: IOMMU Protection flags (READ/WRITE/...) * @type: Type of the reserved region @@ -303,6 +309,7 @@ enum iommu_resv_type { struct iommu_resv_region { struct list_head list; phys_addr_t start; + dma_addr_t dva; size_t length; int prot; enum iommu_resv_type type; @@ -838,6 +845,7 @@ struct iommu_fault_param { * @pci_32bit_workaround: Limit DMA allocations to 32-bit IOVAs * @require_direct: device requires IOMMU_RESV_DIRECT regions * @shadow_on_flush: IOTLB flushes are used to sync shadow tables + * @require_translated: device requires IOMMU_RESV_TRANSLATED regions * * TODO: migrate other per device data pointers under iommu_dev_data, e.g. * struct iommu_group *iommu_group; @@ -853,6 +861,7 @@ struct dev_iommu { u32 pci_32bit_workaround:1; u32 require_direct:1; u32 shadow_on_flush:1; + u32 require_translated:1; }; int iommu_device_register(struct iommu_device *iommu, @@ -938,6 +947,9 @@ extern bool iommu_default_passthrough(void); extern struct iommu_resv_region * iommu_alloc_resv_region(phys_addr_t start, size_t length, int prot, enum iommu_resv_type type, gfp_t gfp); +extern struct iommu_resv_region * +iommu_alloc_resv_region_tr(phys_addr_t start, dma_addr_t dva_start, size_t length, + int prot, enum iommu_resv_type type, gfp_t gfp); extern int iommu_get_group_resv_regions(struct iommu_group *group, struct list_head *head); diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h index bdd2e0652bc30b..53edd69acb9bdb 100644 --- a/include/linux/iopoll.h +++ b/include/linux/iopoll.h @@ -159,7 +159,7 @@ * * This macro does not rely on timekeeping. Hence it is safe to call even when * timekeeping is suspended, at the expense of an underestimation of wall clock - * time, which is rather minimal with a non-zero delay_us. + * time, which is rather minimal with a non-zero @delay_us. * * When available, you'll probably want to use one of the specialized * macros defined below rather than this macro directly. @@ -167,9 +167,9 @@ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either * case, the last read value at @args is stored in @val. */ -#define read_poll_timeout_atomic(op, val, cond, sleep_us, timeout_us, \ - sleep_before_read, args...) \ - poll_timeout_us_atomic((val) = op(args), cond, sleep_us, timeout_us, sleep_before_read) +#define read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, \ + delay_before_read, args...) \ + poll_timeout_us_atomic((val) = op(args), cond, delay_us, timeout_us, delay_before_read) /** * readx_poll_timeout - Periodically poll an address until a condition is met or a timeout occurs diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index b78488df6c989d..102924fd0193c1 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -398,9 +398,6 @@ struct gicv5_its_itt_cfg { void gicv5_init_lpis(u32 max); void gicv5_deinit_lpis(void); -int gicv5_alloc_lpi(void); -void gicv5_free_lpi(u32 lpi); - void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 6b76e7a6f4c224..8c13426f4117a9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -318,7 +318,8 @@ static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop) struct kvm_mmio_fragment { gpa_t gpa; void *data; - unsigned len; + u64 val; + unsigned int len; }; struct kvm_vcpu { @@ -1029,6 +1030,13 @@ static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) return NULL; } +static inline bool kvm_is_vcpu_creation_in_progress(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->lock); + + return kvm->created_vcpus != atomic_read(&kvm->online_vcpus); +} + void kvm_destroy_vcpus(struct kvm *kvm); int kvm_trylock_all_vcpus(struct kvm *kvm); diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h index 382c56a97bba1d..584db296e43b20 100644 --- a/include/linux/lsm_audit.h +++ b/include/linux/lsm_audit.h @@ -94,7 +94,7 @@ struct common_audit_data { #endif char *kmod_name; struct lsm_ioctlop_audit *op; - struct file *file; + const struct file *file; struct lsm_ibpkey_audit *ibpkey; struct lsm_ibendport_audit *ibendport; int reason; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 8c42b4bde09c07..b4958167e38196 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -191,6 +191,9 @@ LSM_HOOK(int, 0, file_permission, struct file *file, int mask) LSM_HOOK(int, 0, file_alloc_security, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_release, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file) +LSM_HOOK(int, 0, backing_file_alloc, struct file *backing_file, + const struct file *user_file) +LSM_HOOK(void, LSM_RET_VOID, backing_file_free, struct file *backing_file) LSM_HOOK(int, 0, file_ioctl, struct file *file, unsigned int cmd, unsigned long arg) LSM_HOOK(int, 0, file_ioctl_compat, struct file *file, unsigned int cmd, @@ -198,6 +201,8 @@ LSM_HOOK(int, 0, file_ioctl_compat, struct file *file, unsigned int cmd, LSM_HOOK(int, 0, mmap_addr, unsigned long addr) LSM_HOOK(int, 0, mmap_file, struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) +LSM_HOOK(int, 0, mmap_backing_file, struct vm_area_struct *vma, + struct file *backing_file, struct file *user_file) LSM_HOOK(int, 0, file_mprotect, struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) LSM_HOOK(int, 0, file_lock, struct file *file, unsigned int cmd) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index d48bf0ad26f4f7..b4f8cad53ddb03 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -104,6 +104,7 @@ struct security_hook_list { struct lsm_blob_sizes { unsigned int lbs_cred; unsigned int lbs_file; + unsigned int lbs_backing_file; unsigned int lbs_ib; unsigned int lbs_inode; unsigned int lbs_sock; diff --git a/include/linux/memory_ordering_model.h b/include/linux/memory_ordering_model.h new file mode 100644 index 00000000000000..267a12ca66307e --- /dev/null +++ b/include/linux/memory_ordering_model.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_MEMORY_ORDERING_MODEL_H +#define __ASM_MEMORY_ORDERING_MODEL_H + +/* Arch hooks to implement the PR_{GET_SET}_MEM_MODEL prctls */ + +struct task_struct; +int arch_prctl_mem_model_get(struct task_struct *t); +int arch_prctl_mem_model_set(struct task_struct *t, unsigned long val); + +#endif diff --git a/include/linux/mm.h b/include/linux/mm.h index abb4963c1f0640..2d6d268a279872 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4078,10 +4078,9 @@ static inline void mmap_action_ioremap_full(struct vm_area_desc *desc, mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc)); } -void mmap_action_prepare(struct mmap_action *action, - struct vm_area_desc *desc); -int mmap_action_complete(struct mmap_action *action, - struct vm_area_struct *vma); +int mmap_action_prepare(struct vm_area_desc *desc); +int mmap_action_complete(struct vm_area_struct *vma, + struct mmap_action *action, bool is_compat); /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index e9e964c20e5307..9dc4750296af99 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -329,6 +329,8 @@ struct mmc_card { #define MMC_QUIRK_BROKEN_CACHE_FLUSH (1<<16) /* Don't flush cache until the write has occurred */ #define MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY (1<<17) /* Disable broken SD poweroff notify support */ #define MMC_QUIRK_NO_UHS_DDR50_TUNING (1<<18) /* Disable DDR50 tuning */ +#define MMC_QUIRK_BROKEN_MDT (1<<19) /* Wrong manufacturing year */ +#define MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME (1<<20) /* Secure erase/trim time is fixed regardless of size */ bool written_flag; /* Indicates eMMC has been written since power on */ bool reenable_cmdq; /* Re-enable Command Queue */ diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 7d22d4c4ea2e7a..8667f72503d958 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -426,14 +426,9 @@ extern char *parse_args(const char *name, void *arg, parse_unknown_fn unknown); /* Called by module remove. */ -#ifdef CONFIG_SYSFS -extern void destroy_params(const struct kernel_param *params, unsigned num); -#else -static inline void destroy_params(const struct kernel_param *params, - unsigned num) -{ -} -#endif /* !CONFIG_SYSFS */ +#ifdef CONFIG_MODULES +void module_destroy_params(const struct kernel_param *params, unsigned int num); +#endif /* All the helper functions */ /* The macros to do compile-time type checking stolen from Jakub diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 6a024cf1c53ac1..f2f80103649d5d 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -290,6 +290,12 @@ SPI_MEM_OP_NO_DUMMY, \ SPI_MEM_OP_NO_DATA) +#define SPINAND_PAGE_READ_PACKED_8D_8D_0_OP(addr) \ + SPI_MEM_OP(SPI_MEM_DTR_OP_PACKED_CMD(0x13, addr >> 16, 8), \ + SPI_MEM_DTR_OP_ADDR(2, addr & 0xffff, 8), \ + SPI_MEM_OP_NO_DUMMY, \ + SPI_MEM_OP_NO_DATA) + #define SPINAND_PAGE_READ_FROM_CACHE_8D_8D_8D_OP(addr, ndummy, buf, len, freq) \ SPI_MEM_OP(SPI_MEM_DTR_OP_RPT_CMD(0x9d, 8), \ SPI_MEM_DTR_OP_ADDR(2, addr, 8), \ @@ -482,6 +488,7 @@ struct spinand_ecc_info { #define SPINAND_HAS_PROG_PLANE_SELECT_BIT BIT(2) #define SPINAND_HAS_READ_PLANE_SELECT_BIT BIT(3) #define SPINAND_NO_RAW_ACCESS BIT(4) +#define SPINAND_ODTR_PACKED_PAGE_READ BIT(5) /** * struct spinand_ondie_ecc_conf - private SPI-NAND on-die ECC engine structure diff --git a/include/linux/mutex.h b/include/linux/mutex.h index ecaa0440f6ec48..f57d2a97da57fd 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -87,12 +87,12 @@ do { \ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) #ifdef CONFIG_DEBUG_LOCK_ALLOC -void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key); +void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key); static inline void __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) { - mutex_init_lockep(lock, name, key); + mutex_init_lockdep(lock, name, key); } #else extern void mutex_init_generic(struct mutex *lock); @@ -146,7 +146,7 @@ static inline void __mutex_init(struct mutex *lock, const char *name, { mutex_rt_init_generic(lock); } -#endif /* !CONFIG_LOCKDEP */ +#endif /* !CONFIG_DEBUG_LOCK_ALLOC */ #endif /* CONFIG_PREEMPT_RT */ #ifdef CONFIG_DEBUG_MUTEXES diff --git a/include/linux/nstree.h b/include/linux/nstree.h index 175e4625bfa6d9..5b64d457288198 100644 --- a/include/linux/nstree.h +++ b/include/linux/nstree.h @@ -61,7 +61,7 @@ static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_t /** * ns_tree_add_raw - Add a namespace to a namespace - * @ns: Namespace to add + * @__ns: Namespace to add * * This function adds a namespace to the appropriate namespace tree * without assigning a id. @@ -70,7 +70,7 @@ static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_t /** * ns_tree_add - Add a namespace to a namespace tree - * @ns: Namespace to add + * @__ns: Namespace to add * * This function assigns a new id to the namespace and adds it to the * appropriate namespace tree and list. @@ -81,7 +81,7 @@ static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_t /** * ns_tree_remove - Remove a namespace from a namespace tree - * @ns: Namespace to remove + * @__ns: Namespace to remove * * This function removes a namespace from the appropriate namespace * tree and list. diff --git a/include/linux/padata.h b/include/linux/padata.h index 765f2778e264a5..b6232bea6edf59 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -149,23 +149,23 @@ struct padata_mt_job { /** * struct padata_instance - The overall control structure. * - * @cpu_online_node: Linkage for CPU online callback. - * @cpu_dead_node: Linkage for CPU offline callback. + * @cpuhp_node: Linkage for CPU hotplug callbacks. * @parallel_wq: The workqueue used for parallel work. * @serial_wq: The workqueue used for serial work. * @pslist: List of padata_shell objects attached to this instance. * @cpumask: User supplied cpumasks for parallel and serial works. + * @validate_cpumask: Internal cpumask used to validate @cpumask during hotplug. * @kobj: padata instance kernel object. * @lock: padata instance lock. * @flags: padata flags. */ struct padata_instance { - struct hlist_node cpu_online_node; - struct hlist_node cpu_dead_node; + struct hlist_node cpuhp_node; struct workqueue_struct *parallel_wq; struct workqueue_struct *serial_wq; struct list_head pslist; struct padata_cpumask cpumask; + cpumask_var_t validate_cpumask; struct kobject kobj; struct mutex lock; u8 flags; diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d512301..57e9463e4347bc 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -575,12 +575,6 @@ struct pci_dev { u8 supported_speeds; /* Supported Link Speeds Vector */ phys_addr_t rom; /* Physical address if not from BAR */ size_t romlen; /* Length if not from BAR */ - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; - unsigned long priv_flags; /* Private flags for the PCI driver */ /* These methods index pci_reset_fn_methods[] */ diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h index 38a82d65e58e96..951d333622685e 100644 --- a/include/linux/pgalloc_tag.h +++ b/include/linux/pgalloc_tag.h @@ -181,7 +181,7 @@ static inline struct alloc_tag *__pgalloc_tag_get(struct page *page) if (get_page_tag_ref(page, &ref, &handle)) { alloc_tag_sub_check(&ref); - if (ref.ct) + if (ref.ct && !is_codetag_empty(&ref)) tag = ct_to_alloc_tag(ref.ct); put_page_tag_ref(handle); } diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 93ba0143ca476a..02f55d6aa89311 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -49,8 +49,8 @@ struct dev_pm_domain_attach_data { const char * const *pd_names; - const u32 num_pd_names; - const u32 pd_flags; + u32 num_pd_names; + u32 pd_flags; }; struct dev_pm_domain_list { @@ -121,6 +121,13 @@ struct dev_pm_domain_list { * powered-off until the ->sync_state() callback is * invoked. This flag informs genpd to allow a * power-off without waiting for ->sync_state(). + * GENPD_FLAG_DEFER_OFF: Defer powerdown if there are any consumer + * device fwlinks indicating that some consumer + * devices have not yet probed. This is useful + * for power domains which are active at boot and + * must not be shut down until all consumers + * complete their probe sequence. + */ #define GENPD_FLAG_PM_CLK (1U << 0) #define GENPD_FLAG_IRQ_SAFE (1U << 1) @@ -133,6 +140,7 @@ struct dev_pm_domain_list { #define GENPD_FLAG_DEV_NAME_FW (1U << 8) #define GENPD_FLAG_NO_SYNC_STATE (1U << 9) #define GENPD_FLAG_NO_STAY_ON (1U << 10) +#define GENPD_FLAG_DEFER_OFF (1U << 11) enum gpd_status { GENPD_STATE_ON = 0, /* PM domain is on */ diff --git a/include/linux/ppp_defs.h b/include/linux/ppp_defs.h index b7e57fdbd4139a..b1d1f46d7d3be5 100644 --- a/include/linux/ppp_defs.h +++ b/include/linux/ppp_defs.h @@ -8,6 +8,7 @@ #define _PPP_DEFS_H_ #include +#include #include #define PPP_FCS(fcs, c) crc_ccitt_byte(fcs, c) @@ -25,4 +26,19 @@ static inline bool ppp_proto_is_valid(u16 proto) return !!((proto & 0x0101) == 0x0001); } +/** + * ppp_skb_is_compressed_proto - checks if PPP protocol in a skb is compressed + * @skb: skb to check + * + * Check if the PPP protocol field is compressed (the least significant + * bit of the most significant octet is 1). skb->data must point to the PPP + * protocol header. + * + * Return: Whether the PPP protocol field is compressed. + */ +static inline bool ppp_skb_is_compressed_proto(const struct sk_buff *skb) +{ + return unlikely(skb->data[0] & 0x01); +} + #endif /* _PPP_DEFS_H_ */ diff --git a/include/linux/printk.h b/include/linux/printk.h index 63d516c873b4c4..f594c1266bfd41 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -801,8 +801,22 @@ static inline void print_hex_dump_debug(const char *prefix_str, int prefix_type, } #endif +#if defined(DEBUG) +#define print_hex_dump_devel(prefix_str, prefix_type, rowsize, \ + groupsize, buf, len, ascii) \ + print_hex_dump(KERN_DEBUG, prefix_str, prefix_type, rowsize, \ + groupsize, buf, len, ascii) +#else +static inline void print_hex_dump_devel(const char *prefix_str, int prefix_type, + int rowsize, int groupsize, + const void *buf, size_t len, bool ascii) +{ +} +#endif + /** - * print_hex_dump_bytes - shorthand form of print_hex_dump() with default params + * print_hex_dump_bytes - shorthand form of print_hex_dump_debug() with default + * params * @prefix_str: string to prefix each line with; * caller supplies trailing spaces for alignment if desired * @prefix_type: controls whether prefix of an offset, address, or none @@ -810,7 +824,7 @@ static inline void print_hex_dump_debug(const char *prefix_str, int prefix_type, * @buf: data blob to dump * @len: number of bytes in the @buf * - * Calls print_hex_dump(), with log level of KERN_DEBUG, + * Calls print_hex_dump_debug(), with log level of KERN_DEBUG, * rowsize of 16, groupsize of 1, and ASCII output included. */ #define print_hex_dump_bytes(prefix_str, prefix_type, buf, len) \ diff --git a/include/linux/quotaops.h b/include/linux/quotaops.h index c334f82ed385a1..f9c0f9d7c9d93a 100644 --- a/include/linux/quotaops.h +++ b/include/linux/quotaops.h @@ -44,14 +44,7 @@ int dquot_initialize(struct inode *inode); bool dquot_initialize_needed(struct inode *inode); void dquot_drop(struct inode *inode); struct dquot *dqget(struct super_block *sb, struct kqid qid); -static inline struct dquot *dqgrab(struct dquot *dquot) -{ - /* Make sure someone else has active reference to dquot */ - WARN_ON_ONCE(!atomic_read(&dquot->dq_count)); - WARN_ON_ONCE(!test_bit(DQ_ACTIVE_B, &dquot->dq_flags)); - atomic_inc(&dquot->dq_count); - return dquot; -} +struct dquot *dqgrab(struct dquot *dquot); static inline bool dquot_is_busy(struct dquot *dquot) { diff --git a/include/linux/randomize_kstack.h b/include/linux/randomize_kstack.h index 1d982dbdd0d0bc..5d3916ca747cc2 100644 --- a/include/linux/randomize_kstack.h +++ b/include/linux/randomize_kstack.h @@ -9,7 +9,6 @@ DECLARE_STATIC_KEY_MAYBE(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, randomize_kstack_offset); -DECLARE_PER_CPU(u32, kstack_offset); /* * Do not use this anywhere else in the kernel. This is used here because @@ -50,15 +49,14 @@ DECLARE_PER_CPU(u32, kstack_offset); * add_random_kstack_offset - Increase stack utilization by previously * chosen random offset * - * This should be used in the syscall entry path when interrupts and - * preempt are disabled, and after user registers have been stored to - * the stack. For testing the resulting entropy, please see: - * tools/testing/selftests/lkdtm/stack-entropy.sh + * This should be used in the syscall entry path after user registers have been + * stored to the stack. Preemption may be enabled. For testing the resulting + * entropy, please see: tools/testing/selftests/lkdtm/stack-entropy.sh */ #define add_random_kstack_offset() do { \ if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, \ &randomize_kstack_offset)) { \ - u32 offset = raw_cpu_read(kstack_offset); \ + u32 offset = current->kstack_offset; \ u8 *ptr = __kstack_alloca(KSTACK_OFFSET_MAX(offset)); \ /* Keep allocation even after "ptr" loses scope. */ \ asm volatile("" :: "r"(ptr) : "memory"); \ @@ -69,9 +67,9 @@ DECLARE_PER_CPU(u32, kstack_offset); * choose_random_kstack_offset - Choose the random offset for the next * add_random_kstack_offset() * - * This should only be used during syscall exit when interrupts and - * preempt are disabled. This position in the syscall flow is done to - * frustrate attacks from userspace attempting to learn the next offset: + * This should only be used during syscall exit. Preemption may be enabled. This + * position in the syscall flow is done to frustrate attacks from userspace + * attempting to learn the next offset: * - Maximize the timing uncertainty visible from userspace: if the * offset is chosen at syscall entry, userspace has much more control * over the timing between choosing offsets. "How long will we be in @@ -85,14 +83,20 @@ DECLARE_PER_CPU(u32, kstack_offset); #define choose_random_kstack_offset(rand) do { \ if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, \ &randomize_kstack_offset)) { \ - u32 offset = raw_cpu_read(kstack_offset); \ + u32 offset = current->kstack_offset; \ offset = ror32(offset, 5) ^ (rand); \ - raw_cpu_write(kstack_offset, offset); \ + current->kstack_offset = offset; \ } \ } while (0) + +static inline void random_kstack_task_init(struct task_struct *tsk) +{ + tsk->kstack_offset = 0; +} #else /* CONFIG_RANDOMIZE_KSTACK_OFFSET */ #define add_random_kstack_offset() do { } while (0) #define choose_random_kstack_offset(rand) do { } while (0) +#define random_kstack_task_init(tsk) do { } while (0) #endif /* CONFIG_RANDOMIZE_KSTACK_OFFSET */ #endif diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 2abba7552605c5..e3bc442256922b 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h @@ -261,6 +261,35 @@ static inline void list_replace_rcu(struct list_head *old, old->prev = LIST_POISON2; } +static inline void __list_splice_rcu(struct list_head *list, + struct list_head *prev, + struct list_head *next) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + + last->next = next; + first->prev = prev; + next->prev = last; + rcu_assign_pointer(list_next_rcu(prev), first); +} + +/** + * list_splice_rcu - splice a non-RCU list into an RCU-protected list, + * designed for stacks. + * @list: the non RCU-protected list to splice + * @head: the place in the existing RCU-protected list to splice + * + * The list pointed to by @head can be RCU-read traversed concurrently with + * this function. + */ +static inline void list_splice_rcu(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice_rcu(list, head, head->next); +} + /** * __list_splice_init_rcu - join an RCU-protected list into an existing list. * @list: the RCU-protected list to splice diff --git a/include/linux/rseq.h b/include/linux/rseq.h index b9d62fc2140dd1..7ef79b25e714b9 100644 --- a/include/linux/rseq.h +++ b/include/linux/rseq.h @@ -9,6 +9,11 @@ void __rseq_handle_slowpath(struct pt_regs *regs); +static __always_inline bool rseq_v2(struct task_struct *t) +{ + return IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY) && likely(t->rseq.event.has_rseq > 1); +} + /* Invoked from resume_user_mode_work() */ static inline void rseq_handle_slowpath(struct pt_regs *regs) { @@ -16,8 +21,7 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs) if (current->rseq.event.slowpath) __rseq_handle_slowpath(regs); } else { - /* '&' is intentional to spare one conditional branch */ - if (current->rseq.event.sched_switch & current->rseq.event.has_rseq) + if (current->rseq.event.sched_switch && current->rseq.event.has_rseq) __rseq_handle_slowpath(regs); } } @@ -30,9 +34,9 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs); */ static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { - if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY)) { - /* '&' is intentional to spare one conditional branch */ - if (current->rseq.event.has_rseq & current->rseq.event.user_irq) + if (rseq_v2(current)) { + /* has_rseq is implied in rseq_v2() */ + if (current->rseq.event.user_irq) __rseq_signal_deliver(ksig->sig, regs); } else { if (current->rseq.event.has_rseq) @@ -50,15 +54,22 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t) { struct rseq_event *ev = &t->rseq.event; - if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY)) { + /* + * Only apply the user_irq optimization for RSEQ ABI V2 registrations. + * Legacy users like TCMalloc rely on the original ABI V1 behaviour + * which updates IDs on every context swtich. + */ + if (rseq_v2(t)) { /* - * Avoid a boat load of conditionals by using simple logic - * to determine whether NOTIFY_RESUME needs to be raised. + * Avoid a boat load of conditionals by using simple logic to + * determine whether TIF_NOTIFY_RESUME or TIF_RSEQ needs to be + * raised. * - * It's required when the CPU or MM CID has changed or - * the entry was from user space. + * It's required when the CPU or MM CID has changed or the entry + * was via interrupt from user space. ev->has_rseq does not have + * to be evaluated here because rseq_v2() implies has_rseq. */ - bool raise = (ev->user_irq | ev->ids_changed) & ev->has_rseq; + bool raise = ev->user_irq | ev->ids_changed; if (raise) { ev->sched_switch = true; @@ -66,6 +77,7 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t) } } else { if (ev->has_rseq) { + t->rseq.event.ids_changed = true; t->rseq.event.sched_switch = true; rseq_raise_notify_resume(t); } @@ -119,6 +131,8 @@ static inline void rseq_virt_userspace_exit(void) static inline void rseq_reset(struct task_struct *t) { + /* Protect against preemption and membarrier IPI */ + guard(irqsave)(); memset(&t->rseq, 0, sizeof(t->rseq)); t->rseq.ids.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; } @@ -159,6 +173,7 @@ static inline unsigned int rseq_alloc_align(void) } #else /* CONFIG_RSEQ */ +static inline bool rseq_v2(struct task_struct *t) { return false; } static inline void rseq_handle_slowpath(struct pt_regs *regs) { } static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { } static inline void rseq_sched_switch_event(struct task_struct *t) { } diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h index c6831c93cd6ee5..413a3543fbe8ed 100644 --- a/include/linux/rseq_entry.h +++ b/include/linux/rseq_entry.h @@ -110,6 +110,20 @@ static __always_inline void rseq_slice_clear_grant(struct task_struct *t) t->rseq.slice.state.granted = false; } +/* + * Open coded, so it can be invoked within a user access region. + * + * This clears the user space state of the time slice extensions field only when + * the task has registered the optimized RSEQ_ABI V2. Some legacy registrations, + * e.g. TCMalloc, have conflicting non-ABI fields in struct RSEQ, which would be + * overwritten by an unconditional write. + */ +#define rseq_slice_clear_user(rseq, efault) \ +do { \ + if (rseq_slice_extension_enabled()) \ + unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); \ +} while (0) + static __always_inline bool rseq_grant_slice_extension(bool work_pending) { struct task_struct *curr = current; @@ -220,10 +234,10 @@ static __always_inline bool rseq_slice_extension_enabled(void) { return false; } static __always_inline bool rseq_arm_slice_extension_timer(void) { return false; } static __always_inline void rseq_slice_clear_grant(struct task_struct *t) { } static __always_inline bool rseq_grant_slice_extension(bool work_pending) { return false; } +#define rseq_slice_clear_user(rseq, efault) do { } while (0) #endif /* !CONFIG_RSEQ_SLICE_EXTENSION */ bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr); -bool rseq_debug_validate_ids(struct task_struct *t); static __always_inline void rseq_note_user_irq_entry(void) { @@ -343,43 +357,6 @@ bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, return false; } -/* - * On debug kernels validate that user space did not mess with it if the - * debug branch is enabled. - */ -bool rseq_debug_validate_ids(struct task_struct *t) -{ - struct rseq __user *rseq = t->rseq.usrptr; - u32 cpu_id, uval, node_id; - - /* - * On the first exit after registering the rseq region CPU ID is - * RSEQ_CPU_ID_UNINITIALIZED and node_id in user space is 0! - */ - node_id = t->rseq.ids.cpu_id != RSEQ_CPU_ID_UNINITIALIZED ? - cpu_to_node(t->rseq.ids.cpu_id) : 0; - - scoped_user_read_access(rseq, efault) { - unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault); - if (cpu_id != t->rseq.ids.cpu_id) - goto die; - unsafe_get_user(uval, &rseq->cpu_id, efault); - if (uval != cpu_id) - goto die; - unsafe_get_user(uval, &rseq->node_id, efault); - if (uval != node_id) - goto die; - unsafe_get_user(uval, &rseq->mm_cid, efault); - if (uval != t->rseq.ids.mm_cid) - goto die; - } - return true; -die: - t->rseq.event.fatal = true; -efault: - return false; -} - #endif /* RSEQ_BUILD_SLOW_PATH */ /* @@ -489,37 +466,50 @@ rseq_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long c * faults in task context are fatal too. */ static rseq_inline -bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, - u32 node_id, u64 *csaddr) +bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, u64 *csaddr) { struct rseq __user *rseq = t->rseq.usrptr; - if (static_branch_unlikely(&rseq_debug_enabled)) { - if (!rseq_debug_validate_ids(t)) - return false; - } - scoped_user_rw_access(rseq, efault) { + /* Validate the R/O fields for debug and optimized mode */ + if (static_branch_unlikely(&rseq_debug_enabled) || rseq_v2(t)) { + u32 cpu_id, uval; + + unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault); + if (cpu_id != t->rseq.ids.cpu_id) + goto die; + unsafe_get_user(uval, &rseq->cpu_id, efault); + if (uval != cpu_id) + goto die; + unsafe_get_user(uval, &rseq->node_id, efault); + if (uval != t->rseq.ids.node_id) + goto die; + unsafe_get_user(uval, &rseq->mm_cid, efault); + if (uval != t->rseq.ids.mm_cid) + goto die; + } + unsafe_put_user(ids->cpu_id, &rseq->cpu_id_start, efault); unsafe_put_user(ids->cpu_id, &rseq->cpu_id, efault); - unsafe_put_user(node_id, &rseq->node_id, efault); + unsafe_put_user(ids->node_id, &rseq->node_id, efault); unsafe_put_user(ids->mm_cid, &rseq->mm_cid, efault); if (csaddr) unsafe_get_user(*csaddr, &rseq->rseq_cs, efault); - /* Open coded, so it's in the same user access region */ - if (rseq_slice_extension_enabled()) { - /* Unconditionally clear it, no point in conditionals */ - unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); - } + /* RSEQ ABI V2 only operations */ + if (rseq_v2(t)) + rseq_slice_clear_user(rseq, efault); } rseq_slice_clear_grant(t); /* Cache the new values */ - t->rseq.ids.cpu_cid = ids->cpu_cid; + t->rseq.ids = *ids; rseq_stat_inc(rseq_stats.ids); rseq_trace_update(t, ids); return true; + +die: + t->rseq.event.fatal = true; efault: return false; } @@ -529,11 +519,11 @@ bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, * is in a critical section. */ static rseq_inline bool rseq_update_usr(struct task_struct *t, struct pt_regs *regs, - struct rseq_ids *ids, u32 node_id) + struct rseq_ids *ids) { u64 csaddr; - if (!rseq_set_ids_get_csaddr(t, ids, node_id, &csaddr)) + if (!rseq_set_ids_get_csaddr(t, ids, &csaddr)) return false; /* @@ -602,6 +592,14 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t * interrupts disabled */ guard(pagefault)(); + /* + * This optimization is only valid when the task registered for the + * optimized RSEQ_ABI_V2 variant. Some legacy users rely on the original + * RSEQ implementation behaviour which unconditionally updated the IDs. + * rseq_sched_switch_event() ensures that legacy registrations always + * have both sched_switch and ids_changed set, which is compatible with + * the historical TIF_NOTIFY_RESUME behaviour. + */ if (likely(!t->rseq.event.ids_changed)) { struct rseq __user *rseq = t->rseq.usrptr; /* @@ -613,11 +611,9 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t scoped_user_rw_access(rseq, efault) { unsafe_get_user(csaddr, &rseq->rseq_cs, efault); - /* Open coded, so it's in the same user access region */ - if (rseq_slice_extension_enabled()) { - /* Unconditionally clear it, no point in conditionals */ - unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); - } + /* RSEQ ABI V2 only operations */ + if (rseq_v2(t)) + rseq_slice_clear_user(rseq, efault); } rseq_slice_clear_grant(t); @@ -630,12 +626,12 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t } struct rseq_ids ids = { - .cpu_id = task_cpu(t), - .mm_cid = task_mm_cid(t), + .cpu_id = task_cpu(t), + .mm_cid = task_mm_cid(t), + .node_id = cpu_to_node(ids.cpu_id), }; - u32 node_id = cpu_to_node(ids.cpu_id); - return rseq_update_usr(t, regs, &ids, node_id); + return rseq_update_usr(t, regs, &ids); efault: return false; } diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h index 0b42045988db00..85739a63e85e6f 100644 --- a/include/linux/rseq_types.h +++ b/include/linux/rseq_types.h @@ -9,6 +9,12 @@ #ifdef CONFIG_RSEQ struct rseq; +/* + * rseq_event::has_rseq contains the ABI version number so preserving it + * in AND operations requires a mask. + */ +#define RSEQ_HAS_RSEQ_VERSION_MASK 0xff + /** * struct rseq_event - Storage for rseq related event management * @all: Compound to initialize and clear the data efficiently @@ -17,7 +23,8 @@ struct rseq; * exit to user * @ids_changed: Indicator that IDs need to be updated * @user_irq: True on interrupt entry from user mode - * @has_rseq: True if the task has a rseq pointer installed + * @has_rseq: Greater than 0 if the task has a rseq pointer installed. + * Contains the RSEQ version number * @error: Compound error code for the slow path to analyze * @fatal: User space data corrupted or invalid * @slowpath: Indicator that slow path processing via TIF_NOTIFY_RESUME @@ -59,8 +66,9 @@ struct rseq_event { * compiler emit a single compare on 64-bit * @cpu_id: The CPU ID which was written last to user space * @mm_cid: The MM CID which was written last to user space + * @node_id: The node ID which was written last to user space * - * @cpu_id and @mm_cid are updated when the data is written to user space. + * @cpu_id, @mm_cid and @node_id are updated when the data is written to user space. */ struct rseq_ids { union { @@ -70,6 +78,7 @@ struct rseq_ids { u32 mm_cid; }; }; + u32 node_id; }; /** diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 3390d21c95dd1b..21ceefc4a49f27 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -30,10 +30,10 @@ do { \ #ifdef CONFIG_DEBUG_SPINLOCK extern void do_raw_read_lock(rwlock_t *lock) __acquires_shared(lock); - extern int do_raw_read_trylock(rwlock_t *lock); + extern int do_raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); extern void do_raw_read_unlock(rwlock_t *lock) __releases_shared(lock); extern void do_raw_write_lock(rwlock_t *lock) __acquires(lock); - extern int do_raw_write_trylock(rwlock_t *lock); +extern int do_raw_write_trylock(rwlock_t *lock) __cond_acquires(true, lock); extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock); #else # define do_raw_read_lock(rwlock) do {__acquire_shared(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0) diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index 61a852609eab4f..9e02a5f28cd1d2 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -23,7 +23,7 @@ void __lockfunc _raw_write_lock_bh(rwlock_t *lock) __acquires(lock); void __lockfunc _raw_read_lock_irq(rwlock_t *lock) __acquires_shared(lock); void __lockfunc _raw_write_lock_irq(rwlock_t *lock) __acquires(lock); unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) - __acquires(lock); + __acquires_shared(lock); unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) __acquires(lock); int __lockfunc _raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); @@ -36,7 +36,7 @@ void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) __releases_shared(lock); void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) __releases(lock); void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) - __releases(lock); + __releases_shared(lock); void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) __releases(lock); @@ -116,6 +116,7 @@ _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) #endif static inline int __raw_read_trylock(rwlock_t *lock) + __cond_acquires_shared(true, lock) { preempt_disable(); if (do_raw_read_trylock(lock)) { @@ -127,6 +128,7 @@ static inline int __raw_read_trylock(rwlock_t *lock) } static inline int __raw_write_trylock(rwlock_t *lock) + __cond_acquires(true, lock) { preempt_disable(); if (do_raw_write_trylock(lock)) { diff --git a/include/linux/sched.h b/include/linux/sched.h index 5a5d3dbc9cdf33..d41e7a8f9c85f6 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -998,6 +998,9 @@ struct task_struct { unsigned sched_rt_mutex:1; #endif + /* Save user-dumpable when mm goes away */ + unsigned user_dumpable:1; + /* Bit to tell TOMOYO we're in execve(): */ unsigned in_execve:1; unsigned in_iowait:1; @@ -1592,6 +1595,10 @@ struct task_struct { unsigned long prev_lowest_stack; #endif +#ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET + u32 kstack_offset; +#endif + #ifdef CONFIG_X86_MCE void __user *mce_vaddr; __u64 mce_kflags; diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h index c40115d4e34dc4..8b46bb607897f4 100644 --- a/include/linux/sched/deadline.h +++ b/include/linux/sched/deadline.h @@ -33,6 +33,15 @@ struct root_domain; extern void dl_add_task_root_domain(struct task_struct *p); extern void dl_clear_root_domain(struct root_domain *rd); extern void dl_clear_root_domain_cpu(int cpu); +/* + * Return whether moving DL task @p to @new_mask requires moving DL + * bandwidth accounting between root domains. This helper is specific to + * DL bandwidth move accounting semantics and is shared by + * cpuset_can_attach() and set_cpus_allowed_dl() so both paths use the + * same source root-domain test. + */ +extern bool dl_task_needs_bw_move(struct task_struct *p, + const struct cpumask *new_mask); extern u64 dl_cookie; extern bool dl_bw_visited(int cpu, u64 cookie); diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index a22248aebcf902..a4835a7de07ee6 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -739,7 +739,7 @@ static inline int thread_group_empty(struct task_struct *p) extern struct sighand_struct *lock_task_sighand(struct task_struct *task, unsigned long *flags) - __acquires(&task->sighand->siglock); + __cond_acquires(nonnull, &task->sighand->siglock); static inline void unlock_task_sighand(struct task_struct *task, unsigned long *flags) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 45c0022b91ced3..6f8a4ae860da8b 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -141,18 +141,30 @@ struct sched_domain { unsigned int span_weight; /* - * Span of all CPUs in this domain. + * See sched_domain_span(), on why flex arrays are broken. * - * NOTE: this field is variable length. (Allocated dynamically - * by attaching extra space to the end of the structure, - * depending on how many CPUs the kernel has booted up with) - */ unsigned long span[]; + */ }; static inline struct cpumask *sched_domain_span(struct sched_domain *sd) { - return to_cpumask(sd->span); + /* + * Turns out that C flexible arrays are fundamentally broken since it + * is allowed for offsetof(*sd, span) < sizeof(*sd), this means that + * structure initialzation *sd = { ... }; which writes every byte + * inside sizeof(*type), will over-write the start of the flexible + * array. + * + * Luckily, the way we allocate sched_domain is by: + * + * sizeof(*sd) + cpumask_size() + * + * this means that we have sufficient space for the whole flex array + * *outside* of sizeof(*sd). So use that, and avoid using sd->span. + */ + unsigned long *bitmap = (void *)sd + sizeof(*sd); + return to_cpumask(bitmap); } extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], diff --git a/include/linux/security.h b/include/linux/security.h index ee88dd2d2d1f71..8d2d4856934ea9 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -472,11 +472,17 @@ int security_file_permission(struct file *file, int mask); int security_file_alloc(struct file *file); void security_file_release(struct file *file); void security_file_free(struct file *file); +int security_backing_file_alloc(struct file *backing_file, + const struct file *user_file); +void security_backing_file_free(struct file *backing_file); int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); int security_file_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg); int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags); +int security_mmap_backing_file(struct vm_area_struct *vma, + struct file *backing_file, + struct file *user_file); int security_mmap_addr(unsigned long addr); int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot); @@ -1141,6 +1147,15 @@ static inline void security_file_release(struct file *file) static inline void security_file_free(struct file *file) { } +static inline int security_backing_file_alloc(struct file *backing_file, + const struct file *user_file) +{ + return 0; +} + +static inline void security_backing_file_free(struct file *backing_file) +{ } + static inline int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -1160,6 +1175,13 @@ static inline int security_mmap_file(struct file *file, unsigned long prot, return 0; } +static inline int security_mmap_backing_file(struct vm_area_struct *vma, + struct file *backing_file, + struct file *user_file) +{ + return 0; +} + static inline int security_mmap_addr(unsigned long addr) { return cap_mmap_addr(addr); diff --git a/include/linux/soc/apple/dockchannel.h b/include/linux/soc/apple/dockchannel.h new file mode 100644 index 00000000000000..0b7093935ddf47 --- /dev/null +++ b/include/linux/soc/apple/dockchannel.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Apple Dockchannel devices + * Copyright (C) The Asahi Linux Contributors + */ +#ifndef _LINUX_APPLE_DOCKCHANNEL_H_ +#define _LINUX_APPLE_DOCKCHANNEL_H_ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_APPLE_DOCKCHANNEL) + +struct dockchannel; + +struct dockchannel *dockchannel_init(struct platform_device *pdev); + +int dockchannel_send(struct dockchannel *dockchannel, const void *buf, size_t count); +int dockchannel_recv(struct dockchannel *dockchannel, void *buf, size_t count); +int dockchannel_await(struct dockchannel *dockchannel, + void (*callback)(void *cookie, size_t avail), + void *cookie, size_t count); + +#endif +#endif diff --git a/drivers/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h similarity index 100% rename from drivers/soc/apple/mailbox.h rename to include/linux/soc/apple/mailbox.h diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index bda3c528b51553..42a0666e76fcc5 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -78,6 +78,13 @@ struct apple_rtkit; struct apple_rtkit *devm_apple_rtkit_init(struct device *dev, void *cookie, const char *mbox_name, int mbox_idx, const struct apple_rtkit_ops *ops); +/* + * Frees internal RTKit state allocated by devm_apple_rtkit_init(). + * + * @dev: Pointer to the device node this coprocessor is assocated with + * @rtk: Internal RTKit state initialized by devm_apple_rtkit_init() + */ +void devm_apple_rtkit_free(struct device *dev, struct apple_rtkit *rtk); /* * Non-devm version of devm_apple_rtkit_init. Must be freed with @@ -179,4 +186,12 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, */ int apple_rtkit_poll(struct apple_rtkit *rtk); +/* + * Checks if an endpoint with a given index exists + * + * @rtk: RTKit reference + * @ep: endpoint to check for + */ +bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep); + #endif /* _LINUX_APPLE_RTKIT_H_ */ diff --git a/include/linux/spi/spi-mem.h b/include/linux/spi/spi-mem.h index 5774e554c0f027..f54c708f4c506c 100644 --- a/include/linux/spi/spi-mem.h +++ b/include/linux/spi/spi-mem.h @@ -28,6 +28,14 @@ .dtr = true, \ } +#define SPI_MEM_DTR_OP_PACKED_CMD(__opcode, __addr, __buswidth) \ + { \ + .nbytes = 2, \ + .opcode = __opcode << 8 | __addr, \ + .buswidth = __buswidth, \ + .dtr = true, \ + } + #define SPI_MEM_OP_ADDR(__nbytes, __val, __buswidth) \ { \ .nbytes = __nbytes, \ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index e1e2f144af9b41..241277cd34cf33 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -178,7 +178,7 @@ do { \ #ifdef CONFIG_DEBUG_SPINLOCK extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock); - extern int do_raw_spin_trylock(raw_spinlock_t *lock); + extern int do_raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(true, lock); extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); #else static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) @@ -189,6 +189,7 @@ static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) } static inline int do_raw_spin_trylock(raw_spinlock_t *lock) + __cond_acquires(true, lock) { int ret = arch_spin_trylock(&(lock)->raw_lock); diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h index 1e84e71ca495e4..3a50976471d711 100644 --- a/include/linux/spinlock_up.h +++ b/include/linux/spinlock_up.h @@ -48,16 +48,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->slock = 1; } -/* - * Read-write spinlocks. No debug version. - */ -#define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) -#define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) -#define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) - #else /* DEBUG_SPINLOCK */ #define arch_spin_is_locked(lock) ((void)(lock), 0) /* for sched/core.c and kernel_lock.c: */ @@ -68,4 +58,14 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) #define arch_spin_is_contended(lock) (((void)(lock), 0)) +/* + * Read-write spinlocks. No debug version. + */ +#define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) +#define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) +#define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) + #endif /* __LINUX_SPINLOCK_UP_H */ diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h index 72820503514cc2..01011113d22631 100644 --- a/include/linux/stop_machine.h +++ b/include/linux/stop_machine.h @@ -99,7 +99,7 @@ static inline void print_stop_info(const char *log_lvl, struct task_struct *task * stop_machine: freeze the machine on all CPUs and run this function * @fn: the function to run * @data: the data ptr to pass to @fn() - * @cpus: the cpus to run @fn() on (NULL = run on each online CPU) + * @cpus: the cpus to run @fn() on (NULL = one unspecified online CPU) * * Description: This causes a thread to be scheduled on every CPU, which * will run with interrupts disabled. Each CPU specified by @cpus will @@ -133,7 +133,7 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus); * stop_machine_cpuslocked: freeze the machine on all CPUs and run this function * @fn: the function to run * @data: the data ptr to pass to @fn() - * @cpus: the cpus to run @fn() on (NULL = run on each online CPU) + * @cpus: the cpus to run @fn() on (NULL = one unspecified online CPU) * * Same as above. Avoids nested calls to cpus_read_lock(). * diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h index eb4bd62df3190c..ab61bed2f7afc9 100644 --- a/include/linux/sunrpc/debug.h +++ b/include/linux/sunrpc/debug.h @@ -38,6 +38,8 @@ extern unsigned int nlm_debug; do { \ ifdebug(fac) \ __sunrpc_printk(fmt, ##__VA_ARGS__); \ + else \ + no_printk(fmt, ##__VA_ARGS__); \ } while (0) # define dfprintk_rcu(fac, fmt, ...) \ @@ -46,15 +48,15 @@ do { \ rcu_read_lock(); \ __sunrpc_printk(fmt, ##__VA_ARGS__); \ rcu_read_unlock(); \ + } else { \ + no_printk(fmt, ##__VA_ARGS__); \ } \ } while (0) -# define RPC_IFDEBUG(x) x #else # define ifdebug(fac) if (0) -# define dfprintk(fac, fmt, ...) do {} while (0) -# define dfprintk_rcu(fac, fmt, ...) do {} while (0) -# define RPC_IFDEBUG(x) +# define dfprintk(fac, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +# define dfprintk_rcu(fac, fmt, ...) no_printk(fmt, ##__VA_ARGS__) #endif /* diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index ccba79ebf8932b..0dbdf3722537f9 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -95,10 +95,7 @@ struct rpc_task { int tk_rpc_status; /* Result of last RPC operation */ unsigned short tk_flags; /* misc flags */ unsigned short tk_timeouts; /* maj timeouts */ - -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) unsigned short tk_pid; /* debugging aid */ -#endif unsigned char tk_priority : 2,/* Task priority */ tk_garb_retry : 2, tk_cred_retry : 2; diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 02bd6ddb627821..8787b3511c86c2 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1283,12 +1283,8 @@ static inline long ksys_lchown(const char __user *filename, uid_t user, AT_SYMLINK_NOFOLLOW); } -int do_sys_ftruncate(unsigned int fd, loff_t length, int small); - -static inline long ksys_ftruncate(unsigned int fd, loff_t length) -{ - return do_sys_ftruncate(fd, length, 1); -} +#define FTRUNCATE_LFS (1u << 0) /* allow truncating > 32-bit */ +int ksys_ftruncate(unsigned int fd, loff_t length, unsigned int flags); int do_sys_truncate(const char __user *pathname, loff_t length); diff --git a/include/linux/tcp.h b/include/linux/tcp.h index f72eef31fa23cc..c44cf9ae8d16f5 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -228,8 +228,7 @@ struct tcp_sock { u32 sacked_out; /* SACK'd packets */ u16 tcp_header_len; /* Bytes of tcp header to send */ u8 scaling_ratio; /* see tcp_win_from_space() */ - u8 chrono_type : 2, /* current chronograph type */ - repair : 1, + u8 repair : 1, tcp_usec_ts : 1, /* TSval values in usec */ is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ @@ -264,6 +263,7 @@ struct tcp_sock { * total number of data bytes sent. */ u32 snd_sml; /* Last byte of the most recently transmitted small packet */ + u8 chrono_type; /* current chronograph type */ u32 chrono_start; /* Start time in jiffies of a TCP chrono */ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ u32 write_seq; /* Tail(+1) of data held in tcp send buffer */ diff --git a/include/linux/tpm_eventlog.h b/include/linux/tpm_eventlog.h index 891368e82558ed..aff8ea2fa98e55 100644 --- a/include/linux/tpm_eventlog.h +++ b/include/linux/tpm_eventlog.h @@ -131,11 +131,16 @@ struct tcg_algorithm_info { }; #ifndef TPM_MEMREMAP -#define TPM_MEMREMAP(start, size) NULL +static inline void *TPM_MEMREMAP(unsigned long start, size_t size) +{ + return NULL; +} #endif #ifndef TPM_MEMUNMAP -#define TPM_MEMUNMAP(start, size) do{} while(0) +static inline void TPM_MEMUNMAP(void *mapping, size_t size) +{ +} #endif /** diff --git a/include/linux/trace_printk.h b/include/linux/trace_printk.h index bb5874097f24e2..2670ec7f426299 100644 --- a/include/linux/trace_printk.h +++ b/include/linux/trace_printk.h @@ -107,7 +107,6 @@ do { \ __trace_printk(_THIS_IP_, fmt, ##args); \ } while (0) -extern __printf(2, 3) int __trace_bprintk(unsigned long ip, const char *fmt, ...); extern __printf(2, 3) diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 4fe63169d5a211..56328601218c5a 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -331,16 +331,21 @@ static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size) #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ -#ifndef ARCH_HAS_NOCACHE_UACCESS +#ifndef ARCH_HAS_NONTEMPORAL_UACCESS static inline __must_check unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, +copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { + if (can_do_masked_user_access()) + from = mask_user_address(from); + else + if (!access_ok(from, n)) + return n; return __copy_from_user_inatomic(to, from, n); } -#endif /* ARCH_HAS_NOCACHE_UACCESS */ +#endif /* ARCH_HAS_NONTEMPORAL_UACCESS */ extern __must_check int check_zeroed_user(const void __user *from, size_t size); diff --git a/include/linux/usb.h b/include/linux/usb.h index 4aab200158517e..60bd4a8e919a00 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -55,7 +55,8 @@ struct ep_device; * @eusb2_isoc_ep_comp: eUSB2 isoc companion descriptor for this endpoint * @urb_list: urbs queued to this endpoint; maintained by usbcore * @hcpriv: for use by HCD; typically holds hardware dma queue head (QH) - * with one or more transfer descriptors (TDs) per urb + * with one or more transfer descriptors (TDs) per urb; must be preserved + * by core while BW is allocated for the endpoint * @ep_dev: ep_device for sysfs info * @extra: descriptors following this endpoint in the configuration * @extralen: how many bytes of "extra" are valid diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 2bfe3baa63f423..782c42d25db168 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -72,9 +72,6 @@ struct vdpa_mgmt_dev; * struct vdpa_device - representation of a vDPA device * @dev: underlying device * @vmap: the metadata passed to upper layer to be used for mapping - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * @config: the configuration ops for this device. * @map: the map ops for this device * @cf_lock: Protects get and set access to configuration layout. @@ -90,7 +87,6 @@ struct vdpa_mgmt_dev; struct vdpa_device { struct device dev; union virtio_map vmap; - const char *driver_override; const struct vdpa_config_ops *config; const struct virtio_map_ops *map; struct rw_semaphore cf_lock; /* Protects get/set config */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index e90859956514a1..ef02a4996d4517 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -72,13 +72,11 @@ struct vfio_device { u8 iommufd_attached:1; #endif u8 cdev_opened:1; -#ifdef CONFIG_DEBUG_FS /* * debug_root is a static property of the vfio_device * which must be set prior to registering the vfio_device. */ struct dentry *debug_root; -#endif }; /** diff --git a/include/linux/wmi.h b/include/linux/wmi.h index 75cb0c7cfe5713..14fb644e1701c2 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -18,16 +18,12 @@ * struct wmi_device - WMI device structure * @dev: Device associated with this WMI device * @setable: True for devices implementing the Set Control Method - * @driver_override: Driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * * This represents WMI devices discovered by the WMI driver core. */ struct wmi_device { struct device dev; bool setable; - const char *driver_override; }; /** diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index a4749f56398fd6..218557a8f7577f 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -512,6 +512,28 @@ __printf(1, 4) struct workqueue_struct * alloc_workqueue_noprof(const char *fmt, unsigned int flags, int max_active, ...); #define alloc_workqueue(...) alloc_hooks(alloc_workqueue_noprof(__VA_ARGS__)) +/** + * devm_alloc_workqueue - Resource-managed allocate a workqueue + * @dev: Device to allocate workqueue for + * @fmt: printf format for the name of the workqueue + * @flags: WQ_* flags + * @max_active: max in-flight work items, 0 for default + * @...: args for @fmt + * + * Resource managed workqueue, see alloc_workqueue() for details. + * + * The workqueue will be automatically destroyed on driver detach. Typically + * this should be used in drivers already relying on devm interafaces. + * + * RETURNS: + * Pointer to the allocated workqueue on success, %NULL on failure. + */ +__printf(2, 5) struct workqueue_struct * +devm_alloc_workqueue_noprof(struct device *dev, const char *fmt, + unsigned int flags, int max_active, ...); +#define devm_alloc_workqueue(...) \ + alloc_hooks(devm_alloc_workqueue_noprof(__VA_ARGS__)) + #ifdef CONFIG_LOCKDEP /** * alloc_workqueue_lockdep_map - allocate a workqueue with user-defined lockdep_map @@ -568,6 +590,8 @@ alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags, int max_active, */ #define alloc_ordered_workqueue(fmt, flags, args...) \ alloc_workqueue(fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args) +#define devm_alloc_ordered_workqueue(dev, fmt, flags, args...) \ + devm_alloc_workqueue(dev, fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args) #define create_workqueue(name) \ alloc_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_PERCPU, 1, (name)) diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h index 85b1fff02fde9a..0c95ead5a29770 100644 --- a/include/linux/ww_mutex.h +++ b/include/linux/ww_mutex.h @@ -181,7 +181,7 @@ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, * data structures. */ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) - __releases(ctx) __acquires_shared(ctx) __no_context_analysis + __must_hold(ctx) { #ifdef DEBUG_WW_MUTEXES lockdep_assert_held(ctx); @@ -199,7 +199,7 @@ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) * mutexes have been released with ww_mutex_unlock. */ static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) - __releases_shared(ctx) __no_context_analysis + __releases(ctx) __no_context_analysis { #ifdef CONFIG_DEBUG_LOCK_ALLOC mutex_release(&ctx->first_lock_dep_map, _THIS_IP_); diff --git a/include/linux/xarray.h b/include/linux/xarray.h index be850174e802e6..64f2a5e06cebcc 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -563,6 +563,8 @@ void *__xa_erase(struct xarray *, unsigned long index); void *__xa_store(struct xarray *, unsigned long index, void *entry, gfp_t); void *__xa_cmpxchg(struct xarray *, unsigned long index, void *old, void *entry, gfp_t); +void *__xa_cmpxchg_raw(struct xarray *, unsigned long index, void *old, + void *entry, gfp_t); int __must_check __xa_insert(struct xarray *, unsigned long index, void *entry, gfp_t); int __must_check __xa_alloc(struct xarray *, u32 *id, void *entry, diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h index 69eed69f7f2656..07a250673950ef 100644 --- a/include/net/bluetooth/bluetooth.h +++ b/include/net/bluetooth/bluetooth.h @@ -457,6 +457,7 @@ struct l2cap_ctrl { }; struct hci_dev; +struct hci_conn; typedef void (*hci_req_complete_t)(struct hci_dev *hdev, u8 status, u16 opcode); typedef void (*hci_req_complete_skb_t)(struct hci_dev *hdev, u8 status, @@ -469,6 +470,9 @@ void hci_req_cmd_complete(struct hci_dev *hdev, u16 opcode, u8 status, int hci_ethtool_ts_info(unsigned int index, int sk_proto, struct kernel_ethtool_ts_info *ts_info); +int hci_conn_setsockopt(struct hci_conn *conn, struct sock *sk, int level, + int optname, sockptr_t optval, unsigned int optlen); + #define HCI_REQ_START BIT(0) #define HCI_REQ_SKB BIT(1) diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index a7bffb908c1ec9..947e7c2b08dd81 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -642,6 +642,10 @@ struct hci_dev { bool aosp_quality_report; #endif +#if IS_ENABLED(CONFIG_BT_BRCMEXT) + bool brcm_capable; +#endif + int (*open)(struct hci_dev *hdev); int (*close)(struct hci_dev *hdev); int (*flush)(struct hci_dev *hdev); @@ -1791,6 +1795,13 @@ static inline void hci_set_aosp_capable(struct hci_dev *hdev) #endif } +static inline void hci_set_brcm_capable(struct hci_dev *hdev) +{ +#if IS_ENABLED(CONFIG_BT_BRCMEXT) + hdev->brcm_capable = true; +#endif +} + static inline void hci_devcd_setup(struct hci_dev *hdev) { #ifdef CONFIG_DEV_COREDUMP diff --git a/include/net/bond_3ad.h b/include/net/bond_3ad.h index c92d4a976246d2..05572c19e14b7a 100644 --- a/include/net/bond_3ad.h +++ b/include/net/bond_3ad.h @@ -243,7 +243,7 @@ typedef struct port { churn_state_t sm_churn_actor_state; churn_state_t sm_churn_partner_state; struct slave *slave; /* pointer to the bond slave that this port belongs to */ - struct aggregator *aggregator; /* pointer to an aggregator that this port related to */ + struct aggregator __rcu *aggregator; /* pointer to an aggregator that this port related to */ struct port *next_port_in_aggregator; /* Next port on the linked list of the parent aggregator */ u32 transaction_id; /* continuous number for identification of Marker PDU's; */ struct lacpdu lacpdu; /* the lacpdu that will be sent for this port */ diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h index a7b7abd66e215c..880a5ec786cfec 100644 --- a/include/net/dropreason-core.h +++ b/include/net/dropreason-core.h @@ -80,6 +80,7 @@ FN(UNHANDLED_PROTO) \ FN(SKB_CSUM) \ FN(SKB_GSO_SEG) \ + FN(SKB_BAD_GSO) \ FN(SKB_UCOPY_FAULT) \ FN(DEV_HDR) \ FN(DEV_READY) \ @@ -130,6 +131,7 @@ FN(DUALPI2_STEP_DROP) \ FN(PSP_INPUT) \ FN(PSP_OUTPUT) \ + FN(RECURSION_LIMIT) \ FNe(MAX) /** @@ -426,6 +428,8 @@ enum skb_drop_reason { SKB_DROP_REASON_SKB_CSUM, /** @SKB_DROP_REASON_SKB_GSO_SEG: gso segmentation error */ SKB_DROP_REASON_SKB_GSO_SEG, + /** @SKB_DROP_REASON_SKB_BAD_GSO: malicious gso packet. */ + SKB_DROP_REASON_SKB_BAD_GSO, /** * @SKB_DROP_REASON_SKB_UCOPY_FAULT: failed to copy data from user space, * e.g., via zerocopy_sg_from_iter() or skb_orphan_frags_rx() @@ -622,6 +626,8 @@ enum skb_drop_reason { SKB_DROP_REASON_PSP_INPUT, /** @SKB_DROP_REASON_PSP_OUTPUT: PSP output checks failed */ SKB_DROP_REASON_PSP_OUTPUT, + /** @SKB_DROP_REASON_RECURSION_LIMIT: Dead loop on virtual device. */ + SKB_DROP_REASON_RECURSION_LIMIT, /** * @SKB_DROP_REASON_MAX: the maximum of core drop reasons, which * shouldn't be used as a real 'reason' - only for tracing code gen diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 359b595f1df936..b99805ee2fd14b 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -162,7 +162,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, dev->name); DEV_STATS_INC(dev, tx_errors); } - kfree_skb(skb); + kfree_skb_reason(skb, SKB_DROP_REASON_RECURSION_LIMIT); return; } diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index a078af283bddbb..743bfa8ad8e368 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -568,6 +568,7 @@ struct mana_port_context { netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev); int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx, bool update_hash, bool update_tab); +int mana_disable_vport_rx(struct mana_port_context *apc); int mana_alloc_queues(struct net_device *ndev); int mana_attach(struct net_device *ndev); diff --git a/include/net/mctp.h b/include/net/mctp.h index c3207ce98f07fc..4c0e4655b6c7b0 100644 --- a/include/net/mctp.h +++ b/include/net/mctp.h @@ -26,6 +26,9 @@ struct mctp_hdr { #define MCTP_VER_MIN 1 #define MCTP_VER_MAX 1 +/* Definitions for ver field */ +#define MCTP_HDR_VER_MASK GENMASK(3, 0) + /* Definitions for flags_seq_tag field */ #define MCTP_HDR_FLAG_SOM BIT(7) #define MCTP_HDR_FLAG_EOM BIT(6) diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index ec8a8ec9c0aa69..3ec41574af776c 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -1216,12 +1216,15 @@ struct nft_stats { struct u64_stats_sync syncp; }; +#define NFT_HOOK_REMOVE (1 << 0) + struct nft_hook { struct list_head list; struct list_head ops_list; struct rcu_head rcu; char ifname[IFNAMSIZ]; u8 ifnamelen; + u8 flags; }; struct nf_hook_ops *nft_hook_find_ops(const struct nft_hook *hook, @@ -1676,6 +1679,16 @@ struct nft_trans { u8 put_net:1; }; +/** + * struct nft_trans_hook - nf_tables hook update in transaction + * @list: used internally + * @hook: struct nft_hook with the device hook + */ +struct nft_trans_hook { + struct list_head list; + struct nft_hook *hook; +}; + /** * struct nft_trans_binding - nf_tables object with binding support in transaction * @nft_trans: base structure, MUST be first member diff --git a/include/net/pie.h b/include/net/pie.h index 01cbc66825a40b..1f3db0c355149b 100644 --- a/include/net/pie.h +++ b/include/net/pie.h @@ -104,7 +104,7 @@ static inline void pie_vars_init(struct pie_vars *vars) vars->dq_tstamp = DTIME_INVALID; vars->accu_prob = 0; vars->dq_count = DQCOUNT_INVALID; - vars->avg_dq_rate = 0; + WRITE_ONCE(vars->avg_dq_rate, 0); } static inline struct pie_skb_cb *get_pie_cb(const struct sk_buff *skb) diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index c3d657359a3d2d..5fc0b1ebaf25c0 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -1170,12 +1170,22 @@ static inline void tcf_kfree_skb_list(struct sk_buff *skb) static inline void qdisc_dequeue_drop(struct Qdisc *q, struct sk_buff *skb, enum skb_drop_reason reason) { + struct Qdisc *root; + DEBUG_NET_WARN_ON_ONCE(!(q->flags & TCQ_F_DEQUEUE_DROPS)); DEBUG_NET_WARN_ON_ONCE(q->flags & TCQ_F_NOLOCK); - tcf_set_drop_reason(skb, reason); - skb->next = q->to_free; - q->to_free = skb; + rcu_read_lock(); + root = qdisc_root_sleeping(q); + + if (root->flags & TCQ_F_DEQUEUE_DROPS) { + tcf_set_drop_reason(skb, reason); + skb->next = root->to_free; + root->to_free = skb; + } else { + kfree_skb_reason(skb, reason); + } + rcu_read_unlock(); } /* Instead of calling kfree_skb() while root qdisc lock is held, diff --git a/include/net/tcp.h b/include/net/tcp.h index 978eea2d5df04f..ebc72dce4134dd 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1465,7 +1465,7 @@ static inline u32 tcp_snd_cwnd(const struct tcp_sock *tp) static inline void tcp_snd_cwnd_set(struct tcp_sock *tp, u32 val) { WARN_ON_ONCE((int)val <= 0); - tp->snd_cwnd = val; + WRITE_ONCE(tp->snd_cwnd, val); } static inline bool tcp_in_slow_start(const struct tcp_sock *tp) @@ -2156,7 +2156,34 @@ enum tcp_chrono { __TCP_CHRONO_MAX, }; -void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type); +static inline void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new) +{ + const u32 now = tcp_jiffies32; + enum tcp_chrono old = tp->chrono_type; + + /* Following WRITE_ONCE()s pair with READ_ONCE()s in + * tcp_get_info_chrono_stats(). + */ + if (old > TCP_CHRONO_UNSPEC) + WRITE_ONCE(tp->chrono_stat[old - 1], + tp->chrono_stat[old - 1] + now - tp->chrono_start); + WRITE_ONCE(tp->chrono_start, now); + WRITE_ONCE(tp->chrono_type, new); +} + +static inline void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type) +{ + struct tcp_sock *tp = tcp_sk(sk); + + /* If there are multiple conditions worthy of tracking in a + * chronograph then the highest priority enum takes precedence + * over the other conditions. So that if something "more interesting" + * starts happening, stop the previous chrono and start a new one. + */ + if (type > tp->chrono_type) + tcp_chrono_set(tp, type); +} + void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type); /* This helper is needed, because skb->tcp_tsorted_anchor uses diff --git a/include/net/tcp_ecn.h b/include/net/tcp_ecn.h index e9a933641636e1..865d5c5a7718db 100644 --- a/include/net/tcp_ecn.h +++ b/include/net/tcp_ecn.h @@ -181,7 +181,7 @@ static inline void tcp_accecn_third_ack(struct sock *sk, tcp_accecn_validate_syn_feedback(sk, ace, sent_ect)) { if ((tcp_accecn_extract_syn_ect(ace) == INET_ECN_CE) && !tp->delivered_ce) - tp->delivered_ce++; + WRITE_ONCE(tp->delivered_ce, 1); } break; } diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 0a8e092c0ea878..e426d451b89329 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -22,6 +22,7 @@ struct ib_umem { u64 iova; size_t length; unsigned long address; + unsigned long dma_attrs; u32 writable : 1; u32 is_odp : 1; u32 is_dmabuf : 1; diff --git a/include/sound/control.h b/include/sound/control.h index e07f6b960641ff..9be6546bf787de 100644 --- a/include/sound/control.h +++ b/include/sound/control.h @@ -14,9 +14,12 @@ #define snd_kcontrol_chip(kcontrol) ((kcontrol)->private_data) struct snd_kcontrol; +struct snd_ctl_file; typedef int (snd_kcontrol_info_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_info * uinfo); typedef int (snd_kcontrol_get_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_value * ucontrol); typedef int (snd_kcontrol_put_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_elem_value * ucontrol); +typedef int (snd_kcontrol_lock_t) (struct snd_kcontrol * kcontrol, struct snd_ctl_file *owner); +typedef void (snd_kcontrol_unlock_t) (struct snd_kcontrol * kcontrol); typedef int (snd_kcontrol_tlv_rw_t)(struct snd_kcontrol *kcontrol, int op_flag, /* SNDRV_CTL_TLV_OP_XXX */ unsigned int size, @@ -55,6 +58,8 @@ struct snd_kcontrol_new { snd_kcontrol_info_t *info; snd_kcontrol_get_t *get; snd_kcontrol_put_t *put; + snd_kcontrol_lock_t *lock; + snd_kcontrol_unlock_t *unlock; union { snd_kcontrol_tlv_rw_t *c; const unsigned int *p; @@ -74,6 +79,8 @@ struct snd_kcontrol { snd_kcontrol_info_t *info; snd_kcontrol_get_t *get; snd_kcontrol_put_t *put; + snd_kcontrol_lock_t *lock; + snd_kcontrol_unlock_t *unlock; union { snd_kcontrol_tlv_rw_t *c; const unsigned int *p; diff --git a/include/sound/cs42l42.h b/include/sound/cs42l42.h index 1bd8eee54f6665..b3657965d49109 100644 --- a/include/sound/cs42l42.h +++ b/include/sound/cs42l42.h @@ -62,6 +62,10 @@ #define CS42L42_INTERNAL_FS_MASK (1 << CS42L42_INTERNAL_FS_SHIFT) #define CS42L42_SFTRAMP_RATE (CS42L42_PAGE_10 + 0x0A) +#define CS42L42_SFTRAMP_ASR_RATE_MASK GENMASK(7, 4) +#define CS42L42_SFTRAMP_ASR_RATE_SHIFT 4 +#define CS42L42_SFTRAMP_DSR_RATE_MASK GENMASK(3, 0) +#define CS42L42_SFTRAMP_DSR_RATE_SHIFT 0 #define CS42L42_SLOW_START_ENABLE (CS42L42_PAGE_10 + 0x0B) #define CS42L42_SLOW_START_EN_MASK GENMASK(6, 4) #define CS42L42_SLOW_START_EN_SHIFT 4 diff --git a/include/sound/pcm.h b/include/sound/pcm.h index a7860c047503a8..a0c987c9bd1fee 100644 --- a/include/sound/pcm.h +++ b/include/sound/pcm.h @@ -1073,6 +1073,7 @@ int snd_interval_ranges(struct snd_interval *i, unsigned int count, int snd_interval_ratnum(struct snd_interval *i, unsigned int rats_count, const struct snd_ratnum *rats, unsigned int *nump, unsigned int *denp); +int snd_interval_rate_bits(struct snd_interval *i, unsigned int rate_bits); void _snd_pcm_hw_params_any(struct snd_pcm_hw_params *params); void _snd_pcm_hw_param_setempty(struct snd_pcm_hw_params *params, snd_pcm_hw_param_t var); diff --git a/include/sound/soc-card.h b/include/sound/soc-card.h index ecc02e955279fd..ef46cac97d9968 100644 --- a/include/sound/soc-card.h +++ b/include/sound/soc-card.h @@ -44,7 +44,7 @@ int snd_soc_card_resume_post(struct snd_soc_card *card); int snd_soc_card_probe(struct snd_soc_card *card); int snd_soc_card_late_probe(struct snd_soc_card *card); -void snd_soc_card_fixup_controls(struct snd_soc_card *card); +int snd_soc_card_fixup_controls(struct snd_soc_card *card); int snd_soc_card_remove(struct snd_soc_card *card); int snd_soc_card_set_bias_level(struct snd_soc_card *card, diff --git a/include/sound/soc-component.h b/include/sound/soc-component.h index 2a2b74b24a6096..0435ba376369ad 100644 --- a/include/sound/soc-component.h +++ b/include/sound/soc-component.h @@ -90,6 +90,10 @@ struct snd_soc_component_driver { struct snd_soc_pcm_runtime *rtd); void (*pcm_destruct)(struct snd_soc_component *component, struct snd_pcm *pcm); + int (*pcm_new)(struct snd_soc_component *component, + struct snd_soc_pcm_runtime *rtd); + void (*pcm_free)(struct snd_soc_component *component, + struct snd_pcm *pcm); /* component wide operations */ int (*set_sysclk)(struct snd_soc_component *component, diff --git a/include/sound/soc.h b/include/sound/soc.h index 7d8376c8e1bede..bce164b1e9fa54 100644 --- a/include/sound/soc.h +++ b/include/sound/soc.h @@ -575,8 +575,14 @@ int snd_soc_get_volsw_sx(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol); +bool snd_soc_control_matches(struct snd_kcontrol *kcontrol, + const char *pattern); int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max); +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active); +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *strval); int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo); int snd_soc_bytes_get(struct snd_kcontrol *kcontrol, @@ -1003,7 +1009,7 @@ struct snd_soc_card { int (*probe)(struct snd_soc_card *card); int (*late_probe)(struct snd_soc_card *card); - void (*fixup_controls)(struct snd_soc_card *card); + int (*fixup_controls)(struct snd_soc_card *card); int (*remove)(struct snd_soc_card *card); /* the pre and post PM functions are used to do any PM work before and @@ -1412,6 +1418,9 @@ struct snd_soc_dai *snd_soc_find_dai( struct snd_soc_dai *snd_soc_find_dai_with_mutex( const struct snd_soc_dai_link_component *dlc); +void soc_pcm_set_dai_params(struct snd_soc_dai *dai, + struct snd_pcm_hw_params *params); + #include static inline diff --git a/include/trace/events/dma_fence.h b/include/trace/events/dma_fence.h index 4814a65b68dcb7..3abba45c0601a4 100644 --- a/include/trace/events/dma_fence.h +++ b/include/trace/events/dma_fence.h @@ -9,37 +9,12 @@ struct dma_fence; -DECLARE_EVENT_CLASS(dma_fence, - - TP_PROTO(struct dma_fence *fence), - - TP_ARGS(fence), - - TP_STRUCT__entry( - __string(driver, dma_fence_driver_name(fence)) - __string(timeline, dma_fence_timeline_name(fence)) - __field(unsigned int, context) - __field(unsigned int, seqno) - ), - - TP_fast_assign( - __assign_str(driver); - __assign_str(timeline); - __entry->context = fence->context; - __entry->seqno = fence->seqno; - ), - - TP_printk("driver=%s timeline=%s context=%u seqno=%u", - __get_str(driver), __get_str(timeline), __entry->context, - __entry->seqno) -); - /* * Safe only for call sites which are guaranteed to not race with fence * signaling,holding the fence->lock and having checked for not signaled, or the * signaling path itself. */ -DECLARE_EVENT_CLASS(dma_fence_unsignaled, +DECLARE_EVENT_CLASS(dma_fence, TP_PROTO(struct dma_fence *fence), @@ -64,14 +39,14 @@ DECLARE_EVENT_CLASS(dma_fence_unsignaled, __entry->seqno) ); -DEFINE_EVENT(dma_fence_unsignaled, dma_fence_emit, +DEFINE_EVENT(dma_fence, dma_fence_emit, TP_PROTO(struct dma_fence *fence), TP_ARGS(fence) ); -DEFINE_EVENT(dma_fence_unsignaled, dma_fence_init, +DEFINE_EVENT(dma_fence, dma_fence_init, TP_PROTO(struct dma_fence *fence), @@ -85,14 +60,14 @@ DEFINE_EVENT(dma_fence, dma_fence_destroy, TP_ARGS(fence) ); -DEFINE_EVENT(dma_fence_unsignaled, dma_fence_enable_signal, +DEFINE_EVENT(dma_fence, dma_fence_enable_signal, TP_PROTO(struct dma_fence *fence), TP_ARGS(fence) ); -DEFINE_EVENT(dma_fence_unsignaled, dma_fence_signaled, +DEFINE_EVENT(dma_fence, dma_fence_signaled, TP_PROTO(struct dma_fence *fence), diff --git a/include/trace/events/mptcp.h b/include/trace/events/mptcp.h index 269d949b20254f..04521acba4832e 100644 --- a/include/trace/events/mptcp.h +++ b/include/trace/events/mptcp.h @@ -219,7 +219,7 @@ TRACE_EVENT(mptcp_rcvbuf_grow, __be32 *p32; __entry->time = time; - __entry->rtt_us = msk->rcvq_space.rtt_us >> 3; + __entry->rtt_us = mptcp_rtt_us_est(msk) >> 3; __entry->copied = msk->rcvq_space.copied; __entry->inq = mptcp_inq_hint(sk); __entry->space = msk->rcvq_space.space; diff --git a/include/trace/events/net.h b/include/trace/events/net.h index fdd9ad474ce3ae..dbc2c5598e35ad 100644 --- a/include/trace/events/net.h +++ b/include/trace/events/net.h @@ -10,6 +10,7 @@ #include #include #include +#include TRACE_EVENT(net_dev_start_xmit, @@ -208,7 +209,8 @@ DECLARE_EVENT_CLASS(net_dev_rx_verbose_template, TP_fast_assign( __assign_str(name); #ifdef CONFIG_NET_RX_BUSY_POLL - __entry->napi_id = skb->napi_id; + __entry->napi_id = napi_id_valid(skb->napi_id) ? + skb->napi_id : 0; #else __entry->napi_id = 0; #endif diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h index 578b8038b21178..573f2df3a2c998 100644 --- a/include/trace/events/rxrpc.h +++ b/include/trace/events/rxrpc.h @@ -37,6 +37,7 @@ EM(rxkad_abort_1_short_encdata, "rxkad1-short-encdata") \ EM(rxkad_abort_1_short_header, "rxkad1-short-hdr") \ EM(rxkad_abort_2_short_check, "rxkad2-short-check") \ + EM(rxkad_abort_2_crypto_unaligned, "rxkad2-crypto-unaligned") \ EM(rxkad_abort_2_short_data, "rxkad2-short-data") \ EM(rxkad_abort_2_short_header, "rxkad2-short-hdr") \ EM(rxkad_abort_2_short_len, "rxkad2-short-len") \ @@ -161,8 +162,6 @@ E_(rxrpc_call_poke_timer_now, "Timer-now") #define rxrpc_skb_traces \ - EM(rxrpc_skb_eaten_by_unshare, "ETN unshare ") \ - EM(rxrpc_skb_eaten_by_unshare_nomem, "ETN unshar-nm") \ EM(rxrpc_skb_get_call_rx, "GET call-rx ") \ EM(rxrpc_skb_get_conn_secured, "GET conn-secd") \ EM(rxrpc_skb_get_conn_work, "GET conn-work") \ @@ -189,6 +188,7 @@ EM(rxrpc_skb_put_purge, "PUT purge ") \ EM(rxrpc_skb_put_purge_oob, "PUT purge-oob") \ EM(rxrpc_skb_put_response, "PUT response ") \ + EM(rxrpc_skb_put_response_copy, "PUT resp-cpy ") \ EM(rxrpc_skb_put_rotate, "PUT rotate ") \ EM(rxrpc_skb_put_unknown, "PUT unknown ") \ EM(rxrpc_skb_see_conn_work, "SEE conn-work") \ @@ -197,6 +197,7 @@ EM(rxrpc_skb_see_recvmsg_oob, "SEE recvm-oob") \ EM(rxrpc_skb_see_reject, "SEE reject ") \ EM(rxrpc_skb_see_rotate, "SEE rotate ") \ + EM(rxrpc_skb_see_unshare_nomem, "SEE unshar-nm") \ E_(rxrpc_skb_see_version, "SEE version ") #define rxrpc_local_traces \ @@ -284,7 +285,6 @@ EM(rxrpc_conn_put_unidle, "PUT unidle ") \ EM(rxrpc_conn_put_work, "PUT work ") \ EM(rxrpc_conn_queue_challenge, "QUE chall ") \ - EM(rxrpc_conn_queue_retry_work, "QUE retry-wk") \ EM(rxrpc_conn_queue_rx_work, "QUE rx-work ") \ EM(rxrpc_conn_see_new_service_conn, "SEE new-svc ") \ EM(rxrpc_conn_see_reap_service, "SEE reap-svc") \ diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h index 1641ae3e6ca060..ab9a9386f7b654 100644 --- a/include/trace/events/timer.h +++ b/include/trace/events/timer.h @@ -218,12 +218,13 @@ TRACE_EVENT(hrtimer_setup, * hrtimer_start - called when the hrtimer is started * @hrtimer: pointer to struct hrtimer * @mode: the hrtimers mode + * @was_armed: Was armed when hrtimer_start*() was invoked */ TRACE_EVENT(hrtimer_start, - TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode), + TP_PROTO(struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed), - TP_ARGS(hrtimer, mode), + TP_ARGS(hrtimer, mode, was_armed), TP_STRUCT__entry( __field( void *, hrtimer ) @@ -231,6 +232,7 @@ TRACE_EVENT(hrtimer_start, __field( s64, expires ) __field( s64, softexpires ) __field( enum hrtimer_mode, mode ) + __field( bool, was_armed ) ), TP_fast_assign( @@ -239,13 +241,14 @@ TRACE_EVENT(hrtimer_start, __entry->expires = hrtimer_get_expires(hrtimer); __entry->softexpires = hrtimer_get_softexpires(hrtimer); __entry->mode = mode; + __entry->was_armed = was_armed; ), TP_printk("hrtimer=%p function=%ps expires=%llu softexpires=%llu " - "mode=%s", __entry->hrtimer, __entry->function, + "mode=%s was_armed=%d", __entry->hrtimer, __entry->function, (unsigned long long) __entry->expires, (unsigned long long) __entry->softexpires, - decode_hrtimer_mode(__entry->mode)) + decode_hrtimer_mode(__entry->mode), __entry->was_armed) ); /** diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index e9b5f79e1ee173..83a96c56b8cad8 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -1568,6 +1568,8 @@ enum { IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE, IFLA_BOND_SLAVE_PRIO, IFLA_BOND_SLAVE_ACTOR_PORT_PRIO, + IFLA_BOND_SLAVE_AD_CHURN_ACTOR_STATE, + IFLA_BOND_SLAVE_AD_CHURN_PARTNER_STATE, __IFLA_BOND_SLAVE_MAX, }; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 1ff16141c8a5b4..05c4344b378adc 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -898,7 +898,8 @@ struct io_uring_buf_reg { __u32 ring_entries; __u16 bgid; __u16 flags; - __u64 resv[3]; + __u32 min_left; + __u32 resv[5]; }; /* argument for IORING_REGISTER_PBUF_STATUS */ diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index 1dafbc552d37de..f63edbe71d542b 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -1052,6 +1052,11 @@ struct iommu_fault_alloc { enum iommu_viommu_type { IOMMU_VIOMMU_TYPE_DEFAULT = 0, IOMMU_VIOMMU_TYPE_ARM_SMMUV3 = 1, + /* + * TEGRA241_CMDQV requirements (otherwise, VCMDQs will not work) + * - Kernel will allocate a VINTF (HYP_OWN=0) to back this VIOMMU. So, + * VMM must wire the HYP_OWN bit to 0 in guest VINTF_CONFIG register + */ IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV = 2, }; diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h index f88fa1f68b770b..d37603efc273fd 100644 --- a/include/uapi/linux/landlock.h +++ b/include/uapi/linux/landlock.h @@ -116,7 +116,9 @@ struct landlock_ruleset_attr { * ``LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF``, this flag only affects * future nested domains, not the one being created. It can also be used * with a @ruleset_fd value of -1 to mute subdomain logs without creating a - * domain. + * domain. When combined with %LANDLOCK_RESTRICT_SELF_TSYNC and a + * @ruleset_fd value of -1, this configuration is propagated to all threads + * of the current process. * * The following flag supports policy enforcement in multithreaded processes: * diff --git a/include/uapi/linux/mii.h b/include/uapi/linux/mii.h index 39f7c44baf5350..61d6edad4b94af 100644 --- a/include/uapi/linux/mii.h +++ b/include/uapi/linux/mii.h @@ -82,7 +82,8 @@ #define ADVERTISE_100BASE4 0x0200 /* Try for 100mbps 4k packets */ #define ADVERTISE_PAUSE_CAP 0x0400 /* Try for pause */ #define ADVERTISE_PAUSE_ASYM 0x0800 /* Try for asymetric pause */ -#define ADVERTISE_RESV 0x1000 /* Unused... */ +#define ADVERTISE_XNP 0x1000 /* Extended Next Page */ +#define ADVERTISE_RESV ADVERTISE_XNP /* Used to be reserved */ #define ADVERTISE_RFAULT 0x2000 /* Say we can detect faults */ #define ADVERTISE_LPACK 0x4000 /* Ack link partners response */ #define ADVERTISE_NPAGE 0x8000 /* Next page bit */ diff --git a/include/uapi/linux/mshv.h b/include/uapi/linux/mshv.h index e0645a34b55bca..32ff92b6342b24 100644 --- a/include/uapi/linux/mshv.h +++ b/include/uapi/linux/mshv.h @@ -357,7 +357,7 @@ struct mshv_vtl_sint_post_msg { struct mshv_vtl_ram_disposition { __u64 start_pfn; - __u64 last_pfn; + __u64 last_pfn; /* last_pfn is excluded from the range [start_pfn, last_pfn) */ }; struct mshv_vtl_set_poll_file { diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index b6ec6f6937195f..1a06e7feb7dc9b 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -6,6 +6,11 @@ /* Values to pass as first argument to prctl() */ +#define PR_GET_MEM_MODEL 0x6d4d444c +#define PR_SET_MEM_MODEL 0x4d4d444c +# define PR_SET_MEM_MODEL_DEFAULT 0 +# define PR_SET_MEM_MODEL_TSO 1 + #define PR_SET_PDEATHSIG 1 /* Second arg is a signal */ #define PR_GET_PDEATHSIG 2 /* Second arg is a ptr to return the signal */ diff --git a/include/uapi/linux/rseq.h b/include/uapi/linux/rseq.h index f69344fe6c0863..ca6fe1f9d05e7e 100644 --- a/include/uapi/linux/rseq.h +++ b/include/uapi/linux/rseq.h @@ -28,7 +28,7 @@ enum rseq_cs_flags_bit { RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0, RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1, RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2, - /* (3) Intentional gap to put new bits into a separate byte */ + /* (3) Intentional gap to keep new bits separate */ /* User read only feature flags */ RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE_BIT = 4, @@ -161,6 +161,9 @@ struct rseq { * - RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT * - RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL * - RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE + * + * It is now used for feature status advertisement by the kernel. + * See: enum rseq_cs_flags_bit for further information. */ __u32 flags; diff --git a/include/video/udlfb.h b/include/video/udlfb.h index 58fb5732831a43..ab34790d57ecd6 100644 --- a/include/video/udlfb.h +++ b/include/video/udlfb.h @@ -56,6 +56,7 @@ struct dlfb_data { spinlock_t damage_lock; struct work_struct damage_work; struct fb_ops ops; + atomic_t mmap_count; /* blit-only rendering path metrics, exposed through sysfs */ atomic_t bytes_rendered; /* raw pixel-bytes driver asked to render */ atomic_t bytes_identical; /* saved effort with backbuffer comparison */ diff --git a/init/Kconfig b/init/Kconfig index 7484cd703bc1ab..8ef220ca61bff8 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -2170,7 +2170,6 @@ config PROFILING config RUST bool "Rust support" depends on HAVE_RUST - depends on RUST_IS_AVAILABLE select EXTENDED_MODVERSIONS if MODVERSIONS depends on !MODVERSIONS || GENDWARFKSYMS depends on !GCC_PLUGIN_RANDSTRUCT diff --git a/init/main.c b/init/main.c index 1cb395dd94e43f..0a1d8529212e97 100644 --- a/init/main.c +++ b/init/main.c @@ -833,7 +833,6 @@ static inline void initcall_debug_enable(void) #ifdef CONFIG_RANDOMIZE_KSTACK_OFFSET DEFINE_STATIC_KEY_MAYBE_RO(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, randomize_kstack_offset); -DEFINE_PER_CPU(u32, kstack_offset); static int __init early_randomize_kstack_offset(char *buf) { diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 7a9f94a0ce6f2f..8cc7b47d30894a 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -1124,7 +1124,8 @@ static inline void io_wq_remove_pending(struct io_wq *wq, if (io_wq_is_hashed(work) && work == wq->hash_tail[hash]) { if (prev) prev_work = container_of(prev, struct io_wq_work, list); - if (prev_work && io_get_work_hash(prev_work) == hash) + if (prev_work && io_wq_is_hashed(prev_work) && + io_get_work_hash(prev_work) == hash) wq->hash_tail[hash] = prev_work; else wq->hash_tail[hash] = NULL; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4d7bcbb9740631..97260bca67e7bb 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2308,6 +2308,10 @@ static __cold void io_ring_exit_work(struct work_struct *work) struct io_tctx_node *node; int ret; + mutex_lock(&ctx->uring_lock); + io_terminate_zcrx(ctx); + mutex_unlock(&ctx->uring_lock); + /* * If we're doing polled IO and end up having requests being * submitted async (out-of-line), then completions can come in while diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 5257b3aad39510..15be9d6eb41274 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -47,7 +47,7 @@ static bool io_kbuf_inc_commit(struct io_buffer_list *bl, int len) this_len = min_t(u32, len, buf_len); buf_len -= this_len; /* Stop looping for invalid buffer length of 0 */ - if (buf_len || !this_len) { + if (buf_len > bl->min_left_sub_one || !this_len) { WRITE_ONCE(buf->addr, READ_ONCE(buf->addr) + this_len); WRITE_ONCE(buf->len, buf_len); return false; @@ -637,6 +637,10 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) if (reg.ring_entries >= 65536) return -EINVAL; + /* minimum left byte count is a property of incremental buffers */ + if (!(reg.flags & IOU_PBUF_RING_INC) && reg.min_left) + return -EINVAL; + bl = io_buffer_get_list(ctx, reg.bgid); if (bl) { /* if mapped buffer ring OR classic exists, don't allow */ @@ -684,6 +688,8 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) bl->mask = reg.ring_entries - 1; bl->flags |= IOBL_BUF_RING; bl->buf_ring = br; + if (reg.min_left) + bl->min_left_sub_one = reg.min_left - 1; if (reg.flags & IOU_PBUF_RING_INC) bl->flags |= IOBL_INC; ret = io_buffer_add_list(ctx, bl, reg.bgid); diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h index bf15e26520d385..1d9600d08e552d 100644 --- a/io_uring/kbuf.h +++ b/io_uring/kbuf.h @@ -33,6 +33,13 @@ struct io_buffer_list { __u16 flags; + /* + * minimum required amount to be left to reuse an incrementally + * consumed buffer. If less than this is left at consumption time, + * buffer is done and head is incremented to the next buffer. + */ + __u32 min_left_sub_one; + struct io_mapped_region region; }; diff --git a/io_uring/napi.c b/io_uring/napi.c index 4a10de03e42693..8d68366a4b9039 100644 --- a/io_uring/napi.c +++ b/io_uring/napi.c @@ -276,6 +276,8 @@ static int io_napi_register_napi(struct io_ring_ctx *ctx, /* clean the napi list for new settings */ io_napi_free(ctx); WRITE_ONCE(ctx->napi_track_mode, napi->op_param); + /* cap NAPI at 10 msec of spin time */ + napi->busy_poll_to = min(10000, napi->busy_poll_to); WRITE_ONCE(ctx->napi_busy_poll_dt, napi->busy_poll_to * NSEC_PER_USEC); WRITE_ONCE(ctx->napi_prefer_busy_poll, !!napi->prefer_busy_poll); return 0; diff --git a/io_uring/poll.c b/io_uring/poll.c index 2e9ee47d74bfbd..ae8da73fd7e3f3 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -93,7 +93,7 @@ static bool io_poll_get_ownership_slowpath(struct io_kiocb *req) */ static inline bool io_poll_get_ownership(struct io_kiocb *req) { - if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) + if (unlikely((unsigned int)atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS)) return io_poll_get_ownership_slowpath(req); return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK); } @@ -415,8 +415,10 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, * disable multishot as there is a circular dependency between * CQ posting and triggering the event. */ - if (mask & EPOLL_URING_WAKE) + if (mask & EPOLL_URING_WAKE) { poll->events |= EPOLLONESHOT; + req->apoll_events |= EPOLLONESHOT; + } /* optional, saves extra locking for removal in tw handler */ if (mask && poll->events & EPOLLONESHOT) { diff --git a/io_uring/register.c b/io_uring/register.c index 05362fe7980447..b299f3d6c93ec5 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -599,10 +599,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) if (tail - old_head > p->sq_entries) goto overflow; for (i = old_head; i < tail; i++) { - unsigned src_head = i & (ctx->sq_entries - 1); - unsigned dst_head = i & (p->sq_entries - 1); - - n.sq_sqes[dst_head] = o.sq_sqes[src_head]; + unsigned index, dst_mask, src_mask; + size_t sq_size; + + index = i; + sq_size = sizeof(struct io_uring_sqe); + src_mask = ctx->sq_entries - 1; + dst_mask = p->sq_entries - 1; + if (ctx->flags & IORING_SETUP_SQE128) { + index <<= 1; + sq_size <<= 1; + src_mask = (ctx->sq_entries << 1) - 1; + dst_mask = (p->sq_entries << 1) - 1; + } + memcpy(&n.sq_sqes[index & dst_mask], &o.sq_sqes[index & src_mask], sq_size); } WRITE_ONCE(n.rings->sq.head, old_head); WRITE_ONCE(n.rings->sq.tail, tail); @@ -619,10 +629,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) goto out; } for (i = old_head; i < tail; i++) { - unsigned src_head = i & (ctx->cq_entries - 1); - unsigned dst_head = i & (p->cq_entries - 1); - - n.rings->cqes[dst_head] = o.rings->cqes[src_head]; + unsigned index, dst_mask, src_mask; + size_t cq_size; + + index = i; + cq_size = sizeof(struct io_uring_cqe); + src_mask = ctx->cq_entries - 1; + dst_mask = p->cq_entries - 1; + if (ctx->flags & IORING_SETUP_CQE32) { + index <<= 1; + cq_size <<= 1; + src_mask = (ctx->cq_entries << 1) - 1; + dst_mask = (p->cq_entries << 1) - 1; + } + memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size); } WRITE_ONCE(n.rings->cq.head, old_head); WRITE_ONCE(n.rings->cq.tail, tail); diff --git a/io_uring/timeout.c b/io_uring/timeout.c index cb61d4862fc65c..e3815e3465dde5 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -449,6 +449,8 @@ int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) return -EINVAL; + if (sqe->addr3 || sqe->__pad2[0]) + return -EINVAL; if (sqe->buf_index || sqe->len || sqe->splice_fd_in) return -EINVAL; @@ -521,6 +523,8 @@ static int __io_timeout_prep(struct io_kiocb *req, unsigned flags; u32 off = READ_ONCE(sqe->off); + if (sqe->addr3 || sqe->__pad2[0]) + return -EINVAL; if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in) return -EINVAL; if (off && is_timeout_link) diff --git a/io_uring/tw.c b/io_uring/tw.c index 2f2b4ac4b1267c..578fc2ae1d0fa7 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -273,8 +273,18 @@ void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags) void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) { - struct llist_node *node = llist_del_all(&ctx->work_llist); + struct llist_node *node; + /* + * Running the work items may utilize ->retry_llist as a means + * for capping the number of task_work entries run at the same + * time. But that list can potentially race with moving the work + * from here, if the task is exiting. As any normal task_work + * running holds ->uring_lock already, just guard this slow path + * with ->uring_lock to avoid racing on ->retry_llist. + */ + guard(mutex)(&ctx->uring_lock); + node = llist_del_all(&ctx->work_llist); __io_fallback_tw(node, false); node = llist_del_all(&ctx->retry_llist); __io_fallback_tw(node, false); diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 62d693287457fe..4eb08c832f0b15 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -565,13 +565,13 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq) if (ifq->area) io_zcrx_free_area(ifq, ifq->area); - free_uid(ifq->user); if (ifq->mm_account) mmdrop(ifq->mm_account); if (ifq->dev) put_device(ifq->dev); io_free_rbuf_ring(ifq); + free_uid(ifq->user); mutex_destroy(&ifq->pp_lock); kfree(ifq); } @@ -586,9 +586,10 @@ static void io_zcrx_return_niov_freelist(struct net_iov *niov) { struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); - spin_lock_bh(&area->freelist_lock); + guard(spinlock_bh)(&area->freelist_lock); + if (WARN_ON_ONCE(area->free_count >= area->nia.num_niovs)) + return; area->freelist[area->free_count++] = net_iov_idx(niov); - spin_unlock_bh(&area->freelist_lock); } static void io_zcrx_return_niov(struct net_iov *niov) @@ -624,12 +625,17 @@ static void io_zcrx_scrub(struct io_zcrx_ifq *ifq) } } -static void zcrx_unregister(struct io_zcrx_ifq *ifq) +static void zcrx_unregister_user(struct io_zcrx_ifq *ifq) { if (refcount_dec_and_test(&ifq->user_refs)) { io_close_queue(ifq); io_zcrx_scrub(ifq); } +} + +static void zcrx_unregister(struct io_zcrx_ifq *ifq) +{ + zcrx_unregister_user(ifq); io_put_zcrx_ifq(ifq); } @@ -885,6 +891,36 @@ static struct net_iov *__io_zcrx_get_free_niov(struct io_zcrx_area *area) return &area->nia.niovs[niov_idx]; } +static inline bool is_zcrx_entry_marked(struct io_ring_ctx *ctx, unsigned long id) +{ + return xa_get_mark(&ctx->zcrx_ctxs, id, XA_MARK_0); +} + +static inline void set_zcrx_entry_mark(struct io_ring_ctx *ctx, unsigned long id) +{ + xa_set_mark(&ctx->zcrx_ctxs, id, XA_MARK_0); +} + +void io_terminate_zcrx(struct io_ring_ctx *ctx) +{ + struct io_zcrx_ifq *ifq; + unsigned long id = 0; + + lockdep_assert_held(&ctx->uring_lock); + + while (1) { + scoped_guard(mutex, &ctx->mmap_lock) + ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); + if (!ifq) + break; + if (WARN_ON_ONCE(is_zcrx_entry_marked(ctx, id))) + break; + set_zcrx_entry_mark(ctx, id); + id++; + zcrx_unregister_user(ifq); + } +} + void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) { struct io_zcrx_ifq *ifq; @@ -896,12 +932,17 @@ void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) unsigned long id = 0; ifq = xa_find(&ctx->zcrx_ctxs, &id, ULONG_MAX, XA_PRESENT); - if (ifq) + if (ifq) { + if (WARN_ON_ONCE(!is_zcrx_entry_marked(ctx, id))) { + ifq = NULL; + break; + } xa_erase(&ctx->zcrx_ctxs, id); + } } if (!ifq) break; - zcrx_unregister(ifq); + io_put_zcrx_ifq(ifq); } xa_destroy(&ctx->zcrx_ctxs); @@ -989,7 +1030,8 @@ static void io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq) { struct io_zcrx_area *area = ifq->area; - spin_lock_bh(&area->freelist_lock); + guard(spinlock_bh)(&area->freelist_lock); + while (area->free_count && pp->alloc.count < PP_ALLOC_CACHE_REFILL) { struct net_iov *niov = __io_zcrx_get_free_niov(area); netmem_ref netmem = net_iov_to_netmem(niov); @@ -998,7 +1040,6 @@ static void io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq) io_zcrx_sync_for_device(pp, niov); net_mp_netmem_place_in_cache(pp, netmem); } - spin_unlock_bh(&area->freelist_lock); } static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp) @@ -1224,10 +1265,10 @@ static struct net_iov *io_alloc_fallback_niov(struct io_zcrx_ifq *ifq) if (area->mem.is_dmabuf) return NULL; - spin_lock_bh(&area->freelist_lock); - if (area->free_count) - niov = __io_zcrx_get_free_niov(area); - spin_unlock_bh(&area->freelist_lock); + scoped_guard(spinlock_bh, &area->freelist_lock) { + if (area->free_count) + niov = __io_zcrx_get_free_niov(area); + } if (niov) page_pool_fragment_netmem(net_iov_to_netmem(niov), 1); diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 32ab95b2cb811a..1ef95adeb725e5 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -71,6 +71,7 @@ int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg); int io_register_zcrx_ifq(struct io_ring_ctx *ctx, struct io_uring_zcrx_ifq_reg __user *arg); void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx); +void io_terminate_zcrx(struct io_ring_ctx *ctx); int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, struct socket *sock, unsigned int flags, unsigned issue_flags, unsigned int *len); @@ -85,6 +86,9 @@ static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx, static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx) { } +static inline void io_terminate_zcrx(struct io_ring_ctx *ctx) +{ +} static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq, struct socket *sock, unsigned int flags, unsigned issue_flags, unsigned int *len) diff --git a/kernel/audit.c b/kernel/audit.c index 5a021605652498..d3a8268998d7a0 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1466,6 +1466,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh, err = audit_list_rules_send(skb, seq); break; case AUDIT_TRIM: + if (audit_enabled == AUDIT_LOCKED) + return -EPERM; audit_trim_trees(); audit_log_common_recv_msg(audit_context(), &ab, AUDIT_CONFIG_CHANGE); @@ -1478,6 +1480,8 @@ static int audit_receive_msg(struct sk_buff *skb, struct nlmsghdr *nlh, size_t msglen = data_len; char *old, *new; + if (audit_enabled == AUDIT_LOCKED) + return -EPERM; err = -EINVAL; if (msglen < 2 * sizeof(u32)) break; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index f6af6a8f68c4f6..bd4c65775c351a 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -2786,7 +2786,7 @@ void __audit_log_capset(const struct cred *new, const struct cred *old) context->capset.pid = task_tgid_nr(current); context->capset.cap.effective = new->cap_effective; - context->capset.cap.inheritable = new->cap_effective; + context->capset.cap.inheritable = new->cap_inheritable; context->capset.cap.permitted = new->cap_permitted; context->capset.cap.ambient = new->cap_ambient; context->type = AUDIT_CAPSET; diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index f355cf1c1a1693..523c3a61063bf3 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -341,6 +341,16 @@ static void arena_vm_open(struct vm_area_struct *vma) refcount_inc(&vml->mmap_count); } +static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr) +{ + return -EINVAL; +} + +static int arena_vm_mremap(struct vm_area_struct *vma) +{ + return -EINVAL; +} + static void arena_vm_close(struct vm_area_struct *vma) { struct bpf_map *map = vma->vm_file->private_data; @@ -417,6 +427,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) static const struct vm_operations_struct arena_vm_ops = { .open = arena_vm_open, + .may_split = arena_vm_may_split, + .mremap = arena_vm_mremap, .close = arena_vm_close, .fault = arena_vm_fault, }; @@ -486,10 +498,11 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) arena->user_vm_end = vma->vm_end; /* * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and - * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid - * potential change of user_vm_start. + * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change + * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from + * being copied into the child process on fork. */ - vm_flags_set(vma, VM_DONTEXPAND); + vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY); vma->vm_ops = &arena_vm_ops; return 0; } @@ -549,6 +562,10 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt u32 uaddr32; int ret, i; + if (node_id != NUMA_NO_NODE && + ((unsigned int)node_id >= nr_node_ids || !node_online(node_id))) + return 0; + if (page_cnt > page_cnt_max) return 0; diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index 33de68c95ad8c1..5e25e03535094c 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -1015,8 +1015,10 @@ static void bpf_fd_array_map_clear(struct bpf_map *map, bool need_defer) struct bpf_array *array = container_of(map, struct bpf_array, map); int i; - for (i = 0; i < array->map.max_entries; i++) + for (i = 0; i < array->map.max_entries; i++) { __fd_array_map_delete_elem(map, &i, need_defer); + cond_resched(); + } } static void prog_array_map_seq_show_elem(struct bpf_map *map, void *key, diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 9c96a4477f81a8..4c6079d2cf28d0 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -393,6 +393,9 @@ int bpf_selem_unlink(struct bpf_local_storage_elem *selem) unsigned long flags; int err; + if (in_nmi()) + return -EOPNOTSUPP; + if (unlikely(!selem_linked_to_storage_lockless(selem))) /* selem has already been unlinked from sk */ return 0; diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 0c4a0c8e6f703e..0aa9378fae4f7a 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -359,8 +359,6 @@ BTF_ID(func, bpf_lsm_sb_umount) BTF_ID(func, bpf_lsm_settime) #ifdef CONFIG_SECURITY_NETWORK -BTF_ID(func, bpf_lsm_inet_conn_established) - BTF_ID(func, bpf_lsm_socket_accept) BTF_ID(func, bpf_lsm_socket_bind) BTF_ID(func, bpf_lsm_socket_connect) @@ -381,7 +379,6 @@ BTF_ID(func, bpf_lsm_syslog) BTF_ID(func, bpf_lsm_task_alloc) BTF_ID(func, bpf_lsm_task_prctl) BTF_ID(func, bpf_lsm_task_setscheduler) -BTF_ID(func, bpf_lsm_task_to_inode) BTF_ID(func, bpf_lsm_userns_create) BTF_SET_END(sleepable_lsm_hooks) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 71f9143fe90f3b..63d075f374591f 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -9019,7 +9019,7 @@ static int btf_check_dtor_kfuncs(struct btf *btf, const struct btf_id_dtor_kfunc if (!t || !btf_type_is_ptr(t)) return -EINVAL; - if (IS_ENABLED(CONFIG_CFI_CLANG)) { + if (IS_ENABLED(CONFIG_CFI)) { /* Ensure the destructor kfunc type matches btf_dtor_kfunc_t */ t = btf_type_by_id(btf, t->type); if (!btf_type_is_void(t)) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7b675a451ec8ef..048d275accae21 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -1487,6 +1487,8 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) * know whether fp here is the clone or the original. */ fp->aux->prog = fp; + if (fp->aux->offload) + fp->aux->offload->prog = fp; bpf_prog_clone_free(fp_other); } diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c index 3d619d01088e3e..cc0a43ebab6b99 100644 --- a/kernel/bpf/devmap.c +++ b/kernel/bpf/devmap.c @@ -665,7 +665,7 @@ int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, for (i = 0; i < dtab->n_buckets; i++) { head = dev_map_index_hash(dtab, i); hlist_for_each_entry_rcu(dst, head, index_hlist, - lockdep_is_held(&dtab->index_lock)) { + rcu_read_lock_bh_held()) { if (!is_valid_dst(dst, xdpf)) continue; @@ -747,7 +747,6 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, struct bpf_dtab_netdev *dst, *last_dst = NULL; int excluded_devices[1+MAX_NEST_DEV]; struct hlist_head *head; - struct hlist_node *next; int num_excluded = 0; unsigned int i; int err; @@ -787,7 +786,7 @@ int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, } else { /* BPF_MAP_TYPE_DEVMAP_HASH */ for (i = 0; i < dtab->n_buckets; i++) { head = dev_map_index_hash(dtab, i); - hlist_for_each_entry_safe(dst, next, head, index_hlist) { + hlist_for_each_entry_rcu(dst, head, index_hlist, rcu_read_lock_bh_held()) { if (is_ifindex_excluded(excluded_devices, num_excluded, dst->dev->ifindex)) continue; diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index bc6bc8bb871d41..3dd9b4924ae4fb 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -1056,7 +1056,7 @@ static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr, for_each_possible_cpu(cpu) { if (cpu == current_cpu) - copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value); + copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value); else /* Since elem is preallocated, we cannot touch special fields */ zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); } @@ -1138,6 +1138,10 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, } else if (fd_htab_map_needs_adjust(htab)) { size = round_up(size, 8); memcpy(htab_elem_value(l_new, key_size), value, size); + } else if (map_flags & BPF_F_LOCK) { + copy_map_value_locked(&htab->map, + htab_elem_value(l_new, key_size), + value, false); } else { copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); } diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 6eb6c82ed2ee1a..d51f1b612f1d94 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -845,7 +845,13 @@ int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args, data->buf = buffers->buf; for (i = 0; i < fmt_size; i++) { - if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) { + unsigned char c = fmt[i]; + + /* + * Permit bytes >= 0x80 in plain text so UTF-8 literals can pass + * through unchanged, while still rejecting ASCII control bytes. + */ + if (isascii(c) && !isprint(c) && !isspace(c)) { err = -EINVAL; goto out; } @@ -867,6 +873,15 @@ int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args, * always access fmt[i + 1], in the worst case it will be a 0 */ i++; + c = fmt[i]; + /* + * The format parser below only understands ASCII conversion + * specifiers and modifiers, so reject non-ASCII after '%'. + */ + if (!isascii(c)) { + err = -EINVAL; + goto out; + } /* skip optional "[0 +-][num]" width formatting field */ while (fmt[i] == '0' || fmt[i] == '+' || fmt[i] == '-' || diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c index 8fca0c64f7b1cd..23267213a17fb7 100644 --- a/kernel/bpf/local_storage.c +++ b/kernel/bpf/local_storage.c @@ -270,7 +270,7 @@ static int cgroup_storage_get_next_key(struct bpf_map *_map, void *key, goto enoent; storage = list_next_entry(storage, list_map); - if (!storage) + if (list_entry_is_head(storage, &map->list, list_map)) goto enoent; } else { storage = list_first_entry(&map->list, diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 0ad97d643bf49e..0d6f5569588c35 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -435,9 +435,8 @@ static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) if (aux->offload) { args->info->ifindex = aux->offload->netdev->ifindex; - net = dev_net(aux->offload->netdev); - get_net(net); - ns = &net->ns; + net = maybe_get_net(dev_net(aux->offload->netdev)); + ns = net ? &net->ns : NULL; } else { args->info->ifindex = 0; ns = NULL; @@ -647,9 +646,8 @@ static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) if (args->offmap->netdev) { args->info->ifindex = args->offmap->netdev->ifindex; - net = dev_net(args->offmap->netdev); - get_net(net); - ns = &net->ns; + net = maybe_get_net(dev_net(args->offmap->netdev)); + ns = net ? &net->ns : NULL; } else { args->info->ifindex = 0; ns = NULL; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 700938782bed2f..ed595159f1c536 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -3754,6 +3754,23 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, tr = prog->aux->dst_trampoline; tgt_prog = prog->aux->dst_prog; } + /* + * It is to prevent modifying struct pt_regs via kprobe_write_ctx=true + * freplace prog. Without this check, kprobe_write_ctx=true freplace + * prog is allowed to attach to kprobe_write_ctx=false kprobe prog, and + * then modify the registers of the kprobe prog's target kernel + * function. + * + * This also blocks the combination of uprobe+freplace, because it is + * unable to recognize the use of the tgt_prog as an uprobe or a kprobe + * by tgt_prog itself. At attach time, uprobe/kprobe is recognized by + * the target perf event flags in __perf_event_set_bpf_prog(). + */ + if (prog->type == BPF_PROG_TYPE_EXT && + prog->aux->kprobe_write_ctx != tgt_prog->aux->kprobe_write_ctx) { + err = -EINVAL; + goto out_unlock; + } err = bpf_link_prime(&link->link.link, &link_primer); if (err) diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index 98d9b4c0daff37..e791ae065c39bf 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include "mmap_unlock_work.h" static const char * const iter_task_type_names[] = { @@ -794,11 +796,20 @@ const struct bpf_func_proto bpf_find_vma_proto = { .arg5_type = ARG_ANYTHING, }; +static inline void bpf_iter_mmput_async(struct mm_struct *mm) +{ +#ifdef CONFIG_MMU + mmput_async(mm); +#else + mmput(mm); +#endif +} + struct bpf_iter_task_vma_kern_data { struct task_struct *task; struct mm_struct *mm; - struct mmap_unlock_irq_work *work; - struct vma_iterator vmi; + struct vm_area_struct snapshot; + u64 next_addr; }; struct bpf_iter_task_vma { @@ -819,12 +830,28 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, struct task_struct *task, u64 addr) { struct bpf_iter_task_vma_kern *kit = (void *)it; - bool irq_work_busy = false; int err; BUILD_BUG_ON(sizeof(struct bpf_iter_task_vma_kern) != sizeof(struct bpf_iter_task_vma)); BUILD_BUG_ON(__alignof__(struct bpf_iter_task_vma_kern) != __alignof__(struct bpf_iter_task_vma)); + if (!IS_ENABLED(CONFIG_PER_VMA_LOCK)) { + kit->data = NULL; + return -EOPNOTSUPP; + } + + /* + * Reject irqs-disabled contexts including NMI. Operations used + * by _next() and _destroy() (vma_end_read, fput, bpf_iter_mmput_async) + * can take spinlocks with IRQs disabled (pi_lock, pool->lock). + * Running from NMI or from a tracepoint that fires with those + * locks held could deadlock. + */ + if (irqs_disabled()) { + kit->data = NULL; + return -EBUSY; + } + /* is_iter_reg_valid_uninit guarantees that kit hasn't been initialized * before, so non-NULL kit->data doesn't point to previously * bpf_mem_alloc'd bpf_iter_task_vma_kern_data @@ -834,38 +861,131 @@ __bpf_kfunc int bpf_iter_task_vma_new(struct bpf_iter_task_vma *it, return -ENOMEM; kit->data->task = get_task_struct(task); + /* + * Safely read task->mm and acquire an mm reference. + * + * Cannot use get_task_mm() because its task_lock() is a + * blocking spin_lock that would deadlock if the target task + * already holds alloc_lock on this CPU (e.g. a softirq BPF + * program iterating a task interrupted while holding its + * alloc_lock). + */ + if (!spin_trylock(&task->alloc_lock)) { + err = -EBUSY; + goto err_cleanup_iter; + } kit->data->mm = task->mm; + if (kit->data->mm && !(task->flags & PF_KTHREAD)) + mmget(kit->data->mm); + else + kit->data->mm = NULL; + spin_unlock(&task->alloc_lock); if (!kit->data->mm) { err = -ENOENT; goto err_cleanup_iter; } - /* kit->data->work == NULL is valid after bpf_mmap_unlock_get_irq_work */ - irq_work_busy = bpf_mmap_unlock_get_irq_work(&kit->data->work); - if (irq_work_busy || !mmap_read_trylock(kit->data->mm)) { - err = -EBUSY; - goto err_cleanup_iter; - } - - vma_iter_init(&kit->data->vmi, kit->data->mm, addr); + kit->data->snapshot.vm_file = NULL; + kit->data->next_addr = addr; return 0; err_cleanup_iter: - if (kit->data->task) - put_task_struct(kit->data->task); + put_task_struct(kit->data->task); bpf_mem_free(&bpf_global_ma, kit->data); /* NULL kit->data signals failed bpf_iter_task_vma initialization */ kit->data = NULL; return err; } +/* + * Find and lock the next VMA at or after data->next_addr. + * + * lock_vma_under_rcu() is a point lookup (mas_walk): it finds the VMA + * containing a given address but cannot iterate. An RCU-protected + * maple tree walk with vma_next() (mas_find) is needed first to locate + * the next VMA's vm_start across any gap. + * + * Between the RCU walk and the lock, the VMA may be removed, shrunk, + * or write-locked. On failure, advance past it using vm_end from the + * RCU walk. SLAB_TYPESAFE_BY_RCU can make vm_end stale, so fall back + * to PAGE_SIZE advancement to guarantee forward progress. + */ +static struct vm_area_struct * +bpf_iter_task_vma_find_next(struct bpf_iter_task_vma_kern_data *data) +{ + struct vm_area_struct *vma; + struct vma_iterator vmi; + unsigned long start, end; + +retry: + rcu_read_lock(); + vma_iter_init(&vmi, data->mm, data->next_addr); + vma = vma_next(&vmi); + if (!vma) { + rcu_read_unlock(); + return NULL; + } + start = vma->vm_start; + end = vma->vm_end; + rcu_read_unlock(); + + vma = lock_vma_under_rcu(data->mm, start); + if (!vma) { + if (end <= data->next_addr) + data->next_addr += PAGE_SIZE; + else + data->next_addr = end; + goto retry; + } + + if (unlikely(vma->vm_end <= data->next_addr)) { + data->next_addr += PAGE_SIZE; + vma_end_read(vma); + goto retry; + } + + return vma; +} + +static void bpf_iter_task_vma_snapshot_reset(struct vm_area_struct *snap) +{ + if (snap->vm_file) { + fput(snap->vm_file); + snap->vm_file = NULL; + } +} + __bpf_kfunc struct vm_area_struct *bpf_iter_task_vma_next(struct bpf_iter_task_vma *it) { struct bpf_iter_task_vma_kern *kit = (void *)it; + struct vm_area_struct *snap, *vma; if (!kit->data) /* bpf_iter_task_vma_new failed */ return NULL; - return vma_next(&kit->data->vmi); + + snap = &kit->data->snapshot; + + bpf_iter_task_vma_snapshot_reset(snap); + + vma = bpf_iter_task_vma_find_next(kit->data); + if (!vma) + return NULL; + + memcpy(snap, vma, sizeof(*snap)); + + /* + * The verifier only trusts vm_mm and vm_file (see + * BTF_TYPE_SAFE_TRUSTED_OR_NULL in verifier.c). Take a reference + * on vm_file; vm_mm is already correct because lock_vma_under_rcu() + * verifies vma->vm_mm == mm. All other pointers are untrusted by + * the verifier and left as-is. + */ + if (snap->vm_file) + get_file(snap->vm_file); + + kit->data->next_addr = vma->vm_end; + vma_end_read(vma); + return snap; } __bpf_kfunc void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it) @@ -873,8 +993,9 @@ __bpf_kfunc void bpf_iter_task_vma_destroy(struct bpf_iter_task_vma *it) struct bpf_iter_task_vma_kern *kit = (void *)it; if (kit->data) { - bpf_mmap_unlock_mm(kit->data->work, kit->data->mm); + bpf_iter_task_vma_snapshot_reset(&kit->data->snapshot); put_task_struct(kit->data->task); + bpf_iter_mmput_async(kit->data->mm); bpf_mem_free(&bpf_global_ma, kit->data); } } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e3814152b52f81..77ddd452b8035a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5241,6 +5241,18 @@ static void check_fastcall_stack_contract(struct bpf_verifier_env *env, } } +static void scrub_special_slot(struct bpf_func_state *state, int spi) +{ + int i; + + /* regular write of data into stack destroys any spilled ptr */ + state->stack[spi].spilled_ptr.type = NOT_INIT; + /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ + if (is_stack_slot_special(&state->stack[spi])) + for (i = 0; i < BPF_REG_SIZE; i++) + scrub_spilled_slot(&state->stack[spi].slot_type[i]); +} + /* check_stack_{read,write}_fixed_off functions track spill/fill of registers, * stack boundary and alignment are checked in check_mem_access() */ @@ -5338,12 +5350,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, } else { u8 type = STACK_MISC; - /* regular write of data into stack destroys any spilled ptr */ - state->stack[spi].spilled_ptr.type = NOT_INIT; - /* Mark slots as STACK_MISC if they belonged to spilled ptr/dynptr/iter. */ - if (is_stack_slot_special(&state->stack[spi])) - for (i = 0; i < BPF_REG_SIZE; i++) - scrub_spilled_slot(&state->stack[spi].slot_type[i]); + scrub_special_slot(state, spi); /* when we zero initialize stack slots mark them as such */ if ((reg && register_is_null(reg)) || @@ -5467,8 +5474,13 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env, } } - /* Erase all other spilled pointers. */ - state->stack[spi].spilled_ptr.type = NOT_INIT; + /* + * Scrub slots if variable-offset stack write goes over spilled pointers. + * Otherwise is_spilled_reg() may == true && spilled_ptr.type == NOT_INIT + * and valid program is rejected by check_stack_read_fixed_off() + * with obscure "invalid size of register fill" message. + */ + scrub_special_slot(state, spi); /* Update the slot type. */ new_type = STACK_MISC; @@ -5988,6 +6000,9 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, int perm_flags; const char *reg_name = ""; + if (base_type(reg->type) != PTR_TO_BTF_ID) + goto bad_type; + if (btf_is_kernel(reg->btf)) { perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU; @@ -6000,7 +6015,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, perm_flags |= MEM_PERCPU; } - if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags)) + if (type_flag(reg->type) & ~perm_flags) goto bad_type; /* We need to verify reg->type and reg->btf, before accessing reg->btf */ @@ -16210,11 +16225,20 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, int err; dst_reg = ®s[insn->dst_reg]; - src_reg = NULL; + if (BPF_SRC(insn->code) == BPF_X) + src_reg = ®s[insn->src_reg]; + else + src_reg = NULL; - if (dst_reg->type == PTR_TO_ARENA) { + /* Case where at least one operand is an arena. */ + if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { struct bpf_insn_aux_data *aux = cur_aux(env); + if (dst_reg->type != PTR_TO_ARENA) + *dst_reg = *src_reg; + + dst_reg->subreg_def = env->insn_idx + 1; + if (BPF_CLASS(insn->code) == BPF_ALU64) /* * 32-bit operations zero upper bits automatically. @@ -16230,7 +16254,6 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, ptr_reg = dst_reg; if (BPF_SRC(insn->code) == BPF_X) { - src_reg = ®s[insn->src_reg]; if (src_reg->type != SCALAR_VALUE) { if (dst_reg->type != SCALAR_VALUE) { /* Combining two pointers by any ALU op yields @@ -16313,7 +16336,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, */ if (env->bpf_capable && (BPF_OP(insn->code) == BPF_ADD || BPF_OP(insn->code) == BPF_SUB) && - dst_reg->id && is_reg_const(src_reg, alu32)) { + dst_reg->id && is_reg_const(src_reg, alu32) && + !(BPF_SRC(insn->code) == BPF_X && insn->src_reg == insn->dst_reg)) { u64 val = reg_const_value(src_reg, alu32); s32 off; @@ -17930,6 +17954,23 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) mark_reg_unknown(env, regs, BPF_REG_0); /* ld_abs load up to 32-bit skb data. */ regs[BPF_REG_0].subreg_def = env->insn_idx + 1; + /* + * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 + * which must be explored by the verifier when in a subprog. + */ + if (env->cur_state->curframe) { + struct bpf_verifier_state *branch; + + mark_reg_scratched(env, BPF_REG_0); + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_known_zero(env, regs, BPF_REG_0); + err = prepare_func_exit(env, &env->insn_idx); + if (err) + return err; + env->insn_idx--; + } return 0; } @@ -18734,7 +18775,7 @@ static struct bpf_iarray *jt_from_subprog(struct bpf_verifier_env *env, static struct bpf_iarray * create_jt(int t, struct bpf_verifier_env *env) { - static struct bpf_subprog_info *subprog; + struct bpf_subprog_info *subprog; int subprog_start, subprog_end; struct bpf_iarray *jt; int i; @@ -18802,9 +18843,14 @@ static int visit_gotox_insn(int t, struct bpf_verifier_env *env) return keep_exploring ? KEEP_EXPLORING : DONE_EXPLORING; } -static int visit_tailcall_insn(struct bpf_verifier_env *env, int t) +/* + * Instructions that can abnormally return from a subprog (tail_call + * upon success, ld_{abs,ind} upon load failure) have a hidden exit + * that the verifier must account for. + */ +static int visit_abnormal_return_insn(struct bpf_verifier_env *env, int t) { - static struct bpf_subprog_info *subprog; + struct bpf_subprog_info *subprog; struct bpf_iarray *jt; if (env->insn_aux_data[t].jt) @@ -18837,6 +18883,13 @@ static int visit_insn(int t, struct bpf_verifier_env *env) /* All non-branch instructions have a single fall-through edge. */ if (BPF_CLASS(insn->code) != BPF_JMP && BPF_CLASS(insn->code) != BPF_JMP32) { + if (BPF_CLASS(insn->code) == BPF_LD && + (BPF_MODE(insn->code) == BPF_ABS || + BPF_MODE(insn->code) == BPF_IND)) { + ret = visit_abnormal_return_insn(env, t); + if (ret) + return ret; + } insn_sz = bpf_is_ldimm64(insn) ? 2 : 1; return push_insn(t, t + insn_sz, FALLTHROUGH, env); } @@ -18881,8 +18934,11 @@ static int visit_insn(int t, struct bpf_verifier_env *env) mark_subprog_might_sleep(env, t); if (bpf_helper_changes_pkt_data(insn->imm)) mark_subprog_changes_pkt_data(env, t); - if (insn->imm == BPF_FUNC_tail_call) - visit_tailcall_insn(env, t); + if (insn->imm == BPF_FUNC_tail_call) { + ret = visit_abnormal_return_insn(env, t); + if (ret) + return ret; + } } else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { struct bpf_kfunc_call_arg_meta meta; @@ -19629,6 +19685,13 @@ static bool check_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) * and r7.id=0 (both independent), without temp IDs both would map old_id=X * to cur_id=0 and pass. With temp IDs: r6 maps X->temp1, r7 tries to map * X->temp2, but X is already mapped to temp1, so the check fails correctly. + * + * When old_id has BPF_ADD_CONST set, the compound id (base | flag) and the + * base id (flag stripped) must both map consistently. Example: old has + * r2.id=A, r3.id=A|flag (r3 = r2 + delta), cur has r2.id=B, r3.id=C|flag + * (r3 derived from unrelated r4). Without the base check, idmap gets two + * independent entries A->B and A|flag->C|flag, missing that A->C conflicts + * with A->B. The base ID cross-check catches this. */ static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) { @@ -19637,7 +19700,15 @@ static bool check_scalar_ids(u32 old_id, u32 cur_id, struct bpf_idmap *idmap) cur_id = cur_id ? cur_id : ++idmap->tmp_id_gen; - return check_ids(old_id, cur_id, idmap); + if (!check_ids(old_id, cur_id, idmap)) + return false; + if (old_id & BPF_ADD_CONST) { + old_id &= ~BPF_ADD_CONST; + cur_id &= ~BPF_ADD_CONST; + if (!check_ids(old_id, cur_id, idmap)) + return false; + } + return true; } static void clean_func_state(struct bpf_verifier_env *env, @@ -24897,7 +24968,7 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) } for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) { - if (st_ops_desc->arg_info[member_idx].info->refcounted) { + if (st_ops_desc->arg_info[member_idx].info[i].refcounted) { has_refcounted_arg = true; break; } diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 4ca3cb993da299..8789ba613ea167 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -278,10 +278,12 @@ static void cgroup_finalize_control(struct cgroup *cgrp, int ret); static void css_task_iter_skip(struct css_task_iter *it, struct task_struct *task); static int cgroup_destroy_locked(struct cgroup *cgrp); +static void cgroup_finish_destroy(struct cgroup *cgrp); +static void kill_css_sync(struct cgroup_subsys_state *css); +static void kill_css_finish(struct cgroup_subsys_state *css); static struct cgroup_subsys_state *css_create(struct cgroup *cgrp, struct cgroup_subsys *ss); static void css_release(struct percpu_ref *ref); -static void kill_css(struct cgroup_subsys_state *css); static int cgroup_addrm_files(struct cgroup_subsys_state *css, struct cgroup *cgrp, struct cftype cfts[], bool is_add); @@ -858,6 +860,16 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated) if (was_populated == cgroup_is_populated(cgrp)) break; + /* + * Subtree just emptied below an offlined cgrp. Fire deferred + * destroy. The transition is one-shot. + */ + if (was_populated && !css_is_online(&cgrp->self)) { + cgroup_get(cgrp); + WARN_ON_ONCE(!queue_work(cgroup_offline_wq, + &cgrp->finish_destroy_work)); + } + cgroup1_check_for_release(cgrp); TRACE_CGROUP_PATH(notify_populated, cgrp, cgroup_is_populated(cgrp)); @@ -2100,6 +2112,16 @@ static int cgroup_reconfigure(struct fs_context *fc) return 0; } +static void cgroup_finish_destroy_work_fn(struct work_struct *work) +{ + struct cgroup *cgrp = container_of(work, struct cgroup, finish_destroy_work); + + cgroup_lock(); + cgroup_finish_destroy(cgrp); + cgroup_unlock(); + cgroup_put(cgrp); +} + static void init_cgroup_housekeeping(struct cgroup *cgrp) { struct cgroup_subsys *ss; @@ -2126,7 +2148,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp) #endif init_waitqueue_head(&cgrp->offline_waitq); - init_waitqueue_head(&cgrp->dying_populated_waitq); + INIT_WORK(&cgrp->finish_destroy_work, cgroup_finish_destroy_work_fn); INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent); } @@ -3436,7 +3458,8 @@ static void cgroup_apply_control_disable(struct cgroup *cgrp) if (css->parent && !(cgroup_ss_mask(dsct) & (1 << ss->id))) { - kill_css(css); + kill_css_sync(css); + kill_css_finish(css); } else if (!css_visible(css)) { css_clear_dir(css); if (ss->css_reset) @@ -3995,33 +4018,41 @@ static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v) static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, size_t nbytes, enum psi_res res) { - struct cgroup_file_ctx *ctx = of->priv; + struct cgroup_file_ctx *ctx; struct psi_trigger *new; struct cgroup *cgrp; struct psi_group *psi; + ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); if (!cgrp) return -ENODEV; - cgroup_get(cgrp); - cgroup_kn_unlock(of->kn); + ctx = of->priv; + if (!ctx) { + ret = -ENODEV; + goto out_unlock; + } /* Allow only one trigger per file descriptor */ if (ctx->psi.trigger) { - cgroup_put(cgrp); - return -EBUSY; + ret = -EBUSY; + goto out_unlock; } psi = cgroup_psi(cgrp); new = psi_trigger_create(psi, buf, res, of->file, of); if (IS_ERR(new)) { - cgroup_put(cgrp); - return PTR_ERR(new); + ret = PTR_ERR(new); + goto out_unlock; } smp_store_release(&ctx->psi.trigger, new); - cgroup_put(cgrp); + +out_unlock: + cgroup_kn_unlock(of->kn); + if (ret) + return ret; return nbytes; } @@ -5558,7 +5589,7 @@ static struct cftype cgroup_psi_files[] = { * css destruction is four-stage process. * * 1. Destruction starts. Killing of the percpu_ref is initiated. - * Implemented in kill_css(). + * Implemented in kill_css_finish(). * * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs * and thus css_tryget_online() is guaranteed to fail, the css can be @@ -5768,16 +5799,6 @@ static void offline_css(struct cgroup_subsys_state *css) RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); wake_up_all(&css->cgroup->offline_waitq); - - css->cgroup->nr_dying_subsys[ss->id]++; - /* - * Parent css and cgroup cannot be freed until after the freeing - * of child css, see css_free_rwork_fn(). - */ - while ((css = css->parent)) { - css->nr_descendants--; - css->cgroup->nr_dying_subsys[ss->id]++; - } } /** @@ -6047,7 +6068,7 @@ int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, umode_t mode) /* * This is called when the refcnt of a css is confirmed to be killed. * css_tryget_online() is now guaranteed to fail. Tell the subsystem to - * initiate destruction and put the css ref from kill_css(). + * initiate destruction and put the css ref from kill_css_finish(). */ static void css_killed_work_fn(struct work_struct *work) { @@ -6079,16 +6100,15 @@ static void css_killed_ref_fn(struct percpu_ref *ref) } /** - * kill_css - destroy a css - * @css: css to destroy + * kill_css_sync - synchronous half of css teardown + * @css: css being killed * - * This function initiates destruction of @css by removing cgroup interface - * files and putting its base reference. ->css_offline() will be invoked - * asynchronously once css_tryget_online() is guaranteed to fail and when - * the reference count reaches zero, @css will be released. + * See cgroup_destroy_locked(). */ -static void kill_css(struct cgroup_subsys_state *css) +static void kill_css_sync(struct cgroup_subsys_state *css) { + struct cgroup_subsys *ss = css->ss; + lockdep_assert_held(&cgroup_mutex); if (css->flags & CSS_DYING) @@ -6108,64 +6128,100 @@ static void kill_css(struct cgroup_subsys_state *css) */ css_clear_dir(css); + css->cgroup->nr_dying_subsys[ss->id]++; + /* + * Parent css and cgroup cannot be freed until after the freeing + * of child css, see css_free_rwork_fn(). + */ + while ((css = css->parent)) { + css->nr_descendants--; + css->cgroup->nr_dying_subsys[ss->id]++; + } +} + +/** + * kill_css_finish - deferred half of css teardown + * @css: css being killed + * + * See cgroup_destroy_locked(). + */ +static void kill_css_finish(struct cgroup_subsys_state *css) +{ + lockdep_assert_held(&cgroup_mutex); + + /* + * Skip on re-entry: cgroup_apply_control_disable() may have killed @css + * earlier. cgroup_destroy_locked() can still walk it because + * offline_css() (which NULLs cgrp->subsys[ssid]) runs async. + */ + if (percpu_ref_is_dying(&css->refcnt)) + return; + /* - * Killing would put the base ref, but we need to keep it alive - * until after ->css_offline(). + * Killing would put the base ref, but we need to keep it alive until + * after ->css_offline(). */ css_get(css); /* - * cgroup core guarantees that, by the time ->css_offline() is - * invoked, no new css reference will be given out via - * css_tryget_online(). We can't simply call percpu_ref_kill() and - * proceed to offlining css's because percpu_ref_kill() doesn't - * guarantee that the ref is seen as killed on all CPUs on return. + * cgroup core guarantees that, by the time ->css_offline() is invoked, + * no new css reference will be given out via css_tryget_online(). We + * can't simply call percpu_ref_kill() and proceed to offlining css's + * because percpu_ref_kill() doesn't guarantee that the ref is seen as + * killed on all CPUs on return. * - * Use percpu_ref_kill_and_confirm() to get notifications as each - * css is confirmed to be seen as killed on all CPUs. + * Use percpu_ref_kill_and_confirm() to get notifications as each css is + * confirmed to be seen as killed on all CPUs. */ percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); } /** - * cgroup_destroy_locked - the first stage of cgroup destruction + * cgroup_destroy_locked - destroy @cgrp (called on rmdir) * @cgrp: cgroup to be destroyed * - * css's make use of percpu refcnts whose killing latency shouldn't be - * exposed to userland and are RCU protected. Also, cgroup core needs to - * guarantee that css_tryget_online() won't succeed by the time - * ->css_offline() is invoked. To satisfy all the requirements, - * destruction is implemented in the following two steps. - * - * s1. Verify @cgrp can be destroyed and mark it dying. Remove all - * userland visible parts and start killing the percpu refcnts of - * css's. Set up so that the next stage will be kicked off once all - * the percpu refcnts are confirmed to be killed. - * - * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the - * rest of destruction. Once all cgroup references are gone, the - * cgroup is RCU-freed. - * - * This function implements s1. After this step, @cgrp is gone as far as - * the userland is concerned and a new cgroup with the same name may be - * created. As cgroup doesn't care about the names internally, this - * doesn't cause any problem. + * Tear down @cgrp on behalf of rmdir. Constraints: + * + * - Userspace: rmdir must succeed when cgroup.procs and friends are empty. + * + * - Kernel: subsystem ->css_offline() must not run while any task in @cgrp's + * subtree is still doing kernel work. A task hidden from cgroup.procs (past + * exit_signals() with signal->live cleared) can still schedule, allocate, and + * consume resources until its final context switch. Dying descendants in the + * subtree can host such tasks too. + * + * - Kernel: css_tryget_online() must fail by the time ->css_offline() runs. + * + * The destruction runs in three parts: + * + * - This function: synchronous user-visible state teardown plus kill_css_sync() + * on each subsystem css. + * + * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on + * each subsystem css. Fires once @cgrp's subtree is fully drained, either + * inline here or from cgroup_update_populated(). + * + * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn -> + * ->css_offline() -> release/free. + * + * Return 0 on success, -EBUSY if a userspace-visible task or an online child + * remains. */ static int cgroup_destroy_locked(struct cgroup *cgrp) - __releases(&cgroup_mutex) __acquires(&cgroup_mutex) { struct cgroup *tcgrp, *parent = cgroup_parent(cgrp); struct cgroup_subsys_state *css; struct cgrp_cset_link *link; + struct css_task_iter it; + struct task_struct *task; int ssid, ret; lockdep_assert_held(&cgroup_mutex); - /* - * Only migration can raise populated from zero and we're already - * holding cgroup_mutex. - */ - if (cgroup_is_populated(cgrp)) + css_task_iter_start(&cgrp->self, 0, &it); + task = css_task_iter_next(&it); + css_task_iter_end(&it); + if (task) return -EBUSY; /* @@ -6189,9 +6245,8 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) link->cset->dead = true; spin_unlock_irq(&css_set_lock); - /* initiate massacre of all css's */ for_each_css(css, ssid, cgrp) - kill_css(css); + kill_css_sync(css); /* clear and remove @cgrp dir, @cgrp has an extra ref on its kn */ css_clear_dir(&cgrp->self); @@ -6222,79 +6277,27 @@ static int cgroup_destroy_locked(struct cgroup *cgrp) /* put the base reference */ percpu_ref_kill(&cgrp->self.refcnt); + if (!cgroup_is_populated(cgrp)) + cgroup_finish_destroy(cgrp); + return 0; }; /** - * cgroup_drain_dying - wait for dying tasks to leave before rmdir - * @cgrp: the cgroup being removed - * - * cgroup.procs and cgroup.threads use css_task_iter which filters out - * PF_EXITING tasks so that userspace doesn't see tasks that have already been - * reaped via waitpid(). However, cgroup_has_tasks() - which tests whether the - * cgroup has non-empty css_sets - is only updated when dying tasks pass through - * cgroup_task_dead() in finish_task_switch(). This creates a window where - * cgroup.procs reads empty but cgroup_has_tasks() is still true, making rmdir - * fail with -EBUSY from cgroup_destroy_locked() even though userspace sees no - * tasks. - * - * This function aligns cgroup_has_tasks() with what userspace can observe. If - * cgroup_has_tasks() but the task iterator sees nothing (all remaining tasks are - * PF_EXITING), we wait for cgroup_task_dead() to finish processing them. As the - * window between PF_EXITING and cgroup_task_dead() is short, the wait is brief. - * - * This function only concerns itself with this cgroup's own dying tasks. - * Whether the cgroup has children is cgroup_destroy_locked()'s problem. - * - * Each cgroup_task_dead() kicks the waitqueue via cset->cgrp_links, and we - * retry the full check from scratch. + * cgroup_finish_destroy - deferred half of @cgrp destruction + * @cgrp: cgroup whose subtree just became empty * - * Must be called with cgroup_mutex held. + * See cgroup_destroy_locked() for the rationale. */ -static int cgroup_drain_dying(struct cgroup *cgrp) - __releases(&cgroup_mutex) __acquires(&cgroup_mutex) +static void cgroup_finish_destroy(struct cgroup *cgrp) { - struct css_task_iter it; - struct task_struct *task; - DEFINE_WAIT(wait); + struct cgroup_subsys_state *css; + int ssid; lockdep_assert_held(&cgroup_mutex); -retry: - if (!cgroup_has_tasks(cgrp)) - return 0; - - /* Same iterator as cgroup.threads - if any task is visible, it's busy */ - css_task_iter_start(&cgrp->self, 0, &it); - task = css_task_iter_next(&it); - css_task_iter_end(&it); - if (task) - return -EBUSY; - - /* - * All remaining tasks are PF_EXITING and will pass through - * cgroup_task_dead() shortly. Wait for a kick and retry. - * - * cgroup_has_tasks() can't transition from false to true while we're - * holding cgroup_mutex, but the true to false transition happens - * under css_set_lock (via cgroup_task_dead()). We must retest and - * prepare_to_wait() under css_set_lock. Otherwise, the transition - * can happen between our first test and prepare_to_wait(), and we - * sleep with no one to wake us. - */ - spin_lock_irq(&css_set_lock); - if (!cgroup_has_tasks(cgrp)) { - spin_unlock_irq(&css_set_lock); - return 0; - } - prepare_to_wait(&cgrp->dying_populated_waitq, &wait, - TASK_UNINTERRUPTIBLE); - spin_unlock_irq(&css_set_lock); - mutex_unlock(&cgroup_mutex); - schedule(); - finish_wait(&cgrp->dying_populated_waitq, &wait); - mutex_lock(&cgroup_mutex); - goto retry; + for_each_css(css, ssid, cgrp) + kill_css_finish(css); } int cgroup_rmdir(struct kernfs_node *kn) @@ -6306,12 +6309,9 @@ int cgroup_rmdir(struct kernfs_node *kn) if (!cgrp) return 0; - ret = cgroup_drain_dying(cgrp); - if (!ret) { - ret = cgroup_destroy_locked(cgrp); - if (!ret) - TRACE_CGROUP_PATH(rmdir, cgrp); - } + ret = cgroup_destroy_locked(cgrp); + if (!ret) + TRACE_CGROUP_PATH(rmdir, cgrp); cgroup_kn_unlock(kn); return ret; @@ -7071,7 +7071,6 @@ void cgroup_task_exit(struct task_struct *tsk) static void do_cgroup_task_dead(struct task_struct *tsk) { - struct cgrp_cset_link *link; struct css_set *cset; unsigned long flags; @@ -7085,11 +7084,6 @@ static void do_cgroup_task_dead(struct task_struct *tsk) if (thread_group_leader(tsk) && atomic_read(&tsk->signal->live)) list_add_tail(&tsk->cg_list, &cset->dying_tasks); - /* kick cgroup_drain_dying() waiters, see cgroup_rmdir() */ - list_for_each_entry(link, &cset->cgrp_links, cgrp_link) - if (waitqueue_active(&link->cgrp->dying_populated_waitq)) - wake_up(&link->cgrp->dying_populated_waitq); - if (dl_task(tsk)) dec_dl_tasks_cs(tsk); diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index fd7d19842ded7d..f7aaf01f7cd5e3 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -167,7 +167,13 @@ struct cpuset { */ int nr_deadline_tasks; int nr_migrate_dl_tasks; + /* DL bandwidth that needs destination reservation for this attach. */ u64 sum_migrate_dl_bw; + /* + * CPU used for temporary DL bandwidth allocation during attach; + * -1 if no DL bandwidth was allocated in the current attach. + */ + int dl_bw_cpu; /* Invalid partition error code, not lock protected */ enum prs_errcode prs_err; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 1335e437098e80..cd79557e3f8e20 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -288,6 +288,7 @@ struct cpuset top_cpuset = { .flags = BIT(CS_CPU_EXCLUSIVE) | BIT(CS_MEM_EXCLUSIVE) | BIT(CS_SCHED_LOAD_BALANCE), .partition_root_state = PRS_ROOT, + .dl_bw_cpu = -1, }; /** @@ -579,6 +580,8 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) if (!trial) return NULL; + trial->dl_bw_cpu = -1; + /* Setup cpumask pointer array */ cpumask_var_t *pmask[4] = { &trial->cpus_allowed, @@ -1715,7 +1718,8 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, */ if (is_partition_valid(parent)) adding = cpumask_and(tmp->addmask, - xcpus, parent->effective_xcpus); + cs->effective_xcpus, + parent->effective_xcpus); if (old_prs > 0) new_prs = -old_prs; @@ -2980,6 +2984,7 @@ static void reset_migrate_dl_data(struct cpuset *cs) { cs->nr_migrate_dl_tasks = 0; cs->sum_migrate_dl_bw = 0; + cs->dl_bw_cpu = -1; } /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ @@ -2989,7 +2994,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) struct cpuset *cs, *oldcs; struct task_struct *task; bool setsched_check; - int ret; + int cpu, ret; /* used later by cpuset_attach() */ cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css)); @@ -3034,29 +3039,31 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) } if (dl_task(task)) { + /* + * Count all migrating DL tasks for cpuset task accounting. + * Only tasks that need a root-domain bandwidth move + * contribute to sum_migrate_dl_bw. + */ cs->nr_migrate_dl_tasks++; - cs->sum_migrate_dl_bw += task->dl.dl_bw; + if (dl_task_needs_bw_move(task, cs->effective_cpus)) + cs->sum_migrate_dl_bw += task->dl.dl_bw; } } - if (!cs->nr_migrate_dl_tasks) + if (!cs->sum_migrate_dl_bw) goto out_success; - if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) { - int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); + cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); + if (unlikely(cpu >= nr_cpu_ids)) { + ret = -EINVAL; + goto out_unlock; + } - if (unlikely(cpu >= nr_cpu_ids)) { - reset_migrate_dl_data(cs); - ret = -EINVAL; - goto out_unlock; - } + ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); + if (ret) + goto out_unlock; - ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); - if (ret) { - reset_migrate_dl_data(cs); - goto out_unlock; - } - } + cs->dl_bw_cpu = cpu; out_success: /* @@ -3064,7 +3071,10 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * changes which zero cpus/mems_allowed. */ cs->attach_in_progress++; + out_unlock: + if (ret) + reset_migrate_dl_data(cs); mutex_unlock(&cpuset_mutex); return ret; } @@ -3080,12 +3090,11 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset) mutex_lock(&cpuset_mutex); dec_attach_in_progress_locked(cs); - if (cs->nr_migrate_dl_tasks) { - int cpu = cpumask_any(cs->effective_cpus); + if (cs->dl_bw_cpu >= 0) + dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); - dl_bw_free(cpu, cs->sum_migrate_dl_bw); + if (cs->nr_migrate_dl_tasks) reset_migrate_dl_data(cs); - } mutex_unlock(&cpuset_mutex); } diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c index 9d95824dc6fa09..003cd8651de603 100644 --- a/kernel/cgroup/dmem.c +++ b/kernel/cgroup/dmem.c @@ -602,6 +602,7 @@ get_cg_pool_unlocked(struct dmemcg_state *cg, struct dmem_cgroup_region *region) pool = NULL; continue; } + pool = ERR_PTR(-ENOMEM); } } diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c index 09258eebb5c748..7d21a0db3cef0e 100644 --- a/kernel/cgroup/rdma.c +++ b/kernel/cgroup/rdma.c @@ -283,7 +283,7 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg, ret = PTR_ERR(rpool); goto err; } else { - new = rpool->resources[index].usage + 1; + new = (s64)rpool->resources[index].usage + 1; if (new > rpool->resources[index].max) { ret = -EAGAIN; goto err; diff --git a/kernel/exit.c b/kernel/exit.c index ede3117fa7d413..e2b63696125422 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -571,6 +571,7 @@ static void exit_mm(void) */ smp_mb__after_spinlock(); local_irq_disable(); + current->user_dumpable = (get_dumpable(mm) == SUID_DUMP_USER); current->mm = NULL; membarrier_update_current_mm(NULL); enter_lazy_tlb(mm, current); @@ -1074,6 +1075,7 @@ void __noreturn make_task_dead(int signr) futex_exit_recursive(tsk); tsk->exit_state = EXIT_DEAD; refcount_inc(&tsk->rcu_users); + preempt_disable(); do_task_dead(); } diff --git a/kernel/fork.c b/kernel/fork.c index bc2bf58b93b652..73622ad0665a07 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -95,6 +95,7 @@ #include #include #include +#include #include #include #include @@ -1948,9 +1949,11 @@ static void rv_task_fork(struct task_struct *p) static bool need_futex_hash_allocate_default(u64 clone_flags) { - if ((clone_flags & (CLONE_THREAD | CLONE_VM)) != (CLONE_THREAD | CLONE_VM)) - return false; - return true; + /* + * Allocate a default futex hash for any sibling that will + * share the parent's mm, except vfork. + */ + return (clone_flags & (CLONE_VM | CLONE_VFORK)) == CLONE_VM; } /* @@ -2233,6 +2236,7 @@ __latent_entropy struct task_struct *copy_process( if (retval) goto bad_fork_cleanup_io; + random_kstack_task_init(p); stackleak_task_init(p); if (pid != &init_struct_pid) { @@ -2336,10 +2340,6 @@ __latent_entropy struct task_struct *copy_process( if (retval) goto bad_fork_cancel_cgroup; - /* - * Allocate a default futex hash for the user process once the first - * thread spawns. - */ if (need_futex_hash_allocate_default(clone_flags)) { retval = futex_hash_allocate_default(); if (retval) @@ -3174,11 +3174,10 @@ int ksys_unshare(unsigned long unshare_flags) new_cred, new_fs); if (err) goto bad_unshare_cleanup_cred; - if (new_cred) { err = set_cred_ucounts(new_cred); if (err) - goto bad_unshare_cleanup_cred; + goto bad_unshare_cleanup_nsproxy; } if (new_fs || new_fd || do_sysvsem || new_cred || new_nsproxy) { @@ -3194,8 +3193,10 @@ int ksys_unshare(unsigned long unshare_flags) shm_init_task(current); } - if (new_nsproxy) + if (new_nsproxy) { switch_task_namespaces(current, new_nsproxy); + new_nsproxy = NULL; + } task_lock(current); @@ -3224,13 +3225,15 @@ int ksys_unshare(unsigned long unshare_flags) perf_event_namespaces(current); +bad_unshare_cleanup_nsproxy: + if (new_nsproxy) + put_nsproxy(new_nsproxy); bad_unshare_cleanup_cred: if (new_cred) put_cred(new_cred); bad_unshare_cleanup_fd: if (new_fd) put_files_struct(new_fd); - bad_unshare_cleanup_fs: if (new_fs) free_fs_struct(new_fs); diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index d818b4d47f1bad..b597cb3d17fc11 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -319,8 +319,11 @@ futex_proxy_trylock_atomic(u32 __user *pifutex, struct futex_hash_bucket *hb1, return -EINVAL; /* Ensure that this does not race against an early wakeup */ - if (!futex_requeue_pi_prepare(top_waiter, NULL)) + if (!futex_requeue_pi_prepare(top_waiter, NULL)) { + plist_del(&top_waiter->list, &hb1->chain); + futex_hb_waiters_dec(hb1); return -EAGAIN; + } /* * Try to take the lock for top_waiter and set the FUTEX_WAITERS bit @@ -722,10 +725,12 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, /* * We were woken prior to requeue by a timeout or a signal. - * Unqueue the futex_q and determine which it was. + * Conditionally unqueue the futex_q and determine which it was. */ - plist_del(&q->list, &hb->chain); - futex_hb_waiters_dec(hb); + if (!plist_node_empty(&q->list)) { + plist_del(&q->list, &hb->chain); + futex_hb_waiters_dec(hb); + } /* Handle spurious wakeups gracefully */ ret = -EWOULDBLOCK; diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index cc68a3692905d8..479c42e08b74a5 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -757,13 +757,18 @@ int kho_add_subtree(const char *name, void *fdt) goto out_pack; } - err = fdt_setprop(root_fdt, off, KHO_FDT_SUB_TREE_PROP_NAME, - &phys, sizeof(phys)); - if (err < 0) - goto out_pack; + fdt_err = fdt_setprop(root_fdt, off, KHO_FDT_SUB_TREE_PROP_NAME, + &phys, sizeof(phys)); + if (fdt_err < 0) + goto out_del_node; WARN_ON_ONCE(kho_debugfs_fdt_add(&kho_out.dbg, name, fdt, false)); + err = 0; + goto out_pack; + +out_del_node: + fdt_del_node(root_fdt, off); out_pack: fdt_pack(root_fdt); diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 2a1d165b3167e2..c867f6c15530d5 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -171,7 +171,7 @@ static __always_inline bool __mutex_unlock_fast(struct mutex *lock) #else /* !CONFIG_DEBUG_LOCK_ALLOC */ -void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key) +void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key) { __mutex_init_generic(lock); @@ -181,7 +181,7 @@ void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_k debug_check_no_locks_freed((void *)lock, sizeof(*lock)); lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_SLEEP); } -EXPORT_SYMBOL(mutex_init_lockep); +EXPORT_SYMBOL(mutex_init_lockdep); #endif /* !CONFIG_DEBUG_LOCK_ALLOC */ static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag) diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index c80902eacd797c..e6c6dd0868873e 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1535,20 +1535,23 @@ static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, * * Must be called with lock->wait_lock held and interrupts disabled. It must * have just failed to try_to_take_rt_mutex(). + * + * When invoked from rt_mutex_start_proxy_lock() waiter::task != current ! */ static void __sched remove_waiter(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) { bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); struct task_struct *owner = rt_mutex_owner(lock); + struct task_struct *waiter_task = waiter->task; struct rt_mutex_base *next_lock; lockdep_assert_held(&lock->wait_lock); - raw_spin_lock(¤t->pi_lock); - rt_mutex_dequeue(lock, waiter); - current->pi_blocked_on = NULL; - raw_spin_unlock(¤t->pi_lock); + scoped_guard(raw_spinlock, &waiter_task->pi_lock) { + rt_mutex_dequeue(lock, waiter); + waiter_task->pi_blocked_on = NULL; + } /* * Only update priority if the waiter was the highest priority @@ -1584,7 +1587,7 @@ static void __sched remove_waiter(struct rt_mutex_base *lock, raw_spin_unlock_irq(&lock->wait_lock); rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, - next_lock, NULL, current); + next_lock, NULL, waiter_task); raw_spin_lock_irq(&lock->wait_lock); } diff --git a/kernel/module/main.c b/kernel/module/main.c index c3ce106c70af16..ef2e2130972fe2 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1408,7 +1408,7 @@ static void free_module(struct module *mod) module_unload_free(mod); /* Free any allocated parameters. */ - destroy_params(mod->kp, mod->num_kp); + module_destroy_params(mod->kp, mod->num_kp); if (is_livepatch_module(mod)) free_module_elf(mod); @@ -3519,7 +3519,7 @@ static int load_module(struct load_info *info, const char __user *uargs, mod_sysfs_teardown(mod); coming_cleanup: mod->state = MODULE_STATE_GOING; - destroy_params(mod->kp, mod->num_kp); + module_destroy_params(mod->kp, mod->num_kp); blocking_notifier_call_chain(&module_notify_list, MODULE_STATE_GOING, mod); klp_module_going(mod); diff --git a/kernel/padata.c b/kernel/padata.c index 8657e6e0c224a2..0d3ea1b68b1f70 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -535,7 +535,8 @@ static void padata_init_reorder_list(struct parallel_data *pd) } /* Allocate and initialize the internal cpumask dependend resources. */ -static struct parallel_data *padata_alloc_pd(struct padata_shell *ps) +static struct parallel_data *padata_alloc_pd(struct padata_shell *ps, + int offlining_cpu) { struct padata_instance *pinst = ps->pinst; struct parallel_data *pd; @@ -561,6 +562,10 @@ static struct parallel_data *padata_alloc_pd(struct padata_shell *ps) cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask); cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask); + if (offlining_cpu >= 0) { + __cpumask_clear_cpu(offlining_cpu, pd->cpumask.pcpu); + __cpumask_clear_cpu(offlining_cpu, pd->cpumask.cbcpu); + } padata_init_reorder_list(pd); padata_init_squeues(pd); @@ -607,11 +612,11 @@ static void __padata_stop(struct padata_instance *pinst) } /* Replace the internal control structure with a new one. */ -static int padata_replace_one(struct padata_shell *ps) +static int padata_replace_one(struct padata_shell *ps, int offlining_cpu) { struct parallel_data *pd_new; - pd_new = padata_alloc_pd(ps); + pd_new = padata_alloc_pd(ps, offlining_cpu); if (!pd_new) return -ENOMEM; @@ -621,7 +626,7 @@ static int padata_replace_one(struct padata_shell *ps) return 0; } -static int padata_replace(struct padata_instance *pinst) +static int padata_replace(struct padata_instance *pinst, int offlining_cpu) { struct padata_shell *ps; int err = 0; @@ -629,7 +634,7 @@ static int padata_replace(struct padata_instance *pinst) pinst->flags |= PADATA_RESET; list_for_each_entry(ps, &pinst->pslist, list) { - err = padata_replace_one(ps); + err = padata_replace_one(ps, offlining_cpu); if (err) break; } @@ -646,9 +651,21 @@ static int padata_replace(struct padata_instance *pinst) /* If cpumask contains no active cpu, we mark the instance as invalid. */ static bool padata_validate_cpumask(struct padata_instance *pinst, - const struct cpumask *cpumask) + const struct cpumask *cpumask, + int offlining_cpu) { - if (!cpumask_intersects(cpumask, cpu_online_mask)) { + cpumask_copy(pinst->validate_cpumask, cpu_online_mask); + + /* + * @offlining_cpu is still in cpu_online_mask, so remove it here for + * validation. Using a sub-CPUHP_TEARDOWN_CPU hotplug state where + * @offlining_cpu wouldn't be in the online mask doesn't work because + * padata_cpu_offline() can fail but such a state doesn't allow failure. + */ + if (offlining_cpu >= 0) + __cpumask_clear_cpu(offlining_cpu, pinst->validate_cpumask); + + if (!cpumask_intersects(cpumask, pinst->validate_cpumask)) { pinst->flags |= PADATA_INVALID; return false; } @@ -664,13 +681,13 @@ static int __padata_set_cpumasks(struct padata_instance *pinst, int valid; int err; - valid = padata_validate_cpumask(pinst, pcpumask); + valid = padata_validate_cpumask(pinst, pcpumask, -1); if (!valid) { __padata_stop(pinst); goto out_replace; } - valid = padata_validate_cpumask(pinst, cbcpumask); + valid = padata_validate_cpumask(pinst, cbcpumask, -1); if (!valid) __padata_stop(pinst); @@ -678,7 +695,7 @@ static int __padata_set_cpumasks(struct padata_instance *pinst, cpumask_copy(pinst->cpumask.pcpu, pcpumask); cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); - err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst); + err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst, -1); if (valid) __padata_start(pinst); @@ -730,36 +747,6 @@ EXPORT_SYMBOL(padata_set_cpumask); #ifdef CONFIG_HOTPLUG_CPU -static int __padata_add_cpu(struct padata_instance *pinst, int cpu) -{ - int err = 0; - - if (cpumask_test_cpu(cpu, cpu_online_mask)) { - err = padata_replace(pinst); - - if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && - padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) - __padata_start(pinst); - } - - return err; -} - -static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) -{ - int err = 0; - - if (!cpumask_test_cpu(cpu, cpu_online_mask)) { - if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || - !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) - __padata_stop(pinst); - - err = padata_replace(pinst); - } - - return err; -} - static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) { return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) || @@ -771,27 +758,39 @@ static int padata_cpu_online(unsigned int cpu, struct hlist_node *node) struct padata_instance *pinst; int ret; - pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node); + pinst = hlist_entry_safe(node, struct padata_instance, cpuhp_node); if (!pinst_has_cpu(pinst, cpu)) return 0; mutex_lock(&pinst->lock); - ret = __padata_add_cpu(pinst, cpu); + + ret = padata_replace(pinst, -1); + + if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu, -1) && + padata_validate_cpumask(pinst, pinst->cpumask.cbcpu, -1)) + __padata_start(pinst); + mutex_unlock(&pinst->lock); return ret; } -static int padata_cpu_dead(unsigned int cpu, struct hlist_node *node) +static int padata_cpu_offline(unsigned int cpu, struct hlist_node *node) { struct padata_instance *pinst; int ret; - pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node); + pinst = hlist_entry_safe(node, struct padata_instance, cpuhp_node); if (!pinst_has_cpu(pinst, cpu)) return 0; mutex_lock(&pinst->lock); - ret = __padata_remove_cpu(pinst, cpu); + + if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu, cpu) || + !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu, cpu)) + __padata_stop(pinst); + + ret = padata_replace(pinst, cpu); + mutex_unlock(&pinst->lock); return ret; } @@ -802,15 +801,14 @@ static enum cpuhp_state hp_online; static void __padata_free(struct padata_instance *pinst) { #ifdef CONFIG_HOTPLUG_CPU - cpuhp_state_remove_instance_nocalls(CPUHP_PADATA_DEAD, - &pinst->cpu_dead_node); - cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node); + cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpuhp_node); #endif WARN_ON(!list_empty(&pinst->pslist)); free_cpumask_var(pinst->cpumask.pcpu); free_cpumask_var(pinst->cpumask.cbcpu); + free_cpumask_var(pinst->validate_cpumask); destroy_workqueue(pinst->serial_wq); destroy_workqueue(pinst->parallel_wq); kfree(pinst); @@ -971,10 +969,10 @@ struct padata_instance *padata_alloc(const char *name) if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) goto err_free_serial_wq; - if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { - free_cpumask_var(pinst->cpumask.pcpu); - goto err_free_serial_wq; - } + if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) + goto err_free_p_mask; + if (!alloc_cpumask_var(&pinst->validate_cpumask, GFP_KERNEL)) + goto err_free_cb_mask; INIT_LIST_HEAD(&pinst->pslist); @@ -982,7 +980,7 @@ struct padata_instance *padata_alloc(const char *name) cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask); if (padata_setup_cpumasks(pinst)) - goto err_free_masks; + goto err_free_v_mask; __padata_start(pinst); @@ -991,18 +989,19 @@ struct padata_instance *padata_alloc(const char *name) #ifdef CONFIG_HOTPLUG_CPU cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, - &pinst->cpu_online_node); - cpuhp_state_add_instance_nocalls_cpuslocked(CPUHP_PADATA_DEAD, - &pinst->cpu_dead_node); + &pinst->cpuhp_node); #endif cpus_read_unlock(); return pinst; -err_free_masks: - free_cpumask_var(pinst->cpumask.pcpu); +err_free_v_mask: + free_cpumask_var(pinst->validate_cpumask); +err_free_cb_mask: free_cpumask_var(pinst->cpumask.cbcpu); +err_free_p_mask: + free_cpumask_var(pinst->cpumask.pcpu); err_free_serial_wq: destroy_workqueue(pinst->serial_wq); err_put_cpus: @@ -1045,7 +1044,7 @@ struct padata_shell *padata_alloc_shell(struct padata_instance *pinst) ps->pinst = pinst; cpus_read_lock(); - pd = padata_alloc_pd(ps); + pd = padata_alloc_pd(ps, -1); cpus_read_unlock(); if (!pd) @@ -1094,31 +1093,24 @@ void __init padata_init(void) int ret; ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online", - padata_cpu_online, NULL); + padata_cpu_online, padata_cpu_offline); if (ret < 0) goto err; hp_online = ret; - - ret = cpuhp_setup_state_multi(CPUHP_PADATA_DEAD, "padata:dead", - NULL, padata_cpu_dead); - if (ret < 0) - goto remove_online_state; #endif possible_cpus = num_possible_cpus(); padata_works = kmalloc_objs(struct padata_work, possible_cpus); if (!padata_works) - goto remove_dead_state; + goto remove_online_state; for (i = 0; i < possible_cpus; ++i) list_add(&padata_works[i].pw_list, &padata_free_works); return; -remove_dead_state: -#ifdef CONFIG_HOTPLUG_CPU - cpuhp_remove_multi_state(CPUHP_PADATA_DEAD); remove_online_state: +#ifdef CONFIG_HOTPLUG_CPU cpuhp_remove_multi_state(hp_online); err: #endif diff --git a/kernel/params.c b/kernel/params.c index 7188a12dbe8647..c6a354d54213df 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -745,15 +745,6 @@ void module_param_sysfs_remove(struct module *mod) } #endif -void destroy_params(const struct kernel_param *params, unsigned num) -{ - unsigned int i; - - for (i = 0; i < num; i++) - if (params[i].ops->free) - params[i].ops->free(params[i].arg); -} - struct module_kobject * __init_or_module lookup_or_create_module_kobject(const char *name) { @@ -985,3 +976,21 @@ static int __init param_sysfs_builtin_init(void) late_initcall(param_sysfs_builtin_init); #endif /* CONFIG_SYSFS */ + +#ifdef CONFIG_MODULES + +/* + * module_destroy_params - free all parameters for one module + * @params: module parameters (array) + * @num: number of module parameters + */ +void module_destroy_params(const struct kernel_param *params, unsigned int num) +{ + unsigned int i; + + for (i = 0; i < num; i++) + if (params[i].ops->free) + params[i].ops->free(params[i].arg); +} + +#endif /* CONFIG_MODULES */ diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index af8d07bafe02a7..d04eae18ea688c 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include "power.h" @@ -110,7 +111,8 @@ bool hibernation_available(void) { return nohibernate == 0 && !security_locked_down(LOCKDOWN_HIBERNATION) && - !secretmem_active() && !cxl_mem_active(); + !secretmem_active() && !cxl_mem_active() && + !of_machine_is_compatible("apple,arm-platform"); } /** diff --git a/kernel/printk/printk_ringbuffer.c b/kernel/printk/printk_ringbuffer.c index 56c8e3d031f498..a3526bdd4e10d4 100644 --- a/kernel/printk/printk_ringbuffer.c +++ b/kernel/printk/printk_ringbuffer.c @@ -1302,10 +1302,6 @@ static const char *get_data(struct prb_data_ring *data_ring, return NULL; } - /* Sanity check. Data-less blocks were handled earlier. */ - if (WARN_ON_ONCE(!data_check_size(data_ring, *data_size) || !*data_size)) - return NULL; - /* A valid data block will always be aligned to the ID size. */ if (WARN_ON_ONCE(blk_lpos->begin != ALIGN(blk_lpos->begin, sizeof(db->id))) || WARN_ON_ONCE(blk_lpos->next != ALIGN(blk_lpos->next, sizeof(db->id)))) { @@ -1319,6 +1315,10 @@ static const char *get_data(struct prb_data_ring *data_ring, /* Subtract block ID space from size to reflect data size. */ *data_size -= sizeof(db->id); + /* Sanity check the max size of the regular data block. */ + if (WARN_ON_ONCE(!data_check_size(data_ring, *data_size))) + return NULL; + return &db->data[0]; } diff --git a/kernel/ptrace.c b/kernel/ptrace.c index 392ec2f75f0138..0e3ab697cff5c3 100644 --- a/kernel/ptrace.c +++ b/kernel/ptrace.c @@ -272,11 +272,24 @@ static bool ptrace_has_cap(struct user_namespace *ns, unsigned int mode) return ns_capable(ns, CAP_SYS_PTRACE); } +static bool task_still_dumpable(struct task_struct *task, unsigned int mode) +{ + struct mm_struct *mm = task->mm; + if (mm) { + if (get_dumpable(mm) == SUID_DUMP_USER) + return true; + return ptrace_has_cap(mm->user_ns, mode); + } + + if (task->user_dumpable) + return true; + return ptrace_has_cap(&init_user_ns, mode); +} + /* Returns 0 on success, -errno on denial. */ static int __ptrace_may_access(struct task_struct *task, unsigned int mode) { const struct cred *cred = current_cred(), *tcred; - struct mm_struct *mm; kuid_t caller_uid; kgid_t caller_gid; @@ -337,11 +350,8 @@ static int __ptrace_may_access(struct task_struct *task, unsigned int mode) * Pairs with a write barrier in commit_creds(). */ smp_rmb(); - mm = task->mm; - if (mm && - ((get_dumpable(mm) != SUID_DUMP_USER) && - !ptrace_has_cap(mm->user_ns, mode))) - return -EPERM; + if (!task_still_dumpable(task, mode)) + return -EPERM; return security_ptrace_access_check(task, mode); } diff --git a/kernel/rseq.c b/kernel/rseq.c index 38d3ef540760f4..e75e3a5e312c8a 100644 --- a/kernel/rseq.c +++ b/kernel/rseq.c @@ -236,11 +236,6 @@ static int __init rseq_debugfs_init(void) } __initcall(rseq_debugfs_init); -static bool rseq_set_ids(struct task_struct *t, struct rseq_ids *ids, u32 node_id) -{ - return rseq_set_ids_get_csaddr(t, ids, node_id, NULL); -} - static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs) { struct rseq __user *urseq = t->rseq.usrptr; @@ -258,14 +253,16 @@ static bool rseq_handle_cs(struct task_struct *t, struct pt_regs *regs) static void rseq_slowpath_update_usr(struct pt_regs *regs) { /* - * Preserve rseq state and user_irq state. The generic entry code - * clears user_irq on the way out, the non-generic entry - * architectures are not having user_irq. + * Preserve has_rseq and user_irq state. The generic entry code clears + * user_irq on the way out, the non-generic entry architectures are not + * setting user_irq. */ - const struct rseq_event evt_mask = { .has_rseq = true, .user_irq = true, }; + const struct rseq_event evt_mask = { + .has_rseq = RSEQ_HAS_RSEQ_VERSION_MASK, + .user_irq = true, + }; struct task_struct *t = current; struct rseq_ids ids; - u32 node_id; bool event; if (unlikely(t->flags & PF_EXITING)) @@ -301,9 +298,9 @@ static void rseq_slowpath_update_usr(struct pt_regs *regs) if (!event) return; - node_id = cpu_to_node(ids.cpu_id); + ids.node_id = cpu_to_node(ids.cpu_id); - if (unlikely(!rseq_update_usr(t, regs, &ids, node_id))) { + if (unlikely(!rseq_update_usr(t, regs, &ids))) { /* * Clear the errors just in case this might survive magically, but * leave the rest intact. @@ -335,8 +332,9 @@ void __rseq_handle_slowpath(struct pt_regs *regs) void __rseq_signal_deliver(int sig, struct pt_regs *regs) { rseq_stat_inc(rseq_stats.signal); + /* - * Don't update IDs, they are handled on exit to user if + * Don't update IDs yet, they are handled on exit to user if * necessary. The important thing is to abort a critical section of * the interrupted context as after this point the instruction * pointer in @regs points to the signal handler. @@ -349,6 +347,13 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs) current->rseq.event.error = 0; force_sigsegv(sig); } + + /* + * In legacy mode, force the update of IDs before returning to user + * space to stay compatible. + */ + if (!rseq_v2(current)) + rseq_force_update(); } /* @@ -384,19 +389,22 @@ void rseq_syscall(struct pt_regs *regs) static bool rseq_reset_ids(void) { - struct rseq_ids ids = { - .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, - .mm_cid = 0, - }; + struct rseq __user *rseq = current->rseq.usrptr; /* * If this fails, terminate it because this leaves the kernel in * stupid state as exit to user space will try to fixup the ids * again. */ - if (rseq_set_ids(current, &ids, 0)) - return true; + scoped_user_rw_access(rseq, efault) { + unsafe_put_user(0, &rseq->cpu_id_start, efault); + unsafe_put_user(RSEQ_CPU_ID_UNINITIALIZED, &rseq->cpu_id, efault); + unsafe_put_user(0, &rseq->node_id, efault); + unsafe_put_user(0, &rseq->mm_cid, efault); + } + return true; +efault: force_sig(SIGSEGV); return false; } @@ -404,70 +412,29 @@ static bool rseq_reset_ids(void) /* The original rseq structure size (including padding) is 32 bytes. */ #define ORIG_RSEQ_SIZE 32 -/* - * sys_rseq - setup restartable sequences for caller thread. - */ -SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig) +static long rseq_register(struct rseq __user * rseq, u32 rseq_len, int flags, u32 sig) { u32 rseqfl = 0; + u8 version = 1; - if (flags & RSEQ_FLAG_UNREGISTER) { - if (flags & ~RSEQ_FLAG_UNREGISTER) - return -EINVAL; - /* Unregister rseq for current thread. */ - if (current->rseq.usrptr != rseq || !current->rseq.usrptr) - return -EINVAL; - if (rseq_len != current->rseq.len) - return -EINVAL; - if (current->rseq.sig != sig) - return -EPERM; - if (!rseq_reset_ids()) - return -EFAULT; - rseq_reset(current); - return 0; - } - - if (unlikely(flags & ~(RSEQ_FLAG_SLICE_EXT_DEFAULT_ON))) - return -EINVAL; - - if (current->rseq.usrptr) { - /* - * If rseq is already registered, check whether - * the provided address differs from the prior - * one. - */ - if (current->rseq.usrptr != rseq || rseq_len != current->rseq.len) - return -EINVAL; - if (current->rseq.sig != sig) - return -EPERM; - /* Already registered. */ - return -EBUSY; - } - - /* - * If there was no rseq previously registered, ensure the provided rseq - * is properly aligned, as communcated to user-space through the ELF - * auxiliary vector AT_RSEQ_ALIGN. If rseq_len is the original rseq - * size, the required alignment is the original struct rseq alignment. - * - * The rseq_len is required to be greater or equal to the original rseq - * size. In order to be valid, rseq_len is either the original rseq size, - * or large enough to contain all supported fields, as communicated to - * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE. - */ - if (rseq_len < ORIG_RSEQ_SIZE || - (rseq_len == ORIG_RSEQ_SIZE && !IS_ALIGNED((unsigned long)rseq, ORIG_RSEQ_SIZE)) || - (rseq_len != ORIG_RSEQ_SIZE && (!IS_ALIGNED((unsigned long)rseq, rseq_alloc_align()) || - rseq_len < offsetof(struct rseq, end)))) - return -EINVAL; if (!access_ok(rseq, rseq_len)) return -EFAULT; - if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION)) { - rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE; - if (rseq_slice_extension_enabled() && - (flags & RSEQ_FLAG_SLICE_EXT_DEFAULT_ON)) - rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED; + /* + * Architectures, which use the generic IRQ entry code (at least) enable + * registrations with a size greater than the original v1 fixed sized + * @rseq_len, which has been validated already to utilize the optimized + * v2 ABI mode which also enables extended RSEQ features beyond MMCID. + */ + if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY) && rseq_len > ORIG_RSEQ_SIZE) + version = 2; + + if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION) && version > 1) { + if (rseq_slice_extension_enabled()) { + rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_AVAILABLE; + if (flags & RSEQ_FLAG_SLICE_EXT_DEFAULT_ON) + rseqfl |= RSEQ_CS_FLAG_SLICE_EXT_ENABLED; + } } scoped_user_write_access(rseq, efault) { @@ -485,7 +452,15 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32 unsafe_put_user(RSEQ_CPU_ID_UNINITIALIZED, &rseq->cpu_id, efault); unsafe_put_user(0U, &rseq->node_id, efault); unsafe_put_user(0U, &rseq->mm_cid, efault); - unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); + + /* + * All fields past mm_cid are only valid for non-legacy v2 + * registrations. + */ + if (version > 1) { + if (IS_ENABLED(CONFIG_RSEQ_SLICE_EXTENSION)) + unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); + } } /* @@ -501,11 +476,10 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32 #endif /* - * If rseq was previously inactive, and has just been - * registered, ensure the cpu_id_start and cpu_id fields - * are updated before returning to user-space. + * Ensure the cpu_id_start and cpu_id fields are updated before + * returning to user-space. */ - current->rseq.event.has_rseq = true; + current->rseq.event.has_rseq = version; rseq_force_update(); return 0; @@ -513,6 +487,80 @@ SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32 return -EFAULT; } +static long rseq_unregister(struct rseq __user * rseq, u32 rseq_len, int flags, u32 sig) +{ + if (flags & ~RSEQ_FLAG_UNREGISTER) + return -EINVAL; + if (current->rseq.usrptr != rseq || !current->rseq.usrptr) + return -EINVAL; + if (rseq_len != current->rseq.len) + return -EINVAL; + if (current->rseq.sig != sig) + return -EPERM; + if (!rseq_reset_ids()) + return -EFAULT; + rseq_reset(current); + return 0; +} + +static long rseq_reregister(struct rseq __user * rseq, u32 rseq_len, u32 sig) +{ + /* + * If rseq is already registered, check whether the provided address + * differs from the prior one. + */ + if (current->rseq.usrptr != rseq || rseq_len != current->rseq.len) + return -EINVAL; + if (current->rseq.sig != sig) + return -EPERM; + /* Already registered. */ + return -EBUSY; +} + +static bool rseq_length_valid(struct rseq __user *rseq, unsigned int rseq_len) +{ + /* + * Ensure the provided rseq is properly aligned, as communicated to + * user-space through the ELF auxiliary vector AT_RSEQ_ALIGN. If + * rseq_len is the original rseq size, the required alignment is the + * original struct rseq alignment. + * + * In order to be valid, rseq_len is either the original rseq size, or + * large enough to contain all supported fields, as communicated to + * user-space through the ELF auxiliary vector AT_RSEQ_FEATURE_SIZE. + */ + if (rseq_len < ORIG_RSEQ_SIZE) + return false; + + if (rseq_len == ORIG_RSEQ_SIZE) + return IS_ALIGNED((unsigned long)rseq, ORIG_RSEQ_SIZE); + + return IS_ALIGNED((unsigned long)rseq, rseq_alloc_align()) && + rseq_len >= offsetof(struct rseq, end); +} + +#define RSEQ_FLAGS_SUPPORTED (RSEQ_FLAG_SLICE_EXT_DEFAULT_ON) + +/* + * sys_rseq - Register or unregister restartable sequences for the caller thread. + */ +SYSCALL_DEFINE4(rseq, struct rseq __user *, rseq, u32, rseq_len, int, flags, u32, sig) +{ + if (flags & RSEQ_FLAG_UNREGISTER) + return rseq_unregister(rseq, rseq_len, flags, sig); + + if (unlikely(flags & ~RSEQ_FLAGS_SUPPORTED)) + return -EINVAL; + + if (current->rseq.usrptr) + return rseq_reregister(rseq, rseq_len, sig); + + if (!rseq_length_valid(rseq, rseq_len)) + return -EINVAL; + + return rseq_register(rseq, rseq_len, flags, sig); +} + #ifdef CONFIG_RSEQ_SLICE_EXTENSION struct slice_timer { struct hrtimer timer; @@ -713,6 +761,8 @@ int rseq_slice_extension_prctl(unsigned long arg2, unsigned long arg3) return -ENOTSUPP; if (!current->rseq.usrptr) return -ENXIO; + if (!rseq_v2(current)) + return -ENOTSUPP; /* No change? */ if (enable == !!current->rseq.slice.state.enabled) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 496dff740dcafe..567b1b1efdb586 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4398,6 +4398,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p) p->se.nr_migrations = 0; p->se.vruntime = 0; p->se.vlag = 0; + p->se.rel_deadline = 0; INIT_LIST_HEAD(&p->se.group_node); /* A delayed task cannot be in clone(). */ @@ -4738,7 +4739,7 @@ void sched_post_fork(struct task_struct *p) scx_post_fork(p); } -unsigned long to_ratio(u64 period, u64 runtime) +u64 to_ratio(u64 period, u64 runtime) { if (runtime == RUNTIME_INF) return BW_UNIT; @@ -6705,23 +6706,6 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) } #endif /* SCHED_PROXY_EXEC */ -static inline void proxy_tag_curr(struct rq *rq, struct task_struct *owner) -{ - if (!sched_proxy_exec()) - return; - /* - * pick_next_task() calls set_next_task() on the chosen task - * at some point, which ensures it is not push/pullable. - * However, the chosen/donor task *and* the mutex owner form an - * atomic pair wrt push/pull. - * - * Make sure owner we run is not pushable. Unfortunately we can - * only deal with that by means of a dequeue/enqueue cycle. :-/ - */ - dequeue_task(rq, owner, DEQUEUE_NOCLOCK | DEQUEUE_SAVE); - enqueue_task(rq, owner, ENQUEUE_NOCLOCK | ENQUEUE_RESTORE); -} - /* * __schedule() is the main scheduler function. * @@ -6874,9 +6858,6 @@ static void __sched notrace __schedule(int sched_mode) */ RCU_INIT_POINTER(rq->curr, next); - if (!task_current_donor(rq, next)) - proxy_tag_curr(rq, next); - /* * The membarrier system call requires each architecture * to have a full memory barrier after updating @@ -6910,10 +6891,6 @@ static void __sched notrace __schedule(int sched_mode) /* Also unlocks the rq: */ rq = context_switch(rq, prev, next, &rf); } else { - /* In case next was already curr but just got blocked_donor */ - if (!task_current_donor(rq, next)) - proxy_tag_curr(rq, next); - rq_unpin_lock(rq, &rf); __balance_callbacks(rq, NULL); raw_spin_rq_unlock_irq(rq); diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 153232dd8276ae..ae9fd211cec1f3 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -461,6 +461,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, unsigned int flags) { struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util); + struct sugov_policy *sg_policy = sg_cpu->sg_policy; unsigned long prev_util = sg_cpu->util; unsigned long max_cap; @@ -482,10 +483,10 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util) sg_cpu->util = prev_util; - cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min, + cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, sg_cpu->util, max_cap); - sg_cpu->sg_policy->last_freq_update_time = time; + sg_policy->last_freq_update_time = time; } static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 674de6a48551b5..af7c1e88e46e7c 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2801,12 +2801,26 @@ static int find_later_rq(struct task_struct *task) static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) { - struct task_struct *p; + struct task_struct *i, *p = NULL; + struct rb_node *next_node; if (!has_pushable_dl_tasks(rq)) return NULL; - p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); + next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); + while (next_node) { + i = __node_2_pdl(next_node); + /* make sure task isn't on_cpu (possible with proxy-exec) */ + if (!task_on_cpu(rq, i)) { + p = i; + break; + } + + next_node = rb_next(next_node); + } + + if (!p) + return NULL; WARN_ON_ONCE(rq->cpu != task_cpu(p)); WARN_ON_ONCE(task_current(rq, p)); @@ -3092,20 +3106,18 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p) static void set_cpus_allowed_dl(struct task_struct *p, struct affinity_context *ctx) { - struct root_domain *src_rd; struct rq *rq; WARN_ON_ONCE(!dl_task(p)); rq = task_rq(p); - src_rd = rq->rd; /* * Migrating a SCHED_DEADLINE task between exclusive * cpusets (different root_domains) entails a bandwidth * update. We already made space for us in the destination * domain (see cpuset_can_attach()). */ - if (!cpumask_intersects(src_rd->span, ctx->new_mask)) { + if (dl_task_needs_bw_move(p, ctx->new_mask)) { struct dl_bw *src_dl_b; src_dl_b = dl_bw_of(cpu_of(rq)); @@ -3122,6 +3134,15 @@ static void set_cpus_allowed_dl(struct task_struct *p, set_cpus_allowed_common(p, ctx); } +bool dl_task_needs_bw_move(struct task_struct *p, + const struct cpumask *new_mask) +{ + if (!dl_task(p)) + return false; + + return !cpumask_intersects(task_rq(p)->rd->span, new_mask); +} + /* Assumes rq->lock is held */ static void rq_online_dl(struct rq *rq) { diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 064eaa76be4b9f..3ac01ea9bfb1aa 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -1093,11 +1093,13 @@ static void dispatch_enqueue(struct scx_sched *sch, struct scx_dispatch_q *dsq, if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } else { - bool was_empty; - - was_empty = list_empty(&dsq->list); + /* + * dsq->list can contain parked BPF iterator cursors, so + * list_empty() here isn't a reliable proxy for "no real + * task in the DSQ". Test dsq->first_task directly. + */ list_add_tail(&p->scx.dsq_list.node, &dsq->list); - if (was_empty && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) + if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN)) rcu_assign_pointer(dsq->first_task, p); } } @@ -2740,7 +2742,7 @@ static void set_cpus_allowed_scx(struct task_struct *p, * designation pointless. Cast it away when calling the operation. */ if (SCX_HAS_OP(sch, set_cpumask)) - SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_cpumask, NULL, + SCX_CALL_OP_TASK(sch, SCX_KF_REST, set_cpumask, task_rq(p), p, (struct cpumask *)p->cpus_ptr); } @@ -3404,7 +3406,7 @@ void scx_cgroup_move_task(struct task_struct *p) */ if (SCX_HAS_OP(sch, cgroup_move) && !WARN_ON_ONCE(!p->scx.cgrp_moving_from)) - SCX_CALL_OP_TASK(sch, SCX_KF_UNLOCKED, cgroup_move, NULL, + SCX_CALL_OP_TASK(sch, SCX_KF_REST, cgroup_move, task_rq(p), p, p->scx.cgrp_moving_from, tg_cgrp(task_group(p))); p->scx.cgrp_moving_from = NULL; @@ -3430,9 +3432,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) void scx_group_set_weight(struct task_group *tg, unsigned long weight) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) && tg->scx.weight != weight) @@ -3446,9 +3449,10 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) void scx_group_set_idle(struct task_group *tg, bool idle) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle)) SCX_CALL_OP(sch, SCX_KF_UNLOCKED, cgroup_set_idle, NULL, @@ -3463,9 +3467,10 @@ void scx_group_set_idle(struct task_group *tg, bool idle) void scx_group_set_bandwidth(struct task_group *tg, u64 period_us, u64 quota_us, u64 burst_us) { - struct scx_sched *sch = scx_root; + struct scx_sched *sch; percpu_down_read(&scx_cgroup_ops_rwsem); + sch = scx_root; if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) && (tg->scx.bw_period_us != period_us || @@ -4005,6 +4010,15 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, struct rq *rq, if (cpumask_empty(donee_mask)) break; + /* + * If an earlier pass placed @p on @donor_dsq from a different + * CPU and the donee hasn't consumed it yet, @p is still on the + * previous CPU and task_rq(@p) != @rq. @p can't be moved + * without its rq locked. Skip. + */ + if (task_rq(p) != rq) + continue; + donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr); if (donee >= nr_cpu_ids) continue; @@ -5326,8 +5340,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) static DEFINE_MUTEX(helper_mutex); struct scx_enable_cmd cmd; - if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN), - cpu_possible_mask)) { + if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) { pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n"); return -EINVAL; } diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c index 44c3a50c542c2a..ba8fcb1ab8b57b 100644 --- a/kernel/sched/ext_idle.c +++ b/kernel/sched/ext_idle.c @@ -459,12 +459,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, preempt_disable(); - /* - * Check whether @prev_cpu is still within the allowed set. If not, - * we can still try selecting a nearby CPU. - */ - is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed); - /* * Determine the subset of CPUs usable by @p within @cpus_allowed. */ @@ -481,6 +475,12 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, } } + /* + * Check whether @prev_cpu is still within the allowed set. If not, + * we can still try selecting a nearby CPU. + */ + is_prev_allowed = cpumask_test_cpu(prev_cpu, allowed); + /* * This is necessary to protect llc_cpus. */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ab4114712be740..3bce48ad0bc5a8 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1007,7 +1007,7 @@ static inline void cancel_protect_slice(struct sched_entity *se) * * Which allows tree pruning through eligibility. */ -static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect) +static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) { struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; struct sched_entity *se = __pick_first_entity(cfs_rq); @@ -1078,11 +1078,6 @@ static struct sched_entity *__pick_eevdf(struct cfs_rq *cfs_rq, bool protect) return best; } -static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) -{ - return __pick_eevdf(cfs_rq, true); -} - struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) { struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root); @@ -5540,11 +5535,11 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); * 4) do not run the "skip" process, if something else is available */ static struct sched_entity * -pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq) +pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) { struct sched_entity *se; - se = pick_eevdf(cfs_rq); + se = pick_eevdf(cfs_rq, protect); if (se->sched_delayed) { dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); /* @@ -8809,7 +8804,7 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f { enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; struct task_struct *donor = rq->donor; - struct sched_entity *se = &donor->se, *pse = &p->se; + struct sched_entity *nse, *se = &donor->se, *pse = &p->se; struct cfs_rq *cfs_rq = task_cfs_rq(donor); int cse_is_idle, pse_is_idle; @@ -8920,11 +8915,18 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f } pick: + nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); + /* If @p has become the most eligible task, force preemption */ + if (nse == pse) + goto preempt; + /* - * If @p has become the most eligible task, force preemption. + * Because p is enqueued, nse being null can only mean that we + * dequeued a delayed task. If there are still entities queued in + * cfs, check if the next one will be p. */ - if (__pick_eevdf(cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT) == pse) - goto preempt; + if (!nse && cfs_rq->nr_queued) + goto pick; if (sched_feat(RUN_TO_PARITY)) update_protect_slice(cfs_rq, se); @@ -8959,7 +8961,7 @@ static struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) throttled |= check_cfs_rq_runtime(cfs_rq); - se = pick_next_entity(rq, cfs_rq); + se = pick_next_entity(rq, cfs_rq, true); if (!se) goto again; cfs_rq = group_cfs_rq(se); diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c index 62344560372526..226a6329f3e928 100644 --- a/kernel/sched/membarrier.c +++ b/kernel/sched/membarrier.c @@ -199,7 +199,16 @@ static void ipi_rseq(void *info) * is negligible. */ smp_mb(); - rseq_sched_switch_event(current); + /* + * Legacy mode requires that IDs are written and the critical section is + * evaluated. V2 optimized mode handles the critical section and IDs are + * only updated if they change as a consequence of preemption after + * return from this IPI. + */ + if (rseq_v2(current)) + rseq_sched_switch_event(current); + else + rseq_force_update(); } static void ipi_sync_rq_state(void *info) diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index f69e1f16d92381..0cbee031858a52 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1853,13 +1853,22 @@ static int find_lowest_rq(struct task_struct *task) static struct task_struct *pick_next_pushable_task(struct rq *rq) { - struct task_struct *p; + struct plist_head *head = &rq->rt.pushable_tasks; + struct task_struct *i, *p = NULL; if (!has_pushable_tasks(rq)) return NULL; - p = plist_first_entry(&rq->rt.pushable_tasks, - struct task_struct, pushable_tasks); + plist_for_each_entry(i, head, pushable_tasks) { + /* make sure task isn't on_cpu (possible with proxy-exec) */ + if (!task_on_cpu(rq, i)) { + p = i; + break; + } + } + + if (!p) + return NULL; BUG_ON(rq->cpu != task_cpu(p)); BUG_ON(task_current(rq, p)); @@ -2652,7 +2661,7 @@ static int tg_rt_schedulable(struct task_group *tg, void *data) { struct rt_schedulable_data *d = data; struct task_group *child; - unsigned long total, sum = 0; + u64 total, sum = 0; u64 period, runtime; period = ktime_to_ns(tg->rt_bandwidth.rt_period); @@ -2676,9 +2685,6 @@ static int tg_rt_schedulable(struct task_group *tg, void *data) tg->rt_bandwidth.rt_runtime && tg_has_rt_tasks(tg)) return -EBUSY; - if (WARN_ON(!rt_group_sched_enabled() && tg != &root_task_group)) - return -EBUSY; - total = to_ratio(period, runtime); /* @@ -2822,6 +2828,8 @@ long sched_group_rt_period(struct task_group *tg) static int sched_rt_global_constraints(void) { int ret = 0; + if (!rt_group_sched_enabled()) + return ret; mutex_lock(&rt_constraints_mutex); ret = __rt_schedulable(NULL, 0, 0); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 1ef9ba480f51d1..0d068b2f371890 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2899,7 +2899,7 @@ extern void init_cfs_throttle_work(struct task_struct *p); #define MAX_BW_BITS (64 - BW_SHIFT) #define MAX_BW ((1ULL << MAX_BW_BITS) - 1) -extern unsigned long to_ratio(u64 period, u64 runtime); +extern u64 to_ratio(u64 period, u64 runtime); extern void init_entity_runnable_average(struct sched_entity *se); extern void post_init_entity_util_avg(struct task_struct *p); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 32dcddaead82d9..2864f43bff6df6 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1643,13 +1643,17 @@ sd_init(struct sched_domain_topology_level *tl, int sd_id, sd_weight, sd_flags = 0; struct cpumask *sd_span; - sd_weight = cpumask_weight(tl->mask(tl, cpu)); + sd_span = sched_domain_span(sd); + cpumask_and(sd_span, cpu_map, tl->mask(tl, cpu)); + sd_weight = cpumask_weight(sd_span); + sd_id = cpumask_first(sd_span); if (tl->sd_flags) sd_flags = (*tl->sd_flags)(); if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, - "wrong sd_flags in topology description\n")) + "wrong sd_flags in topology description\n")) sd_flags &= TOPOLOGY_SD_FLAGS; + sd_flags |= asym_cpu_capacity_classify(sd_span, cpu_map); *sd = (struct sched_domain){ .min_interval = sd_weight, @@ -1686,12 +1690,6 @@ sd_init(struct sched_domain_topology_level *tl, .name = tl->name, }; - sd_span = sched_domain_span(sd); - cpumask_and(sd_span, cpu_map, tl->mask(tl, cpu)); - sd_id = cpumask_first(sd_span); - - sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map); - WARN_ONCE((sd->flags & (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY)) == (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY), "CPU capacity asymmetry not supported on SMT\n"); diff --git a/kernel/sys.c b/kernel/sys.c index 62e842055cc9c7..ec432b89e85406 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -2531,6 +2532,16 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +int __weak arch_prctl_mem_model_get(struct task_struct *t) +{ + return -EINVAL; +} + +int __weak arch_prctl_mem_model_set(struct task_struct *t, unsigned long val) +{ + return -EINVAL; +} + SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, unsigned long, arg4, unsigned long, arg5) { @@ -2544,6 +2555,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_GET_MEM_MODEL: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_get(me); + break; + case PR_SET_MEM_MODEL: + if (arg3 || arg4 || arg5) + return -EINVAL; + error = arch_prctl_mem_model_set(me, arg2); + break; case PR_SET_PDEATHSIG: if (!valid_signal(arg2)) { error = -EINVAL; diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 0cd680ccc7e536..73bd6a6a78935c 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -649,6 +649,7 @@ void taskstats_exit(struct task_struct *tsk, int group_dead) goto err; memcpy(stats, tsk->signal->stats, sizeof(*stats)); + stats->version = TASKSTATS_VERSION; send: send_cpu_listeners(rep_skb, listeners); diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 38570998a19b8c..69379b34bcc923 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -94,6 +94,9 @@ static int __clockevents_switch_state(struct clock_event_device *dev, if (dev->features & CLOCK_EVT_FEAT_DUMMY) return 0; + /* On state transitions clear the forced flag unconditionally */ + dev->next_event_forced = 0; + /* Transition with new state-specific callbacks */ switch (state) { case CLOCK_EVT_STATE_DETACHED: @@ -332,8 +335,10 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, if (delta > (int64_t)dev->min_delta_ns) { delta = min(delta, (int64_t) dev->max_delta_ns); clc = ((unsigned long long) delta * dev->mult) >> dev->shift; - if (!dev->set_next_event((unsigned long) clc, dev)) + if (!dev->set_next_event((unsigned long) clc, dev)) { + dev->next_event_forced = 0; return 0; + } } if (dev->next_event_forced) diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 1e37142fe52f4e..c450b41d4bb591 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -479,17 +479,10 @@ static inline void debug_setup_on_stack(struct hrtimer *timer, clockid_t clockid trace_hrtimer_setup(timer, clockid, mode); } -static inline void debug_activate(struct hrtimer *timer, - enum hrtimer_mode mode) +static inline void debug_activate(struct hrtimer *timer, enum hrtimer_mode mode, bool was_armed) { debug_hrtimer_activate(timer, mode); - trace_hrtimer_start(timer, mode); -} - -static inline void debug_deactivate(struct hrtimer *timer) -{ - debug_hrtimer_deactivate(timer); - trace_hrtimer_cancel(timer); + trace_hrtimer_start(timer, mode, was_armed); } static struct hrtimer_clock_base * @@ -1084,9 +1077,9 @@ EXPORT_SYMBOL_GPL(hrtimer_forward); * Returns true when the new timer is the leftmost timer in the tree. */ static bool enqueue_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, - enum hrtimer_mode mode) + enum hrtimer_mode mode, bool was_armed) { - debug_activate(timer, mode); + debug_activate(timer, mode, was_armed); WARN_ON_ONCE(!base->cpu_base->online); base->cpu_base->active_bases |= 1 << base->index; @@ -1146,6 +1139,8 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, if (state & HRTIMER_STATE_ENQUEUED) { bool reprogram; + debug_hrtimer_deactivate(timer); + /* * Remove the timer and force reprogramming when high * resolution mode is active and the timer is on the current @@ -1154,7 +1149,6 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, * reprogramming happens in the interrupt handler. This is a * rare case and less expensive than a smp call. */ - debug_deactivate(timer); reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); /* @@ -1221,15 +1215,15 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, { struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases); struct hrtimer_clock_base *new_base; - bool force_local, first; + bool force_local, first, was_armed; /* * If the timer is on the local cpu base and is the first expiring * timer then this might end up reprogramming the hardware twice - * (on removal and on enqueue). To avoid that by prevent the - * reprogram on removal, keep the timer local to the current CPU - * and enforce reprogramming after it is queued no matter whether - * it is the new first expiring timer again or not. + * (on removal and on enqueue). To avoid that prevent the reprogram + * on removal, keep the timer local to the current CPU and enforce + * reprogramming after it is queued no matter whether it is the new + * first expiring timer again or not. */ force_local = base->cpu_base == this_cpu_base; force_local &= base->cpu_base->next_timer == timer; @@ -1251,7 +1245,7 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, * avoids programming the underlying clock event twice (once at * removal and once after enqueue). */ - remove_hrtimer(timer, base, true, force_local); + was_armed = remove_hrtimer(timer, base, true, force_local); if (mode & HRTIMER_MODE_REL) tim = ktime_add_safe(tim, __hrtimer_cb_get_time(base->clockid)); @@ -1268,7 +1262,15 @@ static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, new_base = base; } - first = enqueue_hrtimer(timer, new_base, mode); + first = enqueue_hrtimer(timer, new_base, mode, was_armed); + + /* + * If the hrtimer interrupt is running, then it will reevaluate the + * clock bases and reprogram the clock event device. + */ + if (new_base->cpu_base->in_hrtirq) + return false; + if (!force_local) { /* * If the current CPU base is online, then the timer is @@ -1362,8 +1364,11 @@ int hrtimer_try_to_cancel(struct hrtimer *timer) base = lock_hrtimer_base(timer, &flags); - if (!hrtimer_callback_running(timer)) + if (!hrtimer_callback_running(timer)) { ret = remove_hrtimer(timer, base, false, false); + if (ret) + trace_hrtimer_cancel(timer); + } unlock_hrtimer_base(timer, &flags); @@ -1799,7 +1804,7 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, */ if (restart != HRTIMER_NORESTART && !(timer->state & HRTIMER_STATE_ENQUEUED)) - enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS); + enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS, false); /* * Separate the ->running assignment from the ->state assignment. @@ -2279,7 +2284,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, while ((node = timerqueue_getnext(&old_base->active))) { timer = container_of(node, struct hrtimer, node); BUG_ON(hrtimer_callback_running(timer)); - debug_deactivate(timer); + debug_hrtimer_deactivate(timer); /* * Mark it as ENQUEUED not INACTIVE otherwise the @@ -2296,7 +2301,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, * sort out already expired timers and reprogram the * event device. */ - enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS); + enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS, true); } } diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 7e57fa31ee26f1..115e0bf012764e 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -108,6 +108,7 @@ static struct clock_event_device *tick_get_oneshot_wakeup_device(int cpu) static void tick_oneshot_wakeup_handler(struct clock_event_device *wd) { + wd->next_event_forced = 0; /* * If we woke up early and the tick was reprogrammed in the * meantime then this may be spurious but harmless. diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index dcadf1d23b8a31..0afaae4e1a59c5 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -4,6 +4,7 @@ */ #define pr_fmt(fmt) "fprobe: " fmt +#include #include #include #include @@ -78,36 +79,33 @@ static const struct rhashtable_params fprobe_rht_params = { }; /* Node insertion and deletion requires the fprobe_mutex */ -static int insert_fprobe_node(struct fprobe_hlist_node *node) +static int insert_fprobe_node(struct fprobe_hlist_node *node, struct fprobe *fp) { + int ret; + lockdep_assert_held(&fprobe_mutex); - return rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); + ret = rhltable_insert(&fprobe_ip_table, &node->hlist, fprobe_rht_params); + /* Set the fprobe pointer if insertion was successful. */ + if (!ret) + WRITE_ONCE(node->fp, fp); + return ret; } -/* Return true if there are synonims */ -static bool delete_fprobe_node(struct fprobe_hlist_node *node) +static void delete_fprobe_node(struct fprobe_hlist_node *node) { lockdep_assert_held(&fprobe_mutex); - bool ret; - /* Avoid double deleting */ + /* Avoid double deleting and non-inserted nodes */ if (READ_ONCE(node->fp) != NULL) { WRITE_ONCE(node->fp, NULL); rhltable_remove(&fprobe_ip_table, &node->hlist, fprobe_rht_params); } - - rcu_read_lock(); - ret = !!rhltable_lookup(&fprobe_ip_table, &node->addr, - fprobe_rht_params); - rcu_read_unlock(); - - return ret; } /* Check existence of the fprobe */ -static bool is_fprobe_still_exist(struct fprobe *fp) +static bool fprobe_registered(struct fprobe *fp) { struct hlist_head *head; struct fprobe_hlist *fph; @@ -120,7 +118,7 @@ static bool is_fprobe_still_exist(struct fprobe *fp) } return false; } -NOKPROBE_SYMBOL(is_fprobe_still_exist); +NOKPROBE_SYMBOL(fprobe_registered); static int add_fprobe_hash(struct fprobe *fp) { @@ -132,9 +130,6 @@ static int add_fprobe_hash(struct fprobe *fp) if (WARN_ON_ONCE(!fph)) return -EINVAL; - if (is_fprobe_still_exist(fp)) - return -EEXIST; - head = &fprobe_table[hash_ptr(fp, FPROBE_HASH_BITS)]; hlist_add_head_rcu(&fp->hlist_array->hlist, head); return 0; @@ -149,7 +144,7 @@ static int del_fprobe_hash(struct fprobe *fp) if (WARN_ON_ONCE(!fph)) return -EINVAL; - if (!is_fprobe_still_exist(fp)) + if (!fprobe_registered(fp)) return -ENOENT; fph->fp = NULL; @@ -326,9 +321,10 @@ static void fprobe_ftrace_remove_ips(unsigned long *addrs, int num) lockdep_assert_held(&fprobe_mutex); fprobe_ftrace_active--; - if (!fprobe_ftrace_active) + if (!fprobe_ftrace_active) { unregister_ftrace_function(&fprobe_ftrace_ops); - if (num) + ftrace_free_filter(&fprobe_ftrace_ops); + } else if (num) ftrace_set_filter_ips(&fprobe_ftrace_ops, addrs, num, 1, 0); } @@ -337,12 +333,37 @@ static bool fprobe_is_ftrace(struct fprobe *fp) return !fp->exit_handler; } +static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace) +{ + struct rhlist_head *head, *pos; + struct fprobe_hlist_node *node; + struct fprobe *fp; + + guard(rcu)(); + head = rhltable_lookup(&fprobe_ip_table, &ip, + fprobe_rht_params); + if (!head) + return false; + /* We have to check the same type on the list. */ + rhl_for_each_entry_rcu(node, pos, head, hlist) { + if (node->addr != ip) + break; + fp = READ_ONCE(node->fp); + if (likely(fp)) { + if ((!ftrace && fp->exit_handler) || + (ftrace && !fp->exit_handler)) + return true; + } + } + + return false; +} + #ifdef CONFIG_MODULES -static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, - int reset) +static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { - ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); - ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, remove, reset); + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); + ftrace_set_filter_ips(&fprobe_ftrace_ops, ips, cnt, 1, 0); } #endif #else @@ -360,11 +381,33 @@ static bool fprobe_is_ftrace(struct fprobe *fp) return false; } +static bool fprobe_exists_on_hash(unsigned long ip, bool ftrace __maybe_unused) +{ + struct rhlist_head *head, *pos; + struct fprobe_hlist_node *node; + struct fprobe *fp; + + guard(rcu)(); + head = rhltable_lookup(&fprobe_ip_table, &ip, + fprobe_rht_params); + if (!head) + return false; + /* We only need to check fp is there. */ + rhl_for_each_entry_rcu(node, pos, head, hlist) { + if (node->addr != ip) + break; + fp = READ_ONCE(node->fp); + if (likely(fp)) + return true; + } + + return false; +} + #ifdef CONFIG_MODULES -static void fprobe_set_ips(unsigned long *ips, unsigned int cnt, int remove, - int reset) +static void fprobe_remove_ips(unsigned long *ips, unsigned int cnt) { - ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, remove, reset); + ftrace_set_filter_ips(&fprobe_graph_ops.ops, ips, cnt, 1, 0); } #endif #endif /* !CONFIG_DYNAMIC_FTRACE_WITH_ARGS && !CONFIG_DYNAMIC_FTRACE_WITH_REGS */ @@ -482,7 +525,7 @@ static void fprobe_return(struct ftrace_graph_ret *trace, if (!fp) break; curr += FPROBE_HEADER_SIZE_IN_LONG; - if (is_fprobe_still_exist(fp) && !fprobe_disabled(fp)) { + if (fprobe_registered(fp) && !fprobe_disabled(fp)) { if (WARN_ON_ONCE(curr + size > size_words)) break; fp->exit_handler(fp, trace->func, ret_ip, fregs, @@ -529,16 +572,16 @@ static void fprobe_graph_remove_ips(unsigned long *addrs, int num) fprobe_graph_active--; /* Q: should we unregister it ? */ - if (!fprobe_graph_active) + if (!fprobe_graph_active) { unregister_ftrace_graph(&fprobe_graph_ops); - - if (num) + ftrace_free_filter(&fprobe_graph_ops.ops); + } else if (num) ftrace_set_filter_ips(&fprobe_graph_ops.ops, addrs, num, 1, 0); } #ifdef CONFIG_MODULES -#define FPROBE_IPS_BATCH_INIT 8 +#define FPROBE_IPS_BATCH_INIT 128 /* instruction pointer address list */ struct fprobe_addr_list { int index; @@ -546,43 +589,29 @@ struct fprobe_addr_list { unsigned long *addrs; }; -static int fprobe_addr_list_add(struct fprobe_addr_list *alist, unsigned long addr) +static int fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node, + struct fprobe_addr_list *alist) { - unsigned long *addrs; + lockdep_assert_in_rcu_read_lock(); - /* Previously we failed to expand the list. */ - if (alist->index == alist->size) - return -ENOSPC; - - alist->addrs[alist->index++] = addr; - if (alist->index < alist->size) + if (!within_module(node->addr, mod)) return 0; - /* Expand the address list */ - addrs = kcalloc(alist->size * 2, sizeof(*addrs), GFP_KERNEL); - if (!addrs) - return -ENOMEM; - - memcpy(addrs, alist->addrs, alist->size * sizeof(*addrs)); - alist->size *= 2; - kfree(alist->addrs); - alist->addrs = addrs; - - return 0; -} - -static void fprobe_remove_node_in_module(struct module *mod, struct fprobe_hlist_node *node, - struct fprobe_addr_list *alist) -{ - if (!within_module(node->addr, mod)) - return; - if (delete_fprobe_node(node)) - return; + delete_fprobe_node(node); + /* If no address list is available, we can't track this address. */ + if (!alist->addrs) + return 0; /* - * If failed to update alist, just continue to update hlist. - * Therefore, at list user handler will not hit anymore. + * Don't care the type here, because all fprobes on the same + * address must be removed eventually. */ - fprobe_addr_list_add(alist, node->addr); + if (!rhltable_lookup(&fprobe_ip_table, &node->addr, fprobe_rht_params)) { + alist->addrs[alist->index++] = node->addr; + if (alist->index == alist->size) + return -ENOSPC; + } + + return 0; } /* Handle module unloading to manage fprobe_ip_table. */ @@ -593,29 +622,50 @@ static int fprobe_module_callback(struct notifier_block *nb, struct fprobe_hlist_node *node; struct rhashtable_iter iter; struct module *mod = data; + bool retry; if (val != MODULE_STATE_GOING) return NOTIFY_DONE; alist.addrs = kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL); - /* If failed to alloc memory, we can not remove ips from hash. */ - if (!alist.addrs) - return NOTIFY_DONE; + /* + * If failed to alloc memory, ftrace_ops will not be able to remove ips from + * hash, but we can still remove nodes from fprobe_ip_table, so we can avoid + * the potential wrong callback. So just print a warning here and try to + * continue without address list. + */ + WARN_ONCE(!alist.addrs, + "Failed to allocate memory for fprobe_addr_list, ftrace_ops will not be updated"); mutex_lock(&fprobe_mutex); +again: + retry = false; + alist.index = 0; rhltable_walk_enter(&fprobe_ip_table, &iter); do { rhashtable_walk_start(&iter); while ((node = rhashtable_walk_next(&iter)) && !IS_ERR(node)) - fprobe_remove_node_in_module(mod, node, &alist); + if (fprobe_remove_node_in_module(mod, node, &alist) < 0) { + retry = true; + break; + } rhashtable_walk_stop(&iter); - } while (node == ERR_PTR(-EAGAIN)); + } while (node == ERR_PTR(-EAGAIN) && !retry); rhashtable_walk_exit(&iter); + /* Remove any ips from hash table(s) */ + if (alist.index > 0) { + fprobe_remove_ips(alist.addrs, alist.index); + /* + * If we break rhashtable walk loop except for -EAGAIN, we need + * to restart looping from start for safety. Anyway, this is + * not a hotpath. + */ + if (retry) + goto again; + } - if (alist.index > 0) - fprobe_set_ips(alist.addrs, alist.index, 1, 0); mutex_unlock(&fprobe_mutex); kfree(alist.addrs); @@ -759,7 +809,6 @@ static int fprobe_init(struct fprobe *fp, unsigned long *addrs, int num) fp->hlist_array = hlist_array; hlist_array->fp = fp; for (i = 0; i < num; i++) { - hlist_array->array[i].fp = fp; addr = ftrace_location(addrs[i]); if (!addr) { fprobe_fail_cleanup(fp); @@ -823,6 +872,8 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter } EXPORT_SYMBOL_GPL(register_fprobe); +static int unregister_fprobe_nolock(struct fprobe *fp); + /** * register_fprobe_ips() - Register fprobe to ftrace by address. * @fp: A fprobe data structure to be registered. @@ -841,35 +892,33 @@ int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num) struct fprobe_hlist *hlist_array; int ret, i; + guard(mutex)(&fprobe_mutex); + if (fprobe_registered(fp)) + return -EEXIST; + ret = fprobe_init(fp, addrs, num); if (ret) return ret; - mutex_lock(&fprobe_mutex); - - hlist_array = fp->hlist_array; if (fprobe_is_ftrace(fp)) ret = fprobe_ftrace_add_ips(addrs, num); else ret = fprobe_graph_add_ips(addrs, num); - - if (!ret) { - add_fprobe_hash(fp); - for (i = 0; i < hlist_array->size; i++) { - ret = insert_fprobe_node(&hlist_array->array[i]); - if (ret) - break; - } - /* fallback on insert error */ - if (ret) { - for (i--; i >= 0; i--) - delete_fprobe_node(&hlist_array->array[i]); - } + if (ret) { + fprobe_fail_cleanup(fp); + return ret; } - mutex_unlock(&fprobe_mutex); - if (ret) - fprobe_fail_cleanup(fp); + hlist_array = fp->hlist_array; + ret = add_fprobe_hash(fp); + for (i = 0; i < hlist_array->size && !ret; i++) + ret = insert_fprobe_node(&hlist_array->array[i], fp); + + if (ret) { + unregister_fprobe_nolock(fp); + /* In error case, wait for clean up safely. */ + synchronize_rcu(); + } return ret; } @@ -913,37 +962,28 @@ bool fprobe_is_registered(struct fprobe *fp) return true; } -/** - * unregister_fprobe() - Unregister fprobe. - * @fp: A fprobe data structure to be unregistered. - * - * Unregister fprobe (and remove ftrace hooks from the function entries). - * - * Return 0 if @fp is unregistered successfully, -errno if not. - */ -int unregister_fprobe(struct fprobe *fp) +static int unregister_fprobe_nolock(struct fprobe *fp) { - struct fprobe_hlist *hlist_array; + struct fprobe_hlist *hlist_array = fp->hlist_array; unsigned long *addrs = NULL; - int ret = 0, i, count; + int i, count; - mutex_lock(&fprobe_mutex); - if (!fp || !is_fprobe_still_exist(fp)) { - ret = -EINVAL; - goto out; - } - - hlist_array = fp->hlist_array; addrs = kcalloc(hlist_array->size, sizeof(unsigned long), GFP_KERNEL); - if (!addrs) { - ret = -ENOMEM; /* TODO: Fallback to one-by-one loop */ - goto out; - } + /* + * This will remove fprobe_hash_node from the hash table even if + * memory allocation fails. However, ftrace_ops will not be updated. + * Anyway, when the last fprobe is unregistered, ftrace_ops is also + * unregistered. + */ + if (!addrs) + pr_warn("Failed to allocate working array. ftrace_ops may not sync.\n"); /* Remove non-synonim ips from table and hash */ count = 0; for (i = 0; i < hlist_array->size; i++) { - if (!delete_fprobe_node(&hlist_array->array[i])) + delete_fprobe_node(&hlist_array->array[i]); + if (addrs && !fprobe_exists_on_hash(hlist_array->array[i].addr, + fprobe_is_ftrace(fp))) addrs[count++] = hlist_array->array[i].addr; } del_fprobe_hash(fp); @@ -955,12 +995,26 @@ int unregister_fprobe(struct fprobe *fp) kfree_rcu(hlist_array, rcu); fp->hlist_array = NULL; + kfree(addrs); -out: - mutex_unlock(&fprobe_mutex); + return 0; +} - kfree(addrs); - return ret; +/** + * unregister_fprobe() - Unregister fprobe. + * @fp: A fprobe data structure to be unregistered. + * + * Unregister fprobe (and remove ftrace hooks from the function entries). + * + * Return 0 if @fp is unregistered successfully, -errno if not. + */ +int unregister_fprobe(struct fprobe *fp) +{ + guard(mutex)(&fprobe_mutex); + if (!fp || !fprobe_registered(fp)) + return -EINVAL; + + return unregister_fprobe_nolock(fp); } EXPORT_SYMBOL_GPL(unregister_fprobe); diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 170170bd83bd9c..aad2c7254f62b9 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1913,7 +1913,7 @@ static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu) static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) { struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; - struct buffer_page *head_page, *orig_head; + struct buffer_page *head_page, *orig_head, *orig_reader; unsigned long entry_bytes = 0; unsigned long entries = 0; int ret; @@ -1924,16 +1924,17 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) return; orig_head = head_page = cpu_buffer->head_page; + orig_reader = cpu_buffer->reader_page; /* Do the reader page first */ - ret = rb_validate_buffer(cpu_buffer->reader_page->page, cpu_buffer->cpu); + ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu); if (ret < 0) { pr_info("Ring buffer reader page is invalid\n"); goto invalid; } entries += ret; - entry_bytes += local_read(&cpu_buffer->reader_page->page->commit); - local_set(&cpu_buffer->reader_page->entries, ret); + entry_bytes += local_read(&orig_reader->page->commit); + local_set(&orig_reader->entries, ret); ts = head_page->page->time_stamp; @@ -2036,8 +2037,8 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { - /* Reader page has already been done */ - if (head_page == cpu_buffer->reader_page) + /* The original reader page has already been checked/counted. */ + if (head_page == orig_reader) continue; ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu); diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index 6809b370e991d8..d1564db95a8f5a 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c @@ -373,10 +373,10 @@ __init static int init_annotated_branch_stats(void) int ret; ret = register_stat_tracer(&annotated_branch_stats); - if (!ret) { + if (ret) { printk(KERN_WARNING "Warning: could not register " "annotated branches stats\n"); - return 1; + return ret; } return 0; } @@ -438,10 +438,10 @@ __init static int all_annotated_branch_stats(void) int ret; ret = register_stat_tracer(&all_branch_stats); - if (!ret) { + if (ret) { printk(KERN_WARNING "Warning: could not register " "all branches stats\n"); - return 1; + return ret; } return 0; } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 73ea180cad5558..f9c8a4f078ea03 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -1361,12 +1361,14 @@ static const char *hist_field_name(struct hist_field *field, field->flags & HIST_FIELD_FL_VAR_REF) { if (field->system) { static char full_name[MAX_FILTER_STR_VAL]; + int len; + + len = snprintf(full_name, sizeof(full_name), "%s.%s.%s", + field->system, field->event_name, + field->name); + if (len >= sizeof(full_name)) + return NULL; - strcat(full_name, field->system); - strcat(full_name, "."); - strcat(full_name, field->event_name); - strcat(full_name, "."); - strcat(full_name, field->name); field_name = full_name; } else field_name = field->name; diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index a5dbb72528e0c3..058724c41c469d 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -765,6 +765,14 @@ static unsigned int number_of_same_symbols(const char *mod, const char *func_nam if (!mod) kallsyms_on_each_match_symbol(count_symbols, func_name, &ctx.count); + /* + * If the symbol is found in vmlinux, use vmlinux resolution only. + * This prevents module symbols from shadowing vmlinux symbols + * and causing -EADDRNOTAVAIL for unqualified kprobe targets. + */ + if (!mod && ctx.count > 0) + return ctx.count; + module_kallsyms_on_each_symbol(mod, count_mod_symbols, &ctx); return ctx.count; diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c index 5ea5e0d76f00bc..3ea17af6016957 100644 --- a/kernel/trace/trace_printk.c +++ b/kernel/trace/trace_printk.c @@ -197,6 +197,7 @@ struct notifier_block module_trace_bprintk_format_nb = { .notifier_call = module_trace_bprintk_format_notify, }; +__printf(2, 3) int __trace_bprintk(unsigned long ip, const char *fmt, ...) { int ret; diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index e1c73065dae51b..e0d3a0da26af51 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -1523,6 +1523,12 @@ static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size, parg->offset = *size; *size += parg->type->size * (parg->count ?: 1); + if (*size > MAX_PROBE_EVENT_SIZE) { + ret = -E2BIG; + trace_probe_log_err(ctx->offset, EVENT_TOO_BIG); + goto fail; + } + if (parg->count) { len = strlen(parg->type->fmttype) + 6; parg->fmt = kmalloc(len, GFP_KERNEL); diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index 9fc56c93713045..262d8707a3df79 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -38,6 +38,7 @@ #define MAX_BTF_ARGS_LEN 128 #define MAX_DENTRY_ARGS_LEN 256 #define MAX_STRING_SIZE PATH_MAX +#define MAX_PROBE_EVENT_SIZE 3072 /* Reserved field names */ #define FIELD_STRING_IP "__probe_ip" @@ -561,7 +562,8 @@ extern int traceprobe_define_arg_fields(struct trace_event_call *event_call, C(BAD_TYPE4STR, "This type does not fit for string."),\ C(NEED_STRING_TYPE, "$comm and immediate-string only accepts string type"),\ C(TOO_MANY_ARGS, "Too many arguments are specified"), \ - C(TOO_MANY_EARGS, "Too many entry arguments specified"), + C(TOO_MANY_EARGS, "Too many entry arguments specified"), \ + C(EVENT_TOO_BIG, "Event too big (too many fields?)"), #undef C #define C(a, b) TP_ERR_##a diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c index 91905aa19294d2..dffef52a807bc8 100644 --- a/kernel/tracepoint.c +++ b/kernel/tracepoint.c @@ -300,6 +300,8 @@ static int tracepoint_add_func(struct tracepoint *tp, lockdep_is_held(&tracepoints_mutex)); old = func_add(&tp_funcs, func, prio); if (IS_ERR(old)) { + if (tp->ext && tp->ext->unregfunc && !static_key_enabled(&tp->key)) + tp->ext->unregfunc(); WARN_ON_ONCE(warn && PTR_ERR(old) != -ENOMEM); return PTR_ERR(old); } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index c6ea96d5b71672..23053ef54162e5 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -5628,7 +5629,9 @@ static int alloc_and_link_pwqs(struct workqueue_struct *wq) ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]); } - return ret; + if (ret) + goto enomem; + return 0; enomem: if (wq->cpu_pwq) { @@ -5884,6 +5887,21 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt, return NULL; } +__printf(1, 0) +static struct workqueue_struct *alloc_workqueue_va(const char *fmt, + unsigned int flags, + int max_active, + va_list args) +{ + struct workqueue_struct *wq; + + wq = __alloc_workqueue(fmt, flags, max_active, args); + if (wq) + wq_init_lockdep(wq); + + return wq; +} + __printf(1, 4) struct workqueue_struct *alloc_workqueue_noprof(const char *fmt, unsigned int flags, @@ -5893,16 +5911,39 @@ struct workqueue_struct *alloc_workqueue_noprof(const char *fmt, va_list args; va_start(args, max_active); - wq = __alloc_workqueue(fmt, flags, max_active, args); + wq = alloc_workqueue_va(fmt, flags, max_active, args); + va_end(args); + + return wq; +} +EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); + +static void devm_workqueue_release(void *res) +{ + destroy_workqueue(res); +} + +__printf(2, 5) struct workqueue_struct * +devm_alloc_workqueue_noprof(struct device *dev, const char *fmt, + unsigned int flags, int max_active, ...) +{ + struct workqueue_struct *wq; + va_list args; + int ret; + + va_start(args, max_active); + wq = alloc_workqueue_va(fmt, flags, max_active, args); va_end(args); if (!wq) return NULL; - wq_init_lockdep(wq); + ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq); + if (ret) + return NULL; return wq; } -EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); +EXPORT_SYMBOL_GPL(devm_alloc_workqueue_noprof); #ifdef CONFIG_LOCKDEP __printf(1, 5) diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index 58991ab09d84b1..ed1bdcf1f8abfc 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -758,8 +760,115 @@ static __init bool need_page_alloc_tagging(void) return mem_profiling_support; } +#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG +/* + * Track page allocations before page_ext is initialized. + * Some pages are allocated before page_ext becomes available, leaving + * their codetag uninitialized. Track these early PFNs so we can clear + * their codetag refs later to avoid warnings when they are freed. + * + * Early allocations include: + * - Base allocations independent of CPU count + * - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, + * such as trace ring buffers, scheduler per-cpu data) + * + * For simplicity, we fix the size to 8192. + * If insufficient, a warning will be triggered to alert the user. + * + * TODO: Replace fixed-size array with dynamic allocation using + * a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. + */ +#define EARLY_ALLOC_PFN_MAX 8192 + +static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX] __initdata; +static atomic_t early_pfn_count __initdata = ATOMIC_INIT(0); + +static void __init __alloc_tag_add_early_pfn(unsigned long pfn) +{ + int old_idx, new_idx; + + do { + old_idx = atomic_read(&early_pfn_count); + if (old_idx >= EARLY_ALLOC_PFN_MAX) { + pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", + EARLY_ALLOC_PFN_MAX); + return; + } + new_idx = old_idx + 1; + } while (!atomic_try_cmpxchg(&early_pfn_count, &old_idx, new_idx)); + + early_pfns[old_idx] = pfn; +} + +typedef void alloc_tag_add_func(unsigned long pfn); +static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = + RCU_INITIALIZER(__alloc_tag_add_early_pfn); + +void alloc_tag_add_early_pfn(unsigned long pfn) +{ + alloc_tag_add_func *alloc_tag_add; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + rcu_read_lock(); + alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); + if (alloc_tag_add) + alloc_tag_add(pfn); + rcu_read_unlock(); +} + +static void __init clear_early_alloc_pfn_tag_refs(void) +{ + unsigned int i; + + if (static_key_enabled(&mem_profiling_compressed)) + return; + + rcu_assign_pointer(alloc_tag_add_early_pfn_ptr, NULL); + /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ + synchronize_rcu(); + + for (i = 0; i < atomic_read(&early_pfn_count); i++) { + unsigned long pfn = early_pfns[i]; + + if (pfn_valid(pfn)) { + struct page *page = pfn_to_page(pfn); + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(page, &ref, &handle)) { + /* + * An early-allocated page could be freed and reallocated + * after its page_ext is initialized but before we clear it. + * In that case, it already has a valid tag set. + * We should not overwrite that valid tag with CODETAG_EMPTY. + * + * Note: there is still a small race window between checking + * ref.ct and calling set_codetag_empty(). We accept this + * race as it's unlikely and the extra complexity of atomic + * cmpxchg is not worth it for this debug-only code path. + */ + if (ref.ct) { + put_page_tag_ref(handle); + continue; + } + + set_codetag_empty(&ref); + update_page_tag_ref(handle, &ref); + put_page_tag_ref(handle); + } + } + + } +} +#else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ +static inline void __init clear_early_alloc_pfn_tag_refs(void) {} +#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */ + static __init void init_page_alloc_tagging(void) { + clear_early_alloc_pfn_tag_refs(); } struct page_ext_operations page_alloc_tagging_ops = { diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig index 70e7a6016de327..9ddfd1a2975769 100644 --- a/lib/crc/Kconfig +++ b/lib/crc/Kconfig @@ -99,13 +99,8 @@ config CRC_OPTIMIZATIONS config CRC_KUNIT_TEST tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS - depends on KUNIT + depends on KUNIT && (CRC7 || CRC16 || CRC_T10DIF || CRC32 || CRC64) default KUNIT_ALL_TESTS - select CRC7 - select CRC16 - select CRC_T10DIF - select CRC32 - select CRC64 help Unit tests for the CRC library functions. diff --git a/lib/crc/tests/crc_kunit.c b/lib/crc/tests/crc_kunit.c index 9a450e25ac8116..9428cd91362536 100644 --- a/lib/crc/tests/crc_kunit.c +++ b/lib/crc/tests/crc_kunit.c @@ -268,8 +268,7 @@ crc_benchmark(struct kunit *test, } } -/* crc7_be */ - +#if IS_REACHABLE(CONFIG_CRC7) static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len) { /* @@ -294,9 +293,9 @@ static void crc7_be_benchmark(struct kunit *test) { crc_benchmark(test, crc7_be_wrapper); } +#endif /* CONFIG_CRC7 */ -/* crc16 */ - +#if IS_REACHABLE(CONFIG_CRC16) static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len) { return crc16(crc, p, len); @@ -318,9 +317,9 @@ static void crc16_benchmark(struct kunit *test) { crc_benchmark(test, crc16_wrapper); } +#endif /* CONFIG_CRC16 */ -/* crc_t10dif */ - +#if IS_REACHABLE(CONFIG_CRC_T10DIF) static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len) { return crc_t10dif_update(crc, p, len); @@ -342,6 +341,9 @@ static void crc_t10dif_benchmark(struct kunit *test) { crc_benchmark(test, crc_t10dif_wrapper); } +#endif /* CONFIG_CRC_T10DIF */ + +#if IS_REACHABLE(CONFIG_CRC32) /* crc32_le */ @@ -414,6 +416,9 @@ static void crc32c_benchmark(struct kunit *test) { crc_benchmark(test, crc32c_wrapper); } +#endif /* CONFIG_CRC32 */ + +#if IS_REACHABLE(CONFIG_CRC64) /* crc64_be */ @@ -463,24 +468,35 @@ static void crc64_nvme_benchmark(struct kunit *test) { crc_benchmark(test, crc64_nvme_wrapper); } +#endif /* CONFIG_CRC64 */ static struct kunit_case crc_test_cases[] = { +#if IS_REACHABLE(CONFIG_CRC7) KUNIT_CASE(crc7_be_test), KUNIT_CASE(crc7_be_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC16) KUNIT_CASE(crc16_test), KUNIT_CASE(crc16_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC_T10DIF) KUNIT_CASE(crc_t10dif_test), KUNIT_CASE(crc_t10dif_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC32) KUNIT_CASE(crc32_le_test), KUNIT_CASE(crc32_le_benchmark), KUNIT_CASE(crc32_be_test), KUNIT_CASE(crc32_be_benchmark), KUNIT_CASE(crc32c_test), KUNIT_CASE(crc32c_benchmark), +#endif +#if IS_REACHABLE(CONFIG_CRC64) KUNIT_CASE(crc64_be_test), KUNIT_CASE(crc64_be_benchmark), KUNIT_CASE(crc64_nvme_test), KUNIT_CASE(crc64_nvme_benchmark), +#endif {}, }; diff --git a/lib/crypto/mpi/mpicoder.c b/lib/crypto/mpi/mpicoder.c index bf716a03c7045e..9359a58c29ec0a 100644 --- a/lib/crypto/mpi/mpicoder.c +++ b/lib/crypto/mpi/mpicoder.c @@ -347,7 +347,7 @@ MPI mpi_read_raw_from_sgl(struct scatterlist *sgl, unsigned int nbytes) lzeros = 0; len = 0; while (nbytes > 0) { - while (len && !*buff) { + while (len && !*buff && lzeros < nbytes) { lzeros++; len--; buff++; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 0a63c7fba3135c..243662af1af73f 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -277,7 +277,7 @@ static __always_inline size_t copy_from_user_iter_nocache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_inatomic_nocache(to + progress, iter_from, len); + return copy_from_user_inatomic_nontemporal(to + progress, iter_from, len); } size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) @@ -296,7 +296,7 @@ static __always_inline size_t copy_from_user_iter_flushcache(void __user *iter_from, size_t progress, size_t len, void *to, void *priv2) { - return __copy_from_user_flushcache(to + progress, iter_from, len); + return copy_from_user_flushcache(to + progress, iter_from, len); } static __always_inline diff --git a/lib/scatterlist.c b/lib/scatterlist.c index d773720d11bf21..b7fe91ef35b8c6 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1123,8 +1123,7 @@ static ssize_t extract_user_to_sg(struct iov_iter *iter, size_t len, off; /* We decant the page list into the tail of the scatterlist */ - pages = (void *)sgtable->sgl + - array_size(sg_max, sizeof(struct scatterlist)); + pages = (void *)sg + array_size(sg_max, sizeof(struct scatterlist)); pages -= sg_max; do { @@ -1247,7 +1246,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, else page = virt_to_page((void *)kaddr); - sg_set_page(sg, page, len, off); + sg_set_page(sg, page, seg, off); sgtable->nents++; sg++; sg_max--; @@ -1256,6 +1255,7 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter, kaddr += PAGE_SIZE; off = 0; } while (len > 0 && sg_max > 0); + ret -= len; if (maxsize <= 0 || sg_max == 0) break; @@ -1409,7 +1409,7 @@ ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t maxsize, struct sg_table *sgtable, unsigned int sg_max, iov_iter_extraction_t extraction_flags) { - if (maxsize == 0) + if (maxsize == 0 || sg_max == 0) return 0; switch (iov_iter_type(iter)) { diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 0964d53365e611..79fe7d233df1e2 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -185,11 +185,73 @@ static int dmirror_fops_open(struct inode *inode, struct file *filp) return 0; } +static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) +{ + unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; + unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; + unsigned long npages = end_pfn - start_pfn + 1; + unsigned long i; + unsigned long *src_pfns; + unsigned long *dst_pfns; + unsigned int order = 0; + + src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); + dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); + + migrate_device_range(src_pfns, start_pfn, npages); + for (i = 0; i < npages; i++) { + struct page *dpage, *spage; + + spage = migrate_pfn_to_page(src_pfns[i]); + if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) + continue; + + if (WARN_ON(!is_device_private_page(spage) && + !is_device_coherent_page(spage))) + continue; + + order = folio_order(page_folio(spage)); + spage = BACKING_PAGE(spage); + if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { + dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, + order), 0); + } else { + dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); + order = 0; + } + + /* TODO Support splitting here */ + lock_page(dpage); + dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); + if (src_pfns[i] & MIGRATE_PFN_WRITE) + dst_pfns[i] |= MIGRATE_PFN_WRITE; + if (order) + dst_pfns[i] |= MIGRATE_PFN_COMPOUND; + folio_copy(page_folio(dpage), page_folio(spage)); + } + migrate_device_pages(src_pfns, dst_pfns, npages); + migrate_device_finalize(src_pfns, dst_pfns, npages); + kvfree(src_pfns); + kvfree(dst_pfns); +} + static int dmirror_fops_release(struct inode *inode, struct file *filp) { struct dmirror *dmirror = filp->private_data; + struct dmirror_device *mdevice = dmirror->mdevice; + int i; mmu_interval_notifier_remove(&dmirror->notifier); + + if (mdevice->devmem_chunks) { + for (i = 0; i < mdevice->devmem_count; i++) { + struct dmirror_chunk *devmem = + mdevice->devmem_chunks[i]; + + dmirror_device_evict_chunk(devmem); + } + } + xa_destroy(&dmirror->pt); kfree(dmirror); return 0; @@ -1377,56 +1439,6 @@ static int dmirror_snapshot(struct dmirror *dmirror, return ret; } -static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk) -{ - unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT; - unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT; - unsigned long npages = end_pfn - start_pfn + 1; - unsigned long i; - unsigned long *src_pfns; - unsigned long *dst_pfns; - unsigned int order = 0; - - src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - - migrate_device_range(src_pfns, start_pfn, npages); - for (i = 0; i < npages; i++) { - struct page *dpage, *spage; - - spage = migrate_pfn_to_page(src_pfns[i]); - if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) - continue; - - if (WARN_ON(!is_device_private_page(spage) && - !is_device_coherent_page(spage))) - continue; - - order = folio_order(page_folio(spage)); - spage = BACKING_PAGE(spage); - if (src_pfns[i] & MIGRATE_PFN_COMPOUND) { - dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE, - order), 0); - } else { - dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL); - order = 0; - } - - /* TODO Support splitting here */ - lock_page(dpage); - dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)); - if (src_pfns[i] & MIGRATE_PFN_WRITE) - dst_pfns[i] |= MIGRATE_PFN_WRITE; - if (order) - dst_pfns[i] |= MIGRATE_PFN_COMPOUND; - folio_copy(page_folio(dpage), page_folio(spage)); - } - migrate_device_pages(src_pfns, dst_pfns, npages); - migrate_device_finalize(src_pfns, dst_pfns, npages); - kvfree(src_pfns); - kvfree(dst_pfns); -} - /* Removes free pages from the free list so they can't be re-allocated */ static void dmirror_remove_free_pages(struct dmirror_chunk *devmem) { diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 05f74edbc62bf8..7e9c2fa52e35a4 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -40,6 +40,8 @@ obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o obj-$(CONFIG_MIN_HEAP_KUNIT_TEST) += min_heap_kunit.o CFLAGS_overflow_kunit.o = $(call cc-disable-warning, tautological-constant-out-of-range-compare) obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o +# GCC < 12.1 can miscompile errptr() test when branch profiling is enabled. +CFLAGS_printf_kunit.o += -DDISABLE_BRANCH_PROFILING obj-$(CONFIG_PRINTF_KUNIT_TEST) += printf_kunit.o obj-$(CONFIG_RANDSTRUCT_KUNIT_TEST) += randstruct_kunit.o obj-$(CONFIG_SCANF_KUNIT_TEST) += scanf_kunit.o diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c index bb847e5010eb24..d16449bdb8334a 100644 --- a/lib/tests/kunit_iov_iter.c +++ b/lib/tests/kunit_iov_iter.c @@ -42,7 +42,7 @@ static inline u8 pattern(unsigned long x) static void iov_kunit_unmap(void *data) { - vunmap(data); + vfree(data); } static void *__init iov_kunit_create_buffer(struct kunit *test, @@ -53,17 +53,22 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, unsigned long got; void *buffer; - pages = kunit_kcalloc(test, npages, sizeof(struct page *), GFP_KERNEL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); + pages = kzalloc_objs(struct page *, npages, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); *ppages = pages; got = alloc_pages_bulk(GFP_KERNEL, npages, pages); if (got != npages) { release_pages(pages, got); + kvfree(pages); KUNIT_ASSERT_EQ(test, got, npages); } buffer = vmap(pages, npages, VM_MAP | VM_MAP_PUT_PAGES, PAGE_KERNEL); + if (buffer == NULL) { + release_pages(pages, got); + kvfree(pages); + } KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); kunit_add_action_or_reset(test, iov_kunit_unmap, buffer); @@ -369,9 +374,6 @@ static void iov_kunit_destroy_folioq(void *data) for (folioq = data; folioq; folioq = next) { next = folioq->next; - for (int i = 0; i < folioq_nr_slots(folioq); i++) - if (folioq_folio(folioq, i)) - folio_put(folioq_folio(folioq, i)); kfree(folioq); } } diff --git a/lib/ts_kmp.c b/lib/ts_kmp.c index 5520dc28255a85..29466c1803c91b 100644 --- a/lib/ts_kmp.c +++ b/lib/ts_kmp.c @@ -94,8 +94,22 @@ static struct ts_config *kmp_init(const void *pattern, unsigned int len, struct ts_config *conf; struct ts_kmp *kmp; int i; - unsigned int prefix_tbl_len = len * sizeof(unsigned int); - size_t priv_size = sizeof(*kmp) + len + prefix_tbl_len; + unsigned int prefix_tbl_len; + size_t priv_size; + + /* Zero-length patterns would make kmp_find() read beyond kmp->pattern. */ + if (unlikely(!len)) + return ERR_PTR(-EINVAL); + + /* + * kmp->pattern is stored immediately after the prefix_tbl[] table. + * Reject lengths that would wrap while sizing either region. + */ + if (unlikely(check_mul_overflow(len, sizeof(*kmp->prefix_tbl), + &prefix_tbl_len) || + check_add_overflow(sizeof(*kmp), (size_t)len, &priv_size) || + check_add_overflow(priv_size, prefix_tbl_len, &priv_size))) + return ERR_PTR(-EINVAL); conf = alloc_ts_config(priv_size, gfp_mask); if (IS_ERR(conf)) diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540cf1..fe7f18d7194187 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1738,9 +1738,6 @@ void *xa_store(struct xarray *xa, unsigned long index, void *entry, gfp_t gfp) } EXPORT_SYMBOL(xa_store); -static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, - void *old, void *entry, gfp_t gfp); - /** * __xa_cmpxchg() - Conditionally replace an entry in the XArray. * @xa: XArray. @@ -1767,7 +1764,29 @@ void *__xa_cmpxchg(struct xarray *xa, unsigned long index, } EXPORT_SYMBOL(__xa_cmpxchg); -static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, +/** + * __xa_cmpxchg_raw() - Conditionally replace an entry in the XArray. + * @xa: XArray. + * @index: Index into array. + * @old: Old value to test against. + * @entry: New value to place in array. + * @gfp: Memory allocation flags. + * + * You must already be holding the xa_lock when calling this function. + * It will drop the lock if needed to allocate memory, and then reacquire + * it afterwards. + * + * If the entry at @index is the same as @old, replace it with @entry. + * If the return value is equal to @old, then the exchange was successful. + * + * This function is the same as __xa_cmpxchg() except that it does not coerce + * XA_ZERO_ENTRY to NULL on egress. + * + * Context: Any context. Expects xa_lock to be held on entry. May + * release and reacquire xa_lock if @gfp flags permit. + * Return: The old value at this index or xa_err() if an error happened. + */ +void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, void *old, void *entry, gfp_t gfp) { XA_STATE(xas, xa, index); @@ -1787,6 +1806,7 @@ static inline void *__xa_cmpxchg_raw(struct xarray *xa, unsigned long index, return xas_result(&xas, curr); } +EXPORT_SYMBOL(__xa_cmpxchg_raw); /** * __xa_insert() - Store this entry in the XArray if no entry is present. diff --git a/mm/Kconfig b/mm/Kconfig index ebd8ea353687e0..befa8909ae29df 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -572,6 +572,7 @@ config SPLIT_PTE_PTLOCKS depends on !ARM || CPU_CACHE_VIPT depends on !PARISC || PA20 depends on !SPARC32 + depends on !UML config ARCH_ENABLE_SPLIT_PMD_PTLOCK bool diff --git a/mm/backing-dev.c b/mm/backing-dev.c index 7a18fa6c727251..cecbcf9060a653 100644 --- a/mm/backing-dev.c +++ b/mm/backing-dev.c @@ -618,12 +618,13 @@ static void cgwb_release_workfn(struct work_struct *work) wb_shutdown(wb); css_put(wb->memcg_css); - css_put(wb->blkcg_css); - mutex_unlock(&wb->bdi->cgwb_release_mutex); /* triggers blkg destruction if no online users left */ blkcg_unpin_online(wb->blkcg_css); + css_put(wb->blkcg_css); + mutex_unlock(&wb->bdi->cgwb_release_mutex); + fprop_local_destroy_percpu(&wb->memcg_completions); spin_lock_irq(&cgwb_lock); diff --git a/mm/damon/core.c b/mm/damon/core.c index 3e1890d64d067a..3a58c4984f2788 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1368,6 +1368,11 @@ int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) int i; int err = 0; + for (i = 0; i < nr_ctxs; i++) { + if (!is_power_of_2(ctxs[i]->min_region_sz)) + return -EINVAL; + } + mutex_lock(&damon_lock); if ((exclusive && nr_running_ctxs) || (!exclusive && running_exclusive_ctxs)) { @@ -1464,35 +1469,6 @@ int damon_kdamond_pid(struct damon_ctx *ctx) return pid; } -/* - * damon_call_handle_inactive_ctx() - handle DAMON call request that added to - * an inactive context. - * @ctx: The inactive DAMON context. - * @control: Control variable of the call request. - * - * This function is called in a case that @control is added to @ctx but @ctx is - * not running (inactive). See if @ctx handled @control or not, and cleanup - * @control if it was not handled. - * - * Returns 0 if @control was handled by @ctx, negative error code otherwise. - */ -static int damon_call_handle_inactive_ctx( - struct damon_ctx *ctx, struct damon_call_control *control) -{ - struct damon_call_control *c; - - mutex_lock(&ctx->call_controls_lock); - list_for_each_entry(c, &ctx->call_controls, list) { - if (c == control) { - list_del(&control->list); - mutex_unlock(&ctx->call_controls_lock); - return -EINVAL; - } - } - mutex_unlock(&ctx->call_controls_lock); - return 0; -} - /** * damon_call() - Invoke a given function on DAMON worker thread (kdamond). * @ctx: DAMON context to call the function for. @@ -1510,6 +1486,10 @@ static int damon_call_handle_inactive_ctx( * synchronization. The return value of the function will be saved in * &damon_call_control->return_code. * + * Note that this function should be called only after damon_start() with the + * @ctx has succeeded. Otherwise, this function could fall into an indefinite + * wait. + * * Return: 0 on success, negative error code otherwise. */ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) @@ -1520,10 +1500,12 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) INIT_LIST_HEAD(&control->list); mutex_lock(&ctx->call_controls_lock); + if (ctx->call_controls_obsolete) { + mutex_unlock(&ctx->call_controls_lock); + return -ECANCELED; + } list_add_tail(&control->list, &ctx->call_controls); mutex_unlock(&ctx->call_controls_lock); - if (!damon_is_running(ctx)) - return damon_call_handle_inactive_ctx(ctx, control); if (control->repeat) return 0; wait_for_completion(&control->completion); @@ -1551,6 +1533,10 @@ int damon_call(struct damon_ctx *ctx, struct damon_call_control *control) * passed at least one &damos->apply_interval_us, kdamond marks the request as * completed so that damos_walk() can wakeup and return. * + * Note that this function should be called only after damon_start() with the + * @ctx has succeeded. Otherwise, this function could fall into an indefinite + * wait. + * * Return: 0 on success, negative error code otherwise. */ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) @@ -1558,19 +1544,16 @@ int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control) init_completion(&control->completion); control->canceled = false; mutex_lock(&ctx->walk_control_lock); + if (ctx->walk_control_obsolete) { + mutex_unlock(&ctx->walk_control_lock); + return -ECANCELED; + } if (ctx->walk_control) { mutex_unlock(&ctx->walk_control_lock); return -EBUSY; } ctx->walk_control = control; mutex_unlock(&ctx->walk_control_lock); - if (!damon_is_running(ctx)) { - mutex_lock(&ctx->walk_control_lock); - if (ctx->walk_control == control) - ctx->walk_control = NULL; - mutex_unlock(&ctx->walk_control_lock); - return -EINVAL; - } wait_for_completion(&control->completion); if (control->canceled) return -ECANCELED; @@ -2100,12 +2083,24 @@ static inline u64 damos_get_some_mem_psi_total(void) #endif /* CONFIG_PSI */ #ifdef CONFIG_NUMA +static bool invalid_mem_node(int nid) +{ + return nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY); +} + static __kernel_ulong_t damos_get_node_mem_bp( struct damos_quota_goal *goal) { struct sysinfo i; __kernel_ulong_t numerator; + if (invalid_mem_node(goal->nid)) { + if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP) + return 0; + else /* DAMOS_QUOTA_NODE_MEM_FREE_BP */ + return 10000; + } + si_meminfo_node(&i, goal->nid); if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP) numerator = i.totalram - i.freeram; @@ -2122,6 +2117,13 @@ static unsigned long damos_get_node_memcg_used_bp( unsigned long used_pages, numerator; struct sysinfo i; + if (invalid_mem_node(goal->nid)) { + if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP) + return 0; + else /* DAMOS_QUOTA_NODE_MEMCG_FREE_BP */ + return 10000; + } + memcg = mem_cgroup_get_from_id(goal->memcg_id); if (!memcg) { if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP) @@ -2228,7 +2230,8 @@ static unsigned long damos_quota_score(struct damos_quota *quota) /* * Called only if quota->ms, or quota->sz are set, or quota->goals is not empty */ -static void damos_set_effective_quota(struct damos_quota *quota) +static void damos_set_effective_quota(struct damos_quota *quota, + struct damon_ctx *ctx) { unsigned long throughput; unsigned long esz = ULONG_MAX; @@ -2254,6 +2257,7 @@ static void damos_set_effective_quota(struct damos_quota *quota) else throughput = PAGE_SIZE * 1024; esz = min(throughput * quota->ms, esz); + esz = max(ctx->min_region_sz, esz); } if (quota->sz && quota->sz < esz) @@ -2290,11 +2294,12 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) /* First charge window */ if (!quota->total_charged_sz && !quota->charged_from) { quota->charged_from = jiffies; - damos_set_effective_quota(quota); + damos_set_effective_quota(quota, c); } /* New charge window starts */ - if (time_after_eq(jiffies, quota->charged_from + + if (!time_in_range_open(jiffies, quota->charged_from, + quota->charged_from + msecs_to_jiffies(quota->reset_interval))) { if (quota->esz && quota->charged_sz >= quota->esz) s->stat.qt_exceeds++; @@ -2303,7 +2308,7 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) quota->charged_sz = 0; if (trace_damos_esz_enabled()) cached_esz = quota->esz; - damos_set_effective_quota(quota); + damos_set_effective_quota(quota, c); if (trace_damos_esz_enabled() && quota->esz != cached_esz) damos_trace_esz(c, s, quota); } @@ -2751,6 +2756,12 @@ static int kdamond_fn(void *data) pr_debug("kdamond (%d) starts\n", current->pid); + mutex_lock(&ctx->call_controls_lock); + ctx->call_controls_obsolete = false; + mutex_unlock(&ctx->call_controls_lock); + mutex_lock(&ctx->walk_control_lock); + ctx->walk_control_obsolete = false; + mutex_unlock(&ctx->walk_control_lock); complete(&ctx->kdamond_started); kdamond_init_ctx(ctx); @@ -2855,7 +2866,13 @@ static int kdamond_fn(void *data) damon_destroy_targets(ctx); kfree(ctx->regions_score_histogram); + mutex_lock(&ctx->call_controls_lock); + ctx->call_controls_obsolete = true; + mutex_unlock(&ctx->call_controls_lock); kdamond_call(ctx, true); + mutex_lock(&ctx->walk_control_lock); + ctx->walk_control_obsolete = true; + mutex_unlock(&ctx->walk_control_lock); damos_walk_cancel(ctx); pr_debug("kdamond (%d) finishes\n", current->pid); diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c index 7bc5c0b2aea3e1..2e29dbbcaf9433 100644 --- a/mm/damon/lru_sort.c +++ b/mm/damon/lru_sort.c @@ -161,15 +161,6 @@ module_param(monitor_region_end, ulong, 0600); */ static unsigned long addr_unit __read_mostly = 1; -/* - * PID of the DAMON thread - * - * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. - * Else, -1. - */ -static int kdamond_pid __read_mostly = -1; -module_param(kdamond_pid, int, 0400); - static struct damos_stat damon_lru_sort_hot_stat; DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_lru_sort_hot_stat, lru_sort_tried_hot_regions, lru_sorted_hot_regions, @@ -391,12 +382,8 @@ static int damon_lru_sort_turn(bool on) { int err; - if (!on) { - err = damon_stop(&ctx, 1); - if (!err) - kdamond_pid = -1; - return err; - } + if (!on) + return damon_stop(&ctx, 1); err = damon_lru_sort_apply_parameters(); if (err) @@ -405,9 +392,6 @@ static int damon_lru_sort_turn(bool on) err = damon_start(&ctx, 1, true); if (err) return err; - kdamond_pid = damon_kdamond_pid(ctx); - if (kdamond_pid < 0) - return kdamond_pid; return damon_call(ctx, &call_control); } @@ -435,42 +419,83 @@ module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); MODULE_PARM_DESC(addr_unit, "Scale factor for DAMON_LRU_SORT to ops address conversion (default: 1)"); +static bool damon_lru_sort_enabled(void) +{ + if (!ctx) + return false; + return damon_is_running(ctx); +} + static int damon_lru_sort_enabled_store(const char *val, const struct kernel_param *kp) { - bool is_enabled = enabled; - bool enable; int err; - err = kstrtobool(val, &enable); + err = kstrtobool(val, &enabled); if (err) return err; - if (is_enabled == enable) + if (damon_lru_sort_enabled() == enabled) return 0; /* Called before init function. The function will handle this. */ if (!damon_initialized()) - goto set_param_out; + return 0; - err = damon_lru_sort_turn(enable); - if (err) - return err; + return damon_lru_sort_turn(enabled); +} -set_param_out: - enabled = enable; - return err; +static int damon_lru_sort_enabled_load(char *buffer, + const struct kernel_param *kp) +{ + return sprintf(buffer, "%c\n", damon_lru_sort_enabled() ? 'Y' : 'N'); } static const struct kernel_param_ops enabled_param_ops = { .set = damon_lru_sort_enabled_store, - .get = param_get_bool, + .get = damon_lru_sort_enabled_load, }; module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); MODULE_PARM_DESC(enabled, "Enable or disable DAMON_LRU_SORT (default: disabled)"); +static int damon_lru_sort_kdamond_pid_store(const char *val, + const struct kernel_param *kp) +{ + /* + * kdamond_pid is read-only, but kernel command line could write it. + * Do nothing here. + */ + return 0; +} + +static int damon_lru_sort_kdamond_pid_load(char *buffer, + const struct kernel_param *kp) +{ + int kdamond_pid = -1; + + if (ctx) { + kdamond_pid = damon_kdamond_pid(ctx); + if (kdamond_pid < 0) + kdamond_pid = -1; + } + return sprintf(buffer, "%d\n", kdamond_pid); +} + +static const struct kernel_param_ops kdamond_pid_param_ops = { + .set = damon_lru_sort_kdamond_pid_store, + .get = damon_lru_sort_kdamond_pid_load, +}; + +/* + * PID of the DAMON thread + * + * If DAMON_LRU_SORT is enabled, this becomes the PID of the worker thread. + * Else, -1. + */ +module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); + static int __init damon_lru_sort_init(void) { int err; diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c index 43d76f5bed4497..a15cf1e603d2fa 100644 --- a/mm/damon/reclaim.c +++ b/mm/damon/reclaim.c @@ -144,15 +144,6 @@ static unsigned long addr_unit __read_mostly = 1; static bool skip_anon __read_mostly; module_param(skip_anon, bool, 0600); -/* - * PID of the DAMON thread - * - * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. - * Else, -1. - */ -static int kdamond_pid __read_mostly = -1; -module_param(kdamond_pid, int, 0400); - static struct damos_stat damon_reclaim_stat; DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, reclaim_tried_regions, reclaimed_regions, quota_exceeds); @@ -293,12 +284,8 @@ static int damon_reclaim_turn(bool on) { int err; - if (!on) { - err = damon_stop(&ctx, 1); - if (!err) - kdamond_pid = -1; - return err; - } + if (!on) + return damon_stop(&ctx, 1); err = damon_reclaim_apply_parameters(); if (err) @@ -307,9 +294,6 @@ static int damon_reclaim_turn(bool on) err = damon_start(&ctx, 1, true); if (err) return err; - kdamond_pid = damon_kdamond_pid(ctx); - if (kdamond_pid < 0) - return kdamond_pid; return damon_call(ctx, &call_control); } @@ -337,42 +321,83 @@ module_param_cb(addr_unit, &addr_unit_param_ops, &addr_unit, 0600); MODULE_PARM_DESC(addr_unit, "Scale factor for DAMON_RECLAIM to ops address conversion (default: 1)"); +static bool damon_reclaim_enabled(void) +{ + if (!ctx) + return false; + return damon_is_running(ctx); +} + static int damon_reclaim_enabled_store(const char *val, const struct kernel_param *kp) { - bool is_enabled = enabled; - bool enable; int err; - err = kstrtobool(val, &enable); + err = kstrtobool(val, &enabled); if (err) return err; - if (is_enabled == enable) + if (damon_reclaim_enabled() == enabled) return 0; /* Called before init function. The function will handle this. */ if (!damon_initialized()) - goto set_param_out; + return 0; - err = damon_reclaim_turn(enable); - if (err) - return err; + return damon_reclaim_turn(enabled); +} -set_param_out: - enabled = enable; - return err; +static int damon_reclaim_enabled_load(char *buffer, + const struct kernel_param *kp) +{ + return sprintf(buffer, "%c\n", damon_reclaim_enabled() ? 'Y' : 'N'); } static const struct kernel_param_ops enabled_param_ops = { .set = damon_reclaim_enabled_store, - .get = param_get_bool, + .get = damon_reclaim_enabled_load, }; module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); MODULE_PARM_DESC(enabled, "Enable or disable DAMON_RECLAIM (default: disabled)"); +static int damon_reclaim_kdamond_pid_store(const char *val, + const struct kernel_param *kp) +{ + /* + * kdamond_pid is read-only, but kernel command line could write it. + * Do nothing here. + */ + return 0; +} + +static int damon_reclaim_kdamond_pid_load(char *buffer, + const struct kernel_param *kp) +{ + int kdamond_pid = -1; + + if (ctx) { + kdamond_pid = damon_kdamond_pid(ctx); + if (kdamond_pid < 0) + kdamond_pid = -1; + } + return sprintf(buffer, "%d\n", kdamond_pid); +} + +static const struct kernel_param_ops kdamond_pid_param_ops = { + .set = damon_reclaim_kdamond_pid_store, + .get = damon_reclaim_kdamond_pid_load, +}; + +/* + * PID of the DAMON thread + * + * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. + * Else, -1. + */ +module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400); + static int __init damon_reclaim_init(void) { int err; diff --git a/mm/damon/stat.c b/mm/damon/stat.c index 60351a71946063..3951b762cbddf6 100644 --- a/mm/damon/stat.c +++ b/mm/damon/stat.c @@ -19,14 +19,17 @@ static int damon_stat_enabled_store( const char *val, const struct kernel_param *kp); +static int damon_stat_enabled_load(char *buffer, + const struct kernel_param *kp); + static const struct kernel_param_ops enabled_param_ops = { .set = damon_stat_enabled_store, - .get = param_get_bool, + .get = damon_stat_enabled_load, }; static bool enabled __read_mostly = IS_ENABLED( CONFIG_DAMON_STAT_ENABLED_DEFAULT); -module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); +module_param_cb(enabled, &enabled_param_ops, NULL, 0600); MODULE_PARM_DESC(enabled, "Enable of disable DAMON_STAT"); static unsigned long estimated_memory_bandwidth __read_mostly; @@ -255,8 +258,11 @@ static int damon_stat_start(void) if (!damon_stat_context) return -ENOMEM; err = damon_start(&damon_stat_context, 1, true); - if (err) + if (err) { + damon_destroy_ctx(damon_stat_context); + damon_stat_context = NULL; return err; + } damon_stat_last_refresh_jiffies = jiffies; call_control.data = damon_stat_context; @@ -270,17 +276,23 @@ static void damon_stat_stop(void) damon_stat_context = NULL; } +static bool damon_stat_enabled(void) +{ + if (!damon_stat_context) + return false; + return damon_is_running(damon_stat_context); +} + static int damon_stat_enabled_store( const char *val, const struct kernel_param *kp) { - bool is_enabled = enabled; int err; err = kstrtobool(val, &enabled); if (err) return err; - if (is_enabled == enabled) + if (damon_stat_enabled() == enabled) return 0; if (!damon_initialized()) @@ -290,16 +302,17 @@ static int damon_stat_enabled_store( */ return 0; - if (enabled) { - err = damon_stat_start(); - if (err) - enabled = false; - return err; - } + if (enabled) + return damon_stat_start(); damon_stat_stop(); return 0; } +static int damon_stat_enabled_load(char *buffer, const struct kernel_param *kp) +{ + return sprintf(buffer, "%c\n", damon_stat_enabled() ? 'Y' : 'N'); +} + static int __init damon_stat_init(void) { int err = 0; diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index 3a0782e576fab7..9302ad0a603b09 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -533,9 +533,14 @@ static ssize_t memcg_path_show(struct kobject *kobj, { struct damon_sysfs_scheme_filter *filter = container_of(kobj, struct damon_sysfs_scheme_filter, kobj); + int len; - return sysfs_emit(buf, "%s\n", + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; + len = sysfs_emit(buf, "%s\n", filter->memcg_path ? filter->memcg_path : ""); + mutex_unlock(&damon_sysfs_lock); + return len; } static ssize_t memcg_path_store(struct kobject *kobj, @@ -550,8 +555,13 @@ static ssize_t memcg_path_store(struct kobject *kobj, return -ENOMEM; strscpy(path, buf, count + 1); + if (!mutex_trylock(&damon_sysfs_lock)) { + kfree(path); + return -EBUSY; + } kfree(filter->memcg_path); filter->memcg_path = path; + mutex_unlock(&damon_sysfs_lock); return count; } @@ -1187,8 +1197,13 @@ static ssize_t path_show(struct kobject *kobj, { struct damos_sysfs_quota_goal *goal = container_of(kobj, struct damos_sysfs_quota_goal, kobj); + int len; - return sysfs_emit(buf, "%s\n", goal->path ? goal->path : ""); + if (!mutex_trylock(&damon_sysfs_lock)) + return -EBUSY; + len = sysfs_emit(buf, "%s\n", goal->path ? goal->path : ""); + mutex_unlock(&damon_sysfs_lock); + return len; } static ssize_t path_store(struct kobject *kobj, @@ -1203,8 +1218,13 @@ static ssize_t path_store(struct kobject *kobj, return -ENOMEM; strscpy(path, buf, count + 1); + if (!mutex_trylock(&damon_sysfs_lock)) { + kfree(path); + return -EBUSY; + } kfree(goal->path); goal->path = path; + mutex_unlock(&damon_sysfs_lock); return count; } diff --git a/mm/filemap.c b/mm/filemap.c index 3c1e785542dde0..793bf4816ea38f 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -228,7 +228,8 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) page_cache_delete(mapping, folio, shadow); } -void filemap_free_folio(struct address_space *mapping, struct folio *folio) +static void filemap_free_folio(const struct address_space *mapping, + struct folio *folio) { void (*free_folio)(struct folio *); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b298cba853ab95..123d21cded1bfd 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -4456,7 +4456,7 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, goto next; } if (!folio_trylock(folio)) - goto next; + goto requeue; if (!split_folio(folio)) { did_split = true; if (underused) @@ -4465,13 +4465,18 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, } folio_unlock(folio); next: + /* + * If thp_underused() returns false, or if split_folio() + * succeeds, or if split_folio() fails in the case it was + * underused, then consider it used and don't add it back to + * split_queue. + */ if (did_split || !folio_test_partially_mapped(folio)) continue; +requeue: /* - * Only add back to the queue if folio is partially mapped. - * If thp_underused returns false, or if split_folio fails - * in the case it was underused, then consider it used and - * don't add it back to split_queue. + * Add back partially mapped folios, or underused folios that + * we could not lock this round. */ fqueue = folio_split_queue_lock_irqsave(folio, &flags); if (list_empty(&folio->_deferred_list)) { diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 327eaa4074d394..9fda39132d26c1 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -4252,6 +4252,9 @@ static __init int hugetlb_add_param(char *s, int (*setup)(char *)) size_t len; char *p; + if (!s) + return -EINVAL; + if (hugetlb_param_index >= HUGE_MAX_CMDLINE_ARGS) return -EINVAL; diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c index f83ae4998990f6..7693ccefd0c64f 100644 --- a/mm/hugetlb_cma.c +++ b/mm/hugetlb_cma.c @@ -204,6 +204,7 @@ void __init hugetlb_cma_reserve(void) */ per_node = DIV_ROUND_UP(hugetlb_cma_size, nodes_weight(hugetlb_bootmem_nodes)); + per_node = round_up(per_node, PAGE_SIZE << order); pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n", hugetlb_cma_size / SZ_1M, per_node / SZ_1M); } diff --git a/mm/internal.h b/mm/internal.h index cb0af847d7d99d..e1e64b875885ab 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -540,7 +540,6 @@ unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); unsigned find_get_entries(struct address_space *mapping, pgoff_t *start, pgoff_t end, struct folio_batch *fbatch, pgoff_t *indices); -void filemap_free_folio(struct address_space *mapping, struct folio *folio); int truncate_inode_folio(struct address_space *mapping, struct folio *folio); bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end); @@ -1218,6 +1217,16 @@ static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf, } return fpin; } + +static inline bool vma_supports_mlock(const struct vm_area_struct *vma) +{ + if (vma->vm_flags & (VM_SPECIAL | VM_DROPPABLE)) + return false; + if (vma_is_dax(vma) || is_vm_hugetlb_page(vma)) + return false; + return vma != get_gate_vma(current->mm); +} + #else /* !CONFIG_MMU */ static inline void unmap_mapping_folio(struct folio *folio) { } static inline void mlock_new_folio(struct folio *folio) { } @@ -1748,26 +1757,28 @@ int walk_page_range_debug(struct mm_struct *mm, unsigned long start, void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm); int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm); -void remap_pfn_range_prepare(struct vm_area_desc *desc, unsigned long pfn); -int remap_pfn_range_complete(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t pgprot); +int remap_pfn_range_prepare(struct vm_area_desc *desc); +int remap_pfn_range_complete(struct vm_area_struct *vma, + struct mmap_action *action); -static inline void io_remap_pfn_range_prepare(struct vm_area_desc *desc, - unsigned long orig_pfn, unsigned long size) +static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) { + struct mmap_action *action = &desc->action; + const unsigned long orig_pfn = action->remap.start_pfn; + const pgprot_t orig_pgprot = action->remap.pgprot; + const unsigned long size = action->remap.size; const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); + int err; - return remap_pfn_range_prepare(desc, pfn); -} + action->remap.start_pfn = pfn; + action->remap.pgprot = pgprot_decrypted(orig_pgprot); + err = remap_pfn_range_prepare(desc); + if (err) + return err; -static inline int io_remap_pfn_range_complete(struct vm_area_struct *vma, - unsigned long addr, unsigned long orig_pfn, unsigned long size, - pgprot_t orig_prot) -{ - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); - const pgprot_t prot = pgprot_decrypted(orig_prot); - - return remap_pfn_range_complete(vma, addr, pfn, size, prot); + /* Remap does the actual work. */ + action->type = MMAP_REMAP_PFN; + return 0; } #endif /* __MM_INTERNAL_H */ diff --git a/mm/kasan/init.c b/mm/kasan/init.c index f084e7a5df1e5f..9c880f607c6a2d 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -292,7 +292,7 @@ static void kasan_free_pte(pte_t *pte_start, pmd_t *pmd) return; } - pte_free_kernel(&init_mm, (pte_t *)page_to_virt(pmd_page(*pmd))); + pte_free_kernel(&init_mm, pte_start); pmd_clear(pmd); } @@ -307,7 +307,7 @@ static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud) return; } - pmd_free(&init_mm, (pmd_t *)page_to_virt(pud_page(*pud))); + pmd_free(&init_mm, pmd_start); pud_clear(pud); } @@ -322,7 +322,7 @@ static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d) return; } - pud_free(&init_mm, (pud_t *)page_to_virt(p4d_page(*p4d))); + pud_free(&init_mm, pud_start); p4d_clear(p4d); } @@ -337,7 +337,7 @@ static void kasan_free_p4d(p4d_t *p4d_start, pgd_t *pgd) return; } - p4d_free(&init_mm, (p4d_t *)page_to_virt(pgd_page(*pgd))); + p4d_free(&init_mm, p4d_start); pgd_clear(pgd); } diff --git a/mm/memblock.c b/mm/memblock.c index b3ddfdec7a8092..d4a02f1750e91a 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2434,7 +2434,7 @@ int reserve_mem_release_by_name(const char *name) return 0; start = phys_to_virt(map->start); - end = start + map->size - 1; + end = start + map->size; snprintf(buf, sizeof(buf), "reserve_mem:%s", name); free_reserved_area(start, end, 0, buf); map->size = 0; diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c index b8edb9f981d7ff..cfd665a5b78748 100644 --- a/mm/memfd_luo.c +++ b/mm/memfd_luo.c @@ -466,8 +466,13 @@ static int memfd_luo_retrieve_folios(struct file *file, */ for (long j = i + 1; j < nr_folios; j++) { const struct memfd_luo_folio_ser *pfolio = &folios_ser[j]; + phys_addr_t phys; + + if (!pfolio->pfn) + continue; - folio = kho_restore_folio(pfolio->pfn); + phys = PFN_PHYS(pfolio->pfn); + folio = kho_restore_folio(phys); if (folio) folio_put(folio); } diff --git a/mm/memory.c b/mm/memory.c index c65e82c86fed76..e03522c2bea636 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3105,26 +3105,34 @@ static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, } #endif -void remap_pfn_range_prepare(struct vm_area_desc *desc, unsigned long pfn) +int remap_pfn_range_prepare(struct vm_area_desc *desc) { - /* - * We set addr=VMA start, end=VMA end here, so this won't fail, but we - * check it again on complete and will fail there if specified addr is - * invalid. - */ - get_remap_pgoff(vma_desc_is_cow_mapping(desc), desc->start, desc->end, - desc->start, desc->end, pfn, &desc->pgoff); + const struct mmap_action *action = &desc->action; + const unsigned long start = action->remap.start; + const unsigned long end = start + action->remap.size; + const unsigned long pfn = action->remap.start_pfn; + const bool is_cow = vma_desc_is_cow_mapping(desc); + int err; + + err = get_remap_pgoff(is_cow, start, end, desc->start, desc->end, pfn, + &desc->pgoff); + if (err) + return err; + vma_desc_set_flags_mask(desc, VMA_REMAP_FLAGS); + return 0; } -static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size) +static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, + unsigned long size) { - unsigned long end = addr + PAGE_ALIGN(size); + const unsigned long end = addr + PAGE_ALIGN(size); + const bool is_cow = is_cow_mapping(vma->vm_flags); int err; - err = get_remap_pgoff(is_cow_mapping(vma->vm_flags), addr, end, - vma->vm_start, vma->vm_end, pfn, &vma->vm_pgoff); + err = get_remap_pgoff(is_cow, addr, end, vma->vm_start, vma->vm_end, + pfn, &vma->vm_pgoff); if (err) return err; @@ -3157,10 +3165,15 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, } EXPORT_SYMBOL(remap_pfn_range); -int remap_pfn_range_complete(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) +int remap_pfn_range_complete(struct vm_area_struct *vma, + struct mmap_action *action) { - return do_remap_pfn_range(vma, addr, pfn, size, prot); + const unsigned long start = action->remap.start; + const unsigned long pfn = action->remap.start_pfn; + const unsigned long size = action->remap.size; + const pgprot_t prot = action->remap.pgprot; + + return do_remap_pfn_range(vma, start, pfn, size, prot); } /** diff --git a/mm/mempolicy.c b/mm/mempolicy.c index cf92bd6a8226ee..ebe4bc8220b14e 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -3706,18 +3706,19 @@ static ssize_t weighted_interleave_auto_store(struct kobject *kobj, new_wi_state->iw_table[i] = 1; mutex_lock(&wi_state_lock); - if (!input) { - old_wi_state = rcu_dereference_protected(wi_state, - lockdep_is_held(&wi_state_lock)); - if (!old_wi_state) - goto update_wi_state; - if (input == old_wi_state->mode_auto) { - mutex_unlock(&wi_state_lock); - return count; - } + old_wi_state = rcu_dereference_protected(wi_state, + lockdep_is_held(&wi_state_lock)); - memcpy(new_wi_state->iw_table, old_wi_state->iw_table, - nr_node_ids * sizeof(u8)); + if (old_wi_state && input == old_wi_state->mode_auto) { + mutex_unlock(&wi_state_lock); + kfree(new_wi_state); + return count; + } + + if (!input) { + if (old_wi_state) + memcpy(new_wi_state->iw_table, old_wi_state->iw_table, + nr_node_ids * sizeof(u8)); goto update_wi_state; } diff --git a/mm/memremap.c b/mm/memremap.c index ac7be07e3361ae..053842d45cb109 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -454,7 +454,7 @@ void free_zone_device_folio(struct folio *folio) if (WARN_ON_ONCE(!pgmap->ops || !pgmap->ops->folio_free)) break; pgmap->ops->folio_free(folio); - percpu_ref_put_many(&folio->pgmap->ref, nr); + percpu_ref_put_many(&pgmap->ref, nr); break; case MEMORY_DEVICE_GENERIC: diff --git a/mm/migrate.c b/mm/migrate.c index 2c3d489ecf51b6..0cb434599c30cf 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1358,6 +1358,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, int rc; int old_page_state = 0; struct anon_vma *anon_vma = NULL; + bool src_deferred_split = false; + bool src_partially_mapped = false; struct list_head *prev; __migrate_folio_extract(dst, &old_page_state, &anon_vma); @@ -1371,10 +1373,25 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, goto out_unlock_both; } + if (folio_order(src) > 1 && + !data_race(list_empty(&src->_deferred_list))) { + src_deferred_split = true; + src_partially_mapped = folio_test_partially_mapped(src); + } + rc = move_to_new_folio(dst, src, mode); if (rc) goto out; + /* + * Requeue the destination folio on the deferred split queue if + * the source was on the queue. The source is unqueued in + * __folio_migrate_mapping(), so we recorded the state from + * before move_to_new_folio(). + */ + if (src_deferred_split) + deferred_split_folio(dst, src_partially_mapped); + /* * When successful, push dst to LRU immediately: so that if it * turns out to be an mlocked page, remove_migration_ptes() will diff --git a/mm/mlock.c b/mm/mlock.c index 2f699c3497a579..73551c71cebfcd 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -472,10 +472,12 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, int ret = 0; vm_flags_t oldflags = vma->vm_flags; - if (newflags == oldflags || (oldflags & VM_SPECIAL) || - is_vm_hugetlb_page(vma) || vma == get_gate_vma(current->mm) || - vma_is_dax(vma) || vma_is_secretmem(vma) || (oldflags & VM_DROPPABLE)) - /* don't set VM_LOCKED or VM_LOCKONFAULT and don't count */ + if (newflags == oldflags || vma_is_secretmem(vma) || + !vma_supports_mlock(vma)) + /* + * Don't set VM_LOCKED or VM_LOCKONFAULT and don't count. + * For secretmem, don't allow the memory to be unlocked. + */ goto out; vma = vma_modify_flags(vmi, *prev, vma, start, end, &newflags); diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2d4b6f1a554ed2..e92898ad51cd81 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -1289,10 +1289,18 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task, union pgtag_ref_handle handle; union codetag_ref ref; - if (get_page_tag_ref(page, &ref, &handle)) { + if (likely(get_page_tag_ref(page, &ref, &handle))) { alloc_tag_add(&ref, task->alloc_tag, PAGE_SIZE * nr); update_page_tag_ref(handle, &ref); put_page_tag_ref(handle); + } else { + /* + * page_ext is not available yet, record the pfn so we can + * clear the tag ref later when page_ext is initialized. + */ + alloc_tag_add_early_pfn(page_to_pfn(page)); + if (task->alloc_tag) + alloc_tag_set_inaccurate(task->alloc_tag); } } @@ -7798,6 +7806,11 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned */ if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) return NULL; + + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return NULL; + if (!pcp_allowed_order(order)) return NULL; diff --git a/mm/slub.c b/mm/slub.c index 2b2d33cc735cb6..e423afa27d1a46 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5304,6 +5304,10 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) return NULL; + /* On UP, spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return NULL; + retry: if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) return NULL; @@ -6569,16 +6573,6 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, if (!kasan_check_byte(p)) return NULL; - /* - * If reallocation is not necessary (e. g. the new size is less - * than the current allocated size), the current allocation will be - * preserved unless __GFP_THISNODE is set. In the latter case a new - * allocation on the requested node will be attempted. - */ - if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && - nid != page_to_nid(virt_to_page(p))) - goto alloc_new; - if (is_kfence_address(p)) { ks = orig_size = kfence_ksize(p); } else { @@ -6597,6 +6591,16 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, } } + /* + * If reallocation is not necessary (e. g. the new size is less + * than the current allocated size), the current allocation will be + * preserved unless __GFP_THISNODE is set. In the latter case a new + * allocation on the requested node will be attempted. + */ + if (unlikely(flags & __GFP_THISNODE) && nid != NUMA_NO_NODE && + nid != page_to_nid(virt_to_page(p))) + goto alloc_new; + /* If the old object doesn't fit, allocate a bigger one */ if (new_size > ks) goto alloc_new; @@ -6631,7 +6635,7 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, if (ret && p) { /* Disable KASAN checks as the object's redzone is accessed. */ kasan_disable_current(); - memcpy(ret, kasan_reset_tag(p), orig_size ?: ks); + memcpy(ret, kasan_reset_tag(p), min(new_size, (size_t)(orig_size ?: ks))); kasan_enable_current(); } @@ -6865,7 +6869,7 @@ void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long alig if (p) { /* We already know that `p` is not a vmalloc address. */ kasan_disable_current(); - memcpy(n, kasan_reset_tag(p), ksize(p)); + memcpy(n, kasan_reset_tag(p), min(size, ksize(p))); kasan_enable_current(); kfree(p); diff --git a/mm/swapfile.c b/mm/swapfile.c index 94af29d1de8887..90132b74d6a083 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1926,8 +1926,9 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) /* Allocate a slot for hibernation */ swp_entry_t swap_alloc_hibernation_slot(int type) { - struct swap_info_struct *si = swap_type_to_info(type); - unsigned long offset; + struct swap_info_struct *pcp_si, *si = swap_type_to_info(type); + unsigned long pcp_offset, offset = SWAP_ENTRY_INVALID; + struct swap_cluster_info *ci; swp_entry_t entry = {0}; if (!si) @@ -1937,11 +1938,21 @@ swp_entry_t swap_alloc_hibernation_slot(int type) if (get_swap_device_info(si)) { if (si->flags & SWP_WRITEOK) { /* - * Grab the local lock to be compliant - * with swap table allocation. + * Try the local cluster first if it matches the device. If + * not, try grab a new cluster and override local cluster. */ local_lock(&percpu_swap_cluster.lock); - offset = cluster_alloc_swap_entry(si, NULL); + pcp_si = this_cpu_read(percpu_swap_cluster.si[0]); + pcp_offset = this_cpu_read(percpu_swap_cluster.offset[0]); + if (pcp_si == si && pcp_offset) { + ci = swap_cluster_lock(si, pcp_offset); + if (cluster_is_usable(ci, 0)) + offset = alloc_swap_scan_cluster(si, ci, NULL, pcp_offset); + else + swap_cluster_unlock(ci); + } + if (!offset) + offset = cluster_alloc_swap_entry(si, NULL); local_unlock(&percpu_swap_cluster.lock); if (offset) entry = swp_entry(si->type, offset); diff --git a/mm/truncate.c b/mm/truncate.c index 12467c1bd711eb..8617a12cb16956 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -622,6 +622,7 @@ static int folio_launder(struct address_space *mapping, struct folio *folio) int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio, gfp_t gfp) { + void (*free_folio)(struct folio *); int ret; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); @@ -648,9 +649,12 @@ int folio_unmap_invalidate(struct address_space *mapping, struct folio *folio, xa_unlock_irq(&mapping->i_pages); if (mapping_shrinkable(mapping)) inode_lru_list_add(mapping->host); + free_folio = mapping->a_ops->free_folio; spin_unlock(&mapping->host->i_lock); - filemap_free_folio(mapping, folio); + if (free_folio) + free_folio(folio); + folio_put_refs(folio, folio_nr_pages(folio)); return 1; failed: xa_unlock_irq(&mapping->i_pages); diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 927086bb4a3c2e..5590989e18c746 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -573,7 +573,7 @@ static __always_inline ssize_t mfill_atomic_hugetlb( * in the case of shared pmds. fault mutex prevents * races with other faulting threads. */ - idx = linear_page_index(dst_vma, dst_addr); + idx = hugetlb_linear_page_index(dst_vma, dst_addr); mapping = dst_vma->vm_file->f_mapping; hash = hugetlb_fault_mutex_hash(mapping, idx); mutex_lock(&hugetlb_fault_mutex_table[hash]); diff --git a/mm/util.c b/mm/util.c index b05ab6f97e1108..a14de66c945827 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1134,43 +1134,6 @@ void flush_dcache_folio(struct folio *folio) EXPORT_SYMBOL(flush_dcache_folio); #endif -/** - * __compat_vma_mmap() - See description for compat_vma_mmap() - * for details. This is the same operation, only with a specific file operations - * struct which may or may not be the same as vma->vm_file->f_op. - * @f_op: The file operations whose .mmap_prepare() hook is specified. - * @file: The file which backs or will back the mapping. - * @vma: The VMA to apply the .mmap_prepare() hook to. - * Returns: 0 on success or error. - */ -int __compat_vma_mmap(const struct file_operations *f_op, - struct file *file, struct vm_area_struct *vma) -{ - struct vm_area_desc desc = { - .mm = vma->vm_mm, - .file = file, - .start = vma->vm_start, - .end = vma->vm_end, - - .pgoff = vma->vm_pgoff, - .vm_file = vma->vm_file, - .vma_flags = vma->flags, - .page_prot = vma->vm_page_prot, - - .action.type = MMAP_NOTHING, /* Default */ - }; - int err; - - err = f_op->mmap_prepare(&desc); - if (err) - return err; - - mmap_action_prepare(&desc.action, &desc); - set_vma_from_desc(vma, &desc); - return mmap_action_complete(&desc.action, vma); -} -EXPORT_SYMBOL(__compat_vma_mmap); - /** * compat_vma_mmap() - Apply the file's .mmap_prepare() hook to an * existing VMA and execute any requested actions. @@ -1199,7 +1162,38 @@ EXPORT_SYMBOL(__compat_vma_mmap); */ int compat_vma_mmap(struct file *file, struct vm_area_struct *vma) { - return __compat_vma_mmap(file->f_op, file, vma); + struct vm_area_desc desc = { + .mm = vma->vm_mm, + .file = file, + .start = vma->vm_start, + .end = vma->vm_end, + + .pgoff = vma->vm_pgoff, + .vm_file = vma->vm_file, + .vma_flags = vma->flags, + .page_prot = vma->vm_page_prot, + + .action.type = MMAP_NOTHING, /* Default */ + }; + int err; + + err = vfs_mmap_prepare(file, &desc); + if (err) + return err; + + err = mmap_action_prepare(&desc); + if (err) + return err; + + set_vma_from_desc(vma, &desc); + err = mmap_action_complete(vma, &desc.action, + /*is_compat=*/true); + if (err) { + const size_t len = vma_pages(vma) << PAGE_SHIFT; + + do_munmap(current->mm, vma->vm_start, len, NULL); + } + return err; } EXPORT_SYMBOL(compat_vma_mmap); @@ -1283,70 +1277,72 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page) } } -static int mmap_action_finish(struct mmap_action *action, - const struct vm_area_struct *vma, int err) +static int mmap_action_finish(struct vm_area_struct *vma, + struct mmap_action *action, int err, + bool is_compat) { + if (!err && action->success_hook) + err = action->success_hook(vma); + + /* + * If this is invoked from the compatibility layer, post-mmap() hook + * logic will handle cleanup for us. + */ + if (!err || is_compat) + return err; + /* * If an error occurs, unmap the VMA altogether and return an error. We * only clear the newly allocated VMA, since this function is only * invoked if we do NOT merge, so we only clean up the VMA we created. */ - if (err) { - const size_t len = vma_pages(vma) << PAGE_SHIFT; - - do_munmap(current->mm, vma->vm_start, len, NULL); - - if (action->error_hook) { - /* We may want to filter the error. */ - err = action->error_hook(err); - - /* The caller should not clear the error. */ - VM_WARN_ON_ONCE(!err); - } - return err; + if (action->error_hook) { + /* We may want to filter the error. */ + err = action->error_hook(err); + /* The caller should not clear the error. */ + VM_WARN_ON_ONCE(!err); } - - if (action->success_hook) - return action->success_hook(vma); - - return 0; + return err; } #ifdef CONFIG_MMU /** * mmap_action_prepare - Perform preparatory setup for an VMA descriptor * action which need to be performed. - * @desc: The VMA descriptor to prepare for @action. - * @action: The action to perform. + * @desc: The VMA descriptor to prepare for its @desc->action. + * + * Returns: %0 on success, otherwise error. */ -void mmap_action_prepare(struct mmap_action *action, - struct vm_area_desc *desc) +int mmap_action_prepare(struct vm_area_desc *desc) { - switch (action->type) { + switch (desc->action.type) { case MMAP_NOTHING: - break; + return 0; case MMAP_REMAP_PFN: - remap_pfn_range_prepare(desc, action->remap.start_pfn); - break; + return remap_pfn_range_prepare(desc); case MMAP_IO_REMAP_PFN: - io_remap_pfn_range_prepare(desc, action->remap.start_pfn, - action->remap.size); - break; + return io_remap_pfn_range_prepare(desc); } + + WARN_ON_ONCE(1); + return -EINVAL; } EXPORT_SYMBOL(mmap_action_prepare); /** * mmap_action_complete - Execute VMA descriptor action. - * @action: The action to perform. * @vma: The VMA to perform the action upon. + * @action: The action to perform. + * @is_compat: Is this being invoked from the compatibility layer? * * Similar to mmap_action_prepare(). * - * Return: 0 on success, or error, at which point the VMA will be unmapped. + * Return: 0 on success, or error, at which point the VMA will be unmapped if + * !@is_compat. */ -int mmap_action_complete(struct mmap_action *action, - struct vm_area_struct *vma) +int mmap_action_complete(struct vm_area_struct *vma, + struct mmap_action *action, + bool is_compat) { int err = 0; @@ -1354,25 +1350,22 @@ int mmap_action_complete(struct mmap_action *action, case MMAP_NOTHING: break; case MMAP_REMAP_PFN: - err = remap_pfn_range_complete(vma, action->remap.start, - action->remap.start_pfn, action->remap.size, - action->remap.pgprot); + err = remap_pfn_range_complete(vma, action); break; case MMAP_IO_REMAP_PFN: - err = io_remap_pfn_range_complete(vma, action->remap.start, - action->remap.start_pfn, action->remap.size, - action->remap.pgprot); + /* Should have been delegated. */ + WARN_ON_ONCE(1); + err = -EINVAL; break; } - return mmap_action_finish(action, vma, err); + return mmap_action_finish(vma, action, err, is_compat); } EXPORT_SYMBOL(mmap_action_complete); #else -void mmap_action_prepare(struct mmap_action *action, - struct vm_area_desc *desc) +int mmap_action_prepare(struct vm_area_desc *desc) { - switch (action->type) { + switch (desc->action.type) { case MMAP_NOTHING: break; case MMAP_REMAP_PFN: @@ -1380,11 +1373,14 @@ void mmap_action_prepare(struct mmap_action *action, WARN_ON_ONCE(1); /* nommu cannot handle these. */ break; } + + return 0; } EXPORT_SYMBOL(mmap_action_prepare); -int mmap_action_complete(struct mmap_action *action, - struct vm_area_struct *vma) +int mmap_action_complete(struct vm_area_struct *vma, + struct mmap_action *action, + bool is_compat) { int err = 0; @@ -1399,7 +1395,7 @@ int mmap_action_complete(struct mmap_action *action, break; } - return mmap_action_finish(action, vma, err); + return mmap_action_finish(vma, action, err, is_compat); } EXPORT_SYMBOL(mmap_action_complete); #endif diff --git a/mm/vma.c b/mm/vma.c index c8df5f561ad7d1..5cd80cdcf82fa2 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -2589,9 +2589,7 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) vm_stat_account(mm, vma->vm_flags, map->pglen); if (vm_flags & VM_LOCKED) { - if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) || - is_vm_hugetlb_page(vma) || - vma == get_gate_vma(mm)) + if (!vma_supports_mlock(vma)) vm_flags_clear(vma, VM_LOCKED_MASK); else mm->locked_vm += map->pglen; @@ -2613,15 +2611,18 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) vma_set_page_prot(vma); } -static void call_action_prepare(struct mmap_state *map, - struct vm_area_desc *desc) +static int call_action_prepare(struct mmap_state *map, + struct vm_area_desc *desc) { - struct mmap_action *action = &desc->action; + int err; - mmap_action_prepare(action, desc); + err = mmap_action_prepare(desc); + if (err) + return err; - if (action->hide_from_rmap_until_complete) + if (desc->action.hide_from_rmap_until_complete) map->hold_file_rmap_lock = true; + return 0; } /* @@ -2645,7 +2646,9 @@ static int call_mmap_prepare(struct mmap_state *map, if (err) return err; - call_action_prepare(map, desc); + err = call_action_prepare(map, desc); + if (err) + return err; /* Update fields permitted to be changed. */ map->pgoff = desc->pgoff; @@ -2700,13 +2703,12 @@ static bool can_set_ksm_flags_early(struct mmap_state *map) } static int call_action_complete(struct mmap_state *map, - struct vm_area_desc *desc, + struct mmap_action *action, struct vm_area_struct *vma) { - struct mmap_action *action = &desc->action; - int ret; + int err; - ret = mmap_action_complete(action, vma); + err = mmap_action_complete(vma, action, /*is_compat=*/false); /* If we held the file rmap we need to release it. */ if (map->hold_file_rmap_lock) { @@ -2714,7 +2716,14 @@ static int call_action_complete(struct mmap_state *map, i_mmap_unlock_write(file->f_mapping); } - return ret; + + if (err) { + const size_t len = vma_pages(vma) << PAGE_SHIFT; + + do_munmap(current->mm, vma->vm_start, len, NULL); + } + + return err; } static unsigned long __mmap_region(struct file *file, unsigned long addr, @@ -2768,8 +2777,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, __mmap_complete(&map, vma); if (have_mmap_prepare && allocated_new) { - error = call_action_complete(&map, &desc, vma); - + error = call_action_complete(&map, &desc.action, vma); if (error) return error; } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 61caa55a44027b..2c2f74a07f3963 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4361,7 +4361,7 @@ void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long align return NULL; if (p) { - memcpy(n, p, old_size); + memcpy(n, p, min(size, old_size)); vfree(p); } @@ -5416,6 +5416,7 @@ vmap_node_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) { struct vmap_node *vn; + guard(mutex)(&vmap_purge_lock); for_each_vmap_node(vn) decay_va_pool_node(vn, true); diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 2c1430bf8d57cf..79ef9d57f2e16e 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1741,6 +1741,7 @@ static int zs_page_migrate(struct page *newpage, struct page *page, */ d_addr = kmap_local_zpdesc(newzpdesc); copy_page(d_addr, s_addr); + kmsan_copy_page_meta(zpdesc_page(newzpdesc), zpdesc_page(zpdesc)); kunmap_local(d_addr); for (addr = s_addr + offset; addr < s_addr + PAGE_SIZE; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index c40f7d5c4fca58..7aa3af8b10ead0 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -172,39 +172,42 @@ int vlan_dev_set_egress_priority(const struct net_device *dev, u32 skb_prio, u16 vlan_prio) { struct vlan_dev_priv *vlan = vlan_dev_priv(dev); - struct vlan_priority_tci_mapping *mp = NULL; + struct vlan_priority_tci_mapping __rcu **mpp; + struct vlan_priority_tci_mapping *mp; struct vlan_priority_tci_mapping *np; + u32 bucket = skb_prio & 0xF; u32 vlan_qos = (vlan_prio << VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK; /* See if a priority mapping exists.. */ - mp = vlan->egress_priority_map[skb_prio & 0xF]; + mpp = &vlan->egress_priority_map[bucket]; + mp = rtnl_dereference(*mpp); while (mp) { if (mp->priority == skb_prio) { - if (mp->vlan_qos && !vlan_qos) + if (!vlan_qos) { + rcu_assign_pointer(*mpp, rtnl_dereference(mp->next)); vlan->nr_egress_mappings--; - else if (!mp->vlan_qos && vlan_qos) - vlan->nr_egress_mappings++; - mp->vlan_qos = vlan_qos; + kfree_rcu(mp, rcu); + } else { + WRITE_ONCE(mp->vlan_qos, vlan_qos); + } return 0; } - mp = mp->next; + mpp = &mp->next; + mp = rtnl_dereference(*mpp); } /* Create a new mapping then. */ - mp = vlan->egress_priority_map[skb_prio & 0xF]; + if (!vlan_qos) + return 0; + np = kmalloc_obj(struct vlan_priority_tci_mapping); if (!np) return -ENOBUFS; - np->next = mp; np->priority = skb_prio; np->vlan_qos = vlan_qos; - /* Before inserting this element in hash table, make sure all its fields - * are committed to memory. - * coupled with smp_rmb() in vlan_dev_get_egress_qos_mask() - */ - smp_wmb(); - vlan->egress_priority_map[skb_prio & 0xF] = np; + RCU_INIT_POINTER(np->next, rtnl_dereference(vlan->egress_priority_map[bucket])); + rcu_assign_pointer(vlan->egress_priority_map[bucket], np); if (vlan_qos) vlan->nr_egress_mappings++; return 0; @@ -604,11 +607,17 @@ void vlan_dev_free_egress_priority(const struct net_device *dev) int i; for (i = 0; i < ARRAY_SIZE(vlan->egress_priority_map); i++) { - while ((pm = vlan->egress_priority_map[i]) != NULL) { - vlan->egress_priority_map[i] = pm->next; - kfree(pm); + pm = rtnl_dereference(vlan->egress_priority_map[i]); + RCU_INIT_POINTER(vlan->egress_priority_map[i], NULL); + while (pm) { + struct vlan_priority_tci_mapping *next; + + next = rtnl_dereference(pm->next); + kfree_rcu(pm, rcu); + pm = next; } } + vlan->nr_egress_mappings = 0; } static void vlan_dev_uninit(struct net_device *dev) diff --git a/net/8021q/vlan_netlink.c b/net/8021q/vlan_netlink.c index a000b1ef05206a..368d53ca7d8709 100644 --- a/net/8021q/vlan_netlink.c +++ b/net/8021q/vlan_netlink.c @@ -260,13 +260,11 @@ static int vlan_fill_info(struct sk_buff *skb, const struct net_device *dev) goto nla_put_failure; for (i = 0; i < ARRAY_SIZE(vlan->egress_priority_map); i++) { - for (pm = vlan->egress_priority_map[i]; pm; - pm = pm->next) { - if (!pm->vlan_qos) - continue; - + for (pm = rcu_dereference_rtnl(vlan->egress_priority_map[i]); pm; + pm = rcu_dereference_rtnl(pm->next)) { + u16 vlan_qos = READ_ONCE(pm->vlan_qos); m.from = pm->priority; - m.to = (pm->vlan_qos >> 13) & 0x7; + m.to = (vlan_qos >> 13) & 0x7; if (nla_put(skb, IFLA_VLAN_QOS_MAPPING, sizeof(m), &m)) goto nla_put_failure; diff --git a/net/8021q/vlanproc.c b/net/8021q/vlanproc.c index fa67374bda494c..0e424e0895b7e8 100644 --- a/net/8021q/vlanproc.c +++ b/net/8021q/vlanproc.c @@ -262,15 +262,19 @@ static int vlandev_seq_show(struct seq_file *seq, void *offset) vlan->ingress_priority_map[7]); seq_printf(seq, " EGRESS priority mappings: "); + rcu_read_lock(); for (i = 0; i < 16; i++) { - const struct vlan_priority_tci_mapping *mp - = vlan->egress_priority_map[i]; + const struct vlan_priority_tci_mapping *mp = + rcu_dereference(vlan->egress_priority_map[i]); while (mp) { + u16 vlan_qos = READ_ONCE(mp->vlan_qos); + seq_printf(seq, "%u:%d ", - mp->priority, ((mp->vlan_qos >> 13) & 0x7)); - mp = mp->next; + mp->priority, ((vlan_qos >> 13) & 0x7)); + mp = rcu_dereference(mp->next); } } + rcu_read_unlock(); seq_puts(seq, "\n"); return 0; diff --git a/net/batman-adv/bat_iv_ogm.c b/net/batman-adv/bat_iv_ogm.c index f28e9cbf8ad5f2..74ef7dc2b2f981 100644 --- a/net/batman-adv/bat_iv_ogm.c +++ b/net/batman-adv/bat_iv_ogm.c @@ -173,19 +173,12 @@ batadv_iv_ogm_orig_get(struct batadv_priv *bat_priv, const u8 *addr) static struct batadv_neigh_node * batadv_iv_ogm_neigh_new(struct batadv_hard_iface *hard_iface, const u8 *neigh_addr, - struct batadv_orig_node *orig_node, - struct batadv_orig_node *orig_neigh) + struct batadv_orig_node *orig_node) { struct batadv_neigh_node *neigh_node; neigh_node = batadv_neigh_node_get_or_create(orig_node, hard_iface, neigh_addr); - if (!neigh_node) - goto out; - - neigh_node->orig_node = orig_neigh; - -out: return neigh_node; } @@ -335,7 +328,7 @@ static void batadv_iv_ogm_send_to_if(struct batadv_forw_packet *forw_packet, struct batadv_priv *bat_priv = netdev_priv(hard_iface->mesh_iface); const char *fwd_str; u8 packet_num; - s16 buff_pos; + int buff_pos; struct batadv_ogm_packet *batadv_ogm_packet; struct sk_buff *skb; u8 *packet_pos; @@ -906,6 +899,31 @@ static u8 batadv_iv_orig_ifinfo_sum(struct batadv_orig_node *orig_node, return sum; } +/** + * batadv_iv_ogm_neigh_ifinfo_sum() - Get bcast_own sum for a last-hop neighbor + * @bat_priv: the bat priv with all the mesh interface information + * @neigh_node: last-hop neighbor of an originator + * + * Return: Number of replied (rebroadcasted) OGMs for the originator currently + * announced by the neighbor. Returns 0 if the neighbor's originator entry is + * not available anymore. + */ +static u8 batadv_iv_ogm_neigh_ifinfo_sum(struct batadv_priv *bat_priv, + const struct batadv_neigh_node *neigh_node) +{ + struct batadv_orig_node *orig_neigh; + u8 sum; + + orig_neigh = batadv_orig_hash_find(bat_priv, neigh_node->addr); + if (!orig_neigh) + return 0; + + sum = batadv_iv_orig_ifinfo_sum(orig_neigh, neigh_node->if_incoming); + batadv_orig_node_put(orig_neigh); + + return sum; +} + /** * batadv_iv_ogm_orig_update() - use OGM to update corresponding data in an * originator @@ -975,17 +993,9 @@ batadv_iv_ogm_orig_update(struct batadv_priv *bat_priv, } if (!neigh_node) { - struct batadv_orig_node *orig_tmp; - - orig_tmp = batadv_iv_ogm_orig_get(bat_priv, ethhdr->h_source); - if (!orig_tmp) - goto unlock; - neigh_node = batadv_iv_ogm_neigh_new(if_incoming, ethhdr->h_source, - orig_node, orig_tmp); - - batadv_orig_node_put(orig_tmp); + orig_node); if (!neigh_node) goto unlock; } else { @@ -1037,10 +1047,9 @@ batadv_iv_ogm_orig_update(struct batadv_priv *bat_priv, */ if (router_ifinfo && neigh_ifinfo->bat_iv.tq_avg == router_ifinfo->bat_iv.tq_avg) { - sum_orig = batadv_iv_orig_ifinfo_sum(router->orig_node, - router->if_incoming); - sum_neigh = batadv_iv_orig_ifinfo_sum(neigh_node->orig_node, - neigh_node->if_incoming); + sum_orig = batadv_iv_ogm_neigh_ifinfo_sum(bat_priv, router); + sum_neigh = batadv_iv_ogm_neigh_ifinfo_sum(bat_priv, + neigh_node); if (sum_orig >= sum_neigh) goto out; } @@ -1106,7 +1115,6 @@ static bool batadv_iv_ogm_calc_tq(struct batadv_orig_node *orig_node, if (!neigh_node) neigh_node = batadv_iv_ogm_neigh_new(if_incoming, orig_neigh_node->orig, - orig_neigh_node, orig_neigh_node); if (!neigh_node) @@ -1302,6 +1310,32 @@ batadv_iv_ogm_update_seqnos(const struct ethhdr *ethhdr, return ret; } +/** + * batadv_orig_to_direct_router() - get direct next hop neighbor to an orig address + * @bat_priv: the bat priv with all the mesh interface information + * @orig_addr: the originator MAC address to search the best next hop router for + * @if_outgoing: the interface where the OGM should be sent to + * + * Return: A neighbor node which is the best router towards the given originator + * address. Bonding candidates are ignored. + */ +static struct batadv_neigh_node * +batadv_orig_to_direct_router(struct batadv_priv *bat_priv, u8 *orig_addr, + struct batadv_hard_iface *if_outgoing) +{ + struct batadv_neigh_node *neigh_node; + struct batadv_orig_node *orig_node; + + orig_node = batadv_orig_hash_find(bat_priv, orig_addr); + if (!orig_node) + return NULL; + + neigh_node = batadv_orig_router_get(orig_node, if_outgoing); + batadv_orig_node_put(orig_node); + + return neigh_node; +} + /** * batadv_iv_ogm_process_per_outif() - process a batman iv OGM for an outgoing * interface @@ -1372,8 +1406,9 @@ batadv_iv_ogm_process_per_outif(const struct sk_buff *skb, int ogm_offset, router = batadv_orig_router_get(orig_node, if_outgoing); if (router) { - router_router = batadv_orig_router_get(router->orig_node, - if_outgoing); + router_router = batadv_orig_to_direct_router(bat_priv, + router->addr, + if_outgoing); router_ifinfo = batadv_neigh_ifinfo_get(router, if_outgoing); } diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 51fe028b90881e..cec11f1251d66a 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -318,8 +318,8 @@ batadv_bla_del_backbone_claims(struct batadv_bla_backbone_gw *backbone_gw) if (claim->backbone_gw != backbone_gw) continue; - batadv_claim_put(claim); hlist_del_rcu(&claim->hash_entry); + batadv_claim_put(claim); } spin_unlock_bh(list_lock); } @@ -723,6 +723,7 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, if (unlikely(hash_added != 0)) { /* only local changes happened. */ + batadv_backbone_gw_put(backbone_gw); kfree(claim); return; } @@ -1288,6 +1289,13 @@ static void batadv_bla_purge_claims(struct batadv_priv *bat_priv, rcu_read_lock(); hlist_for_each_entry_rcu(claim, head, hash_entry) { + /* only purge claims not currently in the process of being released. + * Such claims could otherwise have a NULL-ptr backbone_gw set because + * they already went through batadv_claim_release() + */ + if (!kref_get_unless_zero(&claim->refcount)) + continue; + backbone_gw = batadv_bla_claim_get_backbone_gw(claim); if (now) goto purge_now; @@ -1313,6 +1321,7 @@ static void batadv_bla_purge_claims(struct batadv_priv *bat_priv, claim->addr, claim->vid); skip: batadv_backbone_gw_put(backbone_gw); + batadv_claim_put(claim); } rcu_read_unlock(); } diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index 3a35aadd8b4191..a4d33ee0fda59e 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -249,6 +249,7 @@ void batadv_mesh_free(struct net_device *mesh_iface) atomic_set(&bat_priv->mesh_state, BATADV_MESH_DEACTIVATING); batadv_purge_outstanding_packets(bat_priv, NULL); + batadv_tp_stop_all(bat_priv); batadv_gw_node_free(bat_priv); diff --git a/net/batman-adv/tp_meter.c b/net/batman-adv/tp_meter.c index 2e42f6b348c83d..066c76113fc433 100644 --- a/net/batman-adv/tp_meter.c +++ b/net/batman-adv/tp_meter.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -365,23 +366,38 @@ static void batadv_tp_vars_put(struct batadv_tp_vars *tp_vars) } /** - * batadv_tp_sender_cleanup() - cleanup sender data and drop and timer - * @bat_priv: the bat priv with all the mesh interface information - * @tp_vars: the private data of the current TP meter session to cleanup + * batadv_tp_list_detach() - remove tp session from mesh session list once + * @tp_vars: the private data of the current TP meter session */ -static void batadv_tp_sender_cleanup(struct batadv_priv *bat_priv, - struct batadv_tp_vars *tp_vars) +static void batadv_tp_list_detach(struct batadv_tp_vars *tp_vars) { - cancel_delayed_work(&tp_vars->finish_work); + bool detached = false; spin_lock_bh(&tp_vars->bat_priv->tp_list_lock); - hlist_del_rcu(&tp_vars->list); + if (!hlist_unhashed(&tp_vars->list)) { + hlist_del_init_rcu(&tp_vars->list); + detached = true; + } spin_unlock_bh(&tp_vars->bat_priv->tp_list_lock); + if (!detached) + return; + + atomic_dec(&tp_vars->bat_priv->tp_num); + /* drop list reference */ batadv_tp_vars_put(tp_vars); +} - atomic_dec(&tp_vars->bat_priv->tp_num); +/** + * batadv_tp_sender_cleanup() - cleanup sender data and drop and timer + * @tp_vars: the private data of the current TP meter session to cleanup + */ +static void batadv_tp_sender_cleanup(struct batadv_tp_vars *tp_vars) +{ + cancel_delayed_work_sync(&tp_vars->finish_work); + + batadv_tp_list_detach(tp_vars); /* kill the timer and remove its reference */ timer_delete_sync(&tp_vars->timer); @@ -886,7 +902,8 @@ static int batadv_tp_send(void *arg) batadv_orig_node_put(orig_node); batadv_tp_sender_end(bat_priv, tp_vars); - batadv_tp_sender_cleanup(bat_priv, tp_vars); + batadv_tp_sender_cleanup(tp_vars); + complete(&tp_vars->finished); batadv_tp_vars_put(tp_vars); @@ -918,7 +935,8 @@ static void batadv_tp_start_kthread(struct batadv_tp_vars *tp_vars) batadv_tp_vars_put(tp_vars); /* cleanup of failed tp meter variables */ - batadv_tp_sender_cleanup(bat_priv, tp_vars); + batadv_tp_sender_cleanup(tp_vars); + complete(&tp_vars->finished); return; } @@ -947,6 +965,13 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, /* look for an already existing test towards this node */ spin_lock_bh(&bat_priv->tp_list_lock); + if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE) { + spin_unlock_bh(&bat_priv->tp_list_lock); + batadv_tp_batctl_error_notify(BATADV_TP_REASON_DST_UNREACHABLE, + dst, bat_priv, session_cookie); + return; + } + tp_vars = batadv_tp_list_find(bat_priv, dst); if (tp_vars) { spin_unlock_bh(&bat_priv->tp_list_lock); @@ -969,6 +994,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, tp_vars = kmalloc_obj(*tp_vars, GFP_ATOMIC); if (!tp_vars) { + atomic_dec(&bat_priv->tp_num); spin_unlock_bh(&bat_priv->tp_list_lock); batadv_dbg(BATADV_DBG_TP_METER, bat_priv, "Meter: %s cannot allocate list elements\n", @@ -1017,6 +1043,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, tp_vars->start_time = jiffies; init_waitqueue_head(&tp_vars->more_bytes); + init_completion(&tp_vars->finished); spin_lock_init(&tp_vars->unacked_lock); INIT_LIST_HEAD(&tp_vars->unacked_list); @@ -1119,14 +1146,7 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t) "Shutting down for inactivity (more than %dms) from %pM\n", BATADV_TP_RECV_TIMEOUT, tp_vars->other_end); - spin_lock_bh(&tp_vars->bat_priv->tp_list_lock); - hlist_del_rcu(&tp_vars->list); - spin_unlock_bh(&tp_vars->bat_priv->tp_list_lock); - - /* drop list reference */ - batadv_tp_vars_put(tp_vars); - - atomic_dec(&bat_priv->tp_num); + batadv_tp_list_detach(tp_vars); spin_lock_bh(&tp_vars->unacked_lock); list_for_each_entry_safe(un, safe, &tp_vars->unacked_list, list) { @@ -1329,9 +1349,12 @@ static struct batadv_tp_vars * batadv_tp_init_recv(struct batadv_priv *bat_priv, const struct batadv_icmp_tp_packet *icmp) { - struct batadv_tp_vars *tp_vars; + struct batadv_tp_vars *tp_vars = NULL; spin_lock_bh(&bat_priv->tp_list_lock); + if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE) + goto out_unlock; + tp_vars = batadv_tp_list_find_session(bat_priv, icmp->orig, icmp->session); if (tp_vars) @@ -1344,8 +1367,10 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv, } tp_vars = kmalloc_obj(*tp_vars, GFP_ATOMIC); - if (!tp_vars) + if (!tp_vars) { + atomic_dec(&bat_priv->tp_num); goto out_unlock; + } ether_addr_copy(tp_vars->other_end, icmp->orig); tp_vars->role = BATADV_TP_RECEIVER; @@ -1464,6 +1489,9 @@ void batadv_tp_meter_recv(struct batadv_priv *bat_priv, struct sk_buff *skb) { struct batadv_icmp_tp_packet *icmp; + if (atomic_read(&bat_priv->mesh_state) != BATADV_MESH_ACTIVE) + goto out; + icmp = (struct batadv_icmp_tp_packet *)skb->data; switch (icmp->subtype) { @@ -1478,9 +1506,57 @@ void batadv_tp_meter_recv(struct batadv_priv *bat_priv, struct sk_buff *skb) "Received unknown TP Metric packet type %u\n", icmp->subtype); } + +out: consume_skb(skb); } +/** + * batadv_tp_stop_all() - stop all currently running tp meter sessions + * @bat_priv: the bat priv with all the mesh interface information + */ +void batadv_tp_stop_all(struct batadv_priv *bat_priv) +{ + struct batadv_tp_vars *tp_vars[BATADV_TP_MAX_NUM]; + struct batadv_tp_vars *tp_var; + size_t count = 0; + size_t i; + + spin_lock_bh(&bat_priv->tp_list_lock); + hlist_for_each_entry(tp_var, &bat_priv->tp_list, list) { + if (WARN_ON_ONCE(count >= BATADV_TP_MAX_NUM)) + break; + + if (!kref_get_unless_zero(&tp_var->refcount)) + continue; + + tp_vars[count++] = tp_var; + } + spin_unlock_bh(&bat_priv->tp_list_lock); + + for (i = 0; i < count; i++) { + tp_var = tp_vars[i]; + + switch (tp_var->role) { + case BATADV_TP_SENDER: + batadv_tp_sender_shutdown(tp_var, + BATADV_TP_REASON_CANCEL); + wake_up(&tp_var->more_bytes); + wait_for_completion(&tp_var->finished); + break; + case BATADV_TP_RECEIVER: + batadv_tp_list_detach(tp_var); + if (timer_shutdown_sync(&tp_var->timer)) + batadv_tp_vars_put(tp_var); + break; + } + + batadv_tp_vars_put(tp_var); + } + + synchronize_net(); +} + /** * batadv_tp_meter_init() - initialize global tp_meter structures */ diff --git a/net/batman-adv/tp_meter.h b/net/batman-adv/tp_meter.h index f0046d366eac65..4e97cd10cd0259 100644 --- a/net/batman-adv/tp_meter.h +++ b/net/batman-adv/tp_meter.h @@ -17,6 +17,7 @@ void batadv_tp_start(struct batadv_priv *bat_priv, const u8 *dst, u32 test_length, u32 *cookie); void batadv_tp_stop(struct batadv_priv *bat_priv, const u8 *dst, u8 return_value); +void batadv_tp_stop_all(struct batadv_priv *bat_priv); void batadv_tp_meter_recv(struct batadv_priv *bat_priv, struct sk_buff *skb); #endif /* _NET_BATMAN_ADV_TP_METER_H_ */ diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index 8fc5fe0e9b0539..daa06f42115429 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1328,6 +1329,9 @@ struct batadv_tp_vars { /** @finish_work: work item for the finishing procedure */ struct delayed_work finish_work; + /** @finished: completion signaled when a sender thread exits */ + struct completion finished; + /** @test_length: test length in milliseconds */ u32 test_length; diff --git a/net/bluetooth/Kconfig b/net/bluetooth/Kconfig index 6b2b65a667008b..0f2a5fbcafc563 100644 --- a/net/bluetooth/Kconfig +++ b/net/bluetooth/Kconfig @@ -110,6 +110,13 @@ config BT_AOSPEXT This options enables support for the Android Open Source Project defined HCI vendor extensions. +config BT_BRCMEXT + bool "Enable Broadcom extensions" + depends on BT + help + This option enables support for the Broadcom defined HCI + vendor extensions. + config BT_DEBUGFS bool "Export Bluetooth internals in debugfs" depends on BT && DEBUG_FS diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile index a7eede7616d856..b4c9013a46cec2 100644 --- a/net/bluetooth/Makefile +++ b/net/bluetooth/Makefile @@ -24,5 +24,6 @@ bluetooth-$(CONFIG_BT_LE) += iso.o bluetooth-$(CONFIG_BT_LEDS) += leds.o bluetooth-$(CONFIG_BT_MSFTEXT) += msft.o bluetooth-$(CONFIG_BT_AOSPEXT) += aosp.o +bluetooth-$(CONFIG_BT_BRCMEXT) += brcm.o bluetooth-$(CONFIG_BT_DEBUGFS) += hci_debugfs.o bluetooth-$(CONFIG_BT_SELFTEST) += selftest.o diff --git a/net/bluetooth/brcm.c b/net/bluetooth/brcm.c new file mode 100644 index 00000000000000..9aa0a265ab3d6b --- /dev/null +++ b/net/bluetooth/brcm.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2026 The Asahi Linux Contributors + */ + +#include +#include + +#include "brcm.h" + +int brcm_set_high_priority(struct hci_dev *hdev, u16 handle, bool enable) +{ + struct sk_buff *skb; + u8 cmd[3]; + + if (!hdev->brcm_capable) + return 0; + + cmd[0] = handle; + cmd[1] = handle >> 8; + cmd[2] = !!enable; + + skb = hci_cmd_sync(hdev, 0xfc57, sizeof(cmd), cmd, HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + kfree_skb(skb); + return 0; +} diff --git a/net/bluetooth/brcm.h b/net/bluetooth/brcm.h new file mode 100644 index 00000000000000..fdaee63bd1d23c --- /dev/null +++ b/net/bluetooth/brcm.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 The Asahi Linux Contributors + */ + +#if IS_ENABLED(CONFIG_BT_BRCMEXT) + +int brcm_set_high_priority(struct hci_dev *hdev, u16 handle, bool enable); + +#else + +static inline int brcm_set_high_priority(struct hci_dev *hdev, u16 handle, bool enable) +{ + return 0; +} + +#endif diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 11d3ad8d255145..02f5e64dd61cb3 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -35,6 +35,7 @@ #include #include +#include "brcm.h" #include "smp.h" #include "eir.h" @@ -2130,6 +2131,9 @@ static int create_big_sync(struct hci_dev *hdev, void *data) u32 flags = 0; int err; + if (!hci_conn_valid(hdev, conn)) + return -ECANCELED; + if (qos->bcast.out.phys == BIT(1)) flags |= MGMT_ADV_FLAG_SEC_2M; @@ -2204,11 +2208,24 @@ static void create_big_complete(struct hci_dev *hdev, void *data, int err) bt_dev_dbg(hdev, "conn %p", conn); + if (err == -ECANCELED) + goto done; + + hci_dev_lock(hdev); + + if (!hci_conn_valid(hdev, conn)) + goto unlock; + if (err) { bt_dev_err(hdev, "Unable to create BIG: %d", err); hci_connect_cfm(conn, err); hci_conn_del(conn); } + +unlock: + hci_dev_unlock(hdev); +done: + hci_conn_put(conn); } struct hci_conn *hci_bind_bis(struct hci_dev *hdev, bdaddr_t *dst, __u8 sid, @@ -2336,10 +2353,11 @@ struct hci_conn *hci_connect_bis(struct hci_dev *hdev, bdaddr_t *dst, BT_BOUND, &data); /* Queue start periodic advertising and create BIG */ - err = hci_cmd_sync_queue(hdev, create_big_sync, conn, + err = hci_cmd_sync_queue(hdev, create_big_sync, hci_conn_get(conn), create_big_complete); if (err < 0) { hci_conn_drop(conn); + hci_conn_put(conn); return ERR_PTR(err); } @@ -3070,6 +3088,32 @@ int hci_conn_set_phy(struct hci_conn *conn, u32 phys) } } +int hci_conn_setsockopt(struct hci_conn *conn, struct sock *sk, int level, + int optname, sockptr_t optval, unsigned int optlen) { + int val; + bool old_high, new_high, changed; + + if (level != SOL_SOCKET) + return 0; + + if (optname != SO_PRIORITY) + return 0; + + if (optlen < sizeof(int)) + return -EINVAL; + + if (copy_from_sockptr(&val, optval, sizeof(val))) + return -EFAULT; + + old_high = sk->sk_priority >= TC_PRIO_INTERACTIVE; + new_high = val >= TC_PRIO_INTERACTIVE; + changed = old_high != new_high; + if (!changed) + return 0; + + return brcm_set_high_priority(conn->hdev, conn->handle, new_high); +} + static int abort_conn_sync(struct hci_dev *hdev, void *data) { struct hci_conn *conn = data; diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 3ebc5e6d45d98a..1b3b9131affaa3 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -3340,8 +3340,6 @@ static void hci_conn_request_evt(struct hci_dev *hdev, void *data, memcpy(conn->dev_class, ev->dev_class, 3); - hci_dev_unlock(hdev); - if (ev->link_type == ACL_LINK || (!(flags & HCI_PROTO_DEFER) && !lmp_esco_capable(hdev))) { struct hci_cp_accept_conn_req cp; @@ -3375,7 +3373,6 @@ static void hci_conn_request_evt(struct hci_dev *hdev, void *data, hci_connect_cfm(conn, 0); } - return; unlock: hci_dev_unlock(hdev); } @@ -5498,9 +5495,11 @@ static void hci_user_passkey_notify_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, ""); + hci_dev_lock(hdev); + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &ev->bdaddr); if (!conn) - return; + goto unlock; conn->passkey_notify = __le32_to_cpu(ev->passkey); conn->passkey_entered = 0; @@ -5509,6 +5508,9 @@ static void hci_user_passkey_notify_evt(struct hci_dev *hdev, void *data, mgmt_user_passkey_notify(hdev, &conn->dst, conn->type, conn->dst_type, conn->passkey_notify, conn->passkey_entered); + +unlock: + hci_dev_unlock(hdev); } static void hci_keypress_notify_evt(struct hci_dev *hdev, void *data, @@ -5519,14 +5521,16 @@ static void hci_keypress_notify_evt(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, ""); + hci_dev_lock(hdev); + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &ev->bdaddr); if (!conn) - return; + goto unlock; switch (ev->type) { case HCI_KEYPRESS_STARTED: conn->passkey_entered = 0; - return; + goto unlock; case HCI_KEYPRESS_ENTERED: conn->passkey_entered++; @@ -5541,13 +5545,16 @@ static void hci_keypress_notify_evt(struct hci_dev *hdev, void *data, break; case HCI_KEYPRESS_COMPLETED: - return; + goto unlock; } if (hci_dev_test_flag(hdev, HCI_MGMT)) mgmt_user_passkey_notify(hdev, &conn->dst, conn->type, conn->dst_type, conn->passkey_notify, conn->passkey_entered); + +unlock: + hci_dev_unlock(hdev); } static void hci_simple_pair_complete_evt(struct hci_dev *hdev, void *data, @@ -7111,9 +7118,29 @@ static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data, continue; } + if (ev->num_bis <= i) { + bt_dev_err(hdev, + "Not enough BIS handles for BIG 0x%2.2x", + ev->handle); + ev->status = HCI_ERROR_UNSPECIFIED; + hci_connect_cfm(conn, ev->status); + hci_conn_del(conn); + continue; + } + if (hci_conn_set_handle(conn, - __le16_to_cpu(ev->bis_handle[i++]))) + __le16_to_cpu(ev->bis_handle[i++]))) { + bt_dev_err(hdev, + "Failed to set BIS handle for BIG 0x%2.2x", + ev->handle); + /* Force error so BIG gets terminated as not all BIS + * could be connected. + */ + ev->status = HCI_ERROR_UNSPECIFIED; + hci_connect_cfm(conn, ev->status); + hci_conn_del(conn); continue; + } conn->state = BT_CONNECTED; set_bit(HCI_CONN_BIG_CREATED, &conn->flags); @@ -7122,7 +7149,10 @@ static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data, hci_iso_setup_path(conn); } - if (!ev->status && !i) + /* If there is an unexpected error or if no BISes have been connected + * for the BIG, terminate it. + */ + if (ev->status == HCI_ERROR_UNSPECIFIED || (!ev->status && !i)) /* If no BISes have been connected for the BIG, * terminate. This is in case all bound connections * have been closed before the BIG creation diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 95c65fece39bdf..0d8053a3fc0a65 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -5473,7 +5473,13 @@ static inline int l2cap_ecred_reconf_rsp(struct l2cap_conn *conn, if (chan->ident != cmd->ident) continue; + l2cap_chan_hold(chan); + l2cap_chan_lock(chan); + l2cap_chan_del(chan, ECONNRESET); + + l2cap_chan_unlock(chan); + l2cap_chan_put(chan); } return 0; @@ -6733,7 +6739,7 @@ static int l2cap_ecred_data_rcv(struct l2cap_chan *chan, struct sk_buff *skb) if (sdu_len > chan->imtu) { BT_ERR("Too big LE L2CAP SDU length: len %u > %u", - skb->len, sdu_len); + sdu_len, chan->imtu); l2cap_send_disconn_req(chan, ECONNRESET); err = -EMSGSIZE; goto failed; diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 71e8c1b45bcee1..ff3bcac4780b1f 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -891,6 +891,16 @@ static int l2cap_sock_setsockopt(struct socket *sock, int level, int optname, BT_DBG("sk %p", sk); + if (level == SOL_SOCKET) { + conn = chan->conn; + if (conn) + err = hci_conn_setsockopt(conn->hcon, sock->sk, level, + optname, optval, optlen); + if (err) + return err; + return sock_setsockopt(sock, level, optname, optval, optlen); + } + if (level == SOL_L2CAP) return l2cap_sock_setsockopt_old(sock, optname, optval, optlen); @@ -1498,6 +1508,9 @@ static struct l2cap_chan *l2cap_sock_new_connection_cb(struct l2cap_chan *chan) { struct sock *sk, *parent = chan->data; + if (!parent) + return NULL; + lock_sock(parent); /* Check for backlog size */ @@ -1657,6 +1670,9 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state, { struct sock *sk = chan->data; + if (!sk) + return; + sk->sk_state = state; if (err) @@ -1758,6 +1774,9 @@ static long l2cap_sock_get_sndtimeo_cb(struct l2cap_chan *chan) { struct sock *sk = chan->data; + if (!sk) + return 0; + return READ_ONCE(sk->sk_sndtimeo); } @@ -1931,6 +1950,9 @@ static struct sock *l2cap_sock_alloc(struct net *net, struct socket *sock, INIT_LIST_HEAD(&l2cap_pi(sk)->rx_busy); + if (sock) + set_bit(SOCK_CUSTOM_SOCKOPT, &sock->flags); + chan = l2cap_chan_create(); if (!chan) { sk_free(sk); diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index b84587811ef4f8..18826d4b9c0bf8 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -1045,7 +1045,8 @@ static int sco_sock_setsockopt(struct socket *sock, int level, int optname, codecs = (void *)buffer; - if (codecs->num_codecs > 1) { + if (codecs->num_codecs != 1 || + optlen < struct_size(codecs, codecs, codecs->num_codecs)) { hci_dev_put(hdev); err = -EINVAL; break; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 178c4738e63bec..2988175a47f26c 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -1120,19 +1120,23 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, switch (skb->protocol) { case htons(ETH_P_IP): - sk->sk_family = AF_INET; - if (sizeof(struct iphdr) <= skb_headlen(skb)) { - sk->sk_rcv_saddr = ip_hdr(skb)->saddr; - sk->sk_daddr = ip_hdr(skb)->daddr; + if (skb_headlen(skb) < sizeof(struct iphdr)) { + ret = -EINVAL; + goto out; } + sk->sk_family = AF_INET; + sk->sk_rcv_saddr = ip_hdr(skb)->saddr; + sk->sk_daddr = ip_hdr(skb)->daddr; break; #if IS_ENABLED(CONFIG_IPV6) case htons(ETH_P_IPV6): - sk->sk_family = AF_INET6; - if (sizeof(struct ipv6hdr) <= skb_headlen(skb)) { - sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; - sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; + if (skb_headlen(skb) < sizeof(struct ipv6hdr)) { + ret = -EINVAL; + goto out; } + sk->sk_family = AF_INET6; + sk->sk_v6_rcv_saddr = ipv6_hdr(skb)->saddr; + sk->sk_v6_daddr = ipv6_hdr(skb)->daddr; break; #endif default: @@ -1156,6 +1160,21 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, skb->ip_summed = CHECKSUM_COMPLETE; } + if (prog->type == BPF_PROG_TYPE_LWT_XMIT) { + if (!ipv6_bpf_stub) { + pr_warn_once("Please test this program with the IPv6 module loaded\n"); + ret = -EOPNOTSUPP; + goto out; + } +#if IS_ENABLED(CONFIG_IPV6) + /* For CONFIG_IPV6=n, ipv6_bpf_stub is NULL which is + * handled by the above if statement. + */ + dst_hold(&net->ipv6.ip6_null_entry->dst); + skb_dst_set(skb, &net->ipv6.ip6_null_entry->dst); +#endif + } + ret = bpf_test_run(prog, skb, repeat, &retval, &duration, false); if (ret) goto out; diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 6b5595868a39c0..7ace0f4941bb61 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -202,11 +202,12 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, f = br_fdb_find_rcu(br, n->ha, vid); if (f) { + const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; if ((p && (p->flags & BR_PROXYARP)) || - (f->dst && (f->dst->flags & BR_PROXYARP_WIFI)) || - br_is_neigh_suppress_enabled(f->dst, vid)) { + (dst && (dst->flags & BR_PROXYARP_WIFI)) || + br_is_neigh_suppress_enabled(dst, vid)) { if (!vid) br_arp_send(br, p, skb->dev, sip, tip, sha, n->ha, sha, 0, 0); @@ -470,9 +471,10 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, f = br_fdb_find_rcu(br, n->ha, vid); if (f) { + const struct net_bridge_port *dst = READ_ONCE(f->dst); bool replied = false; - if (br_is_neigh_suppress_enabled(f->dst, vid)) { + if (br_is_neigh_suppress_enabled(dst, vid)) { if (vid != 0) br_nd_send(br, p, skb, n, skb->vlan_proto, diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c index e2c17f620f009a..6eb3ab69a5140f 100644 --- a/net/bridge/br_fdb.c +++ b/net/bridge/br_fdb.c @@ -236,6 +236,7 @@ struct net_device *br_fdb_find_port(const struct net_device *br_dev, const unsigned char *addr, __u16 vid) { + const struct net_bridge_port *dst; struct net_bridge_fdb_entry *f; struct net_device *dev = NULL; struct net_bridge *br; @@ -248,8 +249,11 @@ struct net_device *br_fdb_find_port(const struct net_device *br_dev, br = netdev_priv(br_dev); rcu_read_lock(); f = br_fdb_find_rcu(br, addr, vid); - if (f && f->dst) - dev = f->dst->dev; + if (f) { + dst = READ_ONCE(f->dst); + if (dst) + dev = dst->dev; + } rcu_read_unlock(); return dev; @@ -346,7 +350,7 @@ static void fdb_delete_local(struct net_bridge *br, vg = nbp_vlan_group(op); if (op != p && ether_addr_equal(op->dev->dev_addr, addr) && (!vid || br_vlan_find(vg, vid))) { - f->dst = op; + WRITE_ONCE(f->dst, op); clear_bit(BR_FDB_ADDED_BY_USER, &f->flags); return; } @@ -357,7 +361,7 @@ static void fdb_delete_local(struct net_bridge *br, /* Maybe bridge device has same hw addr? */ if (p && ether_addr_equal(br->dev->dev_addr, addr) && (!vid || (v && br_vlan_should_use(v)))) { - f->dst = NULL; + WRITE_ONCE(f->dst, NULL); clear_bit(BR_FDB_ADDED_BY_USER, &f->flags); return; } @@ -928,6 +932,7 @@ int br_fdb_test_addr(struct net_device *dev, unsigned char *addr) int br_fdb_fillbuf(struct net_bridge *br, void *buf, unsigned long maxnum, unsigned long skip) { + const struct net_bridge_port *dst; struct net_bridge_fdb_entry *f; struct __fdb_entry *fe = buf; unsigned long delta; @@ -944,7 +949,8 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf, continue; /* ignore pseudo entry for local MAC address */ - if (!f->dst) + dst = READ_ONCE(f->dst); + if (!dst) continue; if (skip) { @@ -956,8 +962,8 @@ int br_fdb_fillbuf(struct net_bridge *br, void *buf, memcpy(fe->mac_addr, f->key.addr.addr, ETH_ALEN); /* due to ABI compat need to split into hi/lo */ - fe->port_no = f->dst->port_no; - fe->port_hi = f->dst->port_no >> 8; + fe->port_no = dst->port_no; + fe->port_hi = dst->port_no >> 8; fe->is_local = test_bit(BR_FDB_LOCAL, &f->flags); if (!test_bit(BR_FDB_STATIC, &f->flags)) { @@ -1083,9 +1089,11 @@ int br_fdb_dump(struct sk_buff *skb, rcu_read_lock(); hlist_for_each_entry_rcu(f, &br->fdb_list, fdb_node) { + const struct net_bridge_port *dst = READ_ONCE(f->dst); + if (*idx < ctx->fdb_idx) goto skip; - if (filter_dev && (!f->dst || f->dst->dev != filter_dev)) { + if (filter_dev && (!dst || dst->dev != filter_dev)) { if (filter_dev != dev) goto skip; /* !f->dst is a special case for bridge @@ -1093,10 +1101,10 @@ int br_fdb_dump(struct sk_buff *skb, * Therefore need a little more filtering * we only want to dump the !f->dst case */ - if (f->dst) + if (dst) goto skip; } - if (!filter_dev && f->dst) + if (!filter_dev && dst) goto skip; err = fdb_fill_info(skb, br, f, diff --git a/net/caif/cfsrvl.c b/net/caif/cfsrvl.c index 171fa32ada85c0..d687fd0b4ed3a8 100644 --- a/net/caif/cfsrvl.c +++ b/net/caif/cfsrvl.c @@ -191,10 +191,20 @@ bool cfsrvl_phyid_match(struct cflayer *layer, int phyid) void caif_free_client(struct cflayer *adap_layer) { + struct cflayer *serv_layer; struct cfsrvl *servl; - if (adap_layer == NULL || adap_layer->dn == NULL) + + if (!adap_layer) + return; + + serv_layer = adap_layer->dn; + if (!serv_layer) return; - servl = container_obj(adap_layer->dn); + + layer_set_dn(adap_layer, NULL); + layer_set_up(serv_layer, NULL); + + servl = container_obj(serv_layer); servl->release(&servl->layer); } EXPORT_SYMBOL(caif_free_client); diff --git a/net/can/raw.c b/net/can/raw.c index eee244ffc31ecc..58a96e933debb5 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -361,6 +361,14 @@ static int raw_notifier(struct notifier_block *nb, unsigned long msg, return NOTIFY_DONE; } +static void raw_sock_destruct(struct sock *sk) +{ + struct raw_sock *ro = raw_sk(sk); + + free_percpu(ro->uniq); + can_sock_destruct(sk); +} + static int raw_init(struct sock *sk) { struct raw_sock *ro = raw_sk(sk); @@ -387,6 +395,8 @@ static int raw_init(struct sock *sk) if (unlikely(!ro->uniq)) return -ENOMEM; + sk->sk_destruct = raw_sock_destruct; + /* set notifier */ spin_lock(&raw_notifier_lock); list_add_tail(&ro->notifier, &raw_notifier_list); @@ -436,7 +446,6 @@ static int raw_release(struct socket *sock) ro->bound = 0; ro->dev = NULL; ro->count = 0; - free_percpu(ro->uniq); sock_orphan(sk); sock->sk = NULL; diff --git a/net/ceph/auth.c b/net/ceph/auth.c index 901b93530b2144..17660bde896be8 100644 --- a/net/ceph/auth.c +++ b/net/ceph/auth.c @@ -245,7 +245,7 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, ac->protocol = 0; ac->ops = NULL; } - if (ac->protocol != protocol) { + if (!ac->protocol) { ret = init_protocol(ac, protocol); if (ret) { pr_err("auth protocol '%s' init failed: %d\n", @@ -257,7 +257,7 @@ int ceph_handle_auth_reply(struct ceph_auth_client *ac, ac->negotiating = false; } - if (result) { + if (result < 0) { pr_err("auth protocol '%s' mauth authentication failed: %d\n", ceph_auth_proto_name(ac->protocol), result); ret = result; diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index 692e0b86882238..9e64e82d0b63bf 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -115,6 +115,11 @@ static int __ceph_x_decrypt(const struct ceph_crypto_key *key, int usage_slot, if (ret) return ret; + if (plaintext_len < sizeof(*hdr)) { + pr_err("%s plaintext too small %d\n", __func__, plaintext_len); + return -EINVAL; + } + hdr = p + ceph_crypt_data_offset(key); if (le64_to_cpu(hdr->magic) != CEPHX_ENC_MAGIC) { pr_err("%s bad magic\n", __func__); diff --git a/net/ceph/crush/crush.c b/net/ceph/crush/crush.c index 254ded0b05f6a1..521aec1d5fc060 100644 --- a/net/ceph/crush/crush.c +++ b/net/ceph/crush/crush.c @@ -47,7 +47,6 @@ int crush_get_bucket_item_weight(const struct crush_bucket *b, int p) void crush_destroy_bucket_uniform(struct crush_bucket_uniform *b) { kfree(b->h.items); - kfree(b); } void crush_destroy_bucket_list(struct crush_bucket_list *b) @@ -55,14 +54,12 @@ void crush_destroy_bucket_list(struct crush_bucket_list *b) kfree(b->item_weights); kfree(b->sum_weights); kfree(b->h.items); - kfree(b); } void crush_destroy_bucket_tree(struct crush_bucket_tree *b) { kfree(b->h.items); kfree(b->node_weights); - kfree(b); } void crush_destroy_bucket_straw(struct crush_bucket_straw *b) @@ -70,14 +67,12 @@ void crush_destroy_bucket_straw(struct crush_bucket_straw *b) kfree(b->straws); kfree(b->item_weights); kfree(b->h.items); - kfree(b); } void crush_destroy_bucket_straw2(struct crush_bucket_straw2 *b) { kfree(b->item_weights); kfree(b->h.items); - kfree(b); } void crush_destroy_bucket(struct crush_bucket *b) @@ -99,6 +94,7 @@ void crush_destroy_bucket(struct crush_bucket *b) crush_destroy_bucket_straw2((struct crush_bucket_straw2 *)b); break; } + kfree(b); } /** diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index d5080530ce0ccb..d2cdc8ee31551e 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -174,6 +174,8 @@ int ceph_monmap_contains(struct ceph_monmap *m, struct ceph_entity_addr *addr) */ static void __send_prepared_auth_request(struct ceph_mon_client *monc, int len) { + BUG_ON(len > monc->m_auth->front_alloc_len); + monc->pending_auth = 1; monc->m_auth->front.iov_len = len; monc->m_auth->hdr.front_len = cpu_to_le32(len); diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index c89e66d4fcb7fe..12658142e5256d 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -389,11 +389,15 @@ static int decode_choose_args(void **p, void *end, struct crush_map *c) goto fail; if (arg->ids_size && - arg->ids_size != c->buckets[bucket_index]->size) + (!c->buckets[bucket_index] || + arg->ids_size != c->buckets[bucket_index]->size)) goto e_inval; } - insert_choose_arg_map(&c->choose_args, arg_map); + if (!__insert_choose_arg_map(&c->choose_args, arg_map)) { + ret = -EEXIST; + goto fail; + } } return 0; @@ -516,6 +520,10 @@ static struct crush_map *crush_decode(void *pbyval, void *end) b->id = ceph_decode_32(p); b->type = ceph_decode_16(p); b->alg = ceph_decode_8(p); + if (b->alg != alg) { + b->alg = 0; + goto bad; + } b->hash = ceph_decode_8(p); b->weight = ceph_decode_32(p); b->size = ceph_decode_32(p); @@ -1702,7 +1710,7 @@ static int osdmap_decode(void **p, void *end, bool msgr2, ceph_decode_need(p, end, 3*sizeof(u32) + map->max_osd*(struct_v >= 5 ? sizeof(u32) : sizeof(u8)) + - sizeof(*map->osd_weight), e_inval); + map->max_osd*sizeof(*map->osd_weight), e_inval); if (ceph_decode_32(p) != map->max_osd) goto e_inval; diff --git a/net/core/dev.c b/net/core/dev.c index 831129f2a69b50..e4fcf09ba2beb7 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4103,15 +4103,16 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d } EXPORT_SYMBOL_GPL(validate_xmit_skb_list); -static void qdisc_pkt_len_segs_init(struct sk_buff *skb) +static enum skb_drop_reason qdisc_pkt_len_segs_init(struct sk_buff *skb) { struct skb_shared_info *shinfo = skb_shinfo(skb); + unsigned int hdr_len, tlen; u16 gso_segs; qdisc_skb_cb(skb)->pkt_len = skb->len; if (!shinfo->gso_size) { qdisc_skb_cb(skb)->pkt_segs = 1; - return; + return SKB_NOT_DROPPED_YET; } qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs; @@ -4119,44 +4120,49 @@ static void qdisc_pkt_len_segs_init(struct sk_buff *skb) /* To get more precise estimation of bytes sent on wire, * we add to pkt_len the headers size of all segments */ - if (skb_transport_header_was_set(skb)) { - unsigned int hdr_len; - - /* mac layer + network layer */ - if (!skb->encapsulation) - hdr_len = skb_transport_offset(skb); - else - hdr_len = skb_inner_transport_offset(skb); - /* + transport layer */ - if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { - const struct tcphdr *th; - struct tcphdr _tcphdr; + /* mac layer + network layer */ + if (!skb->encapsulation) { + if (unlikely(!skb_transport_header_was_set(skb))) + return SKB_NOT_DROPPED_YET; + hdr_len = skb_transport_offset(skb); + } else { + hdr_len = skb_inner_transport_offset(skb); + } + /* + transport layer */ + if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { + const struct tcphdr *th; - th = skb_header_pointer(skb, hdr_len, - sizeof(_tcphdr), &_tcphdr); - if (likely(th)) - hdr_len += __tcp_hdrlen(th); - } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { - struct udphdr _udphdr; + if (!pskb_may_pull(skb, hdr_len + sizeof(struct tcphdr))) + return SKB_DROP_REASON_SKB_BAD_GSO; - if (skb_header_pointer(skb, hdr_len, - sizeof(_udphdr), &_udphdr)) - hdr_len += sizeof(struct udphdr); - } + th = (const struct tcphdr *)(skb->data + hdr_len); + tlen = __tcp_hdrlen(th); + if (tlen < sizeof(*th)) + return SKB_DROP_REASON_SKB_BAD_GSO; + hdr_len += tlen; + if (!pskb_may_pull(skb, hdr_len)) + return SKB_DROP_REASON_SKB_BAD_GSO; + } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { + if (!pskb_may_pull(skb, hdr_len + sizeof(struct udphdr))) + return SKB_DROP_REASON_SKB_BAD_GSO; + hdr_len += sizeof(struct udphdr); + } - if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { - int payload = skb->len - hdr_len; + /* prior pskb_may_pull() might have changed skb->head. */ + shinfo = skb_shinfo(skb); + if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { + int payload = skb->len - hdr_len; - /* Malicious packet. */ - if (payload <= 0) - return; - gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); - shinfo->gso_segs = gso_segs; - qdisc_skb_cb(skb)->pkt_segs = gso_segs; - } - qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; + /* Malicious packet. */ + if (payload <= 0) + return SKB_DROP_REASON_SKB_BAD_GSO; + gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); + shinfo->gso_segs = gso_segs; + qdisc_skb_cb(skb)->pkt_segs = gso_segs; } + qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; + return SKB_NOT_DROPPED_YET; } static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q, @@ -4761,9 +4767,10 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) { struct net_device *dev = skb->dev; struct netdev_queue *txq = NULL; - struct Qdisc *q; - int rc = -ENOMEM; + enum skb_drop_reason reason; + int cpu, rc = -ENOMEM; bool again = false; + struct Qdisc *q; skb_reset_mac_header(skb); skb_assert_len(skb); @@ -4772,6 +4779,12 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) (SKBTX_SCHED_TSTAMP | SKBTX_BPF))) __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); + reason = qdisc_pkt_len_segs_init(skb); + if (unlikely(reason)) { + dev_core_stats_tx_dropped_inc(dev); + kfree_skb_reason(skb, reason); + return -EINVAL; + } /* Disable soft irqs for various locks below. Also * stops preemption for RCU. */ @@ -4779,7 +4792,6 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) skb_update_prio(skb); - qdisc_pkt_len_segs_init(skb); tcx_set_ingress(skb, false); #ifdef CONFIG_NET_EGRESS if (static_branch_unlikely(&egress_needed_key)) { @@ -4832,59 +4844,61 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) * Check this and shot the lock. It is not prone from deadlocks. *Either shot noqueue qdisc, it is even simpler 8) */ - if (dev->flags & IFF_UP) { - int cpu = smp_processor_id(); /* ok because BHs are off */ + if (unlikely(!(dev->flags & IFF_UP))) { + reason = SKB_DROP_REASON_DEV_READY; + goto drop; + } - if (!netif_tx_owned(txq, cpu)) { - bool is_list = false; + cpu = smp_processor_id(); /* ok because BHs are off */ - if (dev_xmit_recursion()) - goto recursion_alert; + if (likely(!netif_tx_owned(txq, cpu))) { + bool is_list = false; - skb = validate_xmit_skb(skb, dev, &again); - if (!skb) - goto out; + if (dev_xmit_recursion()) + goto recursion_alert; - HARD_TX_LOCK(dev, txq, cpu); + skb = validate_xmit_skb(skb, dev, &again); + if (!skb) + goto out; - if (!netif_xmit_stopped(txq)) { - is_list = !!skb->next; + HARD_TX_LOCK(dev, txq, cpu); - dev_xmit_recursion_inc(); - skb = dev_hard_start_xmit(skb, dev, txq, &rc); - dev_xmit_recursion_dec(); + if (!netif_xmit_stopped(txq)) { + is_list = !!skb->next; - /* GSO segments a single SKB into - * a list of frames. TCP expects error - * to mean none of the data was sent. - */ - if (is_list) - rc = NETDEV_TX_OK; - } - HARD_TX_UNLOCK(dev, txq); - if (!skb) /* xmit completed */ - goto out; + dev_xmit_recursion_inc(); + skb = dev_hard_start_xmit(skb, dev, txq, &rc); + dev_xmit_recursion_dec(); - net_crit_ratelimited("Virtual device %s asks to queue packet!\n", - dev->name); - /* NETDEV_TX_BUSY or queue was stopped */ - if (!is_list) - rc = -ENETDOWN; - } else { - /* Recursion is detected! It is possible, - * unfortunately + /* GSO segments a single SKB into a list of frames. + * TCP expects error to mean none of the data was sent. */ -recursion_alert: - net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", - dev->name); - rc = -ENETDOWN; + if (is_list) + rc = NETDEV_TX_OK; } + HARD_TX_UNLOCK(dev, txq); + if (!skb) /* xmit completed */ + goto out; + + net_crit_ratelimited("Virtual device %s asks to queue packet!\n", + dev->name); + /* NETDEV_TX_BUSY or queue was stopped */ + if (!is_list) + rc = -ENETDOWN; + } else { + /* Recursion is detected! It is possible unfortunately. */ +recursion_alert: + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + rc = -ENETDOWN; } + reason = SKB_DROP_REASON_RECURSION_LIMIT; +drop: rcu_read_unlock_bh(); dev_core_stats_tx_dropped_inc(dev); - kfree_skb_list(skb); + kfree_skb_list_reason(skb, reason); return rc; out: rcu_read_unlock_bh(); diff --git a/net/core/filter.c b/net/core/filter.c index 78b548158fb059..d8a853a61b53fc 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -508,7 +508,7 @@ static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp) ((unaligned_ok && offset >= 0) || (!unaligned_ok && offset >= 0 && offset + ip_align >= 0 && - offset + ip_align % size == 0))) { + (offset + ip_align) % size == 0))) { bool ldx_off_ok = offset <= S16_MAX; *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H); @@ -4395,6 +4395,8 @@ u32 xdp_master_redirect(struct xdp_buff *xdp) struct net_device *master, *slave; master = netdev_master_upper_dev_get_rcu(xdp->rxq->dev); + if (unlikely(!(master->flags & IFF_UP))) + return XDP_ABORTED; slave = master->netdev_ops->ndo_xdp_get_xmit_slave(master, xdp); if (slave && slave != xdp->rxq->dev) { /* The target device is different from the receiving device, so @@ -5395,7 +5397,7 @@ static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname, if (val <= 0) return -EINVAL; tp->snd_cwnd_clamp = val; - tp->snd_ssthresh = val; + WRITE_ONCE(tp->snd_ssthresh, val); break; case TCP_BPF_DELACK_MAX: timeout = usecs_to_jiffies(val); @@ -10581,10 +10583,11 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, si->dst_reg, si->dst_reg, \ offsetof(OBJ, OBJ_FIELD)); \ if (si->dst_reg == si->src_reg) { \ - *insn++ = BPF_JMP_A(1); \ + *insn++ = BPF_JMP_A(2); \ *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \ offsetof(struct bpf_sock_ops_kern, \ temp)); \ + *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); \ } \ } while (0) @@ -10618,10 +10621,11 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, si->dst_reg, si->src_reg, \ offsetof(struct bpf_sock_ops_kern, sk));\ if (si->dst_reg == si->src_reg) { \ - *insn++ = BPF_JMP_A(1); \ + *insn++ = BPF_JMP_A(2); \ *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \ offsetof(struct bpf_sock_ops_kern, \ temp)); \ + *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); \ } \ } while (0) diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 1b61bb25ba0e5b..2a98f5fa74eb09 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1374,16 +1374,13 @@ bool __skb_flow_dissect(const struct net *net, break; } - /* least significant bit of the most significant octet - * indicates if protocol field was compressed + /* PFC (compressed 1-byte protocol) frames are not processed. + * A compressed protocol field has the least significant bit of + * the most significant octet set, which will fail the following + * ppp_proto_is_valid(), returning FLOW_DISSECT_RET_OUT_BAD. */ ppp_proto = ntohs(hdr->proto); - if (ppp_proto & 0x0100) { - ppp_proto = ppp_proto >> 8; - nhoff += PPPOE_SES_HLEN - 1; - } else { - nhoff += PPPOE_SES_HLEN; - } + nhoff += PPPOE_SES_HLEN; if (ppp_proto == PPP_IP) { proto = htons(ETH_P_IP); diff --git a/net/core/gro.c b/net/core/gro.c index 31d21de5b15a76..9f8960789b2cfb 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -213,10 +213,12 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) p->data_len += len; p->truesize += delta_truesize; p->len += len; + skb_shinfo(p)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG; if (lp != p) { lp->data_len += len; lp->truesize += delta_truesize; lp->len += len; + skb_shinfo(lp)->flags |= skbinfo->flags & SKBFL_SHARED_FRAG; } NAPI_GRO_CB(skb)->same_flow = 1; return 0; @@ -244,6 +246,8 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) p->truesize += skb->truesize; p->len += skb->len; + skb_shinfo(p)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; + NAPI_GRO_CB(skb)->same_flow = 1; return 0; diff --git a/net/core/neighbour.c b/net/core/neighbour.c index c56a4e7bf790cd..5a9cc7268521cf 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -3211,8 +3211,10 @@ int neigh_xmit(int index, struct net_device *dev, rcu_read_lock(); tbl = rcu_dereference(neigh_tables[index]); - if (!tbl) - goto out_unlock; + if (!tbl) { + rcu_read_unlock(); + goto out_kfree_skb; + } if (index == NEIGH_ARP_TABLE) { u32 key = *((u32 *)addr); @@ -3228,7 +3230,6 @@ int neigh_xmit(int index, struct net_device *dev, goto out_kfree_skb; } err = READ_ONCE(neigh->output)(neigh, skb); -out_unlock: rcu_read_unlock(); } else if (index == NEIGH_LINK_TABLE) { @@ -3238,11 +3239,10 @@ int neigh_xmit(int index, struct net_device *dev, goto out_kfree_skb; err = dev_queue_xmit(skb); } -out: return err; out_kfree_skb: kfree_skb(skb); - goto out; + return err; } EXPORT_SYMBOL(neigh_xmit); diff --git a/net/core/netpoll.c b/net/core/netpoll.c index cd74beffd209ca..84faace50ac281 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -608,14 +608,16 @@ EXPORT_SYMBOL_GPL(__netpoll_setup); /* * Returns a pointer to a string representation of the identifier used * to select the egress interface for the given netpoll instance. buf - * must be a buffer of length at least MAC_ADDR_STR_LEN + 1. + * is used to format np->dev_mac when np->dev_name is empty; bufsz must + * be at least MAC_ADDR_STR_LEN + 1 to fit the formatted MAC address + * and its NUL terminator. */ -static char *egress_dev(struct netpoll *np, char *buf) +static char *egress_dev(struct netpoll *np, char *buf, size_t bufsz) { if (np->dev_name[0]) return np->dev_name; - snprintf(buf, MAC_ADDR_STR_LEN, "%pM", np->dev_mac); + snprintf(buf, bufsz, "%pM", np->dev_mac); return buf; } @@ -645,7 +647,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev) if (!IS_ENABLED(CONFIG_IPV6)) { np_err(np, "IPv6 is not supported %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return -EINVAL; } @@ -667,7 +669,7 @@ static int netpoll_take_ipv6(struct netpoll *np, struct net_device *ndev) } if (err) { np_err(np, "no IPv6 address for %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return err; } @@ -687,14 +689,14 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev) in_dev = __in_dev_get_rtnl(ndev); if (!in_dev) { np_err(np, "no IP address for %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return -EDESTADDRREQ; } ifa = rtnl_dereference(in_dev->ifa_list); if (!ifa) { np_err(np, "no IP address for %s, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); return -EDESTADDRREQ; } @@ -704,6 +706,23 @@ static int netpoll_take_ipv4(struct netpoll *np, struct net_device *ndev) return 0; } +/* + * Test whether the caller left np->local_ip unset, so that + * netpoll_setup() should auto-populate it from the egress device. + * + * np->local_ip is a union of __be32 (IPv4) and struct in6_addr (IPv6), + * so an IPv6 address whose first 4 bytes are zero (e.g. ::1, ::2, + * IPv4-mapped ::ffff:a.b.c.d) must not be tested via the IPv4 arm — + * doing so would misclassify a caller-supplied address as unset and + * silently overwrite it with whatever address the device exposes. + */ +static bool netpoll_local_ip_unset(const struct netpoll *np) +{ + if (np->ipv6) + return ipv6_addr_any(&np->local_ip.in6); + return !np->local_ip.ip; +} + int netpoll_setup(struct netpoll *np) { struct net *net = current->nsproxy->net_ns; @@ -719,7 +738,8 @@ int netpoll_setup(struct netpoll *np) ndev = dev_getbyhwaddr(net, ARPHRD_ETHER, np->dev_mac); if (!ndev) { - np_err(np, "%s doesn't exist, aborting\n", egress_dev(np, buf)); + np_err(np, "%s doesn't exist, aborting\n", + egress_dev(np, buf, sizeof(buf))); err = -ENODEV; goto unlock; } @@ -727,14 +747,14 @@ int netpoll_setup(struct netpoll *np) if (netdev_master_upper_dev_get(ndev)) { np_err(np, "%s is a slave device, aborting\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); err = -EBUSY; goto put; } if (!netif_running(ndev)) { np_info(np, "device %s not up yet, forcing it\n", - egress_dev(np, buf)); + egress_dev(np, buf, sizeof(buf))); err = dev_open(ndev, NULL); if (err) { @@ -747,7 +767,7 @@ int netpoll_setup(struct netpoll *np) rtnl_lock(); } - if (!np->local_ip.ip) { + if (netpoll_local_ip_unset(np)) { if (!np->ipv6) { err = netpoll_take_ipv4(np, ndev); if (err) diff --git a/net/core/page_pool.c b/net/core/page_pool.c index 265a729431bb7d..8171d1173221b4 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -327,6 +327,11 @@ static void page_pool_uninit(struct page_pool *pool) if (!pool->system) free_percpu(pool->recycle_stats); #endif + + if (pool->mp_ops) { + pool->mp_ops->destroy(pool); + static_branch_dec(&page_pool_mem_providers); + } } /** @@ -1126,11 +1131,6 @@ static void __page_pool_destroy(struct page_pool *pool) page_pool_unlist(pool); page_pool_uninit(pool); - if (pool->mp_ops) { - pool->mp_ops->destroy(pool); - static_branch_dec(&page_pool_mem_providers); - } - kfree(pool); } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 69daba3ddaf06b..ad0c7b0b4a506e 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -1572,6 +1572,7 @@ static noinline_for_stack int rtnl_fill_vfinfo(struct sk_buff *skb, port_guid.vf = ivi.vf; memcpy(vf_mac.mac, ivi.mac, sizeof(ivi.mac)); + memset(&vf_broadcast, 0, sizeof(vf_broadcast)); memcpy(vf_broadcast.broadcast, dev->broadcast, dev->addr_len); vf_vlan.vlan = ivi.vlan; vf_vlan.qos = ivi.qos; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 43ee86dcf2eaf3..28bd8304796d7a 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -2258,6 +2258,7 @@ struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, skb_frag_ref(skb, i); } skb_shinfo(n)->nr_frags = i; + skb_shinfo(n)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; } if (skb_has_frag_list(skb)) { @@ -4373,6 +4374,8 @@ int skb_shift(struct sk_buff *tgt, struct sk_buff *skb, int shiftlen) tgt->ip_summed = CHECKSUM_PARTIAL; skb->ip_summed = CHECKSUM_PARTIAL; + skb_shinfo(tgt)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; + skb_len_add(skb, -shiftlen); skb_len_add(tgt, shiftlen); @@ -4983,7 +4986,8 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, skb_copy_from_linear_data_offset(head_skb, offset, skb_put(nskb, hsize), hsize); - skb_shinfo(nskb)->flags |= skb_shinfo(head_skb)->flags & + skb_shinfo(nskb)->flags |= (skb_shinfo(head_skb)->flags | + skb_shinfo(frag_skb)->flags) & SKBFL_SHARED_FRAG; if (skb_zerocopy_clone(nskb, frag_skb, GFP_ATOMIC)) @@ -5000,6 +5004,9 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, nfrags = skb_shinfo(list_skb)->nr_frags; frag = skb_shinfo(list_skb)->frags; frag_skb = list_skb; + + skb_shinfo(nskb)->flags |= skb_shinfo(frag_skb)->flags & SKBFL_SHARED_FRAG; + if (!skb_headlen(list_skb)) { BUG_ON(!nfrags); } else { @@ -5142,7 +5149,7 @@ static const u8 skb_ext_type_len[] = { #endif }; -static __always_inline unsigned int skb_ext_total_length(void) +static __always_inline __no_profile unsigned int skb_ext_total_length(void) { unsigned int l = SKB_EXT_CHUNKSIZEOF(struct skb_ext); int i; @@ -5156,9 +5163,7 @@ static __always_inline unsigned int skb_ext_total_length(void) static void skb_extensions_init(void) { BUILD_BUG_ON(SKB_EXT_NUM > 8); -#if !IS_ENABLED(CONFIG_KCOV_INSTRUMENT_ALL) BUILD_BUG_ON(skb_ext_total_length() > 255); -#endif skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache", SKB_EXT_ALIGN_VALUE * skb_ext_total_length(), @@ -6226,6 +6231,8 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, from_shinfo->frags, from_shinfo->nr_frags * sizeof(skb_frag_t)); to_shinfo->nr_frags += from_shinfo->nr_frags; + if (from_shinfo->nr_frags) + to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG; if (!skb_cloned(from)) from_shinfo->nr_frags = 0; diff --git a/net/dsa/conduit.c b/net/dsa/conduit.c index a1b044467bd6fa..8398d72d7e4d36 100644 --- a/net/dsa/conduit.c +++ b/net/dsa/conduit.c @@ -27,9 +27,7 @@ static int dsa_conduit_get_regs_len(struct net_device *dev) int len; if (ops && ops->get_regs_len) { - netdev_lock_ops(dev); len = ops->get_regs_len(dev); - netdev_unlock_ops(dev); if (len < 0) return len; ret += len; @@ -60,15 +58,11 @@ static void dsa_conduit_get_regs(struct net_device *dev, int len; if (ops && ops->get_regs_len && ops->get_regs) { - netdev_lock_ops(dev); len = ops->get_regs_len(dev); - if (len < 0) { - netdev_unlock_ops(dev); + if (len < 0) return; - } regs->len = len; ops->get_regs(dev, regs, data); - netdev_unlock_ops(dev); data += regs->len; } @@ -115,10 +109,8 @@ static void dsa_conduit_get_ethtool_stats(struct net_device *dev, int count, mcount = 0; if (ops && ops->get_sset_count && ops->get_ethtool_stats) { - netdev_lock_ops(dev); mcount = ops->get_sset_count(dev, ETH_SS_STATS); ops->get_ethtool_stats(dev, stats, data); - netdev_unlock_ops(dev); } list_for_each_entry(dp, &dst->ports, list) { @@ -149,10 +141,8 @@ static void dsa_conduit_get_ethtool_phy_stats(struct net_device *dev, if (count >= 0) phy_ethtool_get_stats(dev->phydev, stats, data); } else if (ops && ops->get_sset_count && ops->get_ethtool_phy_stats) { - netdev_lock_ops(dev); count = ops->get_sset_count(dev, ETH_SS_PHY_STATS); ops->get_ethtool_phy_stats(dev, stats, data); - netdev_unlock_ops(dev); } if (count < 0) @@ -176,13 +166,11 @@ static int dsa_conduit_get_sset_count(struct net_device *dev, int sset) struct dsa_switch_tree *dst = cpu_dp->dst; int count = 0; - netdev_lock_ops(dev); if (sset == ETH_SS_PHY_STATS && dev->phydev && (!ops || !ops->get_ethtool_phy_stats)) count = phy_ethtool_get_sset_count(dev->phydev); else if (ops && ops->get_sset_count) count = ops->get_sset_count(dev, sset); - netdev_unlock_ops(dev); if (count < 0) count = 0; @@ -239,7 +227,6 @@ static void dsa_conduit_get_strings(struct net_device *dev, u32 stringset, struct dsa_switch_tree *dst = cpu_dp->dst; int count, mcount = 0; - netdev_lock_ops(dev); if (stringset == ETH_SS_PHY_STATS && dev->phydev && !ops->get_ethtool_phy_stats) { mcount = phy_ethtool_get_sset_count(dev->phydev); @@ -253,7 +240,6 @@ static void dsa_conduit_get_strings(struct net_device *dev, u32 stringset, mcount = 0; ops->get_strings(dev, stringset, data); } - netdev_unlock_ops(dev); list_for_each_entry(dp, &dst->ports, list) { if (!dsa_port_is_dsa(dp) && !dsa_port_is_cpu(dp)) diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c index 5fb812443a08f2..4366cbac3f06c5 100644 --- a/net/ipv4/ah4.c +++ b/net/ipv4/ah4.c @@ -124,9 +124,14 @@ static void ah_output_done(void *data, int err) struct iphdr *top_iph = ip_hdr(skb); struct ip_auth_hdr *ah = ip_auth_hdr(skb); int ihl = ip_hdrlen(skb); + int seqhi_len = 0; + __be32 *seqhi; + if (x->props.flags & XFRM_STATE_ESN) + seqhi_len = sizeof(*seqhi); iph = AH_SKB_CB(skb)->tmp; - icv = ah_tmp_icv(iph, ihl); + seqhi = (__be32 *)((char *)iph + ihl); + icv = ah_tmp_icv(seqhi, seqhi_len); memcpy(ah->auth_data, icv, ahp->icv_trunc_len); top_iph->tos = iph->tos; @@ -270,12 +275,17 @@ static void ah_input_done(void *data, int err) struct ip_auth_hdr *ah = ip_auth_hdr(skb); int ihl = ip_hdrlen(skb); int ah_hlen = (ah->hdrlen + 2) << 2; + int seqhi_len = 0; + __be32 *seqhi; if (err) goto out; + if (x->props.flags & XFRM_STATE_ESN) + seqhi_len = sizeof(*seqhi); work_iph = AH_SKB_CB(skb)->tmp; - auth_data = ah_tmp_auth(work_iph, ihl); + seqhi = (__be32 *)((char *)work_iph + ihl); + auth_data = ah_tmp_auth(seqhi, seqhi_len); icv = ah_tmp_icv(auth_data, ahp->icv_trunc_len); err = crypto_memneq(icv, auth_data, ahp->icv_trunc_len) ? -EBADMSG : 0; diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 6dfc0bcdef6542..6a5febbdbee493 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -873,7 +873,8 @@ static int esp_input(struct xfrm_state *x, struct sk_buff *skb) nfrags = 1; goto skip_cow; - } else if (!skb_has_frag_list(skb)) { + } else if (!skb_has_frag_list(skb) && + !skb_has_shared_frag(skb)) { nfrags = skb_shinfo(skb)->nr_frags; nfrags++; diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index 4e2a6c70dcd840..ba4b1eec9ea2da 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -64,6 +64,7 @@ #include #include #include +#include #include #include #include @@ -373,7 +374,9 @@ static int icmp_glue_bits(void *from, char *to, int offset, int len, int odd, to, len); skb->csum = csum_block_add(skb->csum, csum, odd); - if (icmp_pointers[icmp_param->data.icmph.type].error) + if (icmp_param->data.icmph.type <= NR_ICMP_TYPES && + icmp_pointers[array_index_nospec(icmp_param->data.icmph.type, + NR_ICMP_TYPES + 1)].error) nf_ct_attach(skb, icmp_param->skb); return 0; } diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index e961936b6be76b..bc987a59a0952c 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1482,16 +1482,19 @@ void inet_csk_listen_stop(struct sock *sk) if (nreq) { refcount_set(&nreq->rsk_refcnt, 1); + rcu_read_lock(); if (inet_csk_reqsk_queue_add(nsk, nreq, child)) { __NET_INC_STATS(sock_net(nsk), LINUX_MIB_TCPMIGRATEREQSUCCESS); reqsk_migrate_reset(req); + READ_ONCE(nsk->sk_data_ready)(nsk); } else { __NET_INC_STATS(sock_net(nsk), LINUX_MIB_TCPMIGRATEREQFAILURE); reqsk_migrate_reset(nreq); __reqsk_free(nreq); } + rcu_read_unlock(); /* inet_csk_reqsk_queue_add() has already * called inet_child_forget() on failure case. diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index e4790cc7b5c2ec..5bcd73cbdb41c0 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -1233,6 +1233,8 @@ static int __ip_append_data(struct sock *sk, if (err < 0) goto error; copy = err; + if (!(flags & MSG_NO_SHARED_FRAGS)) + skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; wmem_alloc_delta += copy; } else if (!zc) { int i = skb_shinfo(skb)->nr_frags; diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index 5683c328990f49..f430d6f0463e7a 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -65,7 +65,7 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, DEV_STATS_INC(dev, tx_errors); } ip_rt_put(rt); - kfree_skb(skb); + kfree_skb_reason(skb, SKB_DROP_REASON_RECURSION_LIMIT); return; } diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c index 1cdd9c28ab2da4..97ead883e4a13b 100644 --- a/net/ipv4/netfilter/arp_tables.c +++ b/net/ipv4/netfilter/arp_tables.c @@ -110,13 +110,25 @@ static inline int arp_packet_match(const struct arphdr *arphdr, arpptr += dev->addr_len; memcpy(&src_ipaddr, arpptr, sizeof(u32)); arpptr += sizeof(u32); - tgt_devaddr = arpptr; - arpptr += dev->addr_len; + + if (IS_ENABLED(CONFIG_FIREWIRE_NET) && dev->type == ARPHRD_IEEE1394) { + if (unlikely(memchr_inv(arpinfo->tgt_devaddr.mask, 0, + sizeof(arpinfo->tgt_devaddr.mask)))) + return 0; + + tgt_devaddr = NULL; + } else { + tgt_devaddr = arpptr; + arpptr += dev->addr_len; + } memcpy(&tgt_ipaddr, arpptr, sizeof(u32)); if (NF_INVF(arpinfo, ARPT_INV_SRCDEVADDR, arp_devaddr_compare(&arpinfo->src_devaddr, src_devaddr, - dev->addr_len)) || + dev->addr_len))) + return 0; + + if (tgt_devaddr && NF_INVF(arpinfo, ARPT_INV_TGTDEVADDR, arp_devaddr_compare(&arpinfo->tgt_devaddr, tgt_devaddr, dev->addr_len))) diff --git a/net/ipv4/netfilter/arpt_mangle.c b/net/ipv4/netfilter/arpt_mangle.c index a4e07e5e9c118d..f65dd339208e8a 100644 --- a/net/ipv4/netfilter/arpt_mangle.c +++ b/net/ipv4/netfilter/arpt_mangle.c @@ -40,6 +40,10 @@ target(struct sk_buff *skb, const struct xt_action_param *par) } arpptr += pln; if (mangle->flags & ARPT_MANGLE_TDEV) { + if (unlikely(IS_ENABLED(CONFIG_FIREWIRE_NET) && + skb->dev->type == ARPHRD_IEEE1394)) + return NF_DROP; + if (ARPT_DEV_ADDR_LEN_MAX < hln || (arpptr + hln > skb_tail_pointer(skb))) return NF_DROP; @@ -47,6 +51,10 @@ target(struct sk_buff *skb, const struct xt_action_param *par) } arpptr += hln; if (mangle->flags & ARPT_MANGLE_TIP) { + if (unlikely(IS_ENABLED(CONFIG_FIREWIRE_NET) && + skb->dev->type == ARPHRD_IEEE1394)) + return NF_DROP; + if (ARPT_MANGLE_ADDR_LEN_MAX < pln || (arpptr + pln > skb_tail_pointer(skb))) return NF_DROP; diff --git a/net/ipv4/netfilter/iptable_nat.c b/net/ipv4/netfilter/iptable_nat.c index a5db7c67d61bef..625a1ca13b1bad 100644 --- a/net/ipv4/netfilter/iptable_nat.c +++ b/net/ipv4/netfilter/iptable_nat.c @@ -79,7 +79,7 @@ static int ipt_nat_register_lookups(struct net *net) while (i) nf_nat_ipv4_unregister_fn(net, &ops[--i]); - kfree(ops); + kfree_rcu(ops, rcu); return ret; } } @@ -100,7 +100,7 @@ static void ipt_nat_unregister_lookups(struct net *net) for (i = 0; i < ARRAY_SIZE(nf_nat_ipv4_ops); i++) nf_nat_ipv4_unregister_fn(net, &ops[i]); - kfree(ops); + kfree_rcu(ops, rcu); } static int iptable_nat_table_init(struct net *net) diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 2c9036c719b686..11a763cbc8482f 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -2466,10 +2466,10 @@ static int replace_nexthop_single(struct net *net, struct nexthop *old, goto err_notify; } - /* When replacing an IPv4 nexthop with an IPv6 nexthop, potentially + /* When replacing a nexthop with one of a different family, potentially * update IPv4 indication in all the groups using the nexthop. */ - if (oldi->family == AF_INET && newi->family == AF_INET6) { + if (oldi->family != newi->family) { list_for_each_entry(nhge, &old->grp_list, nh_list) { struct nexthop *nhp = nhge->nh_parent; struct nh_group *nhg; diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index fc3affd9c8014b..b5f0a65c678645 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -286,7 +286,6 @@ static int cookie_tcp_reqsk_init(struct sock *sk, struct sk_buff *skb, treq->rcv_isn = ntohl(th->seq) - 1; treq->snt_isn = ntohl(th->ack_seq) - 1; treq->syn_tos = TCP_SKB_CB(skb)->ip_dsfield; - treq->req_usec_ts = false; #if IS_ENABLED(CONFIG_MPTCP) treq->is_mptcp = sk_is_mptcp(sk); @@ -349,6 +348,7 @@ struct request_sock *cookie_tcp_reqsk_alloc(const struct request_sock_ops *ops, ireq->wscale_ok = tcp_opt->wscale_ok; ireq->ecn_ok = !!(tcp_opt->rcv_tsecr & TS_OPT_ECN); + treq->req_usec_ts = false; treq->ts_off = tsoff; return req; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 202a4e57a21885..cee51749df16c2 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3458,7 +3458,7 @@ int tcp_disconnect(struct sock *sk, int flags) icsk->icsk_rto = TCP_TIMEOUT_INIT; WRITE_ONCE(icsk->icsk_rto_min, TCP_RTO_MIN); WRITE_ONCE(icsk->icsk_delack_max, TCP_DELACK_MAX); - tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH); tcp_snd_cwnd_set(tp, TCP_INIT_CWND); tp->snd_cwnd_cnt = 0; tp->is_cwnd_limited = 0; @@ -3656,7 +3656,8 @@ static void tcp_enable_tx_delay(struct sock *sk, int val) if (delta && sk->sk_state == TCP_ESTABLISHED) { s64 srtt = (s64)tp->srtt_us + delta; - tp->srtt_us = clamp_t(s64, srtt, 1, ~0U); + WRITE_ONCE(tp->srtt_us, + clamp_t(s64, srtt, 1, ~0U)); /* Note: does not deal with non zero icsk_backoff */ tcp_set_rto(sk); @@ -4225,12 +4226,18 @@ static void tcp_get_info_chrono_stats(const struct tcp_sock *tp, struct tcp_info *info) { u64 stats[__TCP_CHRONO_MAX], total = 0; - enum tcp_chrono i; + enum tcp_chrono i, cur; + /* Following READ_ONCE()s pair with WRITE_ONCE()s in tcp_chrono_set(). + * This is because socket lock might not be owned by us at this point. + * This is best effort, tcp_get_timestamping_opt_stats() can + * see wrong values. A real fix would be too costly for TCP fast path. + */ + cur = READ_ONCE(tp->chrono_type); for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) { - stats[i] = tp->chrono_stat[i - 1]; - if (i == tp->chrono_type) - stats[i] += tcp_jiffies32 - tp->chrono_start; + stats[i] = READ_ONCE(tp->chrono_stat[i - 1]); + if (i == cur) + stats[i] += tcp_jiffies32 - READ_ONCE(tp->chrono_start); stats[i] *= USEC_PER_SEC / HZ; total += stats[i]; } @@ -4462,9 +4469,9 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED, info.tcpi_sndbuf_limited, TCP_NLA_PAD); nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT, - tp->data_segs_out, TCP_NLA_PAD); + READ_ONCE(tp->data_segs_out), TCP_NLA_PAD); nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS, - tp->total_retrans, TCP_NLA_PAD); + READ_ONCE(tp->total_retrans), TCP_NLA_PAD); rate = READ_ONCE(sk->sk_pacing_rate); rate64 = (rate != ~0UL) ? rate : ~0ULL; @@ -4473,37 +4480,42 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk, rate64 = tcp_compute_delivery_rate(tp); nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD); - nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp)); - nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering); - nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp)); + nla_put_u32(stats, TCP_NLA_SND_CWND, READ_ONCE(tp->snd_cwnd)); + nla_put_u32(stats, TCP_NLA_REORDERING, READ_ONCE(tp->reordering)); + nla_put_u32(stats, TCP_NLA_MIN_RTT, data_race(tcp_min_rtt(tp))); nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, READ_ONCE(inet_csk(sk)->icsk_retransmits)); - nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited); - nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh); - nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered); - nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce); - - nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una); + nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, data_race(!!tp->rate_app_limited)); + nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, READ_ONCE(tp->snd_ssthresh)); + nla_put_u32(stats, TCP_NLA_DELIVERED, READ_ONCE(tp->delivered)); + nla_put_u32(stats, TCP_NLA_DELIVERED_CE, READ_ONCE(tp->delivered_ce)); + + nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, + max_t(int, 0, + READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_una))); nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state); - nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent, - TCP_NLA_PAD); - nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans, + nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, READ_ONCE(tp->bytes_sent), TCP_NLA_PAD); - nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups); - nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen); - nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3); - nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash); + nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, + READ_ONCE(tp->bytes_retrans), TCP_NLA_PAD); + nla_put_u32(stats, TCP_NLA_DSACK_DUPS, READ_ONCE(tp->dsack_dups)); + nla_put_u32(stats, TCP_NLA_REORD_SEEN, READ_ONCE(tp->reord_seen)); + nla_put_u32(stats, TCP_NLA_SRTT, READ_ONCE(tp->srtt_us) >> 3); + nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, + READ_ONCE(tp->timeout_rehash)); nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT, - max_t(int, 0, tp->write_seq - tp->snd_nxt)); + max_t(int, 0, + READ_ONCE(tp->write_seq) - READ_ONCE(tp->snd_nxt))); nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns, TCP_NLA_PAD); if (ack_skb) nla_put_u8(stats, TCP_NLA_TTL, tcp_skb_ttl_or_hop_limit(ack_skb)); - nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash); + nla_put_u32(stats, TCP_NLA_REHASH, + READ_ONCE(tp->plb_rehash) + READ_ONCE(tp->timeout_rehash)); return stats; } diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c index 760941e55153e6..3df6160f515670 100644 --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c @@ -896,8 +896,8 @@ static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs) if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { bbr->mode = BBR_DRAIN; /* drain queue we created */ - tcp_sk(sk)->snd_ssthresh = - bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + WRITE_ONCE(tcp_sk(sk)->snd_ssthresh, + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)); } /* fall through to check if in-flight is already small: */ if (bbr->mode == BBR_DRAIN && bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <= @@ -1042,7 +1042,7 @@ __bpf_kfunc static void bbr_init(struct sock *sk) struct bbr *bbr = inet_csk_ca(sk); bbr->prior_cwnd = 0; - tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + WRITE_ONCE(tp->snd_ssthresh, TCP_INFINITE_SSTHRESH); bbr->rtt_cnt = 0; bbr->next_rtt_delivered = tp->delivered; bbr->prev_ca_state = TCP_CA_Open; diff --git a/net/ipv4/tcp_bic.c b/net/ipv4/tcp_bic.c index 58358bf92e1b8a..65444ff142413a 100644 --- a/net/ipv4/tcp_bic.c +++ b/net/ipv4/tcp_bic.c @@ -74,7 +74,7 @@ static void bictcp_init(struct sock *sk) bictcp_reset(ca); if (initial_ssthresh) - tcp_sk(sk)->snd_ssthresh = initial_ssthresh; + WRITE_ONCE(tcp_sk(sk)->snd_ssthresh, initial_ssthresh); } /* diff --git a/net/ipv4/tcp_cdg.c b/net/ipv4/tcp_cdg.c index ceabfd690a2967..0812c390aee564 100644 --- a/net/ipv4/tcp_cdg.c +++ b/net/ipv4/tcp_cdg.c @@ -162,7 +162,7 @@ static void tcp_cdg_hystart_update(struct sock *sk) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTTRAINCWND, tcp_snd_cwnd(tp)); - tp->snd_ssthresh = tcp_snd_cwnd(tp); + WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp)); return; } } @@ -181,7 +181,7 @@ static void tcp_cdg_hystart_update(struct sock *sk) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTDELAYCWND, tcp_snd_cwnd(tp)); - tp->snd_ssthresh = tcp_snd_cwnd(tp); + WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp)); } } } diff --git a/net/ipv4/tcp_cubic.c b/net/ipv4/tcp_cubic.c index 76c23675ae50ab..f891e8d1e54589 100644 --- a/net/ipv4/tcp_cubic.c +++ b/net/ipv4/tcp_cubic.c @@ -136,7 +136,7 @@ __bpf_kfunc static void cubictcp_init(struct sock *sk) bictcp_hystart_reset(sk); if (!hystart && initial_ssthresh) - tcp_sk(sk)->snd_ssthresh = initial_ssthresh; + WRITE_ONCE(tcp_sk(sk)->snd_ssthresh, initial_ssthresh); } __bpf_kfunc static void cubictcp_cwnd_event(struct sock *sk, enum tcp_ca_event event) @@ -423,7 +423,7 @@ static void hystart_update(struct sock *sk, u32 delay) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTTRAINCWND, tcp_snd_cwnd(tp)); - tp->snd_ssthresh = tcp_snd_cwnd(tp); + WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp)); } } } @@ -443,7 +443,7 @@ static void hystart_update(struct sock *sk, u32 delay) NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPHYSTARTDELAYCWND, tcp_snd_cwnd(tp)); - tp->snd_ssthresh = tcp_snd_cwnd(tp); + WRITE_ONCE(tp->snd_ssthresh, tcp_snd_cwnd(tp)); } } } diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c index 03abe0848420d7..6f103038b0152d 100644 --- a/net/ipv4/tcp_dctcp.c +++ b/net/ipv4/tcp_dctcp.c @@ -177,7 +177,7 @@ static void dctcp_react_to_loss(struct sock *sk) struct tcp_sock *tp = tcp_sk(sk); ca->loss_cwnd = tcp_snd_cwnd(tp); - tp->snd_ssthresh = max(tcp_snd_cwnd(tp) >> 1U, 2U); + WRITE_ONCE(tp->snd_ssthresh, max(tcp_snd_cwnd(tp) >> 1U, 2U)); } __bpf_kfunc static void dctcp_state(struct sock *sk, u8 new_state) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index cba89733d1216b..cb4bcc5a857877 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -476,14 +476,14 @@ static bool tcp_accecn_process_option(struct tcp_sock *tp, static void tcp_count_delivered_ce(struct tcp_sock *tp, u32 ecn_count) { - tp->delivered_ce += ecn_count; + WRITE_ONCE(tp->delivered_ce, tp->delivered_ce + ecn_count); } /* Updates the delivered and delivered_ce counts */ static void tcp_count_delivered(struct tcp_sock *tp, u32 delivered, bool ece_ack) { - tp->delivered += delivered; + WRITE_ONCE(tp->delivered, tp->delivered + delivered); if (tcp_ecn_mode_rfc3168(tp) && ece_ack) tcp_count_delivered_ce(tp, delivered); } @@ -1133,7 +1133,7 @@ static void tcp_rtt_estimator(struct sock *sk, long mrtt_us) tcp_bpf_rtt(sk, mrtt_us, srtt); } - tp->srtt_us = max(1U, srtt); + WRITE_ONCE(tp->srtt_us, max(1U, srtt)); } void tcp_update_pacing_rate(struct sock *sk) @@ -1247,7 +1247,7 @@ static u32 tcp_dsack_seen(struct tcp_sock *tp, u32 start_seq, else if (tp->tlp_high_seq && tp->tlp_high_seq == end_seq) state->flag |= FLAG_DSACK_TLP; - tp->dsack_dups += dup_segs; + WRITE_ONCE(tp->dsack_dups, tp->dsack_dups + dup_segs); /* Skip the DSACK if dup segs weren't retransmitted by sender */ if (tp->dsack_dups > tp->total_retrans) return 0; @@ -1294,12 +1294,13 @@ static void tcp_check_sack_reordering(struct sock *sk, const u32 low_seq, tp->sacked_out, tp->undo_marker ? tp->undo_retrans : 0); #endif - tp->reordering = min_t(u32, (metric + mss - 1) / mss, - READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)); + WRITE_ONCE(tp->reordering, + min_t(u32, (metric + mss - 1) / mss, + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering))); } /* This exciting event is worth to be remembered. 8) */ - tp->reord_seen++; + WRITE_ONCE(tp->reord_seen, tp->reord_seen + 1); NET_INC_STATS(sock_net(sk), ts ? LINUX_MIB_TCPTSREORDER : LINUX_MIB_TCPSACKREORDER); } @@ -2440,9 +2441,10 @@ static void tcp_check_reno_reordering(struct sock *sk, const int addend) if (!tcp_limit_reno_sacked(tp)) return; - tp->reordering = min_t(u32, tp->packets_out + addend, - READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering)); - tp->reord_seen++; + WRITE_ONCE(tp->reordering, + min_t(u32, tp->packets_out + addend, + READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_max_reordering))); + WRITE_ONCE(tp->reord_seen, tp->reord_seen + 1); NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPRENOREORDER); } @@ -2566,7 +2568,7 @@ void tcp_enter_loss(struct sock *sk) (icsk->icsk_ca_state == TCP_CA_Loss && !icsk->icsk_retransmits)) { tp->prior_ssthresh = tcp_current_ssthresh(sk); tp->prior_cwnd = tcp_snd_cwnd(tp); - tp->snd_ssthresh = icsk->icsk_ca_ops->ssthresh(sk); + WRITE_ONCE(tp->snd_ssthresh, icsk->icsk_ca_ops->ssthresh(sk)); tcp_ca_event(sk, CA_EVENT_LOSS); tcp_init_undo(tp); } @@ -2580,8 +2582,8 @@ void tcp_enter_loss(struct sock *sk) reordering = READ_ONCE(net->ipv4.sysctl_tcp_reordering); if (icsk->icsk_ca_state <= TCP_CA_Disorder && tp->sacked_out >= reordering) - tp->reordering = min_t(unsigned int, tp->reordering, - reordering); + WRITE_ONCE(tp->reordering, + min_t(unsigned int, tp->reordering, reordering)); tcp_set_ca_state(sk, TCP_CA_Loss); tp->high_seq = tp->snd_nxt; @@ -2859,7 +2861,7 @@ static void tcp_undo_cwnd_reduction(struct sock *sk, bool unmark_loss) tcp_snd_cwnd_set(tp, icsk->icsk_ca_ops->undo_cwnd(sk)); if (tp->prior_ssthresh > tp->snd_ssthresh) { - tp->snd_ssthresh = tp->prior_ssthresh; + WRITE_ONCE(tp->snd_ssthresh, tp->prior_ssthresh); tcp_ecn_withdraw_cwr(tp); } } @@ -2977,7 +2979,7 @@ static void tcp_init_cwnd_reduction(struct sock *sk) tp->prior_cwnd = tcp_snd_cwnd(tp); tp->prr_delivered = 0; tp->prr_out = 0; - tp->snd_ssthresh = inet_csk(sk)->icsk_ca_ops->ssthresh(sk); + WRITE_ONCE(tp->snd_ssthresh, inet_csk(sk)->icsk_ca_ops->ssthresh(sk)); tcp_ecn_queue_cwr(tp); } @@ -3119,7 +3121,7 @@ static void tcp_non_congestion_loss_retransmit(struct sock *sk) if (icsk->icsk_ca_state != TCP_CA_Loss) { tp->high_seq = tp->snd_nxt; - tp->snd_ssthresh = tcp_current_ssthresh(sk); + WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk)); tp->prior_ssthresh = 0; tp->undo_marker = 0; tcp_set_ca_state(sk, TCP_CA_Loss); @@ -3912,7 +3914,7 @@ static void tcp_snd_una_update(struct tcp_sock *tp, u32 ack) sock_owned_by_me((struct sock *)tp); tp->bytes_acked += delta; tcp_snd_sne_update(tp, ack); - tp->snd_una = ack; + WRITE_ONCE(tp->snd_una, ack); } static void tcp_rcv_sne_update(struct tcp_sock *tp, u32 seq) @@ -4286,11 +4288,15 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag) goto old_ack; } - /* If the ack includes data we haven't sent yet, discard - * this segment (RFC793 Section 3.9). + /* If the ack includes data we haven't sent yet, drop the + * segment. RFC 793 Section 3.9 and RFC 5961 Section 5.2 + * require us to send an ACK back in that case. */ - if (after(ack, tp->snd_nxt)) + if (after(ack, tp->snd_nxt)) { + if (!(flag & FLAG_NO_CHALLENGE_ACK)) + tcp_send_challenge_ack(sk, false); return -SKB_DROP_REASON_TCP_ACK_UNSENT_DATA; + } if (after(ack, prior_snd_una)) { flag |= FLAG_SND_UNA_ADVANCED; @@ -6836,7 +6842,7 @@ static bool tcp_rcv_fastopen_synack(struct sock *sk, struct sk_buff *synack, NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFASTOPENACTIVE); /* SYN-data is counted as two separate packets in tcp_ack() */ if (tp->delivered > 1) - --tp->delivered; + WRITE_ONCE(tp->delivered, tp->delivered - 1); } tcp_fastopen_add_skb(sk, synack); @@ -7267,7 +7273,7 @@ tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) SKB_DR_SET(reason, NOT_SPECIFIED); switch (sk->sk_state) { case TCP_SYN_RECV: - tp->delivered++; /* SYN-ACK delivery isn't tracked in tcp_ack */ + WRITE_ONCE(tp->delivered, tp->delivered + 1); /* SYN-ACK delivery isn't tracked in tcp_ack */ if (!tp->srtt_us) tcp_synack_rtt_meas(sk, req); @@ -7295,7 +7301,7 @@ tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) if (sk->sk_socket) sk_wake_async(sk, SOCK_WAKE_IO, POLL_OUT); - tp->snd_una = TCP_SKB_CB(skb)->ack_seq; + WRITE_ONCE(tp->snd_una, TCP_SKB_CB(skb)->ack_seq); tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale; tcp_init_wl(tp, TCP_SKB_CB(skb)->seq); diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c index 06b1d5d3b6df7b..dc0c081fc1f33f 100644 --- a/net/ipv4/tcp_metrics.c +++ b/net/ipv4/tcp_metrics.c @@ -490,13 +490,13 @@ void tcp_init_metrics(struct sock *sk) val = READ_ONCE(net->ipv4.sysctl_tcp_no_ssthresh_metrics_save) ? 0 : tcp_metric_get(tm, TCP_METRIC_SSTHRESH); if (val) { - tp->snd_ssthresh = val; + WRITE_ONCE(tp->snd_ssthresh, val); if (tp->snd_ssthresh > tp->snd_cwnd_clamp) - tp->snd_ssthresh = tp->snd_cwnd_clamp; + WRITE_ONCE(tp->snd_ssthresh, tp->snd_cwnd_clamp); } val = tcp_metric_get(tm, TCP_METRIC_REORDERING); if (val && tp->reordering != val) - tp->reordering = val; + WRITE_ONCE(tp->reordering, val); crtt = tcp_metric_get(tm, TCP_METRIC_RTT); rcu_read_unlock(); diff --git a/net/ipv4/tcp_nv.c b/net/ipv4/tcp_nv.c index a60662f4bdf92c..f345897a68dfcf 100644 --- a/net/ipv4/tcp_nv.c +++ b/net/ipv4/tcp_nv.c @@ -396,8 +396,8 @@ static void tcpnv_acked(struct sock *sk, const struct ack_sample *sample) /* We have enough data to determine we are congested */ ca->nv_allow_cwnd_growth = 0; - tp->snd_ssthresh = - (nv_ssthresh_factor * max_win) >> 3; + WRITE_ONCE(tp->snd_ssthresh, + (nv_ssthresh_factor * max_win) >> 3); if (tcp_snd_cwnd(tp) - max_win > 2) { /* gap > 2, we do exponential cwnd decrease */ int dec; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 326b58ff1118d0..a51186b42be933 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -171,7 +171,7 @@ void tcp_cwnd_restart(struct sock *sk, s32 delta) tcp_ca_event(sk, CA_EVENT_CWND_RESTART); - tp->snd_ssthresh = tcp_current_ssthresh(sk); + WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk)); restart_cwnd = min(restart_cwnd, cwnd); while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd) @@ -1668,8 +1668,10 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, if (skb->len != tcp_header_size) { tcp_event_data_sent(tp, sk); - tp->data_segs_out += tcp_skb_pcount(skb); - tp->bytes_sent += skb->len - tcp_header_size; + WRITE_ONCE(tp->data_segs_out, + tp->data_segs_out + tcp_skb_pcount(skb)); + WRITE_ONCE(tp->bytes_sent, + tp->bytes_sent + skb->len - tcp_header_size); } if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) @@ -2124,7 +2126,7 @@ static void tcp_cwnd_application_limited(struct sock *sk) u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk)); u32 win_used = max(tp->snd_cwnd_used, init_win); if (win_used < tcp_snd_cwnd(tp)) { - tp->snd_ssthresh = tcp_current_ssthresh(sk); + WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk)); tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1); } tp->snd_cwnd_used = 0; @@ -2606,6 +2608,7 @@ static int tcp_clone_payload(struct sock *sk, struct sk_buff *to, todo = min_t(int, skb_frag_size(fragfrom), probe_size - len); len += todo; + skb_shinfo(to)->flags |= skb_shinfo(skb)->flags & SKBFL_SHARED_FRAG; if (lastfrag && skb_frag_page(fragfrom) == skb_frag_page(lastfrag) && skb_frag_off(fragfrom) == skb_frag_off(lastfrag) + @@ -2878,30 +2881,6 @@ static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb, return false; } -static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new) -{ - const u32 now = tcp_jiffies32; - enum tcp_chrono old = tp->chrono_type; - - if (old > TCP_CHRONO_UNSPEC) - tp->chrono_stat[old - 1] += now - tp->chrono_start; - tp->chrono_start = now; - tp->chrono_type = new; -} - -void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type) -{ - struct tcp_sock *tp = tcp_sk(sk); - - /* If there are multiple conditions worthy of tracking in a - * chronograph then the highest priority enum takes precedence - * over the other conditions. So that if something "more interesting" - * starts happening, stop the previous chrono and start a new one. - */ - if (type > tp->chrono_type) - tcp_chrono_set(tp, type); -} - void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type) { struct tcp_sock *tp = tcp_sk(sk); @@ -3648,8 +3627,8 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs); if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN) __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS); - tp->total_retrans += segs; - tp->bytes_retrans += skb->len; + WRITE_ONCE(tp->total_retrans, tp->total_retrans + segs); + WRITE_ONCE(tp->bytes_retrans, tp->bytes_retrans + skb->len); /* make sure skb->data is aligned on arches that require it * and check if ack-trimming & collapsing extended the headroom @@ -4159,7 +4138,7 @@ static void tcp_connect_init(struct sock *sk) tp->snd_wnd = 0; tcp_init_wl(tp, 0); tcp_write_queue_purge(sk); - tp->snd_una = tp->write_seq; + WRITE_ONCE(tp->snd_una, tp->write_seq); tp->snd_sml = tp->write_seq; tp->snd_up = tp->write_seq; WRITE_ONCE(tp->snd_nxt, tp->write_seq); @@ -4652,7 +4631,8 @@ int tcp_rtx_synack(const struct sock *sk, struct request_sock *req) * However in this case, we are dealing with a passive fastopen * socket thus we can change total_retrans value. */ - tcp_sk_rw(sk)->total_retrans++; + WRITE_ONCE(tcp_sk_rw(sk)->total_retrans, + tcp_sk_rw(sk)->total_retrans + 1); } trace_tcp_retransmit_synack(sk, req); WRITE_ONCE(req->num_retrans, req->num_retrans + 1); diff --git a/net/ipv4/tcp_plb.c b/net/ipv4/tcp_plb.c index 4bcf7eff95e390..b7f9b60d8991f2 100644 --- a/net/ipv4/tcp_plb.c +++ b/net/ipv4/tcp_plb.c @@ -79,7 +79,7 @@ void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb) sk_rethink_txhash(sk); plb->consec_cong_rounds = 0; - tcp_sk(sk)->plb_rehash++; + WRITE_ONCE(tcp_sk(sk)->plb_rehash, tcp_sk(sk)->plb_rehash + 1); NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPPLBREHASH); } EXPORT_SYMBOL_GPL(tcp_plb_check_rehash); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 5a14a53a3c9ef9..61631a2dcea7f9 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -50,7 +50,8 @@ static u32 tcp_clamp_rto_to_user_timeout(const struct sock *sk) u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when) { const struct inet_connection_sock *icsk = inet_csk(sk); - u32 remaining, user_timeout; + u32 user_timeout; + s32 remaining; s32 elapsed; user_timeout = READ_ONCE(icsk->icsk_user_timeout); @@ -61,7 +62,7 @@ u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when) if (unlikely(elapsed < 0)) elapsed = 0; remaining = msecs_to_jiffies(user_timeout) - elapsed; - remaining = max_t(u32, remaining, TCP_TIMEOUT_MIN); + remaining = max_t(int, remaining, TCP_TIMEOUT_MIN); return min_t(u32, remaining, when); } @@ -297,7 +298,7 @@ static int tcp_write_timeout(struct sock *sk) } if (sk_rethink_txhash(sk)) { - tp->timeout_rehash++; + WRITE_ONCE(tp->timeout_rehash, tp->timeout_rehash + 1); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPTIMEOUTREHASH); } diff --git a/net/ipv4/tcp_vegas.c b/net/ipv4/tcp_vegas.c index 786848ad37ea8d..3ec7308441a781 100644 --- a/net/ipv4/tcp_vegas.c +++ b/net/ipv4/tcp_vegas.c @@ -240,7 +240,8 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) */ tcp_snd_cwnd_set(tp, min(tcp_snd_cwnd(tp), (u32)target_cwnd + 1)); - tp->snd_ssthresh = tcp_vegas_ssthresh(tp); + WRITE_ONCE(tp->snd_ssthresh, + tcp_vegas_ssthresh(tp)); } else if (tcp_in_slow_start(tp)) { /* Slow start. */ @@ -256,8 +257,8 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) * we slow down. */ tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1); - tp->snd_ssthresh - = tcp_vegas_ssthresh(tp); + WRITE_ONCE(tp->snd_ssthresh, + tcp_vegas_ssthresh(tp)); } else if (diff < alpha) { /* We don't have enough extra packets * in the network, so speed up. @@ -275,7 +276,7 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 acked) else if (tcp_snd_cwnd(tp) > tp->snd_cwnd_clamp) tcp_snd_cwnd_set(tp, tp->snd_cwnd_clamp); - tp->snd_ssthresh = tcp_current_ssthresh(sk); + WRITE_ONCE(tp->snd_ssthresh, tcp_current_ssthresh(sk)); } /* Wipe the slate clean for the next RTT. */ diff --git a/net/ipv4/tcp_westwood.c b/net/ipv4/tcp_westwood.c index c6e97141eef259..b5a42adfd6ca1f 100644 --- a/net/ipv4/tcp_westwood.c +++ b/net/ipv4/tcp_westwood.c @@ -244,11 +244,11 @@ static void tcp_westwood_event(struct sock *sk, enum tcp_ca_event event) switch (event) { case CA_EVENT_COMPLETE_CWR: - tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk); + WRITE_ONCE(tp->snd_ssthresh, tcp_westwood_bw_rttmin(sk)); tcp_snd_cwnd_set(tp, tp->snd_ssthresh); break; case CA_EVENT_LOSS: - tp->snd_ssthresh = tcp_westwood_bw_rttmin(sk); + WRITE_ONCE(tp->snd_ssthresh, tcp_westwood_bw_rttmin(sk)); /* Update RTT_min when next ack arrives */ w->reset_rtt_min = 1; break; diff --git a/net/ipv4/tcp_yeah.c b/net/ipv4/tcp_yeah.c index 18b07ff5d20e6c..74a2538e79e06f 100644 --- a/net/ipv4/tcp_yeah.c +++ b/net/ipv4/tcp_yeah.c @@ -147,7 +147,8 @@ static void tcp_yeah_cong_avoid(struct sock *sk, u32 ack, u32 acked) tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), yeah->reno_count)); - tp->snd_ssthresh = tcp_snd_cwnd(tp); + WRITE_ONCE(tp->snd_ssthresh, + tcp_snd_cwnd(tp)); } if (yeah->reno_count <= 2) diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index cb99a3c27053ef..2551bfaa26810e 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -365,10 +365,10 @@ int udp_v4_get_port(struct sock *sk, unsigned short snum) return udp_lib_get_port(sk, snum, hash2_nulladdr); } -static int compute_score(struct sock *sk, const struct net *net, - __be32 saddr, __be16 sport, - __be32 daddr, unsigned short hnum, - int dif, int sdif) +static __always_inline int +compute_score(struct sock *sk, const struct net *net, + __be32 saddr, __be16 sport, __be32 daddr, + unsigned short hnum, int dif, int sdif) { int score; struct inet_sock *inet; @@ -508,8 +508,8 @@ static struct sock *udp4_lib_lookup2(const struct net *net, continue; /* compute_score is too long of a function to be - * inlined, and calling it again here yields - * measurable overhead for some + * inlined twice here, and calling it uninlined + * here yields measurable overhead for some * workloads. Work around it by jumping * backwards to rescore 'result'. */ diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index cb26beea439824..de1e68199a0145 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -317,14 +317,19 @@ static void ah6_output_done(void *data, int err) struct ipv6hdr *top_iph = ipv6_hdr(skb); struct ip_auth_hdr *ah = ip_auth_hdr(skb); struct tmp_ext *iph_ext; + int seqhi_len = 0; + __be32 *seqhi; extlen = skb_network_header_len(skb) - sizeof(struct ipv6hdr); if (extlen) extlen += sizeof(*iph_ext); + if (x->props.flags & XFRM_STATE_ESN) + seqhi_len = sizeof(*seqhi); iph_base = AH_SKB_CB(skb)->tmp; iph_ext = ah_tmp_ext(iph_base); - icv = ah_tmp_icv(iph_ext, extlen); + seqhi = (__be32 *)((char *)iph_ext + extlen); + icv = ah_tmp_icv(seqhi, seqhi_len); memcpy(ah->auth_data, icv, ahp->icv_trunc_len); memcpy(top_iph, iph_base, IPV6HDR_BASELEN); @@ -471,13 +476,18 @@ static void ah6_input_done(void *data, int err) struct ip_auth_hdr *ah = ip_auth_hdr(skb); int hdr_len = skb_network_header_len(skb); int ah_hlen = ipv6_authlen(ah); + int seqhi_len = 0; + __be32 *seqhi; if (err) goto out; + if (x->props.flags & XFRM_STATE_ESN) + seqhi_len = sizeof(*seqhi); work_iph = AH_SKB_CB(skb)->tmp; auth_data = ah_tmp_auth(work_iph, hdr_len); - icv = ah_tmp_icv(auth_data, ahp->icv_trunc_len); + seqhi = (__be32 *)(auth_data + ahp->icv_trunc_len); + icv = ah_tmp_icv(seqhi, seqhi_len); err = crypto_memneq(icv, auth_data, ahp->icv_trunc_len) ? -EBADMSG : 0; if (err) diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index 9f75313734f8cd..9c06c5a1419dc4 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -915,7 +915,8 @@ static int esp6_input(struct xfrm_state *x, struct sk_buff *skb) nfrags = 1; goto skip_cow; - } else if (!skb_has_frag_list(skb)) { + } else if (!skb_has_frag_list(skb) && + !skb_has_shared_frag(skb)) { nfrags = skb_shinfo(skb)->nr_frags; nfrags++; diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index 95558fd6f447e3..03cbce842c1a7a 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -491,6 +491,7 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb) struct net *net = dev_net(skb->dev); struct inet6_dev *idev; struct ipv6hdr *oldhdr; + unsigned int chdr_len; unsigned char *buf; int accept_rpl_seg; int i, err; @@ -592,8 +593,10 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb) skb_pull(skb, ((hdr->hdrlen + 1) << 3)); skb_postpull_rcsum(skb, oldhdr, sizeof(struct ipv6hdr) + ((hdr->hdrlen + 1) << 3)); - if (unlikely(!hdr->segments_left)) { - if (pskb_expand_head(skb, sizeof(struct ipv6hdr) + ((chdr->hdrlen + 1) << 3), 0, + chdr_len = sizeof(struct ipv6hdr) + ((chdr->hdrlen + 1) << 3); + if (unlikely(!hdr->segments_left || + skb_headroom(skb) < chdr_len + skb->mac_len)) { + if (pskb_expand_head(skb, chdr_len + skb->mac_len, 0, GFP_ATOMIC)) { __IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_OUTDISCARDS); kfree_skb(skb); @@ -603,7 +606,7 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb) oldhdr = ipv6_hdr(skb); } - skb_push(skb, ((chdr->hdrlen + 1) << 3) + sizeof(struct ipv6hdr)); + skb_push(skb, chdr_len); skb_reset_network_header(skb); skb_mac_header_rebuild(skb); skb_set_transport_header(skb, sizeof(struct ipv6hdr)); diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c index d5d23a9296eac8..88356cbfb68b19 100644 --- a/net/ipv6/icmp.c +++ b/net/ipv6/icmp.c @@ -1104,7 +1104,6 @@ static int icmpv6_rcv(struct sk_buff *skb) struct net *net = dev_net_rcu(skb->dev); struct net_device *dev = icmp6_dev(skb); struct inet6_dev *idev = __in6_dev_get(dev); - const struct in6_addr *saddr, *daddr; struct icmp6hdr *hdr; u8 type; @@ -1135,12 +1134,10 @@ static int icmpv6_rcv(struct sk_buff *skb) __ICMP6_INC_STATS(dev_net_rcu(dev), idev, ICMP6_MIB_INMSGS); - saddr = &ipv6_hdr(skb)->saddr; - daddr = &ipv6_hdr(skb)->daddr; - if (skb_checksum_validate(skb, IPPROTO_ICMPV6, ip6_compute_pseudo)) { net_dbg_ratelimited("ICMPv6 checksum failed [%pI6c > %pI6c]\n", - saddr, daddr); + &ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr); goto csum_error; } @@ -1220,7 +1217,8 @@ static int icmpv6_rcv(struct sk_buff *skb) break; net_dbg_ratelimited("icmpv6: msg of unknown type [%pI6c > %pI6c]\n", - saddr, daddr); + &ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr); /* * error of unknown type. diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index dafcc0dcd77a53..0097d4784c7120 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -2261,10 +2261,11 @@ static int ip6erspan_changelink(struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack) { - struct ip6gre_net *ign = net_generic(dev_net(dev), ip6gre_net_id); + struct ip6_tnl *t = netdev_priv(dev); struct __ip6_tnl_parm p; - struct ip6_tnl *t; + struct ip6gre_net *ign; + ign = net_generic(t->net, ip6gre_net_id); t = ip6gre_changelink_common(dev, tb, data, &p, extack); if (IS_ERR(t)) return PTR_ERR(t); diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 8e2a6b28cea7ae..3f14e363c96e20 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -1765,6 +1765,8 @@ static int __ip6_append_data(struct sock *sk, if (err < 0) goto error; copy = err; + if (!(flags & MSG_NO_SHARED_FRAGS)) + skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; wmem_alloc_delta += copy; } else if (!zc) { int i = skb_shinfo(skb)->nr_frags; diff --git a/net/ipv6/netfilter/ip6table_nat.c b/net/ipv6/netfilter/ip6table_nat.c index e119d4f090cc88..5be723232df8f1 100644 --- a/net/ipv6/netfilter/ip6table_nat.c +++ b/net/ipv6/netfilter/ip6table_nat.c @@ -81,7 +81,7 @@ static int ip6t_nat_register_lookups(struct net *net) while (i) nf_nat_ipv6_unregister_fn(net, &ops[--i]); - kfree(ops); + kfree_rcu(ops, rcu); return ret; } } @@ -102,7 +102,7 @@ static void ip6t_nat_unregister_lookups(struct net *net) for (i = 0; i < ARRAY_SIZE(nf_nat_ipv6_ops); i++) nf_nat_ipv6_unregister_fn(net, &ops[i]); - kfree(ops); + kfree_rcu(ops, rcu); } static int ip6table_nat_table_init(struct net *net) diff --git a/net/ipv6/rpl_iptunnel.c b/net/ipv6/rpl_iptunnel.c index c7942cf655671c..4e10adcd70e89d 100644 --- a/net/ipv6/rpl_iptunnel.c +++ b/net/ipv6/rpl_iptunnel.c @@ -287,7 +287,16 @@ static int rpl_input(struct sk_buff *skb) if (!dst) { ip6_route_input(skb); + + /* ip6_route_input() sets a NOREF dst; force a refcount on it + * before caching or further use. + */ + skb_dst_force(skb); dst = skb_dst(skb); + if (unlikely(!dst)) { + err = -ENETUNREACH; + goto drop; + } /* cache only if we don't create a dst reference loop */ if (!dst->error && lwtst != dst->lwtstate) { diff --git a/net/ipv6/seg6_iptunnel.c b/net/ipv6/seg6_iptunnel.c index d6a0f7df90807f..ead677bca49015 100644 --- a/net/ipv6/seg6_iptunnel.c +++ b/net/ipv6/seg6_iptunnel.c @@ -500,7 +500,16 @@ static int seg6_input_core(struct net *net, struct sock *sk, if (!dst) { ip6_route_input(skb); + + /* ip6_route_input() sets a NOREF dst; force a refcount on it + * before caching or further use. + */ + skb_dst_force(skb); dst = skb_dst(skb); + if (unlikely(!dst)) { + err = -ENETUNREACH; + goto drop; + } /* cache only if we don't create a dst reference loop */ if (!dst->error && lwtst != dst->lwtstate) { @@ -715,7 +724,8 @@ static int seg6_build_state(struct net *net, struct nlattr *nla, newts->type = LWTUNNEL_ENCAP_SEG6; newts->flags |= LWTUNNEL_STATE_INPUT_REDIRECT; - if (tuninfo->mode != SEG6_IPTUN_MODE_L2ENCAP) + if (tuninfo->mode != SEG6_IPTUN_MODE_L2ENCAP && + tuninfo->mode != SEG6_IPTUN_MODE_L2ENCAP_RED) newts->flags |= LWTUNNEL_STATE_OUTPUT_REDIRECT; newts->headroom = seg6_lwt_headroom(tuninfo); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 010b909275dd01..301649a63e8a37 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -127,10 +127,11 @@ void udp_v6_rehash(struct sock *sk) udp_lib_rehash(sk, new_hash, new_hash4); } -static int compute_score(struct sock *sk, const struct net *net, - const struct in6_addr *saddr, __be16 sport, - const struct in6_addr *daddr, unsigned short hnum, - int dif, int sdif) +static __always_inline int +compute_score(struct sock *sk, const struct net *net, + const struct in6_addr *saddr, __be16 sport, + const struct in6_addr *daddr, unsigned short hnum, + int dif, int sdif) { int bound_dev_if, score; struct inet_sock *inet; @@ -260,8 +261,8 @@ static struct sock *udp6_lib_lookup2(const struct net *net, continue; /* compute_score is too long of a function to be - * inlined, and calling it again here yields - * measurable overhead for some + * inlined twice here, and calling it uninlined + * here yields measurable overhead for some * workloads. Work around it by jumping * backwards to rescore 'result'. */ diff --git a/net/ipv6/xfrm6_protocol.c b/net/ipv6/xfrm6_protocol.c index ea2f805d3b014c..9b586fcec4850b 100644 --- a/net/ipv6/xfrm6_protocol.c +++ b/net/ipv6/xfrm6_protocol.c @@ -88,8 +88,10 @@ int xfrm6_rcv_encap(struct sk_buff *skb, int nexthdr, __be32 spi, dst = ip6_route_input_lookup(dev_net(skb->dev), skb->dev, &fl6, skb, flags); - if (dst->error) + if (dst->error) { + dst_release(dst); goto drop; + } skb_dst_set(skb, dst); } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 810bea1aacc5e5..53bd98646e33e2 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -6009,7 +6009,8 @@ ieee80211_determine_our_sta_mode(struct ieee80211_sub_if_data *sdata, if (is_5ghz && !(vht_cap.cap & (IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ | - IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ))) { + IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ | + IEEE80211_VHT_CAP_EXT_NSS_BW_MASK))) { conn->bw_limit = IEEE80211_CONN_BW_LIMIT_80; mlme_link_id_dbg(sdata, link_id, "no VHT 160 MHz capability on 5 GHz, limiting to 80 MHz"); @@ -9053,7 +9054,7 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, struct ieee80211_bss *bss = (void *)cbss->priv; struct sta_info *new_sta = NULL; struct ieee80211_link_data *link; - bool have_sta = false; + struct sta_info *have_sta = NULL; bool mlo; int err; u16 new_links; @@ -9072,11 +9073,8 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, mlo = false; } - if (assoc) { - rcu_read_lock(); + if (assoc) have_sta = sta_info_get(sdata, ap_mld_addr); - rcu_read_unlock(); - } if (mlo && !have_sta && WARN_ON(sdata->vif.valid_links || sdata->vif.active_links)) @@ -9239,6 +9237,8 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata, out_release_chan: ieee80211_link_release_channel(link); out_err: + if (mlo && have_sta) + WARN_ON(__sta_info_destroy(have_sta)); ieee80211_vif_set_links(sdata, 0, 0); return err; } diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 11d6c56c9d7eeb..7a8c964b0ae68a 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -4928,7 +4928,7 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx, struct sk_buff *skb = rx->skb; struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); - static ieee80211_rx_result res; + ieee80211_rx_result res; int orig_len = skb->len; int hdrlen = ieee80211_hdrlen(hdr->frame_control); int snap_offs = hdrlen; diff --git a/net/mac80211/util.c b/net/mac80211/util.c index b2e6c8b98381bc..6cceaf4bc0d976 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -3623,11 +3623,11 @@ void ieee80211_dfs_radar_detected_work(struct wiphy *wiphy, struct ieee80211_local *local = container_of(work, struct ieee80211_local, radar_detected_work); struct cfg80211_chan_def chandef; - struct ieee80211_chanctx *ctx; + struct ieee80211_chanctx *ctx, *tmp; lockdep_assert_wiphy(local->hw.wiphy); - list_for_each_entry(ctx, &local->chanctx_list, list) { + list_for_each_entry_safe(ctx, tmp, &local->chanctx_list, list) { if (ctx->replace_state == IEEE80211_CHANCTX_REPLACES_OTHER) continue; diff --git a/net/mctp/route.c b/net/mctp/route.c index 59ad60b885631c..1b0c55b1779aa2 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -441,6 +441,7 @@ static int mctp_dst_input(struct mctp_dst *dst, struct sk_buff *skb) unsigned long f; u8 tag, flags; int rc; + u8 ver; msk = NULL; rc = -EINVAL; @@ -467,7 +468,8 @@ static int mctp_dst_input(struct mctp_dst *dst, struct sk_buff *skb) netid = mctp_cb(skb)->net; skb_pull(skb, sizeof(struct mctp_hdr)); - if (mh->ver != 1) + ver = mh->ver & MCTP_HDR_VER_MASK; + if (ver < MCTP_VER_MIN || ver > MCTP_VER_MAX) goto out; flags = mh->flags_seq_tag & (MCTP_HDR_FLAG_SOM | MCTP_HDR_FLAG_EOM); @@ -1325,6 +1327,7 @@ static int mctp_pkttype_receive(struct sk_buff *skb, struct net_device *dev, struct mctp_dst dst; struct mctp_hdr *mh; int rc; + u8 ver; rcu_read_lock(); mdev = __mctp_dev_get(dev); @@ -1342,7 +1345,8 @@ static int mctp_pkttype_receive(struct sk_buff *skb, struct net_device *dev, /* We have enough for a header; decode and route */ mh = mctp_hdr(skb); - if (mh->ver < MCTP_VER_MIN || mh->ver > MCTP_VER_MAX) + ver = mh->ver & MCTP_HDR_VER_MASK; + if (ver < MCTP_VER_MIN || ver > MCTP_VER_MAX) goto err_drop; /* source must be valid unicast or null; drop reserved ranges and diff --git a/net/mptcp/fastopen.c b/net/mptcp/fastopen.c index 82ec15bcfd7f56..082c46c0f50ee7 100644 --- a/net/mptcp/fastopen.c +++ b/net/mptcp/fastopen.c @@ -12,6 +12,7 @@ void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subf struct sock *sk, *ssk; struct sk_buff *skb; struct tcp_sock *tp; + bool has_rxtstamp; /* on early fallback the subflow context is deleted by * subflow_syn_recv_sock() @@ -40,12 +41,13 @@ void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subf */ tp->copied_seq += skb->len; subflow->ssn_offset += skb->len; + has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; /* Only the sequence delta is relevant */ MPTCP_SKB_CB(skb)->map_seq = -skb->len; MPTCP_SKB_CB(skb)->end_seq = 0; MPTCP_SKB_CB(skb)->offset = 0; - MPTCP_SKB_CB(skb)->has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; + MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; MPTCP_SKB_CB(skb)->cant_coalesce = 1; mptcp_data_lock(sk); diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 57a45669040679..3c152bf66cd5ac 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -16,6 +16,7 @@ struct mptcp_pm_add_entry { struct list_head list; struct mptcp_addr_info addr; u8 retrans_times; + bool timer_done; struct timer_list add_timer; struct mptcp_sock *sock; struct rcu_head rcu; @@ -283,6 +284,9 @@ int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, struct sock *ssk = mptcp_subflow_tcp_sock(subflow); struct mptcp_addr_info local, remote; + if (!__mptcp_subflow_active(subflow)) + continue; + mptcp_local_address((struct sock_common *)ssk, &local); if (!mptcp_addresses_equal(&local, addr, addr->port)) continue; @@ -305,18 +309,31 @@ static unsigned int mptcp_adjust_add_addr_timeout(struct mptcp_sock *msk) const struct net *net = sock_net((struct sock *)msk); unsigned int rto = mptcp_get_add_addr_timeout(net); struct mptcp_subflow_context *subflow; - unsigned int max = 0; + unsigned int max = 0, max_stale = 0; + + if (!rto) + return 0; mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); struct inet_connection_sock *icsk = inet_csk(ssk); - if (icsk->icsk_rto > max) + if (!__mptcp_subflow_active(subflow)) + continue; + + if (unlikely(subflow->stale)) { + if (icsk->icsk_rto > max_stale) + max_stale = icsk->icsk_rto; + } else if (icsk->icsk_rto > max) { max = icsk->icsk_rto; + } } - if (max && max < rto) - rto = max; + if (max) + return min(max, rto); + + if (max_stale) + return min(max_stale, rto); return rto; } @@ -327,26 +344,22 @@ static void mptcp_pm_add_timer(struct timer_list *timer) add_timer); struct mptcp_sock *msk = entry->sock; struct sock *sk = (struct sock *)msk; - unsigned int timeout; + unsigned int timeout = 0; pr_debug("msk=%p\n", msk); - if (!msk) - return; - - if (inet_sk_state_load(sk) == TCP_CLOSE) - return; - - if (!entry->addr.id) - return; + bh_lock_sock(sk); + if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) + goto out; - if (mptcp_pm_should_add_signal_addr(msk)) { - sk_reset_timer(sk, timer, jiffies + TCP_RTO_MAX / 8); + if (sock_owned_by_user(sk)) { + /* Try again later. */ + timeout = HZ / 20; goto out; } timeout = mptcp_adjust_add_addr_timeout(msk); - if (!timeout) + if (!timeout || mptcp_pm_should_add_signal_addr(msk)) goto out; spin_lock_bh(&msk->pm.lock); @@ -359,8 +372,9 @@ static void mptcp_pm_add_timer(struct timer_list *timer) } if (entry->retrans_times < ADD_ADDR_RETRANS_MAX) - sk_reset_timer(sk, timer, - jiffies + (timeout << entry->retrans_times)); + timeout <<= entry->retrans_times; + else + timeout = 0; spin_unlock_bh(&msk->pm.lock); @@ -368,7 +382,13 @@ static void mptcp_pm_add_timer(struct timer_list *timer) mptcp_pm_subflow_established(msk); out: - __sock_put(sk); + if (timeout) + sk_reset_timer(sk, timer, jiffies + timeout); + else + /* if sock_put calls sk_free: avoid waiting for this timer */ + entry->timer_done = true; + bh_unlock_sock(sk); + sock_put(sk); } struct mptcp_pm_add_entry * @@ -431,6 +451,7 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0); reset_timer: + add_entry->timer_done = false; timeout = mptcp_adjust_add_addr_timeout(msk); if (timeout) sk_reset_timer(sk, &add_entry->add_timer, jiffies + timeout); @@ -451,7 +472,8 @@ static void mptcp_pm_free_anno_list(struct mptcp_sock *msk) spin_unlock_bh(&msk->pm.lock); list_for_each_entry_safe(entry, tmp, &free_list, list) { - sk_stop_timer_sync(sk, &entry->add_timer); + if (!entry->timer_done) + sk_stop_timer_sync(sk, &entry->add_timer); kfree_rcu(entry, rcu); } } diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c index 0ebf43be993993..fc818b63752e37 100644 --- a/net/mptcp/pm_kernel.c +++ b/net/mptcp/pm_kernel.c @@ -347,6 +347,8 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) /* check first for announce */ if (msk->pm.add_addr_signaled < endp_signal_max) { + u8 endp_id; + /* due to racing events on both ends we can reach here while * previous add address is still running: if we invoke now * mptcp_pm_announce_addr(), that will fail and the @@ -360,19 +362,20 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) if (!select_signal_address(pernet, msk, &local)) goto subflow; + /* Special case for ID0: set the correct ID */ + endp_id = local.addr.id; + if (endp_id == msk->mpc_endpoint_id) + local.addr.id = 0; + /* If the alloc fails, we are on memory pressure, not worth * continuing, and trying to create subflows. */ if (!mptcp_pm_alloc_anno_list(msk, &local.addr)) return; - __clear_bit(local.addr.id, msk->pm.id_avail_bitmap); + __clear_bit(endp_id, msk->pm.id_avail_bitmap); msk->pm.add_addr_signaled++; - /* Special case for ID0: set the correct ID */ - if (local.addr.id == msk->mpc_endpoint_id) - local.addr.id = 0; - mptcp_pm_announce_addr(msk, &local.addr, false); mptcp_pm_addr_send_ack(msk); @@ -1278,6 +1281,7 @@ static void __reset_counters(struct pm_nl_pernet *pernet) WRITE_ONCE(pernet->endp_signal_max, 0); WRITE_ONCE(pernet->endp_subflow_max, 0); WRITE_ONCE(pernet->endp_laminar_max, 0); + WRITE_ONCE(pernet->endp_fullmesh_max, 0); pernet->endpoints = 0; } diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 614c3f583ca030..8ef967aa80a0b5 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -879,6 +879,32 @@ static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) return moved; } +static void mptcp_rcv_rtt_update(struct mptcp_sock *msk, + struct mptcp_subflow_context *subflow) +{ + const struct tcp_sock *tp = tcp_sk(subflow->tcp_sock); + u32 rtt_us = tp->rcv_rtt_est.rtt_us; + int id; + + /* Update once per subflow per rcvwnd to avoid touching the msk + * too often. + */ + if (!rtt_us || tp->rcv_rtt_est.seq == subflow->prev_rtt_seq) + return; + + subflow->prev_rtt_seq = tp->rcv_rtt_est.seq; + + /* Pairs with READ_ONCE() in mptcp_rtt_us_est(). */ + id = msk->rcv_rtt_est.next_sample; + WRITE_ONCE(msk->rcv_rtt_est.samples[id], rtt_us); + if (++msk->rcv_rtt_est.next_sample == MPTCP_RTT_SAMPLES) + msk->rcv_rtt_est.next_sample = 0; + + /* EWMA among the incoming subflows */ + msk->scaling_ratio = ((msk->scaling_ratio << 3) - msk->scaling_ratio + + tp->scaling_ratio) >> 3; +} + void mptcp_data_ready(struct sock *sk, struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); @@ -892,6 +918,7 @@ void mptcp_data_ready(struct sock *sk, struct sock *ssk) return; mptcp_data_lock(sk); + mptcp_rcv_rtt_update(msk, subflow); if (!sock_owned_by_user(sk)) { /* Wake-up the reader only for in-sequence data */ if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk)) @@ -2077,7 +2104,6 @@ static void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) msk->rcvspace_init = 1; msk->rcvq_space.copied = 0; - msk->rcvq_space.rtt_us = 0; /* initial rcv_space offering made to peer */ msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, @@ -2088,15 +2114,15 @@ static void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. * - * Only difference: Use highest rtt estimate of the subflows in use. + * Only difference: Use lowest rtt estimate of the subflows in use, see + * mptcp_rcv_rtt_update() and mptcp_rtt_us_est(). */ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) { struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; - u8 scaling_ratio = U8_MAX; - u32 time, advmss = 1; - u64 rtt_us, mstamp; + u32 time, rtt_us; + u64 mstamp; msk_owned_by_me(msk); @@ -2111,29 +2137,8 @@ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) mstamp = mptcp_stamp(); time = tcp_stamp_us_delta(mstamp, READ_ONCE(msk->rcvq_space.time)); - rtt_us = msk->rcvq_space.rtt_us; - if (rtt_us && time < (rtt_us >> 3)) - return; - - rtt_us = 0; - mptcp_for_each_subflow(msk, subflow) { - const struct tcp_sock *tp; - u64 sf_rtt_us; - u32 sf_advmss; - - tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); - - sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); - sf_advmss = READ_ONCE(tp->advmss); - - rtt_us = max(sf_rtt_us, rtt_us); - advmss = max(sf_advmss, advmss); - scaling_ratio = min(tp->scaling_ratio, scaling_ratio); - } - - msk->rcvq_space.rtt_us = rtt_us; - msk->scaling_ratio = scaling_ratio; - if (time < (rtt_us >> 3) || rtt_us == 0) + rtt_us = mptcp_rtt_us_est(msk); + if (rtt_us == U32_MAX || time < (rtt_us >> 3)) return; if (msk->rcvq_space.copied <= msk->rcvq_space.space) @@ -3000,6 +3005,7 @@ static void __mptcp_init_sock(struct sock *sk) msk->timer_ival = TCP_RTO_MIN; msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; msk->backlog_len = 0; + mptcp_init_rtt_est(msk); WRITE_ONCE(msk->first, NULL); inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; @@ -3279,7 +3285,8 @@ bool __mptcp_close(struct sock *sk, long timeout) goto cleanup; } - if (mptcp_data_avail(msk) || timeout < 0) { + if (mptcp_data_avail(msk) || timeout < 0 || + (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime)) { /* If the msk has read data, or the caller explicitly ask it, * do the MPTCP equivalent of TCP reset, aka MPTCP fastclose */ @@ -3445,6 +3452,7 @@ static int mptcp_disconnect(struct sock *sk, int flags) msk->bytes_retrans = 0; msk->rcvspace_init = 0; msk->fastclosing = 0; + mptcp_init_rtt_est(msk); /* for fallback's sake */ WRITE_ONCE(msk->ack_seq, 0); @@ -3570,7 +3578,6 @@ struct sock *mptcp_sk_clone_init(const struct sock *sk, * uses the correct data */ mptcp_copy_inaddrs(nsk, ssk); - __mptcp_propagate_sndbuf(nsk, ssk); mptcp_rcv_space_init(msk, ssk); msk->rcvq_space.time = mptcp_stamp(); @@ -4228,6 +4235,7 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, mptcp_graft_subflows(newsk); mptcp_rps_record_subflows(msk); + __mptcp_propagate_sndbuf(newsk, mptcp_subflow_tcp_sock(subflow)); /* Do late cleanup for the first subflow as necessary. Also * deal with bad peers not doing a complete shutdown. diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index ec15e503da8b70..d19c54761c27aa 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -268,6 +268,13 @@ struct mptcp_data_frag { struct page *page; }; +/* Arbitrary compromise between as low as possible to react timely to subflow + * close event and as big as possible to avoid being fouled by biased large + * samples due to peer sending data on a different subflow WRT to the incoming + * ack. + */ +#define MPTCP_RTT_SAMPLES 5 + /* MPTCP connection sock */ struct mptcp_sock { /* inet_connection_sock must be the first member */ @@ -340,11 +347,17 @@ struct mptcp_sock { */ struct mptcp_pm_data pm; struct mptcp_sched_ops *sched; + + /* Most recent rtt_us observed by in use incoming subflows. */ + struct { + u32 samples[MPTCP_RTT_SAMPLES]; + u32 next_sample; + } rcv_rtt_est; + struct { int space; /* bytes copied in last measurement window */ int copied; /* bytes copied in this measurement window */ u64 time; /* start time of measurement window */ - u64 rtt_us; /* last maximum rtt of subflows */ } rcvq_space; u8 scaling_ratio; bool allow_subflows; @@ -422,6 +435,27 @@ static inline struct mptcp_data_frag *mptcp_send_head(const struct sock *sk) return msk->first_pending; } +static inline void mptcp_init_rtt_est(struct mptcp_sock *msk) +{ + int i; + + for (i = 0; i < MPTCP_RTT_SAMPLES; ++i) + msk->rcv_rtt_est.samples[i] = U32_MAX; + msk->rcv_rtt_est.next_sample = 0; + msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; +} + +static inline u32 mptcp_rtt_us_est(const struct mptcp_sock *msk) +{ + u32 rtt_us = READ_ONCE(msk->rcv_rtt_est.samples[0]); + int i; + + /* Lockless access of collected samples. */ + for (i = 1; i < MPTCP_RTT_SAMPLES; ++i) + rtt_us = min(rtt_us, READ_ONCE(msk->rcv_rtt_est.samples[i])); + return rtt_us; +} + static inline struct mptcp_data_frag *mptcp_send_next(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); @@ -523,6 +557,7 @@ struct mptcp_subflow_context { u32 map_data_len; __wsum map_data_csum; u32 map_csum_len; + u32 prev_rtt_seq; u32 request_mptcp : 1, /* send MP_CAPABLE */ request_join : 1, /* send MP_JOIN */ request_bkup : 1, diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index de90a2897d2d8f..1cf608e7357bda 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -159,10 +159,10 @@ static int mptcp_setsockopt_sol_socket_tstamp(struct mptcp_sock *msk, int optnam lock_sock(sk); mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); - bool slow = lock_sock_fast(ssk); - sock_set_timestamp(sk, optname, !!val); - unlock_sock_fast(ssk, slow); + lock_sock(ssk); + sock_set_timestamp(ssk, optname, !!val); + release_sock(ssk); } release_sock(sk); @@ -235,10 +235,10 @@ static int mptcp_setsockopt_sol_socket_timestamping(struct mptcp_sock *msk, mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); - bool slow = lock_sock_fast(ssk); - sock_set_timestamping(sk, optname, timestamping); - unlock_sock_fast(ssk, slow); + lock_sock(ssk); + sock_set_timestamping(ssk, optname, timestamping); + release_sock(ssk); } release_sock(sk); @@ -812,6 +812,10 @@ static int mptcp_setsockopt_all_sf(struct mptcp_sock *msk, int level, if (ret) break; } + + if (!ret) + sockopt_seq_inc(msk); + return ret; } diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 4ff5863aa9fdeb..84566553ac4420 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -581,7 +581,7 @@ static void subflow_finish_connect(struct sock *sk, const struct sk_buff *skb) subflow->backup); if (!subflow_thmac_valid(subflow)) { - MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINACKMAC); + MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_JOINSYNACKMAC); subflow->reset_reason = MPTCP_RST_EMPTCP; goto do_reset; } @@ -908,7 +908,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, if (!subflow_hmac_valid(subflow_req, &mp_opt)) { SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_JOINACKMAC); - subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); + subflow_add_reset_reason(skb, MPTCP_RST_EMPTCP); goto dispose_child; } diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index 3601eb86d02508..7c570f48ade28f 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -102,6 +102,18 @@ __ip_vs_dst_check(struct ip_vs_dest *dest) return dest_dst; } +/* Based on ip_exceeds_mtu(). */ +static bool ip_vs_exceeds_mtu(const struct sk_buff *skb, unsigned int mtu) +{ + if (skb->len <= mtu) + return false; + + if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu)) + return false; + + return true; +} + static inline bool __mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu) { @@ -111,10 +123,9 @@ __mtu_check_toobig_v6(const struct sk_buff *skb, u32 mtu) */ if (IP6CB(skb)->frag_max_size > mtu) return true; /* largest fragment violate MTU */ - } - else if (skb->len > mtu && !skb_is_gso(skb)) { + } else if (ip_vs_exceeds_mtu(skb, mtu)) return true; /* Packet size violate MTU size */ - } + return false; } @@ -232,7 +243,7 @@ static inline bool ensure_mtu_is_adequate(struct netns_ipvs *ipvs, int skb_af, return true; if (unlikely(ip_hdr(skb)->frag_off & htons(IP_DF) && - skb->len > mtu && !skb_is_gso(skb) && + ip_vs_exceeds_mtu(skb, mtu) && !ip_vs_iph_icmp(ipvsh))) { icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu)); diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index 645d2c43ebf7af..7e10fa65cbdd31 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -466,9 +466,13 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, if (!ih) goto out_unlock; - if (ct->proto.sctp.init[dir] && ct->proto.sctp.init[!dir]) - ct->proto.sctp.init[!dir] = 0; - ct->proto.sctp.init[dir] = 1; + /* Do not record INIT matching peer vtag (stale or retransmitted INIT). */ + if (old_state == SCTP_CONNTRACK_NONE || + ct->proto.sctp.vtag[!dir] != ih->init_tag) { + if (ct->proto.sctp.init[dir] && ct->proto.sctp.init[!dir]) + ct->proto.sctp.init[!dir] = 0; + ct->proto.sctp.init[dir] = 1; + } pr_debug("Setting vtag %x for dir %d\n", ih->init_tag, !dir); ct->proto.sctp.vtag[!dir] = ih->init_tag; diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index 939502ff7c8713..81534213f00f34 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -181,6 +181,57 @@ static int sip_parse_addr(const struct nf_conn *ct, const char *cp, return 1; } +/* Parse optional port number after IP address. + * Returns false on malformed input, true otherwise. + * If port is non-NULL, stores parsed port in network byte order. + * If no port is present, sets *port to default SIP port. + */ +static bool sip_parse_port(const char *dptr, const char **endp, + const char *limit, __be16 *port) +{ + unsigned int p = 0; + int len = 0; + + if (dptr >= limit) + return false; + + if (*dptr != ':') { + if (port) + *port = htons(SIP_PORT); + if (endp) + *endp = dptr; + return true; + } + + dptr++; /* skip ':' */ + + while (dptr < limit && isdigit(*dptr)) { + p = p * 10 + (*dptr - '0'); + dptr++; + len++; + if (len > 5) /* max "65535" */ + return false; + } + + if (len == 0) + return false; + + /* reached limit while parsing port */ + if (dptr >= limit) + return false; + + if (p < 1024 || p > 65535) + return false; + + if (port) + *port = htons(p); + + if (endp) + *endp = dptr; + + return true; +} + /* skip ip address. returns its length. */ static int epaddr_len(const struct nf_conn *ct, const char *dptr, const char *limit, int *shift) @@ -193,11 +244,8 @@ static int epaddr_len(const struct nf_conn *ct, const char *dptr, return 0; } - /* Port number */ - if (*dptr == ':') { - dptr++; - dptr += digits_len(ct, dptr, limit, shift); - } + if (!sip_parse_port(dptr, &dptr, limit, NULL)) + return 0; return dptr - aux; } @@ -228,6 +276,51 @@ static int skp_epaddr_len(const struct nf_conn *ct, const char *dptr, return epaddr_len(ct, dptr, limit, shift); } +/* simple_strtoul stops after first non-number character. + * But as we're not dealing with c-strings, we can't rely on + * hitting \r,\n,\0 etc. before moving past end of buffer. + * + * This is a variant of simple_strtoul, but doesn't require + * a c-string. + * + * If value exceeds UINT_MAX, 0 is returned. + */ +static unsigned int sip_strtouint(const char *cp, unsigned int len, char **endp) +{ + const unsigned int max = sizeof("4294967295"); + unsigned int olen = len; + const char *s = cp; + u64 result = 0; + + if (len > max) + len = max; + + while (olen > 0 && isdigit(*s)) { + unsigned int value; + + if (len == 0) + goto err; + + value = *s - '0'; + result = result * 10 + value; + + if (result > UINT_MAX) + goto err; + s++; + len--; + olen--; + } + + if (endp) + *endp = (char *)s; + + return result; +err: + if (endp) + *endp = (char *)cp; + return 0; +} + /* Parse a SIP request line of the form: * * Request-Line = Method SP Request-URI SP SIP-Version CRLF @@ -241,7 +334,6 @@ int ct_sip_parse_request(const struct nf_conn *ct, { const char *start = dptr, *limit = dptr + datalen, *end; unsigned int mlen; - unsigned int p; int shift = 0; /* Skip method and following whitespace */ @@ -267,14 +359,8 @@ int ct_sip_parse_request(const struct nf_conn *ct, if (!sip_parse_addr(ct, dptr, &end, addr, limit, true)) return -1; - if (end < limit && *end == ':') { - end++; - p = simple_strtoul(end, (char **)&end, 10); - if (p < 1024 || p > 65535) - return -1; - *port = htons(p); - } else - *port = htons(SIP_PORT); + if (!sip_parse_port(end, &end, limit, port)) + return -1; if (end == dptr) return 0; @@ -509,7 +595,6 @@ int ct_sip_parse_header_uri(const struct nf_conn *ct, const char *dptr, union nf_inet_addr *addr, __be16 *port) { const char *c, *limit = dptr + datalen; - unsigned int p; int ret; ret = ct_sip_walk_headers(ct, dptr, dataoff ? *dataoff : 0, datalen, @@ -520,14 +605,8 @@ int ct_sip_parse_header_uri(const struct nf_conn *ct, const char *dptr, if (!sip_parse_addr(ct, dptr + *matchoff, &c, addr, limit, true)) return -1; - if (*c == ':') { - c++; - p = simple_strtoul(c, (char **)&c, 10); - if (p < 1024 || p > 65535) - return -1; - *port = htons(p); - } else - *port = htons(SIP_PORT); + if (!sip_parse_port(c, &c, limit, port)) + return -1; if (dataoff) *dataoff = c - dptr; @@ -609,7 +688,7 @@ int ct_sip_parse_numerical_param(const struct nf_conn *ct, const char *dptr, return 0; start += strlen(name); - *val = simple_strtoul(start, &end, 0); + *val = sip_strtouint(start, limit - start, (char **)&end); if (start == end) return -1; if (matchoff && matchlen) { @@ -1065,6 +1144,8 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, mediaoff = sdpoff; for (i = 0; i < ARRAY_SIZE(sdp_media_types); ) { + char *end; + if (ct_sip_get_sdp_header(ct, *dptr, mediaoff, *datalen, SDP_HDR_MEDIA, SDP_HDR_UNSPEC, &mediaoff, &medialen) <= 0) @@ -1080,8 +1161,8 @@ static int process_sdp(struct sk_buff *skb, unsigned int protoff, mediaoff += t->len; medialen -= t->len; - port = simple_strtoul(*dptr + mediaoff, NULL, 10); - if (port == 0) + port = sip_strtouint(*dptr + mediaoff, *datalen - mediaoff, (char **)&end); + if (port == 0 || *dptr + mediaoff == end) continue; if (port < 1024 || port > 65535) { nf_ct_helper_log(skb, ct, "wrong port %u", port); @@ -1255,7 +1336,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, */ if (ct_sip_get_header(ct, *dptr, 0, *datalen, SIP_HDR_EXPIRES, &matchoff, &matchlen) > 0) - expires = simple_strtoul(*dptr + matchoff, NULL, 10); + expires = sip_strtouint(*dptr + matchoff, *datalen - matchoff, NULL); ret = ct_sip_parse_header_uri(ct, *dptr, NULL, *datalen, SIP_HDR_CONTACT, NULL, @@ -1286,6 +1367,10 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, goto store_cseq; } + helper = rcu_dereference(nfct_help(ct)->helper); + if (!helper) + return NF_DROP; + exp = nf_ct_expect_alloc(ct); if (!exp) { nf_ct_helper_log(skb, ct, "cannot alloc expectation"); @@ -1296,10 +1381,6 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, if (sip_direct_signalling) saddr = &ct->tuplehash[!dir].tuple.src.u3; - helper = rcu_dereference(nfct_help(ct)->helper); - if (!helper) - return NF_DROP; - nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); exp->timeout.expires = sip_timeout * HZ; @@ -1359,7 +1440,7 @@ static int process_register_response(struct sk_buff *skb, unsigned int protoff, if (ct_sip_get_header(ct, *dptr, 0, *datalen, SIP_HDR_EXPIRES, &matchoff, &matchlen) > 0) - expires = simple_strtoul(*dptr + matchoff, NULL, 10); + expires = sip_strtouint(*dptr + matchoff, *datalen - matchoff, NULL); while (1) { unsigned int c_expires = expires; @@ -1419,10 +1500,12 @@ static int process_sip_response(struct sk_buff *skb, unsigned int protoff, struct nf_conn *ct = nf_ct_get(skb, &ctinfo); unsigned int matchoff, matchlen, matchend; unsigned int code, cseq, i; + char *end; if (*datalen < strlen("SIP/2.0 200")) return NF_ACCEPT; - code = simple_strtoul(*dptr + strlen("SIP/2.0 "), NULL, 10); + code = sip_strtouint(*dptr + strlen("SIP/2.0 "), + *datalen - strlen("SIP/2.0 "), NULL); if (!code) { nf_ct_helper_log(skb, ct, "cannot get code"); return NF_DROP; @@ -1433,8 +1516,8 @@ static int process_sip_response(struct sk_buff *skb, unsigned int protoff, nf_ct_helper_log(skb, ct, "cannot parse cseq"); return NF_DROP; } - cseq = simple_strtoul(*dptr + matchoff, NULL, 10); - if (!cseq && *(*dptr + matchoff) != '0') { + cseq = sip_strtouint(*dptr + matchoff, *datalen - matchoff, (char **)&end); + if (*dptr + matchoff == end) { nf_ct_helper_log(skb, ct, "cannot get cseq"); return NF_DROP; } @@ -1483,6 +1566,7 @@ static int process_sip_request(struct sk_buff *skb, unsigned int protoff, for (i = 0; i < ARRAY_SIZE(sip_handlers); i++) { const struct sip_handler *handler; + char *end; handler = &sip_handlers[i]; if (handler->request == NULL) @@ -1499,8 +1583,8 @@ static int process_sip_request(struct sk_buff *skb, unsigned int protoff, nf_ct_helper_log(skb, ct, "cannot parse cseq"); return NF_DROP; } - cseq = simple_strtoul(*dptr + matchoff, NULL, 10); - if (!cseq && *(*dptr + matchoff) != '0') { + cseq = sip_strtouint(*dptr + matchoff, *datalen - matchoff, (char **)&end); + if (*dptr + matchoff == end) { nf_ct_helper_log(skb, ct, "cannot get cseq"); return NF_DROP; } @@ -1576,7 +1660,7 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, &matchoff, &matchlen) <= 0) break; - clen = simple_strtoul(dptr + matchoff, (char **)&end, 10); + clen = sip_strtouint(dptr + matchoff, datalen - matchoff, (char **)&end); if (dptr + matchoff == end) break; diff --git a/net/netfilter/nf_nat_amanda.c b/net/netfilter/nf_nat_amanda.c index 98deef6cde6943..8f1054920a8571 100644 --- a/net/netfilter/nf_nat_amanda.c +++ b/net/netfilter/nf_nat_amanda.c @@ -50,7 +50,7 @@ static unsigned int help(struct sk_buff *skb, return NF_DROP; } - sprintf(buffer, "%u", port); + snprintf(buffer, sizeof(buffer), "%u", port); if (!nf_nat_mangle_udp_packet(skb, exp->master, ctinfo, protoff, matchoff, matchlen, buffer, strlen(buffer))) { diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 3b5434e4ec9c19..b30ca94c2bb7ff 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -1228,9 +1228,11 @@ int nf_nat_register_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops, ret = nf_register_net_hooks(net, nat_ops, ops_count); if (ret < 0) { mutex_unlock(&nf_nat_proto_mutex); - for (i = 0; i < ops_count; i++) - kfree(nat_ops[i].priv); - kfree(nat_ops); + for (i = 0; i < ops_count; i++) { + priv = nat_ops[i].priv; + kfree_rcu(priv, rcu_head); + } + kfree_rcu(nat_ops, rcu); return ret; } @@ -1294,7 +1296,7 @@ void nf_nat_unregister_fn(struct net *net, u8 pf, const struct nf_hook_ops *ops, } nat_proto_net->nat_hook_ops = NULL; - kfree(nat_ops); + kfree_rcu(nat_ops, rcu); } unlock: mutex_unlock(&nf_nat_proto_mutex); diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index cf4aeb299bdef2..9fbfc6bff0c221 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -68,25 +68,27 @@ static unsigned int mangle_packet(struct sk_buff *skb, unsigned int protoff, } static int sip_sprintf_addr(const struct nf_conn *ct, char *buffer, + size_t size, const union nf_inet_addr *addr, bool delim) { if (nf_ct_l3num(ct) == NFPROTO_IPV4) - return sprintf(buffer, "%pI4", &addr->ip); + return scnprintf(buffer, size, "%pI4", &addr->ip); else { if (delim) - return sprintf(buffer, "[%pI6c]", &addr->ip6); + return scnprintf(buffer, size, "[%pI6c]", &addr->ip6); else - return sprintf(buffer, "%pI6c", &addr->ip6); + return scnprintf(buffer, size, "%pI6c", &addr->ip6); } } static int sip_sprintf_addr_port(const struct nf_conn *ct, char *buffer, + size_t size, const union nf_inet_addr *addr, u16 port) { if (nf_ct_l3num(ct) == NFPROTO_IPV4) - return sprintf(buffer, "%pI4:%u", &addr->ip, port); + return scnprintf(buffer, size, "%pI4:%u", &addr->ip, port); else - return sprintf(buffer, "[%pI6c]:%u", &addr->ip6, port); + return scnprintf(buffer, size, "[%pI6c]:%u", &addr->ip6, port); } static int map_addr(struct sk_buff *skb, unsigned int protoff, @@ -119,7 +121,7 @@ static int map_addr(struct sk_buff *skb, unsigned int protoff, if (nf_inet_addr_cmp(&newaddr, addr) && newport == port) return 1; - buflen = sip_sprintf_addr_port(ct, buffer, &newaddr, ntohs(newport)); + buflen = sip_sprintf_addr_port(ct, buffer, sizeof(buffer), &newaddr, ntohs(newport)); return mangle_packet(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, buffer, buflen); } @@ -212,7 +214,7 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, &addr, true) > 0 && nf_inet_addr_cmp(&addr, &ct->tuplehash[dir].tuple.src.u3) && !nf_inet_addr_cmp(&addr, &ct->tuplehash[!dir].tuple.dst.u3)) { - buflen = sip_sprintf_addr(ct, buffer, + buflen = sip_sprintf_addr(ct, buffer, sizeof(buffer), &ct->tuplehash[!dir].tuple.dst.u3, true); if (!mangle_packet(skb, protoff, dataoff, dptr, datalen, @@ -229,7 +231,7 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, &addr, false) > 0 && nf_inet_addr_cmp(&addr, &ct->tuplehash[dir].tuple.dst.u3) && !nf_inet_addr_cmp(&addr, &ct->tuplehash[!dir].tuple.src.u3)) { - buflen = sip_sprintf_addr(ct, buffer, + buflen = sip_sprintf_addr(ct, buffer, sizeof(buffer), &ct->tuplehash[!dir].tuple.src.u3, false); if (!mangle_packet(skb, protoff, dataoff, dptr, datalen, @@ -244,10 +246,11 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, if (ct_sip_parse_numerical_param(ct, *dptr, matchend, *datalen, "rport=", &poff, &plen, &n) > 0 && + n >= 1024 && n <= 65535 && htons(n) == ct->tuplehash[dir].tuple.dst.u.udp.port && htons(n) != ct->tuplehash[!dir].tuple.src.u.udp.port) { __be16 p = ct->tuplehash[!dir].tuple.src.u.udp.port; - buflen = sprintf(buffer, "%u", ntohs(p)); + buflen = scnprintf(buffer, sizeof(buffer), "%u", ntohs(p)); if (!mangle_packet(skb, protoff, dataoff, dptr, datalen, poff, plen, buffer, buflen)) { nf_ct_helper_log(skb, ct, "cannot mangle rport"); @@ -418,7 +421,8 @@ static unsigned int nf_nat_sip_expect(struct sk_buff *skb, unsigned int protoff, if (!nf_inet_addr_cmp(&exp->tuple.dst.u3, &exp->saved_addr) || exp->tuple.dst.u.udp.port != exp->saved_proto.udp.port) { - buflen = sip_sprintf_addr_port(ct, buffer, &newaddr, port); + buflen = sip_sprintf_addr_port(ct, buffer, sizeof(buffer), + &newaddr, port); if (!mangle_packet(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, buffer, buflen)) { nf_ct_helper_log(skb, ct, "cannot mangle packet"); @@ -438,8 +442,8 @@ static int mangle_content_len(struct sk_buff *skb, unsigned int protoff, { enum ip_conntrack_info ctinfo; struct nf_conn *ct = nf_ct_get(skb, &ctinfo); + char buffer[sizeof("4294967295")]; unsigned int matchoff, matchlen; - char buffer[sizeof("65536")]; int buflen, c_len; /* Get actual SDP length */ @@ -454,7 +458,7 @@ static int mangle_content_len(struct sk_buff *skb, unsigned int protoff, &matchoff, &matchlen) <= 0) return 0; - buflen = sprintf(buffer, "%u", c_len); + buflen = scnprintf(buffer, sizeof(buffer), "%u", c_len); return mangle_packet(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, buffer, buflen); } @@ -491,7 +495,7 @@ static unsigned int nf_nat_sdp_addr(struct sk_buff *skb, unsigned int protoff, char buffer[INET6_ADDRSTRLEN]; unsigned int buflen; - buflen = sip_sprintf_addr(ct, buffer, addr, false); + buflen = sip_sprintf_addr(ct, buffer, sizeof(buffer), addr, false); if (mangle_sdp_packet(skb, protoff, dataoff, dptr, datalen, sdpoff, type, term, buffer, buflen)) return 0; @@ -509,7 +513,7 @@ static unsigned int nf_nat_sdp_port(struct sk_buff *skb, unsigned int protoff, char buffer[sizeof("nnnnn")]; unsigned int buflen; - buflen = sprintf(buffer, "%u", port); + buflen = scnprintf(buffer, sizeof(buffer), "%u", port); if (!mangle_packet(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, buffer, buflen)) return 0; @@ -529,7 +533,7 @@ static unsigned int nf_nat_sdp_session(struct sk_buff *skb, unsigned int protoff unsigned int buflen; /* Mangle session description owner and contact addresses */ - buflen = sip_sprintf_addr(ct, buffer, addr, false); + buflen = sip_sprintf_addr(ct, buffer, sizeof(buffer), addr, false); if (mangle_sdp_packet(skb, protoff, dataoff, dptr, datalen, sdpoff, SDP_HDR_OWNER, SDP_HDR_MEDIA, buffer, buflen)) return 0; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 8c42247a176c7b..7927cd48798bf3 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -374,6 +374,38 @@ static void nft_netdev_hook_free_rcu(struct nft_hook *hook) call_rcu(&hook->rcu, __nft_netdev_hook_free_rcu); } +static void nft_netdev_hook_unlink_free_rcu(struct nft_hook *hook) +{ + list_del_rcu(&hook->list); + nft_netdev_hook_free_rcu(hook); +} + +static void nft_trans_hook_destroy(struct nft_trans_hook *trans_hook) +{ + list_del(&trans_hook->list); + kfree(trans_hook); +} + +static void nft_netdev_unregister_trans_hook(struct net *net, + const struct nft_table *table, + struct list_head *hook_list) +{ + struct nft_trans_hook *trans_hook, *next; + struct nf_hook_ops *ops; + struct nft_hook *hook; + + list_for_each_entry_safe(trans_hook, next, hook_list, list) { + hook = trans_hook->hook; + + if (!(table->flags & NFT_TABLE_F_DORMANT)) { + list_for_each_entry(ops, &hook->ops_list, list) + nf_unregister_net_hook(net, ops); + } + nft_netdev_hook_unlink_free_rcu(hook); + nft_trans_hook_destroy(trans_hook); + } +} + static void nft_netdev_unregister_hooks(struct net *net, struct list_head *hook_list, bool release_netdev) @@ -384,10 +416,8 @@ static void nft_netdev_unregister_hooks(struct net *net, list_for_each_entry_safe(hook, next, hook_list, list) { list_for_each_entry(ops, &hook->ops_list, list) nf_unregister_net_hook(net, ops); - if (release_netdev) { - list_del(&hook->list); - nft_netdev_hook_free_rcu(hook); - } + if (release_netdev) + nft_netdev_hook_unlink_free_rcu(hook); } } @@ -1994,15 +2024,69 @@ static int nft_nla_put_hook_dev(struct sk_buff *skb, struct nft_hook *hook) return nla_put_string(skb, attr, hook->ifname); } +struct nft_hook_dump_ctx { + struct nft_hook *first; + int n; +}; + +static int nft_dump_basechain_hook_one(struct sk_buff *skb, + struct nft_hook *hook, + struct nft_hook_dump_ctx *dump_ctx) +{ + if (!dump_ctx->first) + dump_ctx->first = hook; + + if (nft_nla_put_hook_dev(skb, hook)) + return -1; + + dump_ctx->n++; + + return 0; +} + +static int nft_dump_basechain_hook_list(struct sk_buff *skb, + const struct net *net, + const struct list_head *hook_list, + struct nft_hook_dump_ctx *dump_ctx) +{ + struct nft_hook *hook; + int err; + + list_for_each_entry_rcu(hook, hook_list, list, + lockdep_commit_lock_is_held(net)) { + err = nft_dump_basechain_hook_one(skb, hook, dump_ctx); + if (err < 0) + return err; + } + + return 0; +} + +static int nft_dump_basechain_trans_hook_list(struct sk_buff *skb, + const struct list_head *trans_hook_list, + struct nft_hook_dump_ctx *dump_ctx) +{ + struct nft_trans_hook *trans_hook; + int err; + + list_for_each_entry(trans_hook, trans_hook_list, list) { + err = nft_dump_basechain_hook_one(skb, trans_hook->hook, dump_ctx); + if (err < 0) + return err; + } + + return 0; +} + static int nft_dump_basechain_hook(struct sk_buff *skb, const struct net *net, int family, const struct nft_base_chain *basechain, - const struct list_head *hook_list) + const struct list_head *hook_list, + const struct list_head *trans_hook_list) { const struct nf_hook_ops *ops = &basechain->ops; - struct nft_hook *hook, *first = NULL; + struct nft_hook_dump_ctx dump_hook_ctx = {}; struct nlattr *nest, *nest_devs; - int n = 0; nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK); if (nest == NULL) @@ -2017,23 +2101,23 @@ static int nft_dump_basechain_hook(struct sk_buff *skb, if (!nest_devs) goto nla_put_failure; - if (!hook_list) + if (!hook_list && !trans_hook_list) hook_list = &basechain->hook_list; - list_for_each_entry_rcu(hook, hook_list, list, - lockdep_commit_lock_is_held(net)) { - if (!first) - first = hook; - - if (nft_nla_put_hook_dev(skb, hook)) - goto nla_put_failure; - n++; + if (hook_list && + nft_dump_basechain_hook_list(skb, net, hook_list, &dump_hook_ctx)) { + goto nla_put_failure; + } else if (trans_hook_list && + nft_dump_basechain_trans_hook_list(skb, trans_hook_list, + &dump_hook_ctx)) { + goto nla_put_failure; } + nla_nest_end(skb, nest_devs); - if (n == 1 && - !hook_is_prefix(first) && - nla_put_string(skb, NFTA_HOOK_DEV, first->ifname)) + if (dump_hook_ctx.n == 1 && + !hook_is_prefix(dump_hook_ctx.first) && + nla_put_string(skb, NFTA_HOOK_DEV, dump_hook_ctx.first->ifname)) goto nla_put_failure; } nla_nest_end(skb, nest); @@ -2047,7 +2131,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, u32 portid, u32 seq, int event, u32 flags, int family, const struct nft_table *table, const struct nft_chain *chain, - const struct list_head *hook_list) + const struct list_head *hook_list, + const struct list_head *trans_hook_list) { struct nlmsghdr *nlh; @@ -2063,7 +2148,7 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, NFTA_CHAIN_PAD)) goto nla_put_failure; - if (!hook_list && + if (!hook_list && !trans_hook_list && (event == NFT_MSG_DELCHAIN || event == NFT_MSG_DESTROYCHAIN)) { nlmsg_end(skb, nlh); @@ -2074,7 +2159,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, const struct nft_base_chain *basechain = nft_base_chain(chain); struct nft_stats __percpu *stats; - if (nft_dump_basechain_hook(skb, net, family, basechain, hook_list)) + if (nft_dump_basechain_hook(skb, net, family, basechain, + hook_list, trans_hook_list)) goto nla_put_failure; if (nla_put_be32(skb, NFTA_CHAIN_POLICY, @@ -2110,7 +2196,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, } static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event, - const struct list_head *hook_list) + const struct list_head *hook_list, + const struct list_head *trans_hook_list) { struct nftables_pernet *nft_net; struct sk_buff *skb; @@ -2130,7 +2217,7 @@ static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event, err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq, event, flags, ctx->family, ctx->table, - ctx->chain, hook_list); + ctx->chain, hook_list, trans_hook_list); if (err < 0) { kfree_skb(skb); goto err; @@ -2176,7 +2263,7 @@ static int nf_tables_dump_chains(struct sk_buff *skb, NFT_MSG_NEWCHAIN, NLM_F_MULTI, table->family, table, - chain, NULL) < 0) + chain, NULL, NULL) < 0) goto done; nl_dump_check_consistent(cb, nlmsg_hdr(skb)); @@ -2230,7 +2317,7 @@ static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info, err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN, - 0, family, table, chain, NULL); + 0, family, table, chain, NULL, NULL); if (err < 0) goto err_fill_chain_info; @@ -2323,10 +2410,8 @@ void nf_tables_chain_destroy(struct nft_chain *chain) if (nft_base_chain_netdev(table->family, basechain->ops.hooknum)) { list_for_each_entry_safe(hook, next, - &basechain->hook_list, list) { - list_del_rcu(&hook->list); - nft_netdev_hook_free_rcu(hook); - } + &basechain->hook_list, list) + nft_netdev_hook_unlink_free_rcu(hook); } module_put(basechain->type->owner); if (rcu_access_pointer(basechain->stats)) { @@ -2395,8 +2480,12 @@ static struct nft_hook *nft_hook_list_find(struct list_head *hook_list, list_for_each_entry(hook, hook_list, list) { if (!strncmp(hook->ifname, this->ifname, - min(hook->ifnamelen, this->ifnamelen))) + min(hook->ifnamelen, this->ifnamelen))) { + if (hook->flags & NFT_HOOK_REMOVE) + continue; + return hook; + } } return NULL; @@ -3026,6 +3115,7 @@ static int nf_tables_updchain(struct nft_ctx *ctx, u8 genmask, u8 policy, list_for_each_entry(ops, &h->ops_list, list) nf_unregister_net_hook(ctx->net, ops); } + /* hook.list is on stack, no need for list_del_rcu() */ list_del(&h->list); nft_netdev_hook_free_rcu(h); } @@ -3154,6 +3244,32 @@ static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info, return nf_tables_addchain(&ctx, family, policy, flags, extack); } +static int nft_trans_delhook(struct nft_hook *hook, + struct list_head *del_list) +{ + struct nft_trans_hook *trans_hook; + + trans_hook = kmalloc_obj(*trans_hook, GFP_KERNEL); + if (!trans_hook) + return -ENOMEM; + + trans_hook->hook = hook; + list_add_tail(&trans_hook->list, del_list); + hook->flags |= NFT_HOOK_REMOVE; + + return 0; +} + +static void nft_trans_delhook_abort(struct list_head *del_list) +{ + struct nft_trans_hook *trans_hook, *next; + + list_for_each_entry_safe(trans_hook, next, del_list, list) { + trans_hook->hook->flags &= ~NFT_HOOK_REMOVE; + nft_trans_hook_destroy(trans_hook); + } +} + static int nft_delchain_hook(struct nft_ctx *ctx, struct nft_base_chain *basechain, struct netlink_ext_ack *extack) @@ -3180,7 +3296,10 @@ static int nft_delchain_hook(struct nft_ctx *ctx, err = -ENOENT; goto err_chain_del_hook; } - list_move(&hook->list, &chain_del_list); + if (nft_trans_delhook(hook, &chain_del_list) < 0) { + err = -ENOMEM; + goto err_chain_del_hook; + } } trans = nft_trans_alloc_chain(ctx, NFT_MSG_DELCHAIN); @@ -3200,7 +3319,7 @@ static int nft_delchain_hook(struct nft_ctx *ctx, return 0; err_chain_del_hook: - list_splice(&chain_del_list, &basechain->hook_list); + nft_trans_delhook_abort(&chain_del_list); nft_chain_release_hook(&chain_hook); return err; @@ -8903,10 +9022,8 @@ static void __nft_unregister_flowtable_net_hooks(struct net *net, list_for_each_entry_safe(hook, next, hook_list, list) { list_for_each_entry(ops, &hook->ops_list, list) nft_unregister_flowtable_ops(net, flowtable, ops); - if (release_netdev) { - list_del(&hook->list); - nft_netdev_hook_free_rcu(hook); - } + if (release_netdev) + nft_netdev_hook_unlink_free_rcu(hook); } } @@ -8977,8 +9094,7 @@ static int nft_register_flowtable_net_hooks(struct net *net, nft_unregister_flowtable_ops(net, flowtable, ops); } - list_del_rcu(&hook->list); - nft_netdev_hook_free_rcu(hook); + nft_netdev_hook_unlink_free_rcu(hook); } return err; @@ -8988,9 +9104,25 @@ static void nft_hooks_destroy(struct list_head *hook_list) { struct nft_hook *hook, *next; - list_for_each_entry_safe(hook, next, hook_list, list) { - list_del_rcu(&hook->list); - nft_netdev_hook_free_rcu(hook); + list_for_each_entry_safe(hook, next, hook_list, list) + nft_netdev_hook_unlink_free_rcu(hook); +} + +static void nft_flowtable_unregister_trans_hook(struct net *net, + struct nft_flowtable *flowtable, + struct list_head *hook_list) +{ + struct nft_trans_hook *trans_hook, *next; + struct nf_hook_ops *ops; + struct nft_hook *hook; + + list_for_each_entry_safe(trans_hook, next, hook_list, list) { + hook = trans_hook->hook; + list_for_each_entry(ops, &hook->ops_list, list) + nft_unregister_flowtable_ops(net, flowtable, ops); + + nft_netdev_hook_unlink_free_rcu(hook); + nft_trans_hook_destroy(trans_hook); } } @@ -9079,8 +9211,7 @@ static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh, nft_unregister_flowtable_ops(ctx->net, flowtable, ops); } - list_del_rcu(&hook->list); - nft_netdev_hook_free_rcu(hook); + nft_netdev_hook_unlink_free_rcu(hook); } return err; @@ -9253,7 +9384,10 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx, err = -ENOENT; goto err_flowtable_del_hook; } - list_move(&hook->list, &flowtable_del_list); + if (nft_trans_delhook(hook, &flowtable_del_list) < 0) { + err = -ENOMEM; + goto err_flowtable_del_hook; + } } trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE, @@ -9274,7 +9408,7 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx, return 0; err_flowtable_del_hook: - list_splice(&flowtable_del_list, &flowtable->hook_list); + nft_trans_delhook_abort(&flowtable_del_list); nft_flowtable_hook_release(&flowtable_hook); return err; @@ -9339,8 +9473,10 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net, u32 portid, u32 seq, int event, u32 flags, int family, struct nft_flowtable *flowtable, - struct list_head *hook_list) + struct list_head *hook_list, + struct list_head *trans_hook_list) { + struct nft_trans_hook *trans_hook; struct nlattr *nest, *nest_devs; struct nft_hook *hook; struct nlmsghdr *nlh; @@ -9357,7 +9493,7 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net, NFTA_FLOWTABLE_PAD)) goto nla_put_failure; - if (!hook_list && + if (!hook_list && !trans_hook_list && (event == NFT_MSG_DELFLOWTABLE || event == NFT_MSG_DESTROYFLOWTABLE)) { nlmsg_end(skb, nlh); @@ -9379,13 +9515,20 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net, if (!nest_devs) goto nla_put_failure; - if (!hook_list) + if (!hook_list && !trans_hook_list) hook_list = &flowtable->hook_list; - list_for_each_entry_rcu(hook, hook_list, list, - lockdep_commit_lock_is_held(net)) { - if (nft_nla_put_hook_dev(skb, hook)) - goto nla_put_failure; + if (hook_list) { + list_for_each_entry_rcu(hook, hook_list, list, + lockdep_commit_lock_is_held(net)) { + if (nft_nla_put_hook_dev(skb, hook)) + goto nla_put_failure; + } + } else if (trans_hook_list) { + list_for_each_entry(trans_hook, trans_hook_list, list) { + if (nft_nla_put_hook_dev(skb, trans_hook->hook)) + goto nla_put_failure; + } } nla_nest_end(skb, nest_devs); nla_nest_end(skb, nest); @@ -9439,7 +9582,7 @@ static int nf_tables_dump_flowtable(struct sk_buff *skb, NFT_MSG_NEWFLOWTABLE, NLM_F_MULTI | NLM_F_APPEND, table->family, - flowtable, NULL) < 0) + flowtable, NULL, NULL) < 0) goto done; nl_dump_check_consistent(cb, nlmsg_hdr(skb)); @@ -9539,7 +9682,7 @@ static int nf_tables_getflowtable(struct sk_buff *skb, err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, NFT_MSG_NEWFLOWTABLE, 0, family, - flowtable, NULL); + flowtable, NULL, NULL); if (err < 0) goto err_fill_flowtable_info; @@ -9552,7 +9695,9 @@ static int nf_tables_getflowtable(struct sk_buff *skb, static void nf_tables_flowtable_notify(struct nft_ctx *ctx, struct nft_flowtable *flowtable, - struct list_head *hook_list, int event) + struct list_head *hook_list, + struct list_head *trans_hook_list, + int event) { struct nftables_pernet *nft_net = nft_pernet(ctx->net); struct sk_buff *skb; @@ -9572,7 +9717,8 @@ static void nf_tables_flowtable_notify(struct nft_ctx *ctx, err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid, ctx->seq, event, flags, - ctx->family, flowtable, hook_list); + ctx->family, flowtable, + hook_list, trans_hook_list); if (err < 0) { kfree_skb(skb); goto err; @@ -9586,13 +9732,8 @@ static void nf_tables_flowtable_notify(struct nft_ctx *ctx, static void nf_tables_flowtable_destroy(struct nft_flowtable *flowtable) { - struct nft_hook *hook, *next; - flowtable->data.type->free(&flowtable->data); - list_for_each_entry_safe(hook, next, &flowtable->hook_list, list) { - list_del_rcu(&hook->list); - nft_netdev_hook_free_rcu(hook); - } + nft_hooks_destroy(&flowtable->hook_list); kfree(flowtable->name); module_put(flowtable->data.type->owner); kfree(flowtable); @@ -10111,9 +10252,7 @@ static void nft_commit_release(struct nft_trans *trans) break; case NFT_MSG_DELCHAIN: case NFT_MSG_DESTROYCHAIN: - if (nft_trans_chain_update(trans)) - nft_hooks_destroy(&nft_trans_chain_hooks(trans)); - else + if (!nft_trans_chain_update(trans)) nf_tables_chain_destroy(nft_trans_chain(trans)); break; case NFT_MSG_DELRULE: @@ -10134,9 +10273,7 @@ static void nft_commit_release(struct nft_trans *trans) break; case NFT_MSG_DELFLOWTABLE: case NFT_MSG_DESTROYFLOWTABLE: - if (nft_trans_flowtable_update(trans)) - nft_hooks_destroy(&nft_trans_flowtable_hooks(trans)); - else + if (!nft_trans_flowtable_update(trans)) nf_tables_flowtable_destroy(nft_trans_flowtable(trans)); break; } @@ -10911,31 +11048,28 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) if (nft_trans_chain_update(trans)) { nft_chain_commit_update(nft_trans_container_chain(trans)); nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, - &nft_trans_chain_hooks(trans)); - list_splice(&nft_trans_chain_hooks(trans), - &nft_trans_basechain(trans)->hook_list); + &nft_trans_chain_hooks(trans), NULL); + list_splice_rcu(&nft_trans_chain_hooks(trans), + &nft_trans_basechain(trans)->hook_list); /* trans destroyed after rcu grace period */ } else { nft_chain_commit_drop_policy(nft_trans_container_chain(trans)); nft_clear(net, nft_trans_chain(trans)); - nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, NULL); + nf_tables_chain_notify(&ctx, NFT_MSG_NEWCHAIN, NULL, NULL); nft_trans_destroy(trans); } break; case NFT_MSG_DELCHAIN: case NFT_MSG_DESTROYCHAIN: if (nft_trans_chain_update(trans)) { - nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN, + nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN, NULL, &nft_trans_chain_hooks(trans)); - if (!(table->flags & NFT_TABLE_F_DORMANT)) { - nft_netdev_unregister_hooks(net, - &nft_trans_chain_hooks(trans), - true); - } + nft_netdev_unregister_trans_hook(net, table, + &nft_trans_chain_hooks(trans)); } else { nft_chain_del(nft_trans_chain(trans)); nf_tables_chain_notify(&ctx, NFT_MSG_DELCHAIN, - NULL); + NULL, NULL); nf_tables_unregister_hook(ctx.net, ctx.table, nft_trans_chain(trans)); } @@ -11041,14 +11175,16 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) nf_tables_flowtable_notify(&ctx, nft_trans_flowtable(trans), &nft_trans_flowtable_hooks(trans), + NULL, NFT_MSG_NEWFLOWTABLE); - list_splice(&nft_trans_flowtable_hooks(trans), - &nft_trans_flowtable(trans)->hook_list); + list_splice_rcu(&nft_trans_flowtable_hooks(trans), + &nft_trans_flowtable(trans)->hook_list); } else { nft_clear(net, nft_trans_flowtable(trans)); nf_tables_flowtable_notify(&ctx, nft_trans_flowtable(trans), NULL, + NULL, NFT_MSG_NEWFLOWTABLE); } nft_trans_destroy(trans); @@ -11058,16 +11194,18 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) if (nft_trans_flowtable_update(trans)) { nf_tables_flowtable_notify(&ctx, nft_trans_flowtable(trans), + NULL, &nft_trans_flowtable_hooks(trans), trans->msg_type); - nft_unregister_flowtable_net_hooks(net, - nft_trans_flowtable(trans), - &nft_trans_flowtable_hooks(trans)); + nft_flowtable_unregister_trans_hook(net, + nft_trans_flowtable(trans), + &nft_trans_flowtable_hooks(trans)); } else { list_del_rcu(&nft_trans_flowtable(trans)->list); nf_tables_flowtable_notify(&ctx, nft_trans_flowtable(trans), NULL, + NULL, trans->msg_type); nft_unregister_flowtable_net_hooks(net, nft_trans_flowtable(trans), @@ -11231,8 +11369,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) case NFT_MSG_DELCHAIN: case NFT_MSG_DESTROYCHAIN: if (nft_trans_chain_update(trans)) { - list_splice(&nft_trans_chain_hooks(trans), - &nft_trans_basechain(trans)->hook_list); + nft_trans_delhook_abort(&nft_trans_chain_hooks(trans)); } else { nft_use_inc_restore(&table->use); nft_clear(trans->net, nft_trans_chain(trans)); @@ -11346,8 +11483,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) case NFT_MSG_DELFLOWTABLE: case NFT_MSG_DESTROYFLOWTABLE: if (nft_trans_flowtable_update(trans)) { - list_splice(&nft_trans_flowtable_hooks(trans), - &nft_trans_flowtable(trans)->hook_list); + nft_trans_delhook_abort(&nft_trans_flowtable_hooks(trans)); } else { nft_use_inc_restore(&table->use); nft_clear(trans->net, nft_trans_flowtable(trans)); diff --git a/net/netfilter/nfnetlink_osf.c b/net/netfilter/nfnetlink_osf.c index 45d9ad231a9204..426b85fca8f75f 100644 --- a/net/netfilter/nfnetlink_osf.c +++ b/net/netfilter/nfnetlink_osf.c @@ -31,26 +31,18 @@ EXPORT_SYMBOL_GPL(nf_osf_fingers); static inline int nf_osf_ttl(const struct sk_buff *skb, int ttl_check, unsigned char f_ttl) { - struct in_device *in_dev = __in_dev_get_rcu(skb->dev); const struct iphdr *ip = ip_hdr(skb); - const struct in_ifaddr *ifa; - int ret = 0; - if (ttl_check == NF_OSF_TTL_TRUE) + switch (ttl_check) { + case NF_OSF_TTL_TRUE: return ip->ttl == f_ttl; - if (ttl_check == NF_OSF_TTL_NOCHECK) - return 1; - else if (ip->ttl <= f_ttl) + break; + case NF_OSF_TTL_NOCHECK: return 1; - - in_dev_for_each_ifa_rcu(ifa, in_dev) { - if (inet_ifa_match(ip->saddr, ifa)) { - ret = (ip->ttl == f_ttl); - break; - } + case NF_OSF_TTL_LESS: + default: + return ip->ttl <= f_ttl; } - - return ret; } struct nf_osf_hdr_ctx { @@ -64,9 +56,9 @@ struct nf_osf_hdr_ctx { static bool nf_osf_match_one(const struct sk_buff *skb, const struct nf_osf_user_finger *f, int ttl_check, - struct nf_osf_hdr_ctx *ctx) + const struct nf_osf_hdr_ctx *ctx) { - const __u8 *optpinit = ctx->optp; + const __u8 *optp = ctx->optp; unsigned int check_WSS = 0; int fmatch = FMATCH_WRONG; int foptsize, optnum; @@ -95,17 +87,17 @@ static bool nf_osf_match_one(const struct sk_buff *skb, check_WSS = f->wss.wc; for (optnum = 0; optnum < f->opt_num; ++optnum) { - if (f->opt[optnum].kind == *ctx->optp) { + if (f->opt[optnum].kind == *optp) { __u32 len = f->opt[optnum].length; - const __u8 *optend = ctx->optp + len; + const __u8 *optend = optp + len; fmatch = FMATCH_OK; - switch (*ctx->optp) { + switch (*optp) { case OSFOPT_MSS: - mss = ctx->optp[3]; + mss = optp[3]; mss <<= 8; - mss |= ctx->optp[2]; + mss |= optp[2]; mss = ntohs((__force __be16)mss); break; @@ -113,7 +105,7 @@ static bool nf_osf_match_one(const struct sk_buff *skb, break; } - ctx->optp = optend; + optp = optend; } else fmatch = FMATCH_OPT_WRONG; @@ -156,9 +148,6 @@ static bool nf_osf_match_one(const struct sk_buff *skb, } } - if (fmatch != FMATCH_OK) - ctx->optp = optpinit; - return fmatch == FMATCH_OK; } @@ -320,6 +309,10 @@ static int nfnl_osf_add_callback(struct sk_buff *skb, if (f->opt_num > ARRAY_SIZE(f->opt)) return -EINVAL; + if (f->wss.wc >= OSF_WSS_MAX || + (f->wss.wc == OSF_WSS_MODULO && f->wss.val == 0)) + return -EINVAL; + for (i = 0; i < f->opt_num; i++) { if (!f->opt[i].length || f->opt[i].length > MAX_IPOPTLEN) return -EINVAL; diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c index d550910aabec9b..af990c600745be 100644 --- a/net/netfilter/nft_bitwise.c +++ b/net/netfilter/nft_bitwise.c @@ -196,7 +196,8 @@ static int nft_bitwise_init_shift(struct nft_bitwise *priv, if (err < 0) return err; - if (priv->data.data[0] >= BITS_PER_TYPE(u32)) { + if (!priv->data.data[0] || + priv->data.data[0] >= BITS_PER_TYPE(u32)) { nft_data_release(&priv->data, desc.type); return -EINVAL; } diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index 04c74ccf9b848a..272ce181180777 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -1381,6 +1381,8 @@ static void nft_ct_expect_obj_eval(struct nft_object *obj, if (nf_ct_expect_related(exp, 0) != 0) regs->verdict.code = NF_DROP; + + nf_ct_expect_put(exp); } static const struct nla_policy nft_ct_expect_policy[NFTA_CT_EXPECT_MAX + 1] = { diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c index 152a9fb4d23af5..256e832f1bb99b 100644 --- a/net/netfilter/nft_fwd_netdev.c +++ b/net/netfilter/nft_fwd_netdev.c @@ -116,6 +116,11 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr, goto out; } iph = ip_hdr(skb); + if (iph->ttl <= 1) { + verdict = NF_DROP; + goto out; + } + ip_decrease_ttl(iph); neigh_table = NEIGH_ARP_TABLE; break; @@ -132,6 +137,11 @@ static void nft_fwd_neigh_eval(const struct nft_expr *expr, goto out; } ip6h = ipv6_hdr(skb); + if (ip6h->hop_limit <= 1) { + verdict = NF_DROP; + goto out; + } + ip6h->hop_limit--; neigh_table = NEIGH_ND_TABLE; break; diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c index 1c0b493ef0a994..bdc2f6c90e2f79 100644 --- a/net/netfilter/nft_osf.c +++ b/net/netfilter/nft_osf.c @@ -28,6 +28,11 @@ static void nft_osf_eval(const struct nft_expr *expr, struct nft_regs *regs, struct nf_osf_data data; struct tcphdr _tcph; + if (nft_pf(pkt) != NFPROTO_IPV4) { + regs->verdict.code = NFT_BREAK; + return; + } + if (pkt->tprot != IPPROTO_TCP) { regs->verdict.code = NFT_BREAK; return; @@ -114,7 +119,6 @@ static int nft_osf_validate(const struct nft_ctx *ctx, switch (ctx->family) { case NFPROTO_IPV4: - case NFPROTO_IPV6: case NFPROTO_INET: hooks = (1 << NF_INET_LOCAL_IN) | (1 << NF_INET_PRE_ROUTING) | diff --git a/net/netfilter/xt_mac.c b/net/netfilter/xt_mac.c index 81649da57ba5d3..bd2354760895d2 100644 --- a/net/netfilter/xt_mac.c +++ b/net/netfilter/xt_mac.c @@ -38,25 +38,37 @@ static bool mac_mt(const struct sk_buff *skb, struct xt_action_param *par) return ret; } -static struct xt_match mac_mt_reg __read_mostly = { - .name = "mac", - .revision = 0, - .family = NFPROTO_UNSPEC, - .match = mac_mt, - .matchsize = sizeof(struct xt_mac_info), - .hooks = (1 << NF_INET_PRE_ROUTING) | (1 << NF_INET_LOCAL_IN) | - (1 << NF_INET_FORWARD), - .me = THIS_MODULE, +static struct xt_match mac_mt_reg[] __read_mostly = { + { + .name = "mac", + .family = NFPROTO_IPV4, + .match = mac_mt, + .matchsize = sizeof(struct xt_mac_info), + .hooks = (1 << NF_INET_PRE_ROUTING) | + (1 << NF_INET_LOCAL_IN) | + (1 << NF_INET_FORWARD), + .me = THIS_MODULE, + }, + { + .name = "mac", + .family = NFPROTO_IPV6, + .match = mac_mt, + .matchsize = sizeof(struct xt_mac_info), + .hooks = (1 << NF_INET_PRE_ROUTING) | + (1 << NF_INET_LOCAL_IN) | + (1 << NF_INET_FORWARD), + .me = THIS_MODULE, + }, }; static int __init mac_mt_init(void) { - return xt_register_match(&mac_mt_reg); + return xt_register_matches(mac_mt_reg, ARRAY_SIZE(mac_mt_reg)); } static void __exit mac_mt_exit(void) { - xt_unregister_match(&mac_mt_reg); + xt_unregister_matches(mac_mt_reg, ARRAY_SIZE(mac_mt_reg)); } module_init(mac_mt_init); diff --git a/net/netfilter/xt_owner.c b/net/netfilter/xt_owner.c index 50332888c8d233..7be2fe22b067e8 100644 --- a/net/netfilter/xt_owner.c +++ b/net/netfilter/xt_owner.c @@ -127,26 +127,39 @@ owner_mt(const struct sk_buff *skb, struct xt_action_param *par) return true; } -static struct xt_match owner_mt_reg __read_mostly = { - .name = "owner", - .revision = 1, - .family = NFPROTO_UNSPEC, - .checkentry = owner_check, - .match = owner_mt, - .matchsize = sizeof(struct xt_owner_match_info), - .hooks = (1 << NF_INET_LOCAL_OUT) | - (1 << NF_INET_POST_ROUTING), - .me = THIS_MODULE, +static struct xt_match owner_mt_reg[] __read_mostly = { + { + .name = "owner", + .revision = 1, + .family = NFPROTO_IPV4, + .checkentry = owner_check, + .match = owner_mt, + .matchsize = sizeof(struct xt_owner_match_info), + .hooks = (1 << NF_INET_LOCAL_OUT) | + (1 << NF_INET_POST_ROUTING), + .me = THIS_MODULE, + }, + { + .name = "owner", + .revision = 1, + .family = NFPROTO_IPV6, + .checkentry = owner_check, + .match = owner_mt, + .matchsize = sizeof(struct xt_owner_match_info), + .hooks = (1 << NF_INET_LOCAL_OUT) | + (1 << NF_INET_POST_ROUTING), + .me = THIS_MODULE, + } }; static int __init owner_mt_init(void) { - return xt_register_match(&owner_mt_reg); + return xt_register_matches(owner_mt_reg, ARRAY_SIZE(owner_mt_reg)); } static void __exit owner_mt_exit(void) { - xt_unregister_match(&owner_mt_reg); + xt_unregister_matches(owner_mt_reg, ARRAY_SIZE(owner_mt_reg)); } module_init(owner_mt_init); diff --git a/net/netfilter/xt_physdev.c b/net/netfilter/xt_physdev.c index 343e65f377d442..130842c35c6fa8 100644 --- a/net/netfilter/xt_physdev.c +++ b/net/netfilter/xt_physdev.c @@ -115,24 +115,33 @@ static int physdev_mt_check(const struct xt_mtchk_param *par) return 0; } -static struct xt_match physdev_mt_reg __read_mostly = { - .name = "physdev", - .revision = 0, - .family = NFPROTO_UNSPEC, - .checkentry = physdev_mt_check, - .match = physdev_mt, - .matchsize = sizeof(struct xt_physdev_info), - .me = THIS_MODULE, +static struct xt_match physdev_mt_reg[] __read_mostly = { + { + .name = "physdev", + .family = NFPROTO_IPV4, + .checkentry = physdev_mt_check, + .match = physdev_mt, + .matchsize = sizeof(struct xt_physdev_info), + .me = THIS_MODULE, + }, + { + .name = "physdev", + .family = NFPROTO_IPV6, + .checkentry = physdev_mt_check, + .match = physdev_mt, + .matchsize = sizeof(struct xt_physdev_info), + .me = THIS_MODULE, + }, }; static int __init physdev_mt_init(void) { - return xt_register_match(&physdev_mt_reg); + return xt_register_matches(physdev_mt_reg, ARRAY_SIZE(physdev_mt_reg)); } static void __exit physdev_mt_exit(void) { - xt_unregister_match(&physdev_mt_reg); + xt_unregister_matches(physdev_mt_reg, ARRAY_SIZE(physdev_mt_reg)); } module_init(physdev_mt_init); diff --git a/net/netfilter/xt_policy.c b/net/netfilter/xt_policy.c index cb6e8279010a4b..b5fa65558318f5 100644 --- a/net/netfilter/xt_policy.c +++ b/net/netfilter/xt_policy.c @@ -63,7 +63,7 @@ match_policy_in(const struct sk_buff *skb, const struct xt_policy_info *info, return 0; for (i = sp->len - 1; i >= 0; i--) { - pos = strict ? i - sp->len + 1 : 0; + pos = strict ? sp->len - i - 1 : 0; if (pos >= info->len) return 0; e = &info->pol[pos]; diff --git a/net/netfilter/xt_realm.c b/net/netfilter/xt_realm.c index 6df485f4403d02..61b2f1e58d1505 100644 --- a/net/netfilter/xt_realm.c +++ b/net/netfilter/xt_realm.c @@ -33,7 +33,7 @@ static struct xt_match realm_mt_reg __read_mostly = { .matchsize = sizeof(struct xt_realm_info), .hooks = (1 << NF_INET_POST_ROUTING) | (1 << NF_INET_FORWARD) | (1 << NF_INET_LOCAL_OUT) | (1 << NF_INET_LOCAL_IN), - .family = NFPROTO_UNSPEC, + .family = NFPROTO_IPV4, .me = THIS_MODULE }; diff --git a/net/netfilter/xt_socket.c b/net/netfilter/xt_socket.c index 76e01f292aaff6..811e53bee4085b 100644 --- a/net/netfilter/xt_socket.c +++ b/net/netfilter/xt_socket.c @@ -168,52 +168,41 @@ static int socket_mt_enable_defrag(struct net *net, int family) static int socket_mt_v1_check(const struct xt_mtchk_param *par) { const struct xt_socket_mtinfo1 *info = (struct xt_socket_mtinfo1 *) par->matchinfo; - int err; - - err = socket_mt_enable_defrag(par->net, par->family); - if (err) - return err; if (info->flags & ~XT_SOCKET_FLAGS_V1) { pr_info_ratelimited("unknown flags 0x%x\n", info->flags & ~XT_SOCKET_FLAGS_V1); return -EINVAL; } - return 0; + + return socket_mt_enable_defrag(par->net, par->family); } static int socket_mt_v2_check(const struct xt_mtchk_param *par) { const struct xt_socket_mtinfo2 *info = (struct xt_socket_mtinfo2 *) par->matchinfo; - int err; - - err = socket_mt_enable_defrag(par->net, par->family); - if (err) - return err; if (info->flags & ~XT_SOCKET_FLAGS_V2) { pr_info_ratelimited("unknown flags 0x%x\n", info->flags & ~XT_SOCKET_FLAGS_V2); return -EINVAL; } - return 0; + + return socket_mt_enable_defrag(par->net, par->family); } static int socket_mt_v3_check(const struct xt_mtchk_param *par) { const struct xt_socket_mtinfo3 *info = (struct xt_socket_mtinfo3 *)par->matchinfo; - int err; - err = socket_mt_enable_defrag(par->net, par->family); - if (err) - return err; if (info->flags & ~XT_SOCKET_FLAGS_V3) { pr_info_ratelimited("unknown flags 0x%x\n", info->flags & ~XT_SOCKET_FLAGS_V3); return -EINVAL; } - return 0; + + return socket_mt_enable_defrag(par->net, par->family); } static void socket_mt_destroy(const struct xt_mtdtor_param *par) diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c index 63f1b721c71d06..ae63c5eb06faa7 100644 --- a/net/nfc/digital_technology.c +++ b/net/nfc/digital_technology.c @@ -424,6 +424,12 @@ static void digital_in_recv_sdd_res(struct nfc_digital_dev *ddev, void *arg, size = 4; } + if (target->nfcid1_len + size > NFC_NFCID1_MAXSIZE) { + PROTOCOL_ERR("4.7.2.1"); + rc = -EPROTO; + goto exit; + } + memcpy(target->nfcid1 + target->nfcid1_len, sdd_res->nfcid1 + offset, size); target->nfcid1_len += size; diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index 366d7566308c9f..db5bc6a878ddb0 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -1091,6 +1091,7 @@ static void nfc_llcp_recv_hdlc(struct nfc_llcp_local *local, if (sk->sk_state == LLCP_CLOSED) { release_sock(sk); nfc_llcp_sock_put(llcp_sock); + return; } /* Pass the payload upstream */ @@ -1182,6 +1183,7 @@ static void nfc_llcp_recv_disc(struct nfc_llcp_local *local, if (sk->sk_state == LLCP_CLOSED) { release_sock(sk); nfc_llcp_sock_put(llcp_sock); + return; } if (sk->sk_state == LLCP_CONNECTED) { diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index e209099218b410..bbbde50fc6498f 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -2184,9 +2184,40 @@ static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb, return err; } +static size_t ovs_vport_cmd_msg_size(void) +{ + size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header)); + + msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_PORT_NO */ + msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_TYPE */ + msgsize += nla_total_size(IFNAMSIZ); /* OVS_VPORT_ATTR_NAME */ + msgsize += nla_total_size(sizeof(u32)); /* OVS_VPORT_ATTR_IFINDEX */ + msgsize += nla_total_size(sizeof(s32)); /* OVS_VPORT_ATTR_NETNSID */ + + /* OVS_VPORT_ATTR_STATS */ + msgsize += nla_total_size_64bit(sizeof(struct ovs_vport_stats)); + + /* OVS_VPORT_ATTR_UPCALL_STATS(OVS_VPORT_UPCALL_ATTR_SUCCESS + + * OVS_VPORT_UPCALL_ATTR_FAIL) + */ + msgsize += nla_total_size(nla_total_size_64bit(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64))); + + /* OVS_VPORT_ATTR_UPCALL_PID */ + msgsize += nla_total_size(nr_cpu_ids * sizeof(u32)); + + /* OVS_VPORT_ATTR_OPTIONS(OVS_TUNNEL_ATTR_DST_PORT + + * OVS_TUNNEL_ATTR_EXTENSION(OVS_VXLAN_EXT_GBP)) + */ + msgsize += nla_total_size(nla_total_size(sizeof(u16)) + + nla_total_size(nla_total_size(0))); + + return msgsize; +} + static struct sk_buff *ovs_vport_cmd_alloc_info(void) { - return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + return genlmsg_new(ovs_vport_cmd_msg_size(), GFP_KERNEL); } /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */ @@ -2196,7 +2227,7 @@ struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net, struct sk_buff *skb; int retval; - skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + skb = ovs_vport_cmd_alloc_info(); if (!skb) return ERR_PTR(-ENOMEM); diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c index 12055af832dc08..a1df551e915bca 100644 --- a/net/openvswitch/vport-netdev.c +++ b/net/openvswitch/vport-netdev.c @@ -196,9 +196,13 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) */ if (vport->dev->reg_state == NETREG_REGISTERED) rtnl_delete_link(vport->dev, 0, NULL); - rtnl_unlock(); + /* We can't put the device reference yet, since it can still be in + * use, but rtnl_unlock()->netdev_run_todo() will block until all + * the references are released, so the RCU call must be before it. + */ call_rcu(&vport->rcu, vport_netdev_free); + rtnl_unlock(); } EXPORT_SYMBOL_GPL(ovs_netdev_tunnel_destroy); diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c index 23f629e94a36ab..56b2e2d1a749f8 100644 --- a/net/openvswitch/vport.c +++ b/net/openvswitch/vport.c @@ -406,6 +406,9 @@ int ovs_vport_set_upcall_portids(struct vport *vport, const struct nlattr *ids) if (!nla_len(ids) || nla_len(ids) % sizeof(u32)) return -EINVAL; + if (nla_len(ids) / sizeof(u32) > nr_cpu_ids) + return -EINVAL; + old = ovsl_dereference(vport->upcall_portids); vport_portids = kmalloc(sizeof(*vport_portids) + nla_len(ids), diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index bb2d88205e5a65..7bbefa8a422f05 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2717,7 +2717,8 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) { struct sk_buff *skb = NULL; struct net_device *dev; - struct virtio_net_hdr *vnet_hdr = NULL; + struct virtio_net_hdr vnet_hdr; + bool has_vnet_hdr = false; struct sockcm_cookie sockc; __be16 proto; int err, reserve = 0; @@ -2818,16 +2819,20 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) hlen = LL_RESERVED_SPACE(dev); tlen = dev->needed_tailroom; if (vnet_hdr_sz) { - vnet_hdr = data; data += vnet_hdr_sz; tp_len -= vnet_hdr_sz; - if (tp_len < 0 || - __packet_snd_vnet_parse(vnet_hdr, tp_len)) { + if (tp_len < 0) { + tp_len = -EINVAL; + goto tpacket_error; + } + memcpy(&vnet_hdr, data - vnet_hdr_sz, sizeof(vnet_hdr)); + if (__packet_snd_vnet_parse(&vnet_hdr, tp_len)) { tp_len = -EINVAL; goto tpacket_error; } copylen = __virtio16_to_cpu(vio_le(), - vnet_hdr->hdr_len); + vnet_hdr.hdr_len); + has_vnet_hdr = true; } copylen = max_t(int, copylen, dev->hard_header_len); skb = sock_alloc_send_skb(&po->sk, @@ -2864,12 +2869,12 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) } } - if (vnet_hdr_sz) { - if (virtio_net_hdr_to_skb(skb, vnet_hdr, vio_le())) { + if (has_vnet_hdr) { + if (virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le())) { tp_len = -EINVAL; goto tpacket_error; } - virtio_net_hdr_set_proto(skb, vnet_hdr); + virtio_net_hdr_set_proto(skb, &vnet_hdr); } skb->destructor = tpacket_destruct_skb; diff --git a/net/phonet/socket.c b/net/phonet/socket.c index 4423d483c630ad..bbd710d95b975c 100644 --- a/net/phonet/socket.c +++ b/net/phonet/socket.c @@ -208,9 +208,15 @@ static int pn_socket_autobind(struct socket *sock) sa.spn_family = AF_PHONET; err = pn_socket_bind(sock, (struct sockaddr_unsized *)&sa, sizeof(struct sockaddr_pn)); - if (err != -EINVAL) + /* + * pn_socket_bind() also returns -EINVAL when sk_state != TCP_CLOSE + * without a prior bind, so -EINVAL alone is not sufficient to infer + * that the socket was already bound. Only treat it as "already + * bound" when the port is non-zero; otherwise propagate the error + * instead of crashing the kernel. + */ + if (err != -EINVAL || unlikely(!pn_port(pn_sk(sock->sk)->sobject))) return err; - BUG_ON(!pn_port(pn_sk(sock->sk)->sobject)); return 0; /* socket was already bound */ } diff --git a/net/psp/psp-nl-gen.c b/net/psp/psp-nl-gen.c index 22a48d0fa378c9..953309952cef75 100644 --- a/net/psp/psp-nl-gen.c +++ b/net/psp/psp-nl-gen.c @@ -76,7 +76,7 @@ static const struct genl_split_ops psp_nl_ops[] = { .post_doit = psp_device_unlock, .policy = psp_dev_set_nl_policy, .maxattr = PSP_A_DEV_PSP_VERSIONS_ENA, - .flags = GENL_CMD_CAP_DO, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = PSP_CMD_KEY_ROTATE, @@ -85,7 +85,7 @@ static const struct genl_split_ops psp_nl_ops[] = { .post_doit = psp_device_unlock, .policy = psp_key_rotate_nl_policy, .maxattr = PSP_A_DEV_ID, - .flags = GENL_CMD_CAP_DO, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, }, { .cmd = PSP_CMD_RX_ASSOC, diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c index d4c04c923c5ac6..956f3ff0c22a19 100644 --- a/net/psp/psp_main.c +++ b/net/psp/psp_main.c @@ -263,15 +263,16 @@ EXPORT_SYMBOL(psp_dev_encapsulate); /* Receive handler for PSP packets. * - * Presently it accepts only already-authenticated packets and does not - * support optional fields, such as virtualization cookies. The caller should - * ensure that skb->data is pointing to the mac header, and that skb->mac_len - * is set. This function does not currently adjust skb->csum (CHECKSUM_COMPLETE - * is not supported). + * Accepts only already-authenticated packets. The full PSP header is + * stripped according to psph->hdrlen; any optional fields it advertises + * (virtualization cookies, etc.) are ignored and discarded along with the + * rest of the header. The caller should ensure that skb->data is pointing + * to the mac header, and that skb->mac_len is set. This function does not + * currently adjust skb->csum (CHECKSUM_COMPLETE is not supported). */ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv) { - int l2_hlen = 0, l3_hlen, encap; + int l2_hlen = 0, l3_hlen, encap, psp_hlen; struct psp_skb_ext *pse; struct psphdr *psph; struct ethhdr *eth; @@ -312,18 +313,36 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv) if (unlikely(uh->dest != htons(PSP_DEFAULT_UDP_PORT))) return -EINVAL; - pse = skb_ext_add(skb, SKB_EXT_PSP); - if (!pse) + psph = (struct psphdr *)(skb->data + l2_hlen + l3_hlen + + sizeof(struct udphdr)); + + /* Strip the full PSP header per psph->hdrlen; VC/options are pulled + * into the linear region only so they can be discarded with the + * rest of the header. + */ + psp_hlen = (psph->hdrlen + 1) * 8; + + if (unlikely(psp_hlen < sizeof(struct psphdr))) + return -EINVAL; + + if (psp_hlen > sizeof(struct psphdr) && + !pskb_may_pull(skb, l2_hlen + l3_hlen + + sizeof(struct udphdr) + psp_hlen)) return -EINVAL; psph = (struct psphdr *)(skb->data + l2_hlen + l3_hlen + sizeof(struct udphdr)); + + pse = skb_ext_add(skb, SKB_EXT_PSP); + if (!pse) + return -EINVAL; + pse->spi = psph->spi; pse->dev_id = dev_id; pse->generation = generation; pse->version = FIELD_GET(PSPHDR_VERFL_VERSION, psph->verfl); - encap = PSP_ENCAP_HLEN; + encap = sizeof(struct udphdr) + psp_hlen; encap += strip_icv ? PSP_TRL_SIZE : 0; if (proto == htons(ETH_P_IP)) { @@ -340,8 +359,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv) ipv6h->payload_len = htons(ntohs(ipv6h->payload_len) - encap); } - memmove(skb->data + PSP_ENCAP_HLEN, skb->data, l2_hlen + l3_hlen); - skb_pull(skb, PSP_ENCAP_HLEN); + memmove(skb->data + sizeof(struct udphdr) + psp_hlen, + skb->data, l2_hlen + l3_hlen); + skb_pull(skb, sizeof(struct udphdr) + psp_hlen); if (strip_icv) pskb_trim(skb, skb->len - PSP_TRL_SIZE); diff --git a/net/psp/psp_nl.c b/net/psp/psp_nl.c index 6afd7707ec12ea..0cc744a6e1c9bc 100644 --- a/net/psp/psp_nl.c +++ b/net/psp/psp_nl.c @@ -305,8 +305,13 @@ int psp_assoc_device_get_locked(const struct genl_split_ops *ops, psd = psp_dev_get_for_sock(socket->sk); if (psd) { - err = psp_dev_check_access(psd, genl_info_net(info)); - if (err) { + /* Extra care needed here, psp_dev_get_for_sock() only gives + * us access to struct psp_dev's memory, which is quite weak. + */ + mutex_lock(&psd->lock); + if (!psp_dev_is_registered(psd) || + psp_dev_check_access(psd, genl_info_net(info))) { + mutex_unlock(&psd->lock); psp_dev_put(psd); psd = NULL; } @@ -319,7 +324,6 @@ int psp_assoc_device_get_locked(const struct genl_split_ops *ops, id = info->attrs[PSP_A_ASSOC_DEV_ID]; if (psd) { - mutex_lock(&psd->lock); if (id && psd->id != nla_get_u32(id)) { mutex_unlock(&psd->lock); NL_SET_ERR_MSG_ATTR(info->extack, id, diff --git a/net/qrtr/ns.c b/net/qrtr/ns.c index 3203b222086034..b3f9bbcf9ab9ba 100644 --- a/net/qrtr/ns.c +++ b/net/qrtr/ns.c @@ -22,8 +22,10 @@ static struct { struct socket *sock; struct sockaddr_qrtr bcast_sq; struct list_head lookups; + u32 lookup_count; struct workqueue_struct *workqueue; struct work_struct work; + void (*saved_data_ready)(struct sock *sk); int local_node; } qrtr_ns; @@ -67,8 +69,19 @@ struct qrtr_server { struct qrtr_node { unsigned int id; struct xarray servers; + u32 server_count; }; +/* Max nodes, server, lookup limits are chosen based on the current platform + * requirements. If the requirement changes in the future, these values can be + * increased. + */ +#define QRTR_NS_MAX_NODES 64 +#define QRTR_NS_MAX_SERVERS 256 +#define QRTR_NS_MAX_LOOKUPS 64 + +static u8 node_count; + static struct qrtr_node *node_get(unsigned int node_id) { struct qrtr_node *node; @@ -77,6 +90,11 @@ static struct qrtr_node *node_get(unsigned int node_id) if (node) return node; + if (node_count >= QRTR_NS_MAX_NODES) { + pr_err_ratelimited("QRTR clients exceed max node limit!\n"); + return NULL; + } + /* If node didn't exist, allocate and insert it to the tree */ node = kzalloc_obj(*node); if (!node) @@ -90,6 +108,8 @@ static struct qrtr_node *node_get(unsigned int node_id) return NULL; } + node_count++; + return node; } @@ -229,6 +249,17 @@ static struct qrtr_server *server_add(unsigned int service, if (!service || !port) return NULL; + node = node_get(node_id); + if (!node) + return NULL; + + /* Make sure the new servers per port are capped at the maximum value */ + old = xa_load(&node->servers, port); + if (!old && node->server_count >= QRTR_NS_MAX_SERVERS) { + pr_err_ratelimited("QRTR client node %u exceeds max server limit!\n", node_id); + return NULL; + } + srv = kzalloc_obj(*srv); if (!srv) return NULL; @@ -238,10 +269,6 @@ static struct qrtr_server *server_add(unsigned int service, srv->node = node_id; srv->port = port; - node = node_get(node_id); - if (!node) - goto err; - /* Delete the old server on the same port */ old = xa_store(&node->servers, port, srv, GFP_KERNEL); if (old) { @@ -252,6 +279,8 @@ static struct qrtr_server *server_add(unsigned int service, } else { kfree(old); } + } else { + node->server_count++; } trace_qrtr_ns_server_add(srv->service, srv->instance, @@ -292,6 +321,7 @@ static int server_del(struct qrtr_node *node, unsigned int port, bool bcast) } kfree(srv); + node->server_count--; return 0; } @@ -341,7 +371,7 @@ static int ctrl_cmd_bye(struct sockaddr_qrtr *from) struct qrtr_node *node; unsigned long index; struct kvec iv; - int ret; + int ret = 0; iv.iov_base = &pkt; iv.iov_len = sizeof(pkt); @@ -356,8 +386,10 @@ static int ctrl_cmd_bye(struct sockaddr_qrtr *from) /* Advertise the removal of this client to all local servers */ local_node = node_get(qrtr_ns.local_node); - if (!local_node) - return 0; + if (!local_node) { + ret = 0; + goto delete_node; + } memset(&pkt, 0, sizeof(pkt)); pkt.cmd = cpu_to_le32(QRTR_TYPE_BYE); @@ -374,10 +406,19 @@ static int ctrl_cmd_bye(struct sockaddr_qrtr *from) ret = kernel_sendmsg(qrtr_ns.sock, &msg, &iv, 1, sizeof(pkt)); if (ret < 0 && ret != -ENODEV) { pr_err("failed to send bye cmd\n"); - return ret; + goto delete_node; } } - return 0; + + /* Ignore -ENODEV */ + ret = 0; + +delete_node: + xa_erase(&nodes, from->sq_node); + kfree(node); + node_count--; + + return ret; } static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, @@ -417,6 +458,7 @@ static int ctrl_cmd_del_client(struct sockaddr_qrtr *from, list_del(&lookup->li); kfree(lookup); + qrtr_ns.lookup_count--; } /* Remove the server belonging to this port but don't broadcast @@ -534,6 +576,11 @@ static int ctrl_cmd_new_lookup(struct sockaddr_qrtr *from, if (from->sq_node != qrtr_ns.local_node) return -EINVAL; + if (qrtr_ns.lookup_count >= QRTR_NS_MAX_LOOKUPS) { + pr_err_ratelimited("QRTR client node exceeds max lookup limit!\n"); + return -ENOSPC; + } + lookup = kzalloc_obj(*lookup); if (!lookup) return -ENOMEM; @@ -542,6 +589,7 @@ static int ctrl_cmd_new_lookup(struct sockaddr_qrtr *from, lookup->service = service; lookup->instance = instance; list_add_tail(&lookup->li, &qrtr_ns.lookups); + qrtr_ns.lookup_count++; memset(&filter, 0, sizeof(filter)); filter.service = service; @@ -582,6 +630,7 @@ static void ctrl_cmd_del_lookup(struct sockaddr_qrtr *from, list_del(&lookup->li); kfree(lookup); + qrtr_ns.lookup_count--; } } @@ -670,7 +719,7 @@ static void qrtr_ns_worker(struct work_struct *work) } if (ret < 0) - pr_err("failed while handling packet from %d:%d", + pr_err_ratelimited("failed while handling packet from %d:%d", sq.sq_node, sq.sq_port); } @@ -709,6 +758,7 @@ int qrtr_ns_init(void) goto err_sock; } + qrtr_ns.saved_data_ready = qrtr_ns.sock->sk->sk_data_ready; qrtr_ns.sock->sk->sk_data_ready = qrtr_ns_data_ready; sq.sq_port = QRTR_PORT_CTRL; @@ -749,6 +799,10 @@ int qrtr_ns_init(void) return 0; err_wq: + write_lock_bh(&qrtr_ns.sock->sk->sk_callback_lock); + qrtr_ns.sock->sk->sk_data_ready = qrtr_ns.saved_data_ready; + write_unlock_bh(&qrtr_ns.sock->sk->sk_callback_lock); + destroy_workqueue(qrtr_ns.workqueue); err_sock: sock_release(qrtr_ns.sock); @@ -758,7 +812,12 @@ EXPORT_SYMBOL_GPL(qrtr_ns_init); void qrtr_ns_remove(void) { + write_lock_bh(&qrtr_ns.sock->sk->sk_callback_lock); + qrtr_ns.sock->sk->sk_data_ready = qrtr_ns.saved_data_ready; + write_unlock_bh(&qrtr_ns.sock->sk->sk_callback_lock); + cancel_work_sync(&qrtr_ns.work); + synchronize_net(); destroy_workqueue(qrtr_ns.workqueue); /* sock_release() expects the two references that were put during diff --git a/net/rds/af_rds.c b/net/rds/af_rds.c index b396c673dfaf62..76f625986a7f24 100644 --- a/net/rds/af_rds.c +++ b/net/rds/af_rds.c @@ -357,7 +357,8 @@ static int rds_cong_monitor(struct rds_sock *rs, sockptr_t optval, int optlen) return ret; } -static int rds_set_transport(struct rds_sock *rs, sockptr_t optval, int optlen) +static int rds_set_transport(struct net *net, struct rds_sock *rs, + sockptr_t optval, int optlen) { int t_type; @@ -373,6 +374,10 @@ static int rds_set_transport(struct rds_sock *rs, sockptr_t optval, int optlen) if (t_type < 0 || t_type >= RDS_TRANS_COUNT) return -EINVAL; + /* RDS/IB is restricted to the initial network namespace */ + if (t_type != RDS_TRANS_TCP && !net_eq(net, &init_net)) + return -EPROTOTYPE; + rs->rs_transport = rds_trans_get(t_type); return rs->rs_transport ? 0 : -ENOPROTOOPT; @@ -433,6 +438,7 @@ static int rds_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, unsigned int optlen) { struct rds_sock *rs = rds_sk_to_rs(sock->sk); + struct net *net = sock_net(sock->sk); int ret; if (level != SOL_RDS) { @@ -461,7 +467,7 @@ static int rds_setsockopt(struct socket *sock, int level, int optname, break; case SO_RDS_TRANSPORT: lock_sock(sock->sk); - ret = rds_set_transport(rs, optval, optlen); + ret = rds_set_transport(net, rs, optval, optlen); release_sock(sock->sk); break; case SO_TIMESTAMP_OLD: diff --git a/net/rds/connection.c b/net/rds/connection.c index 412441aaa298e8..c10b7ed06c49f8 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -701,6 +701,13 @@ void rds_for_each_conn_info(struct socket *sock, unsigned int len, i++, head++) { hlist_for_each_entry_rcu(conn, head, c_hash_node) { + /* Zero the per-item buffer before handing it to the + * visitor so any field the visitor does not write - + * including implicit alignment padding - cannot leak + * stack contents to user space via rds_info_copy(). + */ + memset(buffer, 0, item_len); + /* XXX no c_lock usage.. */ if (!visitor(conn, buffer)) continue; @@ -750,6 +757,13 @@ static void rds_walk_conn_path_info(struct socket *sock, unsigned int len, */ cp = conn->c_path; + /* Zero the per-item buffer for the same reason as + * rds_for_each_conn_info(): any byte the visitor + * does not write (including alignment padding) must + * not leak stack contents via rds_info_copy(). + */ + memset(buffer, 0, item_len); + /* XXX no cp_lock usage.. */ if (!visitor(cp, buffer)) continue; diff --git a/net/rds/ib.c b/net/rds/ib.c index ac6affa33ce759..39f87272e071bf 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -401,8 +401,8 @@ static void rds6_ib_ic_info(struct socket *sock, unsigned int len, * allowed to influence which paths have priority. We could call userspace * asserting this policy "routing". */ -static int rds_ib_laddr_check(struct net *net, const struct in6_addr *addr, - __u32 scope_id) +static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr, + __u32 scope_id) { int ret; struct rdma_cm_id *cm_id; @@ -487,6 +487,26 @@ static int rds_ib_laddr_check(struct net *net, const struct in6_addr *addr, return ret; } +static int rds_ib_laddr_check(struct net *net, const struct in6_addr *addr, + __u32 scope_id) +{ + struct rds_ib_device *rds_ibdev = NULL; + + /* RDS/IB is restricted to the initial network namespace */ + if (!net_eq(net, &init_net)) + return -EPROTOTYPE; + + if (ipv6_addr_v4mapped(addr)) { + rds_ibdev = rds_ib_get_device(addr->s6_addr32[3]); + if (rds_ibdev) { + rds_ib_dev_put(rds_ibdev); + return 0; + } + } + + return rds_ib_laddr_check_cm(net, addr, scope_id); +} + static void rds_ib_unregister_client(void) { ib_unregister_client(&rds_ib_client); diff --git a/net/rds/ib.h b/net/rds/ib.h index 8ef3178ed4d61a..5ff346a1e8baa6 100644 --- a/net/rds/ib.h +++ b/net/rds/ib.h @@ -381,6 +381,7 @@ void rds_ib_cm_connect_complete(struct rds_connection *conn, __rds_ib_conn_error(conn, KERN_WARNING "RDS/IB: " fmt) /* ib_rdma.c */ +struct rds_ib_device *rds_ib_get_device(__be32 ipaddr); int rds_ib_update_ipaddr(struct rds_ib_device *rds_ibdev, struct in6_addr *ipaddr); void rds_ib_add_conn(struct rds_ib_device *rds_ibdev, struct rds_connection *conn); diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c index 2cfec252eeac2f..9594ea245f7fee 100644 --- a/net/rds/ib_rdma.c +++ b/net/rds/ib_rdma.c @@ -43,7 +43,7 @@ struct workqueue_struct *rds_ib_mr_wq; static void rds_ib_odp_mr_worker(struct work_struct *work); -static struct rds_ib_device *rds_ib_get_device(__be32 ipaddr) +struct rds_ib_device *rds_ib_get_device(__be32 ipaddr) { struct rds_ib_device *rds_ibdev; struct rds_ib_ipaddr *i_ipaddr; diff --git a/net/rds/message.c b/net/rds/message.c index eaa6f22601a447..7feb0eb6537db8 100644 --- a/net/rds/message.c +++ b/net/rds/message.c @@ -131,24 +131,34 @@ static void rds_rm_zerocopy_callback(struct rds_sock *rs, */ static void rds_message_purge(struct rds_message *rm) { + struct rds_znotifier *znotifier; unsigned long i, flags; - bool zcopy = false; + bool zcopy; if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags))) return; spin_lock_irqsave(&rm->m_rs_lock, flags); + znotifier = rm->data.op_mmp_znotifier; + rm->data.op_mmp_znotifier = NULL; + zcopy = !!znotifier; + if (rm->m_rs) { struct rds_sock *rs = rm->m_rs; - if (rm->data.op_mmp_znotifier) { - zcopy = true; - rds_rm_zerocopy_callback(rs, rm->data.op_mmp_znotifier); + if (znotifier) { + rds_rm_zerocopy_callback(rs, znotifier); rds_wake_sk_sleep(rs); - rm->data.op_mmp_znotifier = NULL; } sock_put(rds_rs_to_sk(rs)); rm->m_rs = NULL; + } else if (znotifier) { + /* + * Zerocopy can fail before the message is queued on the + * socket, so there is no rs to carry the notification. + */ + mm_unaccount_pinned_pages(&znotifier->z_mmp); + kfree(rds_info_from_znotifier(znotifier)); } spin_unlock_irqrestore(&rm->m_rs_lock, flags); @@ -438,6 +448,7 @@ static int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter * for (i = 0; i < rm->data.op_nents; i++) put_page(sg_page(&rm->data.op_sg[i])); + rm->data.op_nents = 0; mmp = &rm->data.op_mmp_znotifier->z_mmp; mm_unaccount_pinned_pages(mmp); ret = -EFAULT; diff --git a/net/rds/rdma.c b/net/rds/rdma.c index aa6465dc742c2d..61fb6e45281bf1 100644 --- a/net/rds/rdma.c +++ b/net/rds/rdma.c @@ -326,10 +326,6 @@ static int __rds_rdma_map(struct rds_sock *rs, struct rds_get_mr_args *args, if (args->cookie_addr && put_user(cookie, (u64 __user *)(unsigned long)args->cookie_addr)) { - if (!need_odp) { - unpin_user_pages(pages, nr_pages); - kfree(sg); - } ret = -EFAULT; goto out; } diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index 96ecb83c907153..27c2aa2dd023c5 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -1486,7 +1486,6 @@ int rxrpc_server_keyring(struct rxrpc_sock *, sockptr_t, int); void rxrpc_kernel_data_consumed(struct rxrpc_call *, struct sk_buff *); void rxrpc_new_skb(struct sk_buff *, enum rxrpc_skb_trace); void rxrpc_see_skb(struct sk_buff *, enum rxrpc_skb_trace); -void rxrpc_eaten_skb(struct sk_buff *, enum rxrpc_skb_trace); void rxrpc_get_skb(struct sk_buff *, enum rxrpc_skb_trace); void rxrpc_free_skb(struct sk_buff *, enum rxrpc_skb_trace); void rxrpc_purge_queue(struct sk_buff_head *); diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c index fec59d9338b9fb..2b19b252225e55 100644 --- a/net/rxrpc/call_event.c +++ b/net/rxrpc/call_event.c @@ -332,7 +332,27 @@ bool rxrpc_input_call_event(struct rxrpc_call *call) saw_ack |= sp->hdr.type == RXRPC_PACKET_TYPE_ACK; - rxrpc_input_call_packet(call, skb); + if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA && + sp->hdr.securityIndex != 0 && + (skb_cloned(skb) || + skb_has_frag_list(skb) || + skb_has_shared_frag(skb))) { + /* Unshare the packet so that it can be + * modified by in-place decryption. + */ + struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC); + + if (nskb) { + rxrpc_new_skb(nskb, rxrpc_skb_new_unshared); + rxrpc_input_call_packet(call, nskb); + rxrpc_free_skb(nskb, rxrpc_skb_put_call_rx); + } else { + /* OOM - Drop the packet. */ + rxrpc_see_skb(skb, rxrpc_skb_see_unshare_nomem); + } + } else { + rxrpc_input_call_packet(call, skb); + } rxrpc_free_skb(skb, rxrpc_skb_put_call_rx); did_receive = true; } diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c index 9a41ec708aeb9f..442414d90ba1cd 100644 --- a/net/rxrpc/conn_event.c +++ b/net/rxrpc/conn_event.c @@ -240,6 +240,34 @@ static void rxrpc_call_is_secure(struct rxrpc_call *call) rxrpc_notify_socket(call); } +static int rxrpc_verify_response(struct rxrpc_connection *conn, + struct sk_buff *skb) +{ + int ret; + + if (skb_cloned(skb) || skb_has_frag_list(skb) || + skb_has_shared_frag(skb)) { + /* Copy the packet if shared so that we can do in-place + * decryption. + */ + struct sk_buff *nskb = skb_copy(skb, GFP_NOFS); + + if (nskb) { + rxrpc_new_skb(nskb, rxrpc_skb_new_unshared); + ret = conn->security->verify_response(conn, nskb); + rxrpc_free_skb(nskb, rxrpc_skb_put_response_copy); + } else { + /* OOM - Drop the packet. */ + rxrpc_see_skb(skb, rxrpc_skb_see_unshare_nomem); + ret = -ENOMEM; + } + } else { + ret = conn->security->verify_response(conn, skb); + } + + return ret; +} + /* * connection-level Rx packet processor */ @@ -270,7 +298,7 @@ static int rxrpc_process_event(struct rxrpc_connection *conn, } spin_unlock_irq(&conn->state_lock); - ret = conn->security->verify_response(conn, skb); + ret = rxrpc_verify_response(conn, skb); if (ret < 0) return ret; @@ -362,7 +390,6 @@ void rxrpc_process_delayed_final_acks(struct rxrpc_connection *conn, bool force) static void rxrpc_do_process_connection(struct rxrpc_connection *conn) { struct sk_buff *skb; - int ret; if (test_and_clear_bit(RXRPC_CONN_EV_CHALLENGE, &conn->events)) rxrpc_secure_connection(conn); @@ -371,17 +398,8 @@ static void rxrpc_do_process_connection(struct rxrpc_connection *conn) * connection that each one has when we've finished with it */ while ((skb = skb_dequeue(&conn->rx_queue))) { rxrpc_see_skb(skb, rxrpc_skb_see_conn_work); - ret = rxrpc_process_event(conn, skb); - switch (ret) { - case -ENOMEM: - case -EAGAIN: - skb_queue_head(&conn->rx_queue, skb); - rxrpc_queue_conn(conn, rxrpc_conn_queue_retry_work); - break; - default: - rxrpc_free_skb(skb, rxrpc_skb_put_conn_work); - break; - } + rxrpc_process_event(conn, skb); + rxrpc_free_skb(skb, rxrpc_skb_put_conn_work); } } diff --git a/net/rxrpc/io_thread.c b/net/rxrpc/io_thread.c index 69795693192521..dc5184a2fa9d1a 100644 --- a/net/rxrpc/io_thread.c +++ b/net/rxrpc/io_thread.c @@ -192,13 +192,12 @@ static bool rxrpc_extract_abort(struct sk_buff *skb) /* * Process packets received on the local endpoint */ -static bool rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb) +static bool rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff *skb) { struct rxrpc_connection *conn; struct sockaddr_rxrpc peer_srx; struct rxrpc_skb_priv *sp; struct rxrpc_peer *peer = NULL; - struct sk_buff *skb = *_skb; bool ret = false; skb_pull(skb, sizeof(struct udphdr)); @@ -244,25 +243,6 @@ static bool rxrpc_input_packet(struct rxrpc_local *local, struct sk_buff **_skb) return rxrpc_bad_message(skb, rxrpc_badmsg_zero_call); if (sp->hdr.seq == 0) return rxrpc_bad_message(skb, rxrpc_badmsg_zero_seq); - - /* Unshare the packet so that it can be modified for in-place - * decryption. - */ - if (sp->hdr.securityIndex != 0) { - skb = skb_unshare(skb, GFP_ATOMIC); - if (!skb) { - rxrpc_eaten_skb(*_skb, rxrpc_skb_eaten_by_unshare_nomem); - *_skb = NULL; - return just_discard; - } - - if (skb != *_skb) { - rxrpc_eaten_skb(*_skb, rxrpc_skb_eaten_by_unshare); - *_skb = skb; - rxrpc_new_skb(skb, rxrpc_skb_new_unshared); - sp = rxrpc_skb(skb); - } - } break; case RXRPC_PACKET_TYPE_CHALLENGE: @@ -494,7 +474,7 @@ int rxrpc_io_thread(void *data) switch (skb->mark) { case RXRPC_SKB_MARK_PACKET: skb->priority = 0; - if (!rxrpc_input_packet(local, &skb)) + if (!rxrpc_input_packet(local, skb)) rxrpc_reject_packet(local, skb); trace_rxrpc_rx_done(skb->mark, skb->priority); rxrpc_free_skb(skb, rxrpc_skb_put_input); diff --git a/net/rxrpc/key.c b/net/rxrpc/key.c index 6301d79ee35a6c..3ec3d89fdf1403 100644 --- a/net/rxrpc/key.c +++ b/net/rxrpc/key.c @@ -502,6 +502,10 @@ static int rxrpc_preparse(struct key_preparsed_payload *prep) if (v1->security_index != RXRPC_SECURITY_RXKAD) goto error; + ret = -EKEYREJECTED; + if (v1->ticket_length > AFSTOKEN_RK_TIX_MAX) + goto error; + plen = sizeof(*token->kad) + v1->ticket_length; prep->quotalen += plen + sizeof(*token); diff --git a/net/rxrpc/rxgk_app.c b/net/rxrpc/rxgk_app.c index 30275cb5ba3e25..0ef2a29eb69581 100644 --- a/net/rxrpc/rxgk_app.c +++ b/net/rxrpc/rxgk_app.c @@ -214,7 +214,7 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb, ticket_len = ntohl(container.token_len); ticket_offset = token_offset + sizeof(container); - if (xdr_round_up(ticket_len) > token_len - sizeof(container)) + if (ticket_len > xdr_round_down(token_len - sizeof(container))) goto short_packet; _debug("KVNO %u", kvno); @@ -245,6 +245,7 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb, if (ret != -ENOMEM) return rxrpc_abort_conn(conn, skb, ec, ret, rxgk_abort_resp_tok_dec); + return ret; } ret = conn->security->default_decode_ticket(conn, skb, ticket_offset, diff --git a/net/rxrpc/rxgk_common.h b/net/rxrpc/rxgk_common.h index 80164d89e19c03..1e257d7ab8ec1b 100644 --- a/net/rxrpc/rxgk_common.h +++ b/net/rxrpc/rxgk_common.h @@ -34,6 +34,7 @@ struct rxgk_context { }; #define xdr_round_up(x) (round_up((x), sizeof(__be32))) +#define xdr_round_down(x) (round_down((x), sizeof(__be32))) #define xdr_object_len(x) (4 + xdr_round_up(x)) /* diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c index eb7f2769d2b121..cba7935977f0ba 100644 --- a/net/rxrpc/rxkad.c +++ b/net/rxrpc/rxkad.c @@ -510,6 +510,9 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON, rxkad_abort_2_short_header); + /* Don't let the crypto algo see a misaligned length. */ + sp->len = round_down(sp->len, 8); + /* Decrypt the skbuff in-place. TODO: We really want to decrypt * directly into the target buffer. */ @@ -543,8 +546,10 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb, if (sg != _sg) kfree(sg); if (ret < 0) { - WARN_ON_ONCE(ret != -ENOMEM); - return ret; + if (ret == -ENOMEM) + return ret; + return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON, + rxkad_abort_2_crypto_unaligned); } /* Extract the decrypted packet length */ @@ -1136,7 +1141,7 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, struct rxrpc_crypt session_key; struct key *server_key; time64_t expiry; - void *ticket; + void *ticket = NULL; u32 version, kvno, ticket_len, level; __be32 csum; int ret, i; @@ -1162,13 +1167,13 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, ret = -ENOMEM; response = kzalloc_obj(struct rxkad_response, GFP_NOFS); if (!response) - goto temporary_error; + goto error; if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), response, sizeof(*response)) < 0) { - rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO, - rxkad_abort_resp_short); - goto protocol_error; + ret = rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO, + rxkad_abort_resp_short); + goto error; } version = ntohl(response->version); @@ -1178,62 +1183,62 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, trace_rxrpc_rx_response(conn, sp->hdr.serial, version, kvno, ticket_len); if (version != RXKAD_VERSION) { - rxrpc_abort_conn(conn, skb, RXKADINCONSISTENCY, -EPROTO, - rxkad_abort_resp_version); - goto protocol_error; + ret = rxrpc_abort_conn(conn, skb, RXKADINCONSISTENCY, -EPROTO, + rxkad_abort_resp_version); + goto error; } if (ticket_len < 4 || ticket_len > MAXKRB5TICKETLEN) { - rxrpc_abort_conn(conn, skb, RXKADTICKETLEN, -EPROTO, - rxkad_abort_resp_tkt_len); - goto protocol_error; + ret = rxrpc_abort_conn(conn, skb, RXKADTICKETLEN, -EPROTO, + rxkad_abort_resp_tkt_len); + goto error; } if (kvno >= RXKAD_TKT_TYPE_KERBEROS_V5) { - rxrpc_abort_conn(conn, skb, RXKADUNKNOWNKEY, -EPROTO, - rxkad_abort_resp_unknown_tkt); - goto protocol_error; + ret = rxrpc_abort_conn(conn, skb, RXKADUNKNOWNKEY, -EPROTO, + rxkad_abort_resp_unknown_tkt); + goto error; } /* extract the kerberos ticket and decrypt and decode it */ ret = -ENOMEM; ticket = kmalloc(ticket_len, GFP_NOFS); if (!ticket) - goto temporary_error_free_resp; + goto error; if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header) + sizeof(*response), ticket, ticket_len) < 0) { - rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO, - rxkad_abort_resp_short_tkt); - goto protocol_error; + ret = rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO, + rxkad_abort_resp_short_tkt); + goto error; } ret = rxkad_decrypt_ticket(conn, server_key, skb, ticket, ticket_len, &session_key, &expiry); if (ret < 0) - goto temporary_error_free_ticket; + goto error; /* use the session key from inside the ticket to decrypt the * response */ ret = rxkad_decrypt_response(conn, response, &session_key); if (ret < 0) - goto temporary_error_free_ticket; + goto error; if (ntohl(response->encrypted.epoch) != conn->proto.epoch || ntohl(response->encrypted.cid) != conn->proto.cid || ntohl(response->encrypted.securityIndex) != conn->security_ix) { - rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, - rxkad_abort_resp_bad_param); - goto protocol_error_free; + ret = rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, + rxkad_abort_resp_bad_param); + goto error; } csum = response->encrypted.checksum; response->encrypted.checksum = 0; rxkad_calc_response_checksum(response); if (response->encrypted.checksum != csum) { - rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, - rxkad_abort_resp_bad_checksum); - goto protocol_error_free; + ret = rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, + rxkad_abort_resp_bad_checksum); + goto error; } for (i = 0; i < RXRPC_MAXCALLS; i++) { @@ -1241,38 +1246,38 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, u32 counter = READ_ONCE(conn->channels[i].call_counter); if (call_id > INT_MAX) { - rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, - rxkad_abort_resp_bad_callid); - goto protocol_error_free; + ret = rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, + rxkad_abort_resp_bad_callid); + goto error; } if (call_id < counter) { - rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, - rxkad_abort_resp_call_ctr); - goto protocol_error_free; + ret = rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, + rxkad_abort_resp_call_ctr); + goto error; } if (call_id > counter) { if (conn->channels[i].call) { - rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, + ret = rxrpc_abort_conn(conn, skb, RXKADSEALEDINCON, -EPROTO, rxkad_abort_resp_call_state); - goto protocol_error_free; + goto error; } conn->channels[i].call_counter = call_id; } } if (ntohl(response->encrypted.inc_nonce) != conn->rxkad.nonce + 1) { - rxrpc_abort_conn(conn, skb, RXKADOUTOFSEQUENCE, -EPROTO, - rxkad_abort_resp_ooseq); - goto protocol_error_free; + ret = rxrpc_abort_conn(conn, skb, RXKADOUTOFSEQUENCE, -EPROTO, + rxkad_abort_resp_ooseq); + goto error; } level = ntohl(response->encrypted.level); if (level > RXRPC_SECURITY_ENCRYPT) { - rxrpc_abort_conn(conn, skb, RXKADLEVELFAIL, -EPROTO, - rxkad_abort_resp_level); - goto protocol_error_free; + ret = rxrpc_abort_conn(conn, skb, RXKADLEVELFAIL, -EPROTO, + rxkad_abort_resp_level); + goto error; } conn->security_level = level; @@ -1280,31 +1285,12 @@ static int rxkad_verify_response(struct rxrpc_connection *conn, * this the connection security can be handled in exactly the same way * as for a client connection */ ret = rxrpc_get_server_data_key(conn, &session_key, expiry, kvno); - if (ret < 0) - goto temporary_error_free_ticket; - - kfree(ticket); - kfree(response); - _leave(" = 0"); - return 0; - -protocol_error_free: - kfree(ticket); -protocol_error: - kfree(response); - key_put(server_key); - return -EPROTO; -temporary_error_free_ticket: +error: kfree(ticket); -temporary_error_free_resp: kfree(response); -temporary_error: - /* Ignore the response packet if we got a temporary error such as - * ENOMEM. We just want to send the challenge again. Note that we - * also come out this way if the ticket decryption fails. - */ key_put(server_key); + _leave(" = %d", ret); return ret; } diff --git a/net/rxrpc/skbuff.c b/net/rxrpc/skbuff.c index 3bcd6ee803960b..e2169d1a14b5fd 100644 --- a/net/rxrpc/skbuff.c +++ b/net/rxrpc/skbuff.c @@ -46,15 +46,6 @@ void rxrpc_get_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) skb_get(skb); } -/* - * Note the dropping of a ref on a socket buffer by the core. - */ -void rxrpc_eaten_skb(struct sk_buff *skb, enum rxrpc_skb_trace why) -{ - int n = atomic_inc_return(&rxrpc_n_rx_skbs); - trace_rxrpc_skb(skb, 0, n, why); -} - /* * Note the destruction of a socket buffer. */ diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 7d5e50c921a07d..6158e13c98d358 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -328,9 +328,13 @@ static int tcf_ct_flow_table_get(struct net *net, struct tcf_ct_params *params) int err = -ENOMEM; mutex_lock(&zones_mutex); - ct_ft = rhashtable_lookup_fast(&zones_ht, &key, zones_params); - if (ct_ft && refcount_inc_not_zero(&ct_ft->ref)) + rcu_read_lock(); + ct_ft = rhashtable_lookup(&zones_ht, &key, zones_params); + if (ct_ft && refcount_inc_not_zero(&ct_ft->ref)) { + rcu_read_unlock(); goto out_unlock; + } + rcu_read_unlock(); ct_ft = kzalloc_obj(*ct_ft); if (!ct_ft) diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index 05e0b14b577314..2c5a7a321a9438 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -354,7 +354,7 @@ static int tcf_blockcast_redir(struct sk_buff *skb, struct tcf_mirred *m, goto assign_prev; tcf_mirred_to_dev(skb, m, dev_prev, - dev_is_mac_header_xmit(dev), + dev_is_mac_header_xmit(dev_prev), mirred_eaction, retval); assign_prev: dev_prev = dev; diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c index 23884ef8b80cea..646a730dca93c9 100644 --- a/net/sched/cls_fw.c +++ b/net/sched/cls_fw.c @@ -74,9 +74,13 @@ TC_INDIRECT_SCOPE int fw_classify(struct sk_buff *skb, } } } else { - struct Qdisc *q = tcf_block_q(tp->chain->block); + struct Qdisc *q; /* Old method: classify the packet using its skb mark. */ + if (tcf_block_shared(tp->chain->block)) + return -1; + + q = tcf_block_q(tp->chain->block); if (id && (TC_H_MAJ(id) == 0 || !(TC_H_MAJ(id ^ q->handle)))) { res->classid = id; diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index 9efe23f8371b3c..7033f859a3948a 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -619,7 +619,7 @@ static bool cake_update_flowkeys(struct flow_keys *keys, } port = rev ? tuple.src.u.all : tuple.dst.u.all; if (port != keys->ports.dst) { - port = keys->ports.dst; + keys->ports.dst = port; upd = true; } } @@ -813,7 +813,7 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb, i++, k = (k + 1) % CAKE_SET_WAYS) { if (q->tags[outer_hash + k] == flow_hash) { if (i) - q->way_hits++; + WRITE_ONCE(q->way_hits, q->way_hits + 1); if (!q->flows[outer_hash + k].set) { /* need to increment host refcnts */ @@ -831,7 +831,7 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb, for (i = 0; i < CAKE_SET_WAYS; i++, k = (k + 1) % CAKE_SET_WAYS) { if (!q->flows[outer_hash + k].set) { - q->way_misses++; + WRITE_ONCE(q->way_misses, q->way_misses + 1); allocate_src = cake_dsrc(flow_mode); allocate_dst = cake_ddst(flow_mode); goto found; @@ -841,7 +841,7 @@ static u32 cake_hash(struct cake_tin_data *q, const struct sk_buff *skb, /* With no empty queues, default to the original * queue, accept the collision, update the host tags. */ - q->way_collisions++; + WRITE_ONCE(q->way_collisions, q->way_collisions + 1); allocate_src = cake_dsrc(flow_mode); allocate_dst = cake_ddst(flow_mode); @@ -1379,9 +1379,9 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off) len -= off; if (qd->max_netlen < len) - qd->max_netlen = len; + WRITE_ONCE(qd->max_netlen, len); if (qd->min_netlen > len) - qd->min_netlen = len; + WRITE_ONCE(qd->min_netlen, len); len += q->rate_overhead; @@ -1401,9 +1401,9 @@ static u32 cake_calc_overhead(struct cake_sched_data *qd, u32 len, u32 off) } if (qd->max_adjlen < len) - qd->max_adjlen = len; + WRITE_ONCE(qd->max_adjlen, len); if (qd->min_adjlen > len) - qd->min_adjlen = len; + WRITE_ONCE(qd->min_adjlen, len); return len; } @@ -1416,7 +1416,7 @@ static u32 cake_overhead(struct cake_sched_data *q, const struct sk_buff *skb) u16 segs = qdisc_pkt_segs(skb); u32 len = qdisc_pkt_len(skb); - q->avg_netoff = cake_ewma(q->avg_netoff, off << 16, 8); + WRITE_ONCE(q->avg_netoff, cake_ewma(q->avg_netoff, off << 16, 8)); if (segs == 1) return cake_calc_overhead(q, len, off); @@ -1590,16 +1590,17 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free) } if (cobalt_queue_full(&flow->cvars, &b->cparams, now)) - b->unresponsive_flow_count++; + WRITE_ONCE(b->unresponsive_flow_count, + b->unresponsive_flow_count + 1); len = qdisc_pkt_len(skb); q->buffer_used -= skb->truesize; b->backlogs[idx] -= len; - b->tin_backlog -= len; + WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); sch->qstats.backlog -= len; flow->dropped++; - b->tin_dropped++; + WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1); if (q->config->rate_flags & CAKE_FLAG_INGRESS) cake_advance_shaper(q, b, skb, now, true); @@ -1795,7 +1796,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, } if (unlikely(len > b->max_skblen)) - b->max_skblen = len; + WRITE_ONCE(b->max_skblen, len); if (qdisc_pkt_segs(skb) > 1 && q->config->rate_flags & CAKE_FLAG_SPLIT_GSO) { struct sk_buff *segs, *nskb; @@ -1819,15 +1820,15 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, numsegs++; slen += segs->len; q->buffer_used += segs->truesize; - b->packets++; + WRITE_ONCE(b->packets, b->packets + 1); } /* stats */ - b->bytes += slen; b->backlogs[idx] += slen; - b->tin_backlog += slen; sch->qstats.backlog += slen; q->avg_window_bytes += slen; + WRITE_ONCE(b->bytes, b->bytes + slen); + WRITE_ONCE(b->tin_backlog, b->tin_backlog + slen); qdisc_tree_reduce_backlog(sch, 1-numsegs, len-slen); consume_skb(skb); @@ -1843,10 +1844,10 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, ack = cake_ack_filter(q, flow); if (ack) { - b->ack_drops++; + WRITE_ONCE(b->ack_drops, b->ack_drops + 1); sch->qstats.drops++; ack_pkt_len = qdisc_pkt_len(ack); - b->bytes += ack_pkt_len; + WRITE_ONCE(b->bytes, b->bytes + ack_pkt_len); q->buffer_used += skb->truesize - ack->truesize; if (q->config->rate_flags & CAKE_FLAG_INGRESS) cake_advance_shaper(q, b, ack, now, true); @@ -1859,12 +1860,12 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, } /* stats */ - b->packets++; - b->bytes += len - ack_pkt_len; + WRITE_ONCE(b->packets, b->packets + 1); b->backlogs[idx] += len - ack_pkt_len; - b->tin_backlog += len - ack_pkt_len; sch->qstats.backlog += len - ack_pkt_len; q->avg_window_bytes += len - ack_pkt_len; + WRITE_ONCE(b->bytes, b->bytes + len - ack_pkt_len); + WRITE_ONCE(b->tin_backlog, b->tin_backlog + len - ack_pkt_len); } if (q->overflow_timeout) @@ -1894,9 +1895,9 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, u64 b = q->avg_window_bytes * (u64)NSEC_PER_SEC; b = div64_u64(b, window_interval); - q->avg_peak_bandwidth = - cake_ewma(q->avg_peak_bandwidth, b, - b > q->avg_peak_bandwidth ? 2 : 8); + WRITE_ONCE(q->avg_peak_bandwidth, + cake_ewma(q->avg_peak_bandwidth, b, + b > q->avg_peak_bandwidth ? 2 : 8)); q->avg_window_bytes = 0; q->avg_window_begin = now; @@ -1917,11 +1918,11 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (!flow->set) { list_add_tail(&flow->flowchain, &b->new_flows); } else { - b->decaying_flow_count--; + WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count - 1); list_move_tail(&flow->flowchain, &b->new_flows); } flow->set = CAKE_SET_SPARSE; - b->sparse_flow_count++; + WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count + 1); flow->deficit = cake_get_flow_quantum(b, flow, q->config->flow_mode); } else if (flow->set == CAKE_SET_SPARSE_WAIT) { @@ -1929,15 +1930,15 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, * in the bulk rotation. */ flow->set = CAKE_SET_BULK; - b->sparse_flow_count--; - b->bulk_flow_count++; + WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1); + WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count + 1); cake_inc_srchost_bulk_flow_count(b, flow, q->config->flow_mode); cake_inc_dsthost_bulk_flow_count(b, flow, q->config->flow_mode); } if (q->buffer_used > q->buffer_max_used) - q->buffer_max_used = q->buffer_used; + WRITE_ONCE(q->buffer_max_used, q->buffer_used); if (q->buffer_used <= q->buffer_limit) return NET_XMIT_SUCCESS; @@ -1977,7 +1978,7 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch) skb = dequeue_head(flow); len = qdisc_pkt_len(skb); b->backlogs[q->cur_flow] -= len; - b->tin_backlog -= len; + WRITE_ONCE(b->tin_backlog, b->tin_backlog - len); sch->qstats.backlog -= len; q->buffer_used -= skb->truesize; sch->q.qlen--; @@ -2042,7 +2043,7 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch) cake_configure_rates(sch, new_rate, true); q->last_checked_active = now; - q->active_queues = num_active_qs; + WRITE_ONCE(q->active_queues, num_active_qs); } begin: @@ -2149,8 +2150,8 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch) */ if (flow->set == CAKE_SET_SPARSE) { if (flow->head) { - b->sparse_flow_count--; - b->bulk_flow_count++; + WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1); + WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count + 1); cake_inc_srchost_bulk_flow_count(b, flow, q->config->flow_mode); cake_inc_dsthost_bulk_flow_count(b, flow, q->config->flow_mode); @@ -2177,7 +2178,8 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch) if (!skb) { /* this queue was actually empty */ if (cobalt_queue_empty(&flow->cvars, &b->cparams, now)) - b->unresponsive_flow_count--; + WRITE_ONCE(b->unresponsive_flow_count, + b->unresponsive_flow_count - 1); if (flow->cvars.p_drop || flow->cvars.count || ktime_before(now, flow->cvars.drop_next)) { @@ -2187,32 +2189,32 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch) list_move_tail(&flow->flowchain, &b->decaying_flows); if (flow->set == CAKE_SET_BULK) { - b->bulk_flow_count--; + WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count - 1); cake_dec_srchost_bulk_flow_count(b, flow, q->config->flow_mode); cake_dec_dsthost_bulk_flow_count(b, flow, q->config->flow_mode); - b->decaying_flow_count++; + WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count + 1); } else if (flow->set == CAKE_SET_SPARSE || flow->set == CAKE_SET_SPARSE_WAIT) { - b->sparse_flow_count--; - b->decaying_flow_count++; + WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1); + WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count + 1); } flow->set = CAKE_SET_DECAYING; } else { /* remove empty queue from the flowchain */ list_del_init(&flow->flowchain); if (flow->set == CAKE_SET_SPARSE || - flow->set == CAKE_SET_SPARSE_WAIT) - b->sparse_flow_count--; - else if (flow->set == CAKE_SET_BULK) { - b->bulk_flow_count--; + flow->set == CAKE_SET_SPARSE_WAIT) { + WRITE_ONCE(b->sparse_flow_count, b->sparse_flow_count - 1); + } else if (flow->set == CAKE_SET_BULK) { + WRITE_ONCE(b->bulk_flow_count, b->bulk_flow_count - 1); cake_dec_srchost_bulk_flow_count(b, flow, q->config->flow_mode); cake_dec_dsthost_bulk_flow_count(b, flow, q->config->flow_mode); - } else - b->decaying_flow_count--; - + } else { + WRITE_ONCE(b->decaying_flow_count, b->decaying_flow_count - 1); + } flow->set = CAKE_SET_NONE; } goto begin; @@ -2234,7 +2236,7 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch) b->tin_deficit -= len; } flow->dropped++; - b->tin_dropped++; + WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1); qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb)); qdisc_qstats_drop(sch); qdisc_dequeue_drop(sch, skb, reason); @@ -2242,17 +2244,19 @@ static struct sk_buff *cake_dequeue(struct Qdisc *sch) goto retry; } - b->tin_ecn_mark += !!flow->cvars.ecn_marked; + WRITE_ONCE(b->tin_ecn_mark, b->tin_ecn_mark + !!flow->cvars.ecn_marked); qdisc_bstats_update(sch, skb); WRITE_ONCE(q->last_active, now); /* collect delay stats */ delay = ktime_to_ns(ktime_sub(now, cobalt_get_enqueue_time(skb))); - b->avge_delay = cake_ewma(b->avge_delay, delay, 8); - b->peak_delay = cake_ewma(b->peak_delay, delay, - delay > b->peak_delay ? 2 : 8); - b->base_delay = cake_ewma(b->base_delay, delay, - delay < b->base_delay ? 2 : 8); + WRITE_ONCE(b->avge_delay, cake_ewma(b->avge_delay, delay, 8)); + WRITE_ONCE(b->peak_delay, + cake_ewma(b->peak_delay, delay, + delay > b->peak_delay ? 2 : 8)); + WRITE_ONCE(b->base_delay, + cake_ewma(b->base_delay, delay, + delay < b->base_delay ? 2 : 8)); len = cake_advance_shaper(q, b, skb, now, false); flow->deficit -= len; @@ -2329,9 +2333,9 @@ static void cake_set_rate(struct cake_tin_data *b, u64 rate, u32 mtu, u8 rate_shft = 0; u64 rate_ns = 0; - b->flow_quantum = 1514; if (rate) { - b->flow_quantum = max(min(rate >> 12, 1514ULL), 300ULL); + WRITE_ONCE(b->flow_quantum, + max(min(rate >> 12, 1514ULL), 300ULL)); rate_shft = 34; rate_ns = ((u64)NSEC_PER_SEC) << rate_shft; rate_ns = div64_u64(rate_ns, max(MIN_RATE, rate)); @@ -2339,9 +2343,11 @@ static void cake_set_rate(struct cake_tin_data *b, u64 rate, u32 mtu, rate_ns >>= 1; rate_shft--; } - } /* else unlimited, ie. zero delay */ - - b->tin_rate_bps = rate; + } else { + /* else unlimited, ie. zero delay */ + WRITE_ONCE(b->flow_quantum, 1514); + } + WRITE_ONCE(b->tin_rate_bps, rate); b->tin_rate_ns = rate_ns; b->tin_rate_shft = rate_shft; @@ -2350,10 +2356,11 @@ static void cake_set_rate(struct cake_tin_data *b, u64 rate, u32 mtu, byte_target_ns = (byte_target * rate_ns) >> rate_shft; - b->cparams.target = max((byte_target_ns * 3) / 2, target_ns); - b->cparams.interval = max(rtt_est_ns + - b->cparams.target - target_ns, - b->cparams.target * 2); + WRITE_ONCE(b->cparams.target, + max((byte_target_ns * 3) / 2, target_ns)); + WRITE_ONCE(b->cparams.interval, + max(rtt_est_ns + b->cparams.target - target_ns, + b->cparams.target * 2)); b->cparams.mtu_time = byte_target_ns; b->cparams.p_inc = 1 << 24; /* 1/256 */ b->cparams.p_dec = 1 << 20; /* 1/4096 */ @@ -2611,25 +2618,27 @@ static void cake_reconfigure(struct Qdisc *sch) { struct cake_sched_data *qd = qdisc_priv(sch); struct cake_sched_config *q = qd->config; + u32 buffer_limit; cake_configure_rates(sch, qd->config->rate_bps, false); if (q->buffer_config_limit) { - qd->buffer_limit = q->buffer_config_limit; + buffer_limit = q->buffer_config_limit; } else if (q->rate_bps) { u64 t = q->rate_bps * q->interval; do_div(t, USEC_PER_SEC / 4); - qd->buffer_limit = max_t(u32, t, 4U << 20); + buffer_limit = max_t(u32, t, 4U << 20); } else { - qd->buffer_limit = ~0; + buffer_limit = ~0; } sch->flags &= ~TCQ_F_CAN_BYPASS; - qd->buffer_limit = min(qd->buffer_limit, - max(sch->limit * psched_mtu(qdisc_dev(sch)), - q->buffer_config_limit)); + WRITE_ONCE(qd->buffer_limit, + min(buffer_limit, + max(sch->limit * psched_mtu(qdisc_dev(sch)), + q->buffer_config_limit))); } static int cake_config_change(struct cake_sched_config *q, struct nlattr *opt, @@ -2774,10 +2783,10 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt, return ret; if (overhead_changed) { - qd->max_netlen = 0; - qd->max_adjlen = 0; - qd->min_netlen = ~0; - qd->min_adjlen = ~0; + WRITE_ONCE(qd->max_netlen, 0); + WRITE_ONCE(qd->max_adjlen, 0); + WRITE_ONCE(qd->min_netlen, ~0); + WRITE_ONCE(qd->min_adjlen, ~0); } if (qd->tins) { @@ -2995,15 +3004,15 @@ static int cake_dump_stats(struct Qdisc *sch, struct gnet_dump *d) goto nla_put_failure; \ } while (0) - PUT_STAT_U64(CAPACITY_ESTIMATE64, q->avg_peak_bandwidth); - PUT_STAT_U32(MEMORY_LIMIT, q->buffer_limit); - PUT_STAT_U32(MEMORY_USED, q->buffer_max_used); - PUT_STAT_U32(AVG_NETOFF, ((q->avg_netoff + 0x8000) >> 16)); - PUT_STAT_U32(MAX_NETLEN, q->max_netlen); - PUT_STAT_U32(MAX_ADJLEN, q->max_adjlen); - PUT_STAT_U32(MIN_NETLEN, q->min_netlen); - PUT_STAT_U32(MIN_ADJLEN, q->min_adjlen); - PUT_STAT_U32(ACTIVE_QUEUES, q->active_queues); + PUT_STAT_U64(CAPACITY_ESTIMATE64, READ_ONCE(q->avg_peak_bandwidth)); + PUT_STAT_U32(MEMORY_LIMIT, READ_ONCE(q->buffer_limit)); + PUT_STAT_U32(MEMORY_USED, READ_ONCE(q->buffer_max_used)); + PUT_STAT_U32(AVG_NETOFF, ((READ_ONCE(q->avg_netoff) + 0x8000) >> 16)); + PUT_STAT_U32(MAX_NETLEN, READ_ONCE(q->max_netlen)); + PUT_STAT_U32(MAX_ADJLEN, READ_ONCE(q->max_adjlen)); + PUT_STAT_U32(MIN_NETLEN, READ_ONCE(q->min_netlen)); + PUT_STAT_U32(MIN_ADJLEN, READ_ONCE(q->min_adjlen)); + PUT_STAT_U32(ACTIVE_QUEUES, READ_ONCE(q->active_queues)); #undef PUT_STAT_U32 #undef PUT_STAT_U64 @@ -3029,38 +3038,38 @@ static int cake_dump_stats(struct Qdisc *sch, struct gnet_dump *d) if (!ts) goto nla_put_failure; - PUT_TSTAT_U64(THRESHOLD_RATE64, b->tin_rate_bps); - PUT_TSTAT_U64(SENT_BYTES64, b->bytes); - PUT_TSTAT_U32(BACKLOG_BYTES, b->tin_backlog); + PUT_TSTAT_U64(THRESHOLD_RATE64, READ_ONCE(b->tin_rate_bps)); + PUT_TSTAT_U64(SENT_BYTES64, READ_ONCE(b->bytes)); + PUT_TSTAT_U32(BACKLOG_BYTES, READ_ONCE(b->tin_backlog)); PUT_TSTAT_U32(TARGET_US, - ktime_to_us(ns_to_ktime(b->cparams.target))); + ktime_to_us(ns_to_ktime(READ_ONCE(b->cparams.target)))); PUT_TSTAT_U32(INTERVAL_US, - ktime_to_us(ns_to_ktime(b->cparams.interval))); + ktime_to_us(ns_to_ktime(READ_ONCE(b->cparams.interval)))); - PUT_TSTAT_U32(SENT_PACKETS, b->packets); - PUT_TSTAT_U32(DROPPED_PACKETS, b->tin_dropped); - PUT_TSTAT_U32(ECN_MARKED_PACKETS, b->tin_ecn_mark); - PUT_TSTAT_U32(ACKS_DROPPED_PACKETS, b->ack_drops); + PUT_TSTAT_U32(SENT_PACKETS, READ_ONCE(b->packets)); + PUT_TSTAT_U32(DROPPED_PACKETS, READ_ONCE(b->tin_dropped)); + PUT_TSTAT_U32(ECN_MARKED_PACKETS, READ_ONCE(b->tin_ecn_mark)); + PUT_TSTAT_U32(ACKS_DROPPED_PACKETS, READ_ONCE(b->ack_drops)); PUT_TSTAT_U32(PEAK_DELAY_US, - ktime_to_us(ns_to_ktime(b->peak_delay))); + ktime_to_us(ns_to_ktime(READ_ONCE(b->peak_delay)))); PUT_TSTAT_U32(AVG_DELAY_US, - ktime_to_us(ns_to_ktime(b->avge_delay))); + ktime_to_us(ns_to_ktime(READ_ONCE(b->avge_delay)))); PUT_TSTAT_U32(BASE_DELAY_US, - ktime_to_us(ns_to_ktime(b->base_delay))); + ktime_to_us(ns_to_ktime(READ_ONCE(b->base_delay)))); - PUT_TSTAT_U32(WAY_INDIRECT_HITS, b->way_hits); - PUT_TSTAT_U32(WAY_MISSES, b->way_misses); - PUT_TSTAT_U32(WAY_COLLISIONS, b->way_collisions); + PUT_TSTAT_U32(WAY_INDIRECT_HITS, READ_ONCE(b->way_hits)); + PUT_TSTAT_U32(WAY_MISSES, READ_ONCE(b->way_misses)); + PUT_TSTAT_U32(WAY_COLLISIONS, READ_ONCE(b->way_collisions)); - PUT_TSTAT_U32(SPARSE_FLOWS, b->sparse_flow_count + - b->decaying_flow_count); - PUT_TSTAT_U32(BULK_FLOWS, b->bulk_flow_count); - PUT_TSTAT_U32(UNRESPONSIVE_FLOWS, b->unresponsive_flow_count); - PUT_TSTAT_U32(MAX_SKBLEN, b->max_skblen); + PUT_TSTAT_U32(SPARSE_FLOWS, READ_ONCE(b->sparse_flow_count) + + READ_ONCE(b->decaying_flow_count)); + PUT_TSTAT_U32(BULK_FLOWS, READ_ONCE(b->bulk_flow_count)); + PUT_TSTAT_U32(UNRESPONSIVE_FLOWS, READ_ONCE(b->unresponsive_flow_count)); + PUT_TSTAT_U32(MAX_SKBLEN, READ_ONCE(b->max_skblen)); - PUT_TSTAT_U32(FLOW_QUANTUM, b->flow_quantum); + PUT_TSTAT_U32(FLOW_QUANTUM, READ_ONCE(b->flow_quantum)); nla_nest_end(d->skb, ts); } @@ -3298,10 +3307,10 @@ static int cake_mq_change(struct Qdisc *sch, struct nlattr *opt, struct cake_sched_data *qd = qdisc_priv(chld); if (overhead_changed) { - qd->max_netlen = 0; - qd->max_adjlen = 0; - qd->min_netlen = ~0; - qd->min_adjlen = ~0; + WRITE_ONCE(qd->max_netlen, 0); + WRITE_ONCE(qd->max_adjlen, 0); + WRITE_ONCE(qd->min_netlen, ~0); + WRITE_ONCE(qd->min_adjlen, ~0); } if (qd->tins) { diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c index 94df8e741a9791..2875bcdb18a413 100644 --- a/net/sched/sch_choke.c +++ b/net/sched/sch_choke.c @@ -229,7 +229,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* Draw a packet at random from queue and compare flow */ if (choke_match_random(q, skb, &idx)) { - q->stats.matched++; + WRITE_ONCE(q->stats.matched, q->stats.matched + 1); choke_drop_by_idx(sch, idx, to_free); goto congestion_drop; } @@ -241,11 +241,13 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, qdisc_qstats_overlimit(sch); if (use_harddrop(q) || !use_ecn(q) || !INET_ECN_set_ce(skb)) { - q->stats.forced_drop++; + WRITE_ONCE(q->stats.forced_drop, + q->stats.forced_drop + 1); goto congestion_drop; } - q->stats.forced_mark++; + WRITE_ONCE(q->stats.forced_mark, + q->stats.forced_mark + 1); } else if (++q->vars.qcount) { if (red_mark_probability(p, &q->vars, q->vars.qavg)) { q->vars.qcount = 0; @@ -253,11 +255,13 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, qdisc_qstats_overlimit(sch); if (!use_ecn(q) || !INET_ECN_set_ce(skb)) { - q->stats.prob_drop++; + WRITE_ONCE(q->stats.prob_drop, + q->stats.prob_drop + 1); goto congestion_drop; } - q->stats.prob_mark++; + WRITE_ONCE(q->stats.prob_mark, + q->stats.prob_mark + 1); } } else q->vars.qR = red_random(p); @@ -272,7 +276,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch, return NET_XMIT_SUCCESS; } - q->stats.pdrop++; + WRITE_ONCE(q->stats.pdrop, q->stats.pdrop + 1); return qdisc_drop(skb, sch, to_free); congestion_drop: @@ -461,10 +465,12 @@ static int choke_dump_stats(struct Qdisc *sch, struct gnet_dump *d) { struct choke_sched_data *q = qdisc_priv(sch); struct tc_choke_xstats st = { - .early = q->stats.prob_drop + q->stats.forced_drop, - .marked = q->stats.prob_mark + q->stats.forced_mark, - .pdrop = q->stats.pdrop, - .matched = q->stats.matched, + .early = READ_ONCE(q->stats.prob_drop) + + READ_ONCE(q->stats.forced_drop), + .marked = READ_ONCE(q->stats.prob_mark) + + READ_ONCE(q->stats.forced_mark), + .pdrop = READ_ONCE(q->stats.pdrop), + .matched = READ_ONCE(q->stats.matched), }; return gnet_stats_copy_app(d, &st, sizeof(st)); diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index 6d7e6389758dc8..d200c08ce50a1e 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -872,11 +872,35 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt, old_backlog = sch->qstats.backlog; while (qdisc_qlen(sch) > sch->limit || q->memory_used > q->memory_limit) { - struct sk_buff *skb = qdisc_dequeue_internal(sch, true); + struct sk_buff *skb = NULL; - q->memory_used -= skb->truesize; - qdisc_qstats_backlog_dec(sch, skb); - rtnl_qdisc_drop(skb, sch); + if (qdisc_qlen(sch) > qdisc_qlen(q->l_queue)) { + skb = qdisc_dequeue_internal(sch, true); + if (unlikely(!skb)) { + WARN_ON_ONCE(1); + break; + } + q->memory_used -= skb->truesize; + rtnl_qdisc_drop(skb, sch); + } else if (qdisc_qlen(q->l_queue)) { + skb = qdisc_dequeue_internal(q->l_queue, true); + if (unlikely(!skb)) { + WARN_ON_ONCE(1); + break; + } + /* L-queue packets are counted in both sch and + * l_queue on enqueue; qdisc_dequeue_internal() + * handled l_queue, so we further account for sch. + */ + --sch->q.qlen; + qdisc_qstats_backlog_dec(sch, skb); + q->memory_used -= skb->truesize; + rtnl_qdisc_drop(skb, q->l_queue); + qdisc_qstats_drop(sch); + } else { + WARN_ON_ONCE(1); + break; + } } qdisc_tree_reduce_backlog(sch, old_qlen - qdisc_qlen(sch), old_backlog - sch->qstats.backlog); diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 8181b52dd9a8aa..84b84e3ad80d6f 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -585,6 +585,8 @@ static int fq_codel_dump_stats(struct Qdisc *sch, struct gnet_dump *d) }; struct list_head *pos; + sch_tree_lock(sch); + st.qdisc_stats.maxpacket = q->cstats.maxpacket; st.qdisc_stats.drop_overlimit = q->drop_overlimit; st.qdisc_stats.ecn_mark = q->cstats.ecn_mark; @@ -593,7 +595,6 @@ static int fq_codel_dump_stats(struct Qdisc *sch, struct gnet_dump *d) st.qdisc_stats.memory_usage = q->memory_usage; st.qdisc_stats.drop_overmemory = q->drop_overmemory; - sch_tree_lock(sch); list_for_each(pos, &q->new_flows) st.qdisc_stats.new_flows_len++; diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index d8ac3519e93799..66ec15998ce054 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -509,18 +509,19 @@ static int fq_pie_dump(struct Qdisc *sch, struct sk_buff *skb) static int fq_pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d) { struct fq_pie_sched_data *q = qdisc_priv(sch); - struct tc_fq_pie_xstats st = { - .packets_in = q->stats.packets_in, - .overlimit = q->stats.overlimit, - .overmemory = q->overmemory, - .dropped = q->stats.dropped, - .ecn_mark = q->stats.ecn_mark, - .new_flow_count = q->new_flow_count, - .memory_usage = q->memory_usage, - }; + struct tc_fq_pie_xstats st = { 0 }; struct list_head *pos; sch_tree_lock(sch); + + st.packets_in = q->stats.packets_in; + st.overlimit = q->stats.overlimit; + st.overmemory = q->overmemory; + st.dropped = q->stats.dropped; + st.ecn_mark = q->stats.ecn_mark; + st.new_flow_count = q->new_flow_count; + st.memory_usage = q->memory_usage; + list_for_each(pos, &q->new_flows) st.new_flows_len++; diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index 95e5d9bfd9c8c0..96021f52d835b5 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -198,7 +198,8 @@ static struct hh_flow_state *seek_list(const u32 hash, return NULL; list_del(&flow->flowchain); kfree(flow); - q->hh_flows_current_cnt--; + WRITE_ONCE(q->hh_flows_current_cnt, + q->hh_flows_current_cnt - 1); } else if (flow->hash_id == hash) { return flow; } @@ -226,7 +227,7 @@ static struct hh_flow_state *alloc_new_hh(struct list_head *head, } if (q->hh_flows_current_cnt >= q->hh_flows_limit) { - q->hh_flows_overlimit++; + WRITE_ONCE(q->hh_flows_overlimit, q->hh_flows_overlimit + 1); return NULL; } /* Create new entry. */ @@ -234,7 +235,7 @@ static struct hh_flow_state *alloc_new_hh(struct list_head *head, if (!flow) return NULL; - q->hh_flows_current_cnt++; + WRITE_ONCE(q->hh_flows_current_cnt, q->hh_flows_current_cnt + 1); INIT_LIST_HEAD(&flow->flowchain); list_add_tail(&flow->flowchain, head); @@ -309,7 +310,7 @@ static enum wdrr_bucket_idx hhf_classify(struct sk_buff *skb, struct Qdisc *sch) return WDRR_BUCKET_FOR_NON_HH; flow->hash_id = hash; flow->hit_timestamp = now; - q->hh_flows_total_cnt++; + WRITE_ONCE(q->hh_flows_total_cnt, q->hh_flows_total_cnt + 1); /* By returning without updating counters in q->hhf_arrays, * we implicitly implement "shielding" (see Optimization O1). @@ -403,7 +404,7 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch, return NET_XMIT_SUCCESS; prev_backlog = sch->qstats.backlog; - q->drop_overlimit++; + WRITE_ONCE(q->drop_overlimit, q->drop_overlimit + 1); /* Return Congestion Notification only if we dropped a packet from this * bucket. */ @@ -686,10 +687,10 @@ static int hhf_dump_stats(struct Qdisc *sch, struct gnet_dump *d) { struct hhf_sched_data *q = qdisc_priv(sch); struct tc_hhf_xstats st = { - .drop_overlimit = q->drop_overlimit, - .hh_overlimit = q->hh_flows_overlimit, - .hh_tot_count = q->hh_flows_total_cnt, - .hh_cur_count = q->hh_flows_current_cnt, + .drop_overlimit = READ_ONCE(q->drop_overlimit), + .hh_overlimit = READ_ONCE(q->hh_flows_overlimit), + .hh_tot_count = READ_ONCE(q->hh_flows_total_cnt), + .hh_cur_count = READ_ONCE(q->hh_flows_current_cnt), }; return gnet_stats_copy_app(d, &st, sizeof(st)); diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index 20df1c08b1e9d0..bc18e1976b6e07 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -227,10 +227,10 @@ static bool loss_4state(struct netem_sched_data *q) if (rnd < clg->a4) { clg->state = LOST_IN_GAP_PERIOD; return true; - } else if (clg->a4 < rnd && rnd < clg->a1 + clg->a4) { + } else if (rnd < clg->a1 + clg->a4) { clg->state = LOST_IN_BURST_PERIOD; return true; - } else if (clg->a1 + clg->a4 < rnd) { + } else { clg->state = TX_IN_GAP_PERIOD; } @@ -247,9 +247,9 @@ static bool loss_4state(struct netem_sched_data *q) case LOST_IN_BURST_PERIOD: if (rnd < clg->a3) clg->state = TX_IN_BURST_PERIOD; - else if (clg->a3 < rnd && rnd < clg->a2 + clg->a3) { + else if (rnd < clg->a2 + clg->a3) { clg->state = TX_IN_GAP_PERIOD; - } else if (clg->a2 + clg->a3 < rnd) { + } else { clg->state = LOST_IN_BURST_PERIOD; return true; } @@ -524,7 +524,7 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch, 1 << get_random_u32_below(8); } - if (unlikely(q->t_len >= sch->limit)) { + if (unlikely(sch->q.qlen >= sch->limit)) { /* re-link segs, so that qdisc_drop_all() frees them all */ skb->next = segs; qdisc_drop_all(skb, sch, to_free); @@ -659,9 +659,8 @@ static void get_slot_next(struct netem_sched_data *q, u64 now) if (!q->slot_dist) next_delay = q->slot_config.min_delay + - (get_random_u32() * - (q->slot_config.max_delay - - q->slot_config.min_delay) >> 32); + mul_u64_u32_shr(q->slot_config.max_delay - q->slot_config.min_delay, + get_random_u32(), 32); else next_delay = tabledist(q->slot_config.dist_delay, (s32)(q->slot_config.dist_jitter), @@ -827,6 +826,39 @@ static int get_dist_table(struct disttable **tbl, const struct nlattr *attr) return 0; } +static int validate_time(const struct nlattr *attr, const char *name, + struct netlink_ext_ack *extack) +{ + if (nla_get_s64(attr) < 0) { + NL_SET_ERR_MSG_ATTR_FMT(extack, attr, "negative %s", name); + return -EINVAL; + } + return 0; +} + +static int validate_slot(const struct nlattr *attr, struct netlink_ext_ack *extack) +{ + const struct tc_netem_slot *c = nla_data(attr); + + if (c->min_delay < 0 || c->max_delay < 0) { + NL_SET_ERR_MSG_ATTR(extack, attr, "negative slot delay"); + return -EINVAL; + } + if (c->min_delay > c->max_delay) { + NL_SET_ERR_MSG_ATTR(extack, attr, "slot min delay greater than max delay"); + return -EINVAL; + } + if (c->dist_delay < 0 || c->dist_jitter < 0) { + NL_SET_ERR_MSG_ATTR(extack, attr, "negative dist delay"); + return -EINVAL; + } + if (c->max_packets < 0 || c->max_bytes < 0) { + NL_SET_ERR_MSG_ATTR(extack, attr, "negative slot limit"); + return -EINVAL; + } + return 0; +} + static void get_slot(struct netem_sched_data *q, const struct nlattr *attr) { const struct tc_netem_slot *c = nla_data(attr); @@ -1040,6 +1072,24 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt, goto table_free; } + if (tb[TCA_NETEM_SLOT]) { + ret = validate_slot(tb[TCA_NETEM_SLOT], extack); + if (ret) + goto table_free; + } + + if (tb[TCA_NETEM_LATENCY64]) { + ret = validate_time(tb[TCA_NETEM_LATENCY64], "latency", extack); + if (ret) + goto table_free; + } + + if (tb[TCA_NETEM_JITTER64]) { + ret = validate_time(tb[TCA_NETEM_JITTER64], "jitter", extack); + if (ret) + goto table_free; + } + sch_tree_lock(sch); /* backup q->clg and q->loss_model */ old_clg = q->clg; @@ -1112,11 +1162,10 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt, /* capping jitter to the range acceptable by tabledist() */ q->jitter = min_t(s64, abs(q->jitter), INT_MAX); - if (tb[TCA_NETEM_PRNG_SEED]) + if (tb[TCA_NETEM_PRNG_SEED]) { q->prng.seed = nla_get_u64(tb[TCA_NETEM_PRNG_SEED]); - else - q->prng.seed = get_random_u64(); - prandom_seed_state(&q->prng.prng_state, q->prng.seed); + prandom_seed_state(&q->prng.prng_state, q->prng.seed); + } unlock: sch_tree_unlock(sch); @@ -1139,6 +1188,9 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt, return -EINVAL; q->loss_model = CLG_RANDOM; + q->prng.seed = get_random_u64(); + prandom_seed_state(&q->prng.prng_state, q->prng.seed); + ret = netem_change(sch, opt, extack); if (ret) pr_info("netem: change failed\n"); diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c index 0a377313b6a9d2..40149edecbd5aa 100644 --- a/net/sched/sch_pie.c +++ b/net/sched/sch_pie.c @@ -90,7 +90,7 @@ static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, bool enqueue = false; if (unlikely(qdisc_qlen(sch) >= sch->limit)) { - q->stats.overlimit++; + WRITE_ONCE(q->stats.overlimit, q->stats.overlimit + 1); goto out; } @@ -104,7 +104,7 @@ static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, /* If packet is ecn capable, mark it if drop probability * is lower than 10%, else drop it. */ - q->stats.ecn_mark++; + WRITE_ONCE(q->stats.ecn_mark, q->stats.ecn_mark + 1); enqueue = true; } @@ -114,15 +114,15 @@ static int pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (!q->params.dq_rate_estimator) pie_set_enqueue_time(skb); - q->stats.packets_in++; + WRITE_ONCE(q->stats.packets_in, q->stats.packets_in + 1); if (qdisc_qlen(sch) > q->stats.maxq) - q->stats.maxq = qdisc_qlen(sch); + WRITE_ONCE(q->stats.maxq, qdisc_qlen(sch)); return qdisc_enqueue_tail(skb, sch); } out: - q->stats.dropped++; + WRITE_ONCE(q->stats.dropped, q->stats.dropped + 1); q->vars.accu_prob = 0; return qdisc_drop_reason(skb, sch, to_free, reason); } @@ -219,16 +219,14 @@ void pie_process_dequeue(struct sk_buff *skb, struct pie_params *params, * packet timestamp. */ if (!params->dq_rate_estimator) { - vars->qdelay = now - pie_get_enqueue_time(skb); + WRITE_ONCE(vars->qdelay, + backlog ? now - pie_get_enqueue_time(skb) : 0); if (vars->dq_tstamp != DTIME_INVALID) dtime = now - vars->dq_tstamp; vars->dq_tstamp = now; - if (backlog == 0) - vars->qdelay = 0; - if (dtime == 0) return; @@ -267,11 +265,11 @@ void pie_process_dequeue(struct sk_buff *skb, struct pie_params *params, count = count / dtime; if (vars->avg_dq_rate == 0) - vars->avg_dq_rate = count; + WRITE_ONCE(vars->avg_dq_rate, count); else - vars->avg_dq_rate = + WRITE_ONCE(vars->avg_dq_rate, (vars->avg_dq_rate - - (vars->avg_dq_rate >> 3)) + (count >> 3); + (vars->avg_dq_rate >> 3)) + (count >> 3)); /* If the queue has receded below the threshold, we hold * on to the last drain rate calculated, else we reset @@ -376,12 +374,12 @@ void pie_calculate_probability(struct pie_params *params, struct pie_vars *vars, if (qdelay > (PSCHED_NS2TICKS(250 * NSEC_PER_MSEC))) delta += MAX_PROB / (100 / 2); - vars->prob += delta; + WRITE_ONCE(vars->prob, vars->prob + delta); if (delta > 0) { /* prevent overflow */ if (vars->prob < oldprob) { - vars->prob = MAX_PROB; + WRITE_ONCE(vars->prob, MAX_PROB); /* Prevent normalization error. If probability is at * maximum value already, we normalize it here, and * skip the check to do a non-linear drop in the next @@ -392,7 +390,7 @@ void pie_calculate_probability(struct pie_params *params, struct pie_vars *vars, } else { /* prevent underflow */ if (vars->prob > oldprob) - vars->prob = 0; + WRITE_ONCE(vars->prob, 0); } /* Non-linear drop in probability: Reduce drop probability quickly if @@ -401,9 +399,9 @@ void pie_calculate_probability(struct pie_params *params, struct pie_vars *vars, if (qdelay == 0 && qdelay_old == 0 && update_prob) /* Reduce drop probability to 98.4% */ - vars->prob -= vars->prob / 64; + WRITE_ONCE(vars->prob, vars->prob - vars->prob / 64); - vars->qdelay = qdelay; + WRITE_ONCE(vars->qdelay, qdelay); vars->backlog_old = backlog; /* We restart the measurement cycle if the following conditions are met @@ -501,22 +499,22 @@ static int pie_dump_stats(struct Qdisc *sch, struct gnet_dump *d) { struct pie_sched_data *q = qdisc_priv(sch); struct tc_pie_xstats st = { - .prob = q->vars.prob << BITS_PER_BYTE, - .delay = ((u32)PSCHED_TICKS2NS(q->vars.qdelay)) / + .prob = READ_ONCE(q->vars.prob) << BITS_PER_BYTE, + .delay = ((u32)PSCHED_TICKS2NS(READ_ONCE(q->vars.qdelay))) / NSEC_PER_USEC, - .packets_in = q->stats.packets_in, - .overlimit = q->stats.overlimit, - .maxq = q->stats.maxq, - .dropped = q->stats.dropped, - .ecn_mark = q->stats.ecn_mark, + .packets_in = READ_ONCE(q->stats.packets_in), + .overlimit = READ_ONCE(q->stats.overlimit), + .maxq = READ_ONCE(q->stats.maxq), + .dropped = READ_ONCE(q->stats.dropped), + .ecn_mark = READ_ONCE(q->stats.ecn_mark), }; /* avg_dq_rate is only valid if dq_rate_estimator is enabled */ - st.dq_rate_estimating = q->params.dq_rate_estimator; + st.dq_rate_estimating = READ_ONCE(q->params.dq_rate_estimator); /* unscale and return dq_rate in bytes per sec */ - if (q->params.dq_rate_estimator) - st.avg_dq_rate = q->vars.avg_dq_rate * + if (st.dq_rate_estimating) + st.avg_dq_rate = READ_ONCE(q->vars.avg_dq_rate) * (PSCHED_TICKS_PER_SEC) >> PIE_SCALE; return gnet_stats_copy_app(d, &st, sizeof(st)); diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 479c42d11083f0..86651a68d40150 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -90,17 +90,20 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, case RED_PROB_MARK: qdisc_qstats_overlimit(sch); if (!red_use_ecn(q)) { - q->stats.prob_drop++; + WRITE_ONCE(q->stats.prob_drop, + q->stats.prob_drop + 1); goto congestion_drop; } if (INET_ECN_set_ce(skb)) { - q->stats.prob_mark++; + WRITE_ONCE(q->stats.prob_mark, + q->stats.prob_mark + 1); skb = tcf_qevent_handle(&q->qe_mark, sch, skb, to_free, &ret); if (!skb) return NET_XMIT_CN | ret; } else if (!red_use_nodrop(q)) { - q->stats.prob_drop++; + WRITE_ONCE(q->stats.prob_drop, + q->stats.prob_drop + 1); goto congestion_drop; } @@ -111,17 +114,20 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, reason = SKB_DROP_REASON_QDISC_OVERLIMIT; qdisc_qstats_overlimit(sch); if (red_use_harddrop(q) || !red_use_ecn(q)) { - q->stats.forced_drop++; + WRITE_ONCE(q->stats.forced_drop, + q->stats.forced_drop + 1); goto congestion_drop; } if (INET_ECN_set_ce(skb)) { - q->stats.forced_mark++; + WRITE_ONCE(q->stats.forced_mark, + q->stats.forced_mark + 1); skb = tcf_qevent_handle(&q->qe_mark, sch, skb, to_free, &ret); if (!skb) return NET_XMIT_CN | ret; } else if (!red_use_nodrop(q)) { - q->stats.forced_drop++; + WRITE_ONCE(q->stats.forced_drop, + q->stats.forced_drop + 1); goto congestion_drop; } @@ -135,7 +141,8 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, sch->qstats.backlog += len; sch->q.qlen++; } else if (net_xmit_drop_count(ret)) { - q->stats.pdrop++; + WRITE_ONCE(q->stats.pdrop, + q->stats.pdrop + 1); qdisc_qstats_drop(sch); } return ret; @@ -155,7 +162,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch) struct red_sched_data *q = qdisc_priv(sch); struct Qdisc *child = q->qdisc; - skb = child->dequeue(child); + skb = qdisc_dequeue_peeked(child); if (skb) { qdisc_bstats_update(sch, skb); qdisc_qstats_backlog_dec(sch, skb); @@ -463,9 +470,13 @@ static int red_dump_stats(struct Qdisc *sch, struct gnet_dump *d) dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_RED, &hw_stats_request); } - st.early = q->stats.prob_drop + q->stats.forced_drop; - st.pdrop = q->stats.pdrop; - st.marked = q->stats.prob_mark + q->stats.forced_mark; + st.early = READ_ONCE(q->stats.prob_drop) + + READ_ONCE(q->stats.forced_drop); + + st.pdrop = READ_ONCE(q->stats.pdrop); + + st.marked = READ_ONCE(q->stats.prob_mark) + + READ_ONCE(q->stats.forced_mark); return gnet_stats_copy_app(d, &st, sizeof(st)); } diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c index d2835f1168e1dc..00286c930b8de7 100644 --- a/net/sched/sch_sfb.c +++ b/net/sched/sch_sfb.c @@ -130,7 +130,7 @@ static void increment_one_qlen(u32 sfbhash, u32 slot, struct sfb_sched_data *q) sfbhash >>= SFB_BUCKET_SHIFT; if (b[hash].qlen < 0xFFFF) - b[hash].qlen++; + WRITE_ONCE(b[hash].qlen, b[hash].qlen + 1); b += SFB_NUMBUCKETS; /* next level */ } } @@ -159,7 +159,7 @@ static void decrement_one_qlen(u32 sfbhash, u32 slot, sfbhash >>= SFB_BUCKET_SHIFT; if (b[hash].qlen > 0) - b[hash].qlen--; + WRITE_ONCE(b[hash].qlen, b[hash].qlen - 1); b += SFB_NUMBUCKETS; /* next level */ } } @@ -179,12 +179,12 @@ static void decrement_qlen(const struct sk_buff *skb, struct sfb_sched_data *q) static void decrement_prob(struct sfb_bucket *b, struct sfb_sched_data *q) { - b->p_mark = prob_minus(b->p_mark, q->decrement); + WRITE_ONCE(b->p_mark, prob_minus(b->p_mark, q->decrement)); } static void increment_prob(struct sfb_bucket *b, struct sfb_sched_data *q) { - b->p_mark = prob_plus(b->p_mark, q->increment); + WRITE_ONCE(b->p_mark, prob_plus(b->p_mark, q->increment)); } static void sfb_zero_all_buckets(struct sfb_sched_data *q) @@ -202,11 +202,14 @@ static u32 sfb_compute_qlen(u32 *prob_r, u32 *avgpm_r, const struct sfb_sched_da const struct sfb_bucket *b = &q->bins[q->slot].bins[0][0]; for (i = 0; i < SFB_LEVELS * SFB_NUMBUCKETS; i++) { - if (qlen < b->qlen) - qlen = b->qlen; - totalpm += b->p_mark; - if (prob < b->p_mark) - prob = b->p_mark; + u32 b_qlen = READ_ONCE(b->qlen); + u32 b_mark = READ_ONCE(b->p_mark); + + if (qlen < b_qlen) + qlen = b_qlen; + totalpm += b_mark; + if (prob < b_mark) + prob = b_mark; b++; } *prob_r = prob; @@ -295,7 +298,8 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (unlikely(sch->q.qlen >= q->limit)) { qdisc_qstats_overlimit(sch); - q->stats.queuedrop++; + WRITE_ONCE(q->stats.queuedrop, + q->stats.queuedrop + 1); goto drop; } @@ -348,7 +352,8 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (unlikely(minqlen >= q->max)) { qdisc_qstats_overlimit(sch); - q->stats.bucketdrop++; + WRITE_ONCE(q->stats.bucketdrop, + q->stats.bucketdrop + 1); goto drop; } @@ -374,7 +379,8 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } if (sfb_rate_limit(skb, q)) { qdisc_qstats_overlimit(sch); - q->stats.penaltydrop++; + WRITE_ONCE(q->stats.penaltydrop, + q->stats.penaltydrop + 1); goto drop; } goto enqueue; @@ -390,14 +396,17 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, * In either case, we want to start dropping packets. */ if (r < (p_min - SFB_MAX_PROB / 2) * 2) { - q->stats.earlydrop++; + WRITE_ONCE(q->stats.earlydrop, + q->stats.earlydrop + 1); goto drop; } } if (INET_ECN_set_ce(skb)) { - q->stats.marked++; + WRITE_ONCE(q->stats.marked, + q->stats.marked + 1); } else { - q->stats.earlydrop++; + WRITE_ONCE(q->stats.earlydrop, + q->stats.earlydrop + 1); goto drop; } } @@ -410,7 +419,8 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch, sch->q.qlen++; increment_qlen(&cb, q); } else if (net_xmit_drop_count(ret)) { - q->stats.childdrop++; + WRITE_ONCE(q->stats.childdrop, + q->stats.childdrop + 1); qdisc_qstats_drop(sch); } return ret; @@ -599,12 +609,12 @@ static int sfb_dump_stats(struct Qdisc *sch, struct gnet_dump *d) { struct sfb_sched_data *q = qdisc_priv(sch); struct tc_sfb_xstats st = { - .earlydrop = q->stats.earlydrop, - .penaltydrop = q->stats.penaltydrop, - .bucketdrop = q->stats.bucketdrop, - .queuedrop = q->stats.queuedrop, - .childdrop = q->stats.childdrop, - .marked = q->stats.marked, + .earlydrop = READ_ONCE(q->stats.earlydrop), + .penaltydrop = READ_ONCE(q->stats.penaltydrop), + .bucketdrop = READ_ONCE(q->stats.bucketdrop), + .queuedrop = READ_ONCE(q->stats.queuedrop), + .childdrop = READ_ONCE(q->stats.childdrop), + .marked = READ_ONCE(q->stats.marked), }; st.maxqlen = sfb_compute_qlen(&st.maxprob, &st.avgprob, q); diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index f721c03514f600..3c85ef1ef4818c 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -634,7 +634,7 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch, queue = skb_get_queue_mapping(skb); child = q->qdiscs[queue]; - if (unlikely(!child)) + if (unlikely(child == &noop_qdisc)) return qdisc_drop(skb, sch, to_free); if (taprio_skb_exceeds_queue_max_sdu(sch, skb)) { @@ -717,7 +717,7 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq, int len; u8 tc; - if (unlikely(!child)) + if (unlikely(child == &noop_qdisc)) return NULL; if (TXTIME_ASSIST_IS_ENABLED(q->flags)) @@ -972,11 +972,12 @@ static enum hrtimer_restart advance_sched(struct hrtimer *timer) } if (should_change_schedules(admin, oper, end_time)) { - /* Set things so the next time this runs, the new - * schedule runs. - */ - end_time = sched_base_time(admin); switch_schedules(q, &admin, &oper); + /* After changing schedules, the next entry is the first one + * in the new schedule, with a pre-calculated end_time. + */ + next = list_first_entry(&oper->entries, struct sched_entry, list); + end_time = next->end_time; } next->end_time = end_time; @@ -2183,6 +2184,9 @@ static int taprio_graft(struct Qdisc *sch, unsigned long cl, if (!dev_queue) return -EINVAL; + if (!new) + new = &noop_qdisc; + if (dev->flags & IFF_UP) dev_deactivate(dev); @@ -2196,14 +2200,14 @@ static int taprio_graft(struct Qdisc *sch, unsigned long cl, *old = q->qdiscs[cl - 1]; if (FULL_OFFLOAD_IS_ENABLED(q->flags)) { WARN_ON_ONCE(dev_graft_qdisc(dev_queue, new) != *old); - if (new) + if (new != &noop_qdisc) qdisc_refcount_inc(new); - if (*old) + if (*old && *old != &noop_qdisc) qdisc_put(*old); } q->qdiscs[cl - 1] = new; - if (new) + if (new != &noop_qdisc) new->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT; if (dev->flags & IFF_UP) diff --git a/net/sctp/inqueue.c b/net/sctp/inqueue.c index f5a7d5a3875555..a024c08432471d 100644 --- a/net/sctp/inqueue.c +++ b/net/sctp/inqueue.c @@ -201,6 +201,7 @@ struct sctp_chunk *sctp_inq_pop(struct sctp_inq *queue) cb->chunk = head_cb->chunk; cb->af = head_cb->af; + cb->encap_port = head_cb->encap_port; } } diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c index 53a5c027f8e318..cd15b695607ebe 100644 --- a/net/sctp/ipv6.c +++ b/net/sctp/ipv6.c @@ -261,9 +261,11 @@ static int sctp_v6_xmit(struct sk_buff *skb, struct sctp_transport *t) skb_set_inner_ipproto(skb, IPPROTO_SCTP); label = ip6_make_flowlabel(sock_net(sk), skb, fl6->flowlabel, true, fl6); + local_bh_disable(); udp_tunnel6_xmit_skb(dst, sk, skb, NULL, &fl6->saddr, &fl6->daddr, tclass, ip6_dst_hoplimit(dst), label, sctp_sk(sk)->udp_port, t->encap_port, false, 0); + local_bh_enable(); return 0; } diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index 828a59b8e7bf8d..5800e7ee7ea022 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -1070,10 +1070,12 @@ static inline int sctp_v4_xmit(struct sk_buff *skb, struct sctp_transport *t) skb_reset_inner_mac_header(skb); skb_reset_inner_transport_header(skb); skb_set_inner_ipproto(skb, IPPROTO_SCTP); + local_bh_disable(); udp_tunnel_xmit_skb(dst_rtable(dst), sk, skb, fl4->saddr, fl4->daddr, dscp, ip4_dst_hoplimit(dst), df, sctp_sk(sk)->udp_port, t->encap_port, false, false, 0); + local_bh_enable(); return 0; } diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 7b823d75914191..8e89a870780c49 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -1556,6 +1556,12 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( /* Tag the variable length parameters. */ chunk->param_hdr.v = skb_pull(chunk->skb, sizeof(struct sctp_inithdr)); + if (asoc->state >= SCTP_STATE_ESTABLISHED) { + /* Discard INIT matching peer vtag after handshake completion (stale INIT). */ + if (ntohl(chunk->subh.init_hdr->init_tag) == asoc->peer.i.init_tag) + return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); + } + /* Verify the INIT chunk before processing it. */ err_chunk = NULL; if (!sctp_verify_init(net, ep, asoc, chunk->chunk_hdr->type, diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 05fb00c9c33576..aeffa10ff2d34a 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -1986,6 +1986,15 @@ static int sctp_sendmsg(struct sock *sk, struct msghdr *msg, size_t msg_len) goto out_unlock; iov_iter_revert(&msg->msg_iter, err); + + /* sctp_sendmsg_to_asoc() may have released the socket + * lock (sctp_wait_for_sndbuf), during which other + * associations on ep->asocs could have been peeled + * off or freed. @asoc itself is revalidated by the + * base.dead and base.sk checks in sctp_wait_for_sndbuf, + * so re-derive the cached cursor from it. + */ + tmp = list_next_entry(asoc, asocs); } goto out_unlock; @@ -4855,8 +4864,9 @@ static struct sock *sctp_clone_sock(struct sock *sk, if (!newsk) return ERR_PTR(err); - /* sk_clone() sets refcnt to 2 */ + /* sk_clone() sets refcnt to 2 and increments sockets_allocated */ sock_put(newsk); + sk_sockets_allocated_dec(newsk); newinet = inet_sk(newsk); newsp = sctp_sk(newsk); @@ -7033,7 +7043,7 @@ static int sctp_getsockopt_peer_auth_chunks(struct sock *sk, int len, /* See if the user provided enough room for all the data */ num_chunks = ntohs(ch->param_hdr.length) - sizeof(struct sctp_paramhdr); - if (len < num_chunks) + if (len < sizeof(struct sctp_authchunks) + num_chunks) return -EINVAL; if (copy_to_user(to, ch->chunks, num_chunks)) diff --git a/net/smc/smc_clc.c b/net/smc/smc_clc.c index c38fc7bf0a7efb..014d527d5462db 100644 --- a/net/smc/smc_clc.c +++ b/net/smc/smc_clc.c @@ -788,8 +788,8 @@ int smc_clc_wait_msg(struct smc_sock *smc, void *buf, int buflen, dclc = (struct smc_clc_msg_decline *)clcm; reason_code = SMC_CLC_DECL_PEERDECL; smc->peer_diagnosis = ntohl(dclc->peer_diagnosis); - if (((struct smc_clc_msg_decline *)buf)->hdr.typev2 & - SMC_FIRST_CONTACT_MASK) { + if ((dclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK) && + smc->conn.lgr) { smc->conn.lgr->sync_err = 1; smc_lgr_terminate_sched(smc->conn.lgr); } diff --git a/net/strparser/strparser.c b/net/strparser/strparser.c index fe0e76fdd1f1cd..a23f4b4dfc6733 100644 --- a/net/strparser/strparser.c +++ b/net/strparser/strparser.c @@ -45,6 +45,14 @@ static void strp_abort_strp(struct strparser *strp, int err) strp->stopped = 1; + if (strp->skb_head) { + kfree_skb(strp->skb_head); + strp->skb_head = NULL; + } + + strp->skb_nextp = NULL; + strp->need_bytes = 0; + if (strp->sk) { struct sock *sk = strp->sk; diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 9b623849723ed0..f2d72181a6fe86 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -414,7 +414,6 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) struct ib_qp_init_attr qp_attr; struct ib_device *dev; int ret = 0; - RPC_IFDEBUG(struct sockaddr *sap); listen_rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt); clear_bit(XPT_CONN, &xprt->xpt_flags); @@ -560,18 +559,20 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) goto errout; } -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) - dprintk("svcrdma: new connection accepted on device %s:\n", dev->name); - sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.src_addr; - dprintk(" local address : %pIS:%u\n", sap, rpc_get_port(sap)); - sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr; - dprintk(" remote address : %pIS:%u\n", sap, rpc_get_port(sap)); - dprintk(" max_sge : %d\n", newxprt->sc_max_send_sges); - dprintk(" sq_depth : %d\n", newxprt->sc_sq_depth); - dprintk(" rdma_rw_ctxs : %d\n", ctxts); - dprintk(" max_requests : %d\n", newxprt->sc_max_requests); - dprintk(" ord : %d\n", conn_param.initiator_depth); -#endif + if (IS_ENABLED(CONFIG_SUNRPC_DEBUG)) { + struct sockaddr *sap; + + dprintk("svcrdma: new connection accepted on device %s:\n", dev->name); + sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.src_addr; + dprintk(" local address : %pIS:%u\n", sap, rpc_get_port(sap)); + sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr; + dprintk(" remote address : %pIS:%u\n", sap, rpc_get_port(sap)); + dprintk(" max_sge : %d\n", newxprt->sc_max_send_sges); + dprintk(" sq_depth : %d\n", newxprt->sc_sq_depth); + dprintk(" rdma_rw_ctxs : %d\n", ctxts); + dprintk(" max_requests : %d\n", newxprt->sc_max_requests); + dprintk(" ord : %d\n", conn_param.initiator_depth); + } return &newxprt->sc_xprt; diff --git a/net/tipc/msg.c b/net/tipc/msg.c index 76284fc538ebdd..b0bba0feef5644 100644 --- a/net/tipc/msg.c +++ b/net/tipc/msg.c @@ -177,8 +177,20 @@ int tipc_buf_append(struct sk_buff **headbuf, struct sk_buff **buf) if (fragid == LAST_FRAGMENT) { TIPC_SKB_CB(head)->validated = 0; - if (unlikely(!tipc_msg_validate(&head))) + + /* If the reassembled skb has been freed in + * tipc_msg_validate() because of an invalid truesize, + * then head will point to a newly allocated reassembled + * skb, while *headbuf points to freed reassembled skb. + * In such cases, correct *headbuf for freeing the newly + * allocated reassembled skb later. + */ + if (unlikely(!tipc_msg_validate(&head))) { + if (head != *headbuf) + *headbuf = head; goto err; + } + *buf = head; TIPC_SKB_CB(head)->tail = NULL; *headbuf = NULL; diff --git a/net/tls/tls.h b/net/tls/tls.h index 2f86baeb71fcb9..a1d8467bece337 100644 --- a/net/tls/tls.h +++ b/net/tls/tls.h @@ -188,6 +188,7 @@ int tls_strp_dev_init(void); void tls_strp_dev_exit(void); void tls_strp_done(struct tls_strparser *strp); +void __tls_strp_done(struct tls_strparser *strp); void tls_strp_stop(struct tls_strparser *strp); int tls_strp_init(struct tls_strparser *strp, struct sock *sk); void tls_strp_data_ready(struct tls_strparser *strp); diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index 98e12f0ff57e51..c72e8831762730 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -624,6 +624,12 @@ void tls_strp_done(struct tls_strparser *strp) WARN_ON(!strp->stopped); cancel_work_sync(&strp->work); + __tls_strp_done(strp); +} + +/* For setup error paths where the strparser was initialized but never armed. */ +void __tls_strp_done(struct tls_strparser *strp) +{ tls_strp_anchor_free(strp); } diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index 83e78a3d1e651e..23a31646d0387d 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -2625,8 +2625,12 @@ void tls_sw_free_ctx_rx(struct tls_context *tls_ctx) void tls_sw_free_resources_rx(struct sock *sk) { struct tls_context *tls_ctx = tls_get_ctx(sk); + struct tls_sw_context_rx *ctx; + + ctx = tls_sw_ctx_rx(tls_ctx); tls_sw_release_resources_rx(sk); + __tls_strp_done(&ctx->strp); tls_sw_free_ctx_rx(tls_ctx); } diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index b23c33df8b4654..001f6602a6659d 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -1964,16 +1964,19 @@ static void unix_peek_fds(struct scm_cookie *scm, struct sk_buff *skb) static void unix_destruct_scm(struct sk_buff *skb) { - struct scm_cookie scm; + struct scm_cookie scm = {}; + + swap(scm.pid, UNIXCB(skb).pid); - memset(&scm, 0, sizeof(scm)); - scm.pid = UNIXCB(skb).pid; if (UNIXCB(skb).fp) unix_detach_fds(&scm, skb); - /* Alas, it calls VFS */ - /* So fscking what? fput() had been SMP-safe since the last Summer */ scm_destroy(&scm); +} + +static void unix_wfree(struct sk_buff *skb) +{ + unix_destruct_scm(skb); sock_wfree(skb); } @@ -1989,7 +1992,7 @@ static int unix_scm_to_skb(struct scm_cookie *scm, struct sk_buff *skb, bool sen if (scm->fp && send_fds) err = unix_attach_fds(scm, skb); - skb->destructor = unix_destruct_scm; + skb->destructor = unix_wfree; return err; } @@ -2066,6 +2069,13 @@ static void scm_stat_del(struct sock *sk, struct sk_buff *skb) } } +static void unix_orphan_scm(struct sock *sk, struct sk_buff *skb) +{ + scm_stat_del(sk, skb); + unix_destruct_scm(skb); + skb->destructor = sock_wfree; +} + /* * Send AF_UNIX data. */ @@ -2679,10 +2689,16 @@ static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor) int err; mutex_lock(&u->iolock); + skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err); - mutex_unlock(&u->iolock); - if (!skb) + if (!skb) { + mutex_unlock(&u->iolock); return err; + } + + unix_orphan_scm(sk, skb); + + mutex_unlock(&u->iolock); return recv_actor(sk, skb); } @@ -2882,6 +2898,9 @@ static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor) #endif spin_unlock(&queue->lock); + + unix_orphan_scm(sk, skb); + mutex_unlock(&u->iolock); return recv_actor(sk, skb); @@ -3300,6 +3319,9 @@ static int unix_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) struct sk_buff *skb; int answ = 0; + if (sk->sk_type != SOCK_STREAM) + return -EOPNOTSUPP; + mutex_lock(&u->iolock); skb = skb_peek(&sk->sk_receive_queue); @@ -3731,15 +3753,15 @@ static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v) struct bpf_prog *prog; struct sock *sk = v; uid_t uid; - bool slow; int ret; if (v == SEQ_START_TOKEN) return 0; - slow = lock_sock_fast(sk); + lock_sock(sk); + unix_state_lock(sk); - if (unlikely(sk_unhashed(sk))) { + if (unlikely(sock_flag(sk, SOCK_DEAD))) { ret = SEQ_SKIP; goto unlock; } @@ -3749,7 +3771,8 @@ static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v) prog = bpf_iter_get_info(&meta, false); ret = unix_prog_seq_show(prog, &meta, v, uid); unlock: - unlock_sock_fast(sk, slow); + unix_state_unlock(sk); + release_sock(sk); return ret; } diff --git a/net/unix/unix_bpf.c b/net/unix/unix_bpf.c index e0d30d6d22acbc..57f3124c9d8db9 100644 --- a/net/unix/unix_bpf.c +++ b/net/unix/unix_bpf.c @@ -185,6 +185,9 @@ int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool r */ if (!psock->sk_pair) { sk_pair = unix_peer(sk); + if (unlikely(!sk_pair)) + return -EINVAL; + sock_hold(sk_pair); psock->sk_pair = sk_pair; } diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index d912ed2f012a3e..08f4dfb9782c28 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1951,12 +1951,12 @@ static void vsock_update_buffer_size(struct vsock_sock *vsk, const struct vsock_transport *transport, u64 val) { - if (val > vsk->buffer_max_size) - val = vsk->buffer_max_size; - if (val < vsk->buffer_min_size) val = vsk->buffer_min_size; + if (val > vsk->buffer_max_size) + val = vsk->buffer_max_size; + if (val != vsk->buffer_size && transport && transport->notify_buffer_size) transport->notify_buffer_size(vsk, &val); diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index 069386a7455795..d5b0fd0a889723 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -375,10 +375,10 @@ static void hvs_open_connection(struct vmbus_channel *chan) } else { sndbuf = max_t(int, sk->sk_sndbuf, RINGBUFFER_HVS_SND_SIZE); sndbuf = min_t(int, sndbuf, RINGBUFFER_HVS_MAX_SIZE); - sndbuf = ALIGN(sndbuf, HV_HYP_PAGE_SIZE); + sndbuf = VMBUS_RING_SIZE(sndbuf); rcvbuf = max_t(int, sk->sk_rcvbuf, RINGBUFFER_HVS_RCV_SIZE); rcvbuf = min_t(int, rcvbuf, RINGBUFFER_HVS_MAX_SIZE); - rcvbuf = ALIGN(rcvbuf, HV_HYP_PAGE_SIZE); + rcvbuf = VMBUS_RING_SIZE(rcvbuf); } chan->max_pkt_size = HVS_MAX_PKT_SIZE; @@ -694,7 +694,6 @@ static ssize_t hvs_stream_enqueue(struct vsock_sock *vsk, struct msghdr *msg, static s64 hvs_stream_has_data(struct vsock_sock *vsk) { struct hvsock *hvs = vsk->trans; - bool need_refill; s64 ret; if (hvs->recv_data_len > 0) @@ -702,9 +701,31 @@ static s64 hvs_stream_has_data(struct vsock_sock *vsk) switch (hvs_channel_readable_payload(hvs->chan)) { case 1: - need_refill = !hvs->recv_desc; - if (!need_refill) - return -EIO; + if (hvs->recv_desc) { + /* Here hvs->recv_data_len is 0, so hvs->recv_desc must + * be NULL unless it points to the 0-byte-payload FIN + * packet or a malformed/short packet: see + * hvs_update_recv_data(). + * + * If hvs->recv_desc points to the FIN packet, here all + * the payload has been dequeued and the peer_shutdown + * flag is set, but hvs_channel_readable_payload() still + * returns 1, because the VMBus ringbuffer's read_index + * is not updated for the FIN packet: + * hvs_stream_dequeue() -> hv_pkt_iter_next() updates + * the cached priv_read_index but has no opportunity to + * update the read_index in hv_pkt_iter_close() as + * hvs_stream_has_data() returns 0 for the FIN packet, + * so it won't get dequeued. + * + * In case hvs->recv_desc points to a malformed/short + * packet, return -EIO. + */ + if (!(vsk->peer_shutdown & SEND_SHUTDOWN)) + return -EIO; + + return 0; + } hvs->recv_desc = hv_pkt_iter_first(hvs->chan); if (!hvs->recv_desc) diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index 8a9fb23c6e853d..0d0265f770adad 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -75,6 +75,7 @@ static bool virtio_transport_can_zcopy(const struct virtio_transport *t_ops, static int virtio_transport_init_zcopy_skb(struct vsock_sock *vsk, struct sk_buff *skb, struct msghdr *msg, + size_t pkt_len, bool zerocopy) { struct ubuf_info *uarg; @@ -83,12 +84,10 @@ static int virtio_transport_init_zcopy_skb(struct vsock_sock *vsk, uarg = msg->msg_ubuf; net_zcopy_get(uarg); } else { - struct iov_iter *iter = &msg->msg_iter; struct ubuf_info_msgzc *uarg_zc; uarg = msg_zerocopy_realloc(sk_vsock(vsk), - iter->count, - NULL, false); + pkt_len, NULL, false); if (!uarg) return -1; @@ -139,27 +138,6 @@ static void virtio_transport_init_hdr(struct sk_buff *skb, hdr->fwd_cnt = cpu_to_le32(0); } -static void virtio_transport_copy_nonlinear_skb(const struct sk_buff *skb, - void *dst, - size_t len) -{ - struct iov_iter iov_iter = { 0 }; - struct kvec kvec; - size_t to_copy; - - kvec.iov_base = dst; - kvec.iov_len = len; - - iov_iter.iter_type = ITER_KVEC; - iov_iter.kvec = &kvec; - iov_iter.nr_segs = 1; - - to_copy = min_t(size_t, len, skb->len); - - skb_copy_datagram_iter(skb, VIRTIO_VSOCK_SKB_CB(skb)->offset, - &iov_iter, to_copy); -} - /* Packet capture */ static struct sk_buff *virtio_transport_build_skb(void *opaque) { @@ -169,12 +147,12 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque) struct sk_buff *skb; size_t payload_len; - /* A packet could be split to fit the RX buffer, so we can retrieve - * the payload length from the header and the buffer pointer taking - * care of the offset in the original packet. + /* A packet could be split to fit the RX buffer, so we use + * the payload length from the header, which has been updated + * by the sender to reflect the fragment size. */ pkt_hdr = virtio_vsock_hdr(pkt); - payload_len = pkt->len; + payload_len = le32_to_cpu(pkt_hdr->len); skb = alloc_skb(sizeof(*hdr) + sizeof(*pkt_hdr) + payload_len, GFP_ATOMIC); @@ -217,12 +195,18 @@ static struct sk_buff *virtio_transport_build_skb(void *opaque) skb_put_data(skb, pkt_hdr, sizeof(*pkt_hdr)); if (payload_len) { - if (skb_is_nonlinear(pkt)) { - void *data = skb_put(skb, payload_len); - - virtio_transport_copy_nonlinear_skb(pkt, data, payload_len); - } else { - skb_put_data(skb, pkt->data, payload_len); + struct iov_iter iov_iter; + struct kvec kvec; + void *data = skb_put(skb, payload_len); + + kvec.iov_base = data; + kvec.iov_len = payload_len; + iov_iter_kvec(&iov_iter, ITER_DEST, &kvec, 1, payload_len); + + if (skb_copy_datagram_iter(pkt, VIRTIO_VSOCK_SKB_CB(pkt)->offset, + &iov_iter, payload_len)) { + kfree_skb(skb); + return NULL; } } @@ -400,11 +384,17 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, * each iteration. If this is last skb for this buffer * and MSG_ZEROCOPY mode is in use - we must allocate * completion for the current syscall. + * + * Pass pkt_len because msg iter is already consumed + * by virtio_transport_fill_skb(), so iter->count + * can not be used for RLIMIT_MEMLOCK pinned-pages + * accounting done by msg_zerocopy_realloc(). */ if (info->msg && info->msg->msg_flags & MSG_ZEROCOPY && skb_len == rest_len && info->op == VIRTIO_VSOCK_OP_RW) { if (virtio_transport_init_zcopy_skb(vsk, skb, info->msg, + pkt_len, can_zcopy)) { kfree_skb(skb); ret = -ENOMEM; @@ -547,9 +537,8 @@ virtio_transport_stream_do_peek(struct vsock_sock *vsk, skb_queue_walk(&vvs->rx_queue, skb) { size_t bytes; - bytes = len - total; - if (bytes > skb->len) - bytes = skb->len; + bytes = min_t(size_t, len - total, + skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset); spin_unlock_bh(&vvs->rx_lock); @@ -1560,8 +1549,6 @@ virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb, return -ENOMEM; } - sk_acceptq_added(sk); - lock_sock_nested(child, SINGLE_DEPTH_NESTING); child->sk_state = TCP_ESTABLISHED; @@ -1583,6 +1570,7 @@ virtio_transport_recv_listen(struct sock *sk, struct sk_buff *skb, return ret; } + sk_acceptq_added(sk); if (virtio_transport_space_update(child, skb)) child->sk_write_space(child); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 1748d374abcab3..686014d394298c 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -818,17 +818,17 @@ int __xfrm_state_delete(struct xfrm_state *x) spin_lock(&net->xfrm.xfrm_state_lock); list_del(&x->km.all); - hlist_del_rcu(&x->bydst); - hlist_del_rcu(&x->bysrc); - if (x->km.seq) - hlist_del_rcu(&x->byseq); + hlist_del_init_rcu(&x->bydst); + hlist_del_init_rcu(&x->bysrc); + if (!hlist_unhashed(&x->byseq)) + hlist_del_init_rcu(&x->byseq); if (!hlist_unhashed(&x->state_cache)) hlist_del_rcu(&x->state_cache); if (!hlist_unhashed(&x->state_cache_input)) hlist_del_rcu(&x->state_cache_input); - if (x->id.spi) - hlist_del_rcu(&x->byspi); + if (!hlist_unhashed(&x->byspi)) + hlist_del_init_rcu(&x->byspi); net->xfrm.state_num--; xfrm_nat_keepalive_state_updated(x); spin_unlock(&net->xfrm.xfrm_state_lock); diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index d56450f6166912..38a90e5ee3d935 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -3323,6 +3323,7 @@ const int xfrm_msg_min[XFRM_NR_MSGTYPES] = { [XFRM_MSG_GETSADINFO - XFRM_MSG_BASE] = sizeof(u32), [XFRM_MSG_NEWSPDINFO - XFRM_MSG_BASE] = sizeof(u32), [XFRM_MSG_GETSPDINFO - XFRM_MSG_BASE] = sizeof(u32), + [XFRM_MSG_MAPPING - XFRM_MSG_BASE] = XMSGSIZE(xfrm_user_mapping), [XFRM_MSG_SETDEFAULT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_default), [XFRM_MSG_GETDEFAULT - XFRM_MSG_BASE] = XMSGSIZE(xfrm_userpolicy_default), }; diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters index fd2fd1c3cb9a51..eeb8c15e709c38 100644 --- a/rust/bindgen_parameters +++ b/rust/bindgen_parameters @@ -12,9 +12,15 @@ # Packed type cannot transitively contain a `#[repr(align)]` type. --opaque-type alt_instr +--opaque-type snd_codec_options +--opaque-type snd_codec +--opaque-type snd_compr_params --opaque-type x86_msi_data --opaque-type x86_msi_addr_lo +# Packed types cannot have larger alignment than the maximal natural aligment of menbers +--opaque-type snd_dec_flac + # `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2, # commit 2aed6b021680 ("context: Escape the try keyword properly"). --opaque-type kunit_try_catch diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 083cc44aa952c2..db5c552ac5cb69 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -31,9 +31,14 @@ #include #include #include +#include #include #include +#include +#include #include +#include +#include #include #include #include @@ -48,9 +53,13 @@ #include #include #include +#include #include #include +#include +#include #include +#include #include #include #include @@ -58,15 +67,22 @@ #include #include #include +#include +#include #include #include +#include #include +#include #include #include #include #include #include +#include +#include #include +#include #include #include #include @@ -81,6 +97,8 @@ #include #include #include +#include +#include #include #include #include @@ -88,6 +106,9 @@ #include #include #include +#include +#include +#include #include /* @@ -113,8 +134,16 @@ const gfp_t RUST_CONST_HELPER___GFP_ZERO = __GFP_ZERO; const gfp_t RUST_CONST_HELPER___GFP_HIGHMEM = ___GFP_HIGHMEM; const gfp_t RUST_CONST_HELPER___GFP_NOWARN = ___GFP_NOWARN; const blk_features_t RUST_CONST_HELPER_BLK_FEAT_ROTATIONAL = BLK_FEAT_ROTATIONAL; +const uint32_t RUST_CONST_HELPER_DRM_EXEC_INTERRUPTIBLE_WAIT = DRM_EXEC_INTERRUPTIBLE_WAIT; const fop_flags_t RUST_CONST_HELPER_FOP_UNSIGNED_OFFSET = FOP_UNSIGNED_OFFSET; +const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; + +const u32 BINDINGS_IIO_CHAN_INFO_RAW = IIO_CHAN_INFO_RAW; +const u32 BINDINGS_IIO_CHAN_INFO_PROCESSED = IIO_CHAN_INFO_PROCESSED; +const u32 BINDINGS_IIO_ANGL = IIO_ANGL; +const u32 BINDINGS_IIO_LIGHT = IIO_LIGHT; + const xa_mark_t RUST_CONST_HELPER_XA_PRESENT = XA_PRESENT; const gfp_t RUST_CONST_HELPER_XA_FLAGS_ALLOC = XA_FLAGS_ALLOC; diff --git a/rust/helpers/device.c b/rust/helpers/device.c index a8ab931a9bd123..89cc49cc29d814 100644 --- a/rust/helpers/device.c +++ b/rust/helpers/device.c @@ -25,3 +25,13 @@ __rust_helper void rust_helper_dev_set_drvdata(struct device *dev, void *data) { dev_set_drvdata(dev, data); } + +void rust_helper_device_lock(struct device *dev) +{ + device_lock(dev); +} + +void rust_helper_device_unlock(struct device *dev) +{ + device_unlock(dev); +} diff --git a/rust/helpers/dma-fence.c b/rust/helpers/dma-fence.c new file mode 100644 index 00000000000000..6491016262934b --- /dev/null +++ b/rust/helpers/dma-fence.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#ifdef CONFIG_DMA_SHARED_BUFFER + +void rust_helper_dma_fence_get(struct dma_fence *fence) +{ + dma_fence_get(fence); +} + +void rust_helper_dma_fence_put(struct dma_fence *fence) +{ + dma_fence_put(fence); +} + +struct dma_fence_chain *rust_helper_dma_fence_chain_alloc(void) +{ + return dma_fence_chain_alloc(); +} + +void rust_helper_dma_fence_chain_free(struct dma_fence_chain *chain) +{ + dma_fence_chain_free(chain); +} + +void rust_helper_dma_fence_set_error(struct dma_fence *fence, int error) +{ + dma_fence_set_error(fence, error); +} + +#endif diff --git a/rust/helpers/dma-mapping.c b/rust/helpers/dma-mapping.c new file mode 100644 index 00000000000000..0d795b1b0738dc --- /dev/null +++ b/rust/helpers/dma-mapping.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) +{ + return dma_mapping_error(dev, dma_addr); +} diff --git a/rust/helpers/dma-resv.c b/rust/helpers/dma-resv.c new file mode 100644 index 00000000000000..05501cb814513b --- /dev/null +++ b/rust/helpers/dma-resv.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +int rust_helper_dma_resv_lock(struct dma_resv *obj, struct ww_acquire_ctx *ctx) +{ + return dma_resv_lock(obj, ctx); +} + +void rust_helper_dma_resv_unlock(struct dma_resv *obj) +{ + dma_resv_unlock(obj); +} diff --git a/rust/helpers/drm.c b/rust/helpers/drm.c index fe226f7b53ef0c..65f3f22b0e1d2f 100644 --- a/rust/helpers/drm.c +++ b/rust/helpers/drm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #ifdef CONFIG_DRM @@ -21,4 +22,57 @@ rust_helper_drm_vma_node_offset_addr(struct drm_vma_offset_node *node) return drm_vma_node_offset_addr(node); } -#endif +#ifdef CONFIG_DRM_GEM_SHMEM_HELPER +__rust_helper void +rust_helper_drm_gem_shmem_object_free(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_free(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, + const struct drm_gem_object *obj) +{ + drm_gem_shmem_object_print_info(p, indent, obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_pin(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_pin(obj); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_unpin(struct drm_gem_object *obj) +{ + drm_gem_shmem_object_unpin(obj); +} + +__rust_helper struct sg_table * +rust_helper_drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) +{ + return drm_gem_shmem_object_get_sg_table(obj); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_vmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + return drm_gem_shmem_object_vmap(obj, map); +} + +__rust_helper void +rust_helper_drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, + struct iosys_map *map) +{ + drm_gem_shmem_object_vunmap(obj, map); +} + +__rust_helper int +rust_helper_drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) +{ + return drm_gem_shmem_object_mmap(obj, vma); +} + +#endif /* CONFIG_DRM_GEM_SHMEM_HELPER */ +#endif /* CONFIG_DRM */ diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c new file mode 100644 index 00000000000000..1ed5b5841f68c2 --- /dev/null +++ b/rust/helpers/drm_gpuvm.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_DRM +#ifdef CONFIG_DRM_GPUVM + +struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj) +{ + return drm_gpuvm_get(obj); +} + +void rust_helper_drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) +{ + return drm_gpuvm_exec_unlock(vm_exec); +} + +void rust_helper_drm_gpuva_init_from_op(struct drm_gpuva *va, struct drm_gpuva_op_map *op) +{ + drm_gpuva_init_from_op(va, op); +} + +struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) +{ + return drm_gpuvm_bo_get(vm_bo); +} + +bool rust_helper_drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm) +{ + return drm_gpuvm_immediate_mode(gpuvm); +} + +bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj) +{ + return drm_gpuvm_is_extobj(gpuvm, obj); +} + +#endif +#endif diff --git a/rust/helpers/drm_syncobj.c b/rust/helpers/drm_syncobj.c new file mode 100644 index 00000000000000..9e14c989edfd72 --- /dev/null +++ b/rust/helpers/drm_syncobj.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#ifdef CONFIG_DRM + +void rust_helper_drm_syncobj_get(struct drm_syncobj *obj) +{ + drm_syncobj_get(obj); +} + +void rust_helper_drm_syncobj_put(struct drm_syncobj *obj) +{ + drm_syncobj_put(obj); +} + +struct dma_fence *rust_helper_drm_syncobj_fence_get(struct drm_syncobj *syncobj) +{ + return drm_syncobj_fence_get(syncobj); +} + +#endif diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index a3c42e51f00a09..218393fd01f32b 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -28,11 +28,17 @@ #include "cred.c" #include "device.c" #include "dma.c" +#include "dma-fence.c" +#include "dma-mapping.c" +#include "dma-resv.c" #include "drm.c" +#include "drm_gpuvm.c" +#include "drm_syncobj.c" #include "err.c" #include "irq.c" #include "fs.c" #include "io.c" +#include "iosys_map.c" #include "jump_label.c" #include "kunit.c" #include "maple_tree.c" diff --git a/rust/helpers/io.c b/rust/helpers/io.c index 397810864a2439..ec57586d988f67 100644 --- a/rust/helpers/io.c +++ b/rust/helpers/io.c @@ -19,6 +19,16 @@ __rust_helper void rust_helper_iounmap(void __iomem *addr) iounmap(addr); } +__rust_helper void rust_helper_memcpy_fromio(void *to, const void __iomem *from, long count) +{ + memcpy_fromio(to, from, count); +} + +__rust_helper void rust_helper_memcpy_toio(void __iomem *to, const void *from, size_t count) +{ + memcpy_toio(to, from, count); +} + __rust_helper u8 rust_helper_readb(const void __iomem *addr) { return readb(addr); diff --git a/rust/helpers/iosys_map.c b/rust/helpers/iosys_map.c new file mode 100644 index 00000000000000..f99598367147ff --- /dev/null +++ b/rust/helpers/iosys_map.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +void rust_helper_iosys_map_memcpy_to(struct iosys_map *dst, size_t dst_offset, + const void *src, size_t len) +{ + iosys_map_memcpy_to(dst, dst_offset, src, len); +} + +void rust_helper_iosys_map_memcpy_from(void *dst, const struct iosys_map *src, + size_t src_offset, size_t len) +{ + iosys_map_memcpy_from(dst, src, src_offset, len); +} + +void rust_helper_iosys_map_memset(struct iosys_map *dst, size_t offset, + int value, size_t len) +{ + iosys_map_memset(dst, offset, value, len); +} diff --git a/rust/helpers/of.c b/rust/helpers/of.c index 8f62ca69e8ba54..da4ae35163c6ff 100644 --- a/rust/helpers/of.c +++ b/rust/helpers/of.c @@ -1,8 +1,29 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include __rust_helper bool rust_helper_is_of_node(const struct fwnode_handle *fwnode) { return is_of_node(fwnode); } + +const struct of_device_id *rust_helper_of_match_device( + const struct of_device_id *matches, const struct device *dev) +{ + return of_match_device(matches, dev); +} + +#ifdef CONFIG_OF +bool rust_helper_of_node_is_root(const struct device_node *np) +{ + return of_node_is_root(np); +} +#endif + +struct device_node *rust_helper_of_parse_phandle(const struct device_node *np, + const char *phandle_name, + int index) +{ + return of_parse_phandle(np, phandle_name, index); +} diff --git a/rust/helpers/page.c b/rust/helpers/page.c index f8463fbed2a267..7ab81a8cad8947 100644 --- a/rust/helpers/page.c +++ b/rust/helpers/page.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include #include @@ -26,3 +27,28 @@ __rust_helper int rust_helper_page_to_nid(const struct page *page) return page_to_nid(page); } #endif + +struct page *rust_helper_phys_to_page(phys_addr_t phys) +{ + return phys_to_page(phys); +} + +phys_addr_t rust_helper_page_to_phys(struct page *page) +{ + return page_to_phys(page); +} + +unsigned long rust_helper_phys_to_pfn(phys_addr_t phys) +{ + return __phys_to_pfn(phys); +} + +struct page *rust_helper_pfn_to_page(unsigned long pfn) +{ + return pfn_to_page(pfn); +} + +bool rust_helper_pfn_valid(unsigned long pfn) +{ + return pfn_valid(pfn); +} diff --git a/rust/helpers/xarray.c b/rust/helpers/xarray.c index 08979b3043410f..77bb186b8f72a3 100644 --- a/rust/helpers/xarray.c +++ b/rust/helpers/xarray.c @@ -2,6 +2,11 @@ #include +__rust_helper void *rust_helper_xa_zero_entry(void) +{ + return XA_ZERO_ENTRY; +} + __rust_helper int rust_helper_xa_err(void *entry) { return xa_err(entry); diff --git a/rust/kernel/addr.rs b/rust/kernel/addr.rs new file mode 100644 index 00000000000000..06aff10a033235 --- /dev/null +++ b/rust/kernel/addr.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel core address types. + +use bindings; +use core::ffi; + +/// A physical memory address (which may be wider than the CPU pointer size) +pub type PhysicalAddr = bindings::phys_addr_t; +/// A DMA memory address (which may be narrower than `PhysicalAddr` on some systems) +pub type DmaAddr = bindings::dma_addr_t; +/// A physical resource size, typically the same width as `PhysicalAddr` +pub type ResourceSize = bindings::resource_size_t; +/// A raw page frame number, not to be confused with the C `pfn_t` which also encodes flags. +pub type Pfn = ffi::c_ulong; diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index e38720349dcf7e..719927971f5796 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -18,6 +18,8 @@ pub use self::kvec::KVec; pub use self::kvec::VVec; pub use self::kvec::Vec; +use crate::types::declare_flags_type; + /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; @@ -25,45 +27,13 @@ pub struct AllocError; use crate::error::{code::EINVAL, Result}; use core::{alloc::Layout, ptr::NonNull}; -/// Flags to be used when allocating memory. -/// -/// They can be combined with the operators `|`, `&`, and `!`. -/// -/// Values can be used from the [`flags`] module. -#[derive(Clone, Copy, PartialEq)] -pub struct Flags(u32); - -impl Flags { - /// Get the raw representation of this flag. - pub(crate) fn as_raw(self) -> u32 { - self.0 - } - - /// Check whether `flags` is contained in `self`. - pub fn contains(self, flags: Flags) -> bool { - (self & flags) == flags - } -} - -impl core::ops::BitOr for Flags { - type Output = Self; - fn bitor(self, rhs: Self) -> Self::Output { - Self(self.0 | rhs.0) - } -} - -impl core::ops::BitAnd for Flags { - type Output = Self; - fn bitand(self, rhs: Self) -> Self::Output { - Self(self.0 & rhs.0) - } -} - -impl core::ops::Not for Flags { - type Output = Self; - fn not(self) -> Self::Output { - Self(!self.0) - } +declare_flags_type! { + /// Flags to be used when allocating memory. + /// + /// They can be combined with the operators `|`, `&`, and `!`. + /// + /// Values can be used from the [`flags`] module. + pub struct Flags(u32); } /// Allocation flags. diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 63bfb91b36712a..1a53412c458dd3 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -1,4 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +// FIXME +#![allow(clippy::undocumented_unsafe_blocks)] //! Allocator support. //! diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index 622b3529edfcbc..2684598cde4cbe 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -682,6 +682,16 @@ where } } +impl AsRef for Box +where + T: ?Sized, + A: Allocator, +{ + fn as_ref(&self) -> &T { + self + } +} + /// # Examples /// /// ``` diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index ac8d6f763ae81d..6cc48330515dd1 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -19,12 +19,15 @@ use core::{ ops::DerefMut, ops::Index, ops::IndexMut, + ops::{Range, RangeBounds}, ptr, ptr::NonNull, slice, slice::SliceIndex, }; +mod drain; +use self::drain::Drain; mod errors; pub use self::errors::{InsertError, PushError, RemoveError}; @@ -192,6 +195,19 @@ where self.len } + /// Forcefully sets `self.len` to `new_len`. + /// + /// # Safety + /// + /// - `new_len` must be less than or equal to [`Self::capacity`]. + /// - If `new_len` is greater than `self.len`, all elements within the interval + /// [`self.len`,`new_len`) must be initialized. + #[inline] + pub unsafe fn set_len(&mut self, new_len: usize) { + debug_assert!(new_len <= self.capacity()); + self.len = new_len; + } + /// Increments `self.len` by `additional`. /// /// # Safety @@ -733,6 +749,76 @@ where } self.truncate(num_kept); } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + /// # Examples + /// + /// ``` + /// let mut v = vec![1, 2, 3]; + /// let u: Vec<_> = v.drain(1..).collect(); + /// assert_eq!(v, &[1]); + /// assert_eq!(u, &[2, 3]); + /// + /// // A full range clears the vector, like `clear()` does + /// v.drain(..); + /// assert_eq!(v, &[]); + /// ``` + pub fn drain(&mut self, range: R) -> Drain<'_, T, A> + where + R: RangeBounds, + { + let len = self.len(); + let Range { start, end } = slice::range(range, ..len); + + unsafe { + // set self.vec length's to start, to be safe in case Drain is leaked + self.set_len(start); + let range_slice = slice::from_raw_parts(self.as_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// This does not preserve ordering of the remaining elements, but is *O*(1). + /// If you need to preserve the element order, use [`remove`] instead. + pub fn swap_remove(&mut self, index: usize) -> T { + if index > self.len() { + panic!("Index out of range"); + } + // SAFETY: index is in range + // self.len() - 1 is in range since at last 1 element exists + unsafe { + let old = ptr::read(self.as_ptr().add(index)); + let last = ptr::read(self.as_ptr().add(self.len() - 1)); + ptr::write(self.as_mut_ptr().add(index), last); + self.dec_len(1); + old + } + } } impl Vec { @@ -1399,3 +1485,51 @@ mod tests { } } } + +// #[stable(feature = "array_try_from_vec", since = "1.48.0")] +impl TryFrom> for [T; N] { + type Error = Vec; + + /// Gets the entire contents of the `Vec` as an array, + /// if its size exactly matches that of the requested array. + /// + /// # Examples + /// + /// ``` + /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3])); + /// assert_eq!(>::new().try_into(), Ok([])); + /// ``` + /// + /// If the length doesn't match, the input comes back in `Err`: + /// ``` + /// let r: Result<[i32; 4], _> = (0..10).collect::>().try_into(); + /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9])); + /// ``` + /// + /// If you're fine with just getting a prefix of the `Vec`, + /// you can call [`.truncate(N)`](Vec::truncate) first. + /// ``` + /// let mut v = String::from("hello world").into_bytes(); + /// v.sort(); + /// v.truncate(2); + /// let [a, b]: [_; 2] = v.try_into().unwrap(); + /// assert_eq!(a, b' '); + /// assert_eq!(b, b'd'); + /// ``` + fn try_from(mut vec: Vec) -> Result<[T; N], Vec> { + if vec.len() != N { + return Err(vec); + } + + // SAFETY: `.set_len(0)` is always sound. + unsafe { vec.dec_len(vec.len()) }; + + // SAFETY: A `Vec`'s pointer is always aligned properly, and + // the alignment the array needs is the same as the items. + // We checked earlier that we have sufficient items. + // The items will not double-drop as the `set_len` + // tells the `Vec` not to also drop them. + let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) }; + Ok(array) + } +} diff --git a/rust/kernel/alloc/kvec/drain.rs b/rust/kernel/alloc/kvec/drain.rs new file mode 100644 index 00000000000000..035878fd112843 --- /dev/null +++ b/rust/kernel/alloc/kvec/drain.rs @@ -0,0 +1,181 @@ +//! Rust standard library vendored code. +//! +//! The contents of this file come from the Rust standard library, hosted in +//! the repository, licensed under +//! "Apache-2.0 OR MIT" and adapted for kernel use. For copyright details, +//! see . +#![allow(clippy::undocumented_unsafe_blocks)] + +use core::fmt; +use core::iter::FusedIterator; +use core::mem::{self, SizedTypeProperties}; +use core::ptr::{self, NonNull}; +use core::slice::{self}; + +use super::{Allocator, Vec}; + +/// A draining iterator for `Vec`. +/// +/// This `struct` is created by [`Vec::drain`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let iter: std::vec::Drain<'_, _> = v.drain(..); +/// ``` +// #[stable(feature = "drain", since = "1.6.0")] +pub struct Drain< + 'a, + T, + A: Allocator, + // #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global, +> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +// #[stable(feature = "collection_debug", since = "1.17.0")] +impl fmt::Debug for Drain<'_, T, A> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() + } +} + +impl<'a, T, A: Allocator> Drain<'a, T, A> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + // #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } +} + +// #[stable(feature = "vec_drain_as_slice", since = "1.46.0")] +impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +unsafe impl Sync for Drain<'_, T, A> {} +// #[stable(feature = "drain", since = "1.6.0")] +unsafe impl Send for Drain<'_, T, A> {} + +// #[stable(feature = "drain", since = "1.6.0")] +impl Iterator for Drain<'_, T, A> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter + .next() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +impl DoubleEndedIterator for Drain<'_, T, A> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter + .next_back() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +// #[stable(feature = "drain", since = "1.6.0")] +impl Drop for Drain<'_, T, A> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>); + + impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.set_len(start + self.0.tail_len); + } + } + } + } + + let iter = mem::take(&mut self.iter); + let drop_len = iter.len(); + + let mut vec = self.vec; + + if T::IS_ZST { + // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + unsafe { + let vec = vec.as_mut(); + let old_len = vec.len(); + vec.set_len(old_len + drop_len + self.tail_len); + vec.truncate(old_len + self.tail_len); + } + + return; + } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // it also gets touched by vec::Splice which may turn it into a dangling pointer + // which would make it and the vec pointer point to different allocations which would + // lead to invalid pointer arithmetic below. + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + #[cfg(not(version("1.87")))] + let drop_offset = drop_ptr.sub_ptr(vec_ptr); + #[cfg(version("1.87"))] + let drop_offset = drop_ptr.offset_from_unsigned(vec_ptr); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +// #[stable(feature = "fused", since = "1.26.0")] +impl FusedIterator for Drain<'_, T, A> {} diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index f5adee48d40cb8..d8d26870bea2eb 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -1257,18 +1257,17 @@ impl Registration { /// # Safety /// /// - This function may only be called from the cpufreq C infrastructure. + /// - The pointer arguments must be valid pointers. unsafe extern "C" fn adjust_perf_callback( - cpu: c_uint, + ptr: *mut bindings::cpufreq_policy, min_perf: c_ulong, target_perf: c_ulong, capacity: c_ulong, ) { - // SAFETY: The C API guarantees that `cpu` refers to a valid CPU number. - let cpu_id = unsafe { CpuId::from_u32_unchecked(cpu) }; - - if let Ok(mut policy) = PolicyCpu::from_cpu(cpu_id) { - T::adjust_perf(&mut policy, min_perf, target_perf, capacity); - } + // SAFETY: The `ptr` is guaranteed to be valid by the contract with the C code for the + // lifetime of `policy`. + let policy = unsafe { Policy::from_raw_mut(ptr) }; + T::adjust_perf(policy, min_perf, target_perf, capacity); } /// Driver's `get_intermediate` callback. diff --git a/rust/kernel/devcoredump.rs b/rust/kernel/devcoredump.rs new file mode 100644 index 00000000000000..a4a42d862f63b5 --- /dev/null +++ b/rust/kernel/devcoredump.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device coredump support. +//! +//! C header: [`include/linux/devcoredump.h`](../../../../include/linux/devcoredump.h) + +use crate::{ + alloc, bindings, device, error::from_result, prelude::Result, time::Jiffies, + types::ForeignOwnable, ThisModule, +}; + +use core::ops::Deref; + +/// The default timeout for device coredumps. +pub const DEFAULT_TIMEOUT: Jiffies = bindings::DEVCD_TIMEOUT as Jiffies; + +/// Trait to implement reading from a device coredump. +/// +/// Users must implement this trait to provide device coredump support. +pub trait DevCoreDump { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn read(&self, buf: &mut [u8], offset: usize) -> Result; +} + +unsafe extern "C" fn read_callback< + 'a, + T: ForeignOwnable: Deref>, + D: DevCoreDump, +>( + buffer: *mut crate::ffi::c_char, + offset: bindings::loff_t, + count: usize, + data: *mut crate::ffi::c_void, + _datalen: usize, +) -> isize { + // SAFETY: This pointer came from into_foreign() below. + let coredump = unsafe { T::borrow(data.cast()) }; + // SAFETY: The caller guarantees `buffer` points to at least `count` bytes. + let buf = unsafe { core::slice::from_raw_parts_mut(buffer, count) }; + + from_result(|| Ok(coredump.read(buf, offset.try_into()?)?.try_into()?)) +} + +unsafe extern "C" fn free_callback< + 'a, + T: ForeignOwnable: Deref>, + D: DevCoreDump, +>( + data: *mut crate::ffi::c_void, +) { + // SAFETY: This pointer came from into_foreign() below. + unsafe { + T::from_foreign(data.cast()); + } +} + +/// Registers a coredump for the given device. +pub fn dev_coredump<'a, T: ForeignOwnable: Deref>, D: DevCoreDump>( + dev: &device::Device, + module: &'static ThisModule, + coredump: T, + gfp: alloc::Flags, + timeout: Jiffies, +) { + // SAFETY: Call upholds dev_coredumpm lifetime requirements. + unsafe { + bindings::dev_coredumpm_timeout( + dev.as_raw(), + module.0, + coredump.into_foreign() as *mut _, + 0, + gfp.as_raw(), + Some(read_callback::<'a, T, D>), + Some(free_callback::<'a, T, D>), + timeout, + ) + } +} diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 94e0548e76871d..95d6c4b076de70 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -4,6 +4,7 @@ //! //! C header: [`include/linux/device.h`](srctree/include/linux/device.h) +use crate::of; use crate::{ bindings, fmt, @@ -20,6 +21,8 @@ use core::{ ptr, // }; +use crate::types::NotThreadSafe; + pub mod property; // Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`. @@ -335,13 +338,13 @@ impl Device { impl Device { /// Obtain the raw `struct device *`. - pub(crate) fn as_raw(&self) -> *mut bindings::device { + pub fn as_raw(&self) -> *mut bindings::device { self.0.get() } /// Returns a reference to the parent device, if any. #[cfg_attr(not(CONFIG_AUXILIARY_BUS), expect(dead_code))] - pub(crate) fn parent(&self) -> Option<&Device> { + pub fn parent(&self) -> Option<&Device> { // SAFETY: // - By the type invariant `self.as_raw()` is always valid. // - The parent device is only ever set at device creation. @@ -358,6 +361,13 @@ impl Device { } } + /// Returns the driver_data pointer. + pub fn get_drvdata(&self) -> *mut T { + // SAFETY: dev_get_drvdata returns a field of the device, + // pointer to which is valid by type invariant + unsafe { bindings::dev_get_drvdata(self.as_raw()) as *mut T } + } + /// Convert a raw C `struct device` pointer to a `&'a Device`. /// /// # Safety @@ -371,6 +381,13 @@ impl Device { unsafe { &*ptr.cast() } } + /// Gets the OpenFirmware node attached to this device + pub fn of_node(&self) -> Option { + let ptr = self.0.get(); + // SAFETY: This is safe as long as of_node is NULL or valid. + unsafe { of::Node::get_from_raw((*ptr).of_node) } + } + /// Prints an emergency-level message (level 0) prefixed with device information. /// /// More details are available from [`dev_emerg`]. @@ -489,6 +506,58 @@ impl Device { // defined as a `#[repr(transparent)]` wrapper around `fwnode_handle`. Some(unsafe { &*fwnode_handle.cast() }) } + + /// Locks the [`Device`] for exclusive access. + pub fn lock(&self) -> Guard<'_, Ctx> { + // SAFETY: `self` is always valid by the type invariant. + unsafe { bindings::device_lock(self.as_raw()) }; + + Guard { + dev: self, + _not_send: NotThreadSafe, + } + } + + /// ensure Device is bound + pub fn is_bound(&self) -> Option> { + let guard = self.lock(); + if !unsafe { bindings::device_is_bound(self.as_raw()) } { + return None; + } + Some(guard) + } + + /// excute closure while the device is bound + pub fn while_bound_with(&self, f: F) -> Result + where + F: FnOnce(&Device) -> Result, + { + let _guard = self.lock(); + if unsafe { !bindings::device_is_bound(self.as_raw()) } { + return Err(ENODEV); + } + let ptr: *const Self = self; + let ptr = ptr.cast::>(); + f(unsafe { &*ptr }) + } +} + +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, Ctx: DeviceContext = Normal> { + dev: &'a Device, + _not_send: NotThreadSafe, +} + +impl Drop for Guard<'_, Ctx> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::device_unlock(self.dev.as_raw()) }; + } } // SAFETY: `Device` is a transparent wrapper of a type that doesn't depend on `Device`'s generic diff --git a/rust/kernel/device/property.rs b/rust/kernel/device/property.rs index 5aead835fbbc02..272fdcb482ce25 100644 --- a/rust/kernel/device/property.rs +++ b/rust/kernel/device/property.rs @@ -59,7 +59,7 @@ impl FwNode { } /// Obtain the raw `struct fwnode_handle *`. - pub(crate) fn as_raw(&self) -> *mut bindings::fwnode_handle { + pub fn as_raw(&self) -> *mut bindings::fwnode_handle { self.0.get() } diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index a396f84357394c..7f483f96ff2510 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -250,9 +250,6 @@ pub mod attrs { /// Specifies that writes to the mapping may be buffered to improve performance. pub const DMA_ATTR_WRITE_COMBINE: Attrs = Attrs(bindings::DMA_ATTR_WRITE_COMBINE); - /// Lets the platform to avoid creating a kernel virtual mapping for the allocated buffer. - pub const DMA_ATTR_NO_KERNEL_MAPPING: Attrs = Attrs(bindings::DMA_ATTR_NO_KERNEL_MAPPING); - /// Allows platform code to skip synchronization of the CPU cache for the given buffer assuming /// that it has been already transferred to 'device' domain. pub const DMA_ATTR_SKIP_CPU_SYNC: Attrs = Attrs(bindings::DMA_ATTR_SKIP_CPU_SYNC); diff --git a/rust/kernel/dma_buf.rs b/rust/kernel/dma_buf.rs new file mode 100644 index 00000000000000..318518ff0b28f9 --- /dev/null +++ b/rust/kernel/dma_buf.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA buffer API +//! +//! C header: [`include/linux/dma-buf.h`](srctree/include/linux/dma-buf.h) + +use bindings; +use kernel::types::*; + +/// A DMA buffer object. +/// +/// # Invariants +/// +/// The data layout of this type is equivalent to that of `struct dma_buf`. +#[repr(transparent)] +pub struct DmaBuf(Opaque); + +// SAFETY: `struct dma_buf` is thread-safe +unsafe impl Send for DmaBuf {} +// SAFETY: `struct dma_buf` is thread-safe +unsafe impl Sync for DmaBuf {} + +impl DmaBuf { + /// Convert from a `*mut bindings::dma_buf` to a [`DmaBuf`]. + /// + /// # Safety + /// + /// The caller guarantees that `self_ptr` points to a valid initialized `struct dma_buf` for the + /// duration of the lifetime of `'a`, and promises to not violate rust's data aliasing rules + /// using the reference provided by this function. + pub(crate) unsafe fn as_ref<'a>(self_ptr: *mut bindings::dma_buf) -> &'a Self { + // SAFETY: Our data layout is equivalent to `dma_buf` . + unsafe { &*self_ptr.cast() } + } + + pub(crate) fn as_raw(&self) -> *mut bindings::dma_buf { + self.0.get() + } +} diff --git a/rust/kernel/dma_fence.rs b/rust/kernel/dma_fence.rs new file mode 100644 index 00000000000000..47e612f4c52d10 --- /dev/null +++ b/rust/kernel/dma_fence.rs @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DMA fence abstraction. +//! +//! C header: [`include/linux/dma_fence.h`](../../include/linux/dma_fence.h) + +use crate::{ + bindings, + prelude::*, + sync::LockClassKey, + types::Opaque, +}; +use core::ops::{Deref, DerefMut}; +use core::ptr::addr_of_mut; +use core::sync::atomic::{AtomicU64, Ordering}; + +mod private { + /// Marker that a trait cannot be implemented outside of this mod + pub trait Sealed {} +} + +/// Any kind of DMA Fence Object +/// +/// # Invariants +/// raw() returns a valid pointer to a dma_fence and we own a reference to it. +pub trait RawDmaFence: private::Sealed { + /// Returns the raw `struct dma_fence` pointer. + fn raw(&self) -> *mut bindings::dma_fence; + + /// Returns the raw `struct dma_fence` pointer and consumes the object. + /// + /// The caller is responsible for dropping the reference. + fn into_raw(self) -> *mut bindings::dma_fence + where + Self: Sized, + { + let ptr = self.raw(); + core::mem::forget(self); + ptr + } + + /// Advances this fence to the chain node which will signal this sequence number. + /// If no sequence number is provided, this returns `self` again. + /// If the seqno has already been signaled, returns None. + fn chain_find_seqno(self, seqno: u64) -> Result> + where + Self: Sized, + { + let mut ptr = self.into_raw(); + + // SAFETY: This will safely fail if this DmaFence is not a chain. + // `ptr` is valid per the type invariant. + let ret = unsafe { bindings::dma_fence_chain_find_seqno(&mut ptr, seqno) }; + + if ret != 0 { + // SAFETY: This is either an owned reference or NULL, dma_fence_put can handle both. + unsafe { bindings::dma_fence_put(ptr) }; + Err(Error::from_errno(ret)) + } else if ptr.is_null() { + Ok(None) + } else { + // SAFETY: ptr is valid and non-NULL as checked above. + Ok(Some(unsafe { Fence::from_raw(ptr) })) + } + } + + /// Signal completion of this fence + fn signal(&self) { + // SAFETY: Safe to call on any valid dma_fence object + unsafe { bindings::dma_fence_signal(self.raw()) }; + } + + /// Set the error flag on this fence + fn set_error(&self, err: Error) { + // SAFETY: Safe to call on any valid dma_fence object + unsafe { bindings::dma_fence_set_error(self.raw(), err.to_errno()) }; + } +} + +/// A generic DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +pub struct Fence { + ptr: *mut bindings::dma_fence, +} + +impl Fence { + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// The caller must own a reference to the dma_fence, which is transferred to the new object. + pub(crate) unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Fence { + Fence { ptr } + } + + /// Create a new Fence object from a raw pointer to a dma_fence. + /// + /// # Safety + /// Takes a borrowed reference to the dma_fence, and increments the reference count. + pub(crate) unsafe fn get_raw(ptr: *mut bindings::dma_fence) -> Fence { + // SAFETY: Pointer is valid per the safety contract + unsafe { bindings::dma_fence_get(ptr) }; + Fence { ptr } + } + + /// Create a new Fence object from a RawDmaFence. + pub fn from_fence(fence: &dyn RawDmaFence) -> Fence { + // SAFETY: Pointer is valid per the RawDmaFence contract + unsafe { Self::get_raw(fence.raw()) } + } +} + +impl private::Sealed for Fence {} + +impl RawDmaFence for Fence { + fn raw(&self) -> *mut bindings::dma_fence { + self.ptr + } +} + +impl Drop for Fence { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::dma_fence_put(self.ptr) }; + } +} + +impl Clone for Fence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.ptr); + Self::from_raw(self.ptr) + } + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for Fence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for Fence {} + +/// Trait which must be implemented by driver-specific fence objects. +#[vtable] +pub trait FenceOps: Sized + Send + Sync { + /// Returns the driver name. This is a callback to allow drivers to compute the name at + /// runtime, without having it to store permanently for each fence, or build a cache of + /// some sort. + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Return the name of the context this fence belongs to. This is a callback to allow drivers + /// to compute the name at runtime, without having it to store permanently for each fence, or + /// build a cache of some sort. + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr; + + /// Enable software signaling of fence. + fn enable_signaling(self: &FenceObject) -> bool { + false + } + + /// Peek whether the fence is signaled, as a fastpath optimization for e.g. dma_fence_wait() or + /// dma_fence_add_callback(). + fn signaled(self: &FenceObject) -> bool { + false + } +} + +unsafe extern "C" fn get_driver_name_cb( + fence: *mut bindings::dma_fence, +) -> *const crate::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_driver_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn get_timeline_name_cb( + fence: *mut bindings::dma_fence, +) -> *const crate::ffi::c_char { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::get_timeline_name(unsafe { &mut *p }).as_char_ptr() +} + +unsafe extern "C" fn enable_signaling_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::enable_signaling(unsafe { &mut *p }) +} + +unsafe extern "C" fn signaled_cb(fence: *mut bindings::dma_fence) -> bool { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: The caller is responsible for passing a valid dma_fence subtype + T::signaled(unsafe { &mut *p }) +} + +unsafe extern "C" fn release_cb(fence: *mut bindings::dma_fence) { + // SAFETY: All of our fences are FenceObject. + let p = unsafe { crate::container_of!(fence, FenceObject, fence) as *mut FenceObject }; + + // SAFETY: p is never used after this + unsafe { + core::ptr::drop_in_place(&mut (*p).inner); + } + + // SAFETY: All of our fences are allocated using kmalloc, so this is safe. + unsafe { bindings::dma_fence_free(fence) }; +} + +/// A driver-specific DMA Fence Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence and we own a reference to it. +#[repr(C)] +pub struct FenceObject { + fence: bindings::dma_fence, + lock: Opaque, + inner: T, +} + +impl FenceObject { + const SIZE: usize = core::mem::size_of::(); + + const VTABLE: bindings::dma_fence_ops = bindings::dma_fence_ops { + get_driver_name: Some(get_driver_name_cb::), + get_timeline_name: Some(get_timeline_name_cb::), + enable_signaling: if T::HAS_ENABLE_SIGNALING { + Some(enable_signaling_cb::) + } else { + None + }, + signaled: if T::HAS_SIGNALED { + Some(signaled_cb::) + } else { + None + }, + wait: None, // Deprecated + release: Some(release_cb::), + set_deadline: None, + }; +} + +impl Deref for FenceObject { + type Target = T; + + fn deref(&self) -> &T { + &self.inner + } +} + +impl DerefMut for FenceObject { + fn deref_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +impl private::Sealed for FenceObject {} +impl RawDmaFence for FenceObject { + fn raw(&self) -> *mut bindings::dma_fence { + &self.fence as *const _ as *mut _ + } +} + +/// A unique reference to a driver-specific fence object +pub struct UniqueFence(*mut FenceObject); + +impl Deref for UniqueFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { &*self.0 } + } +} + +impl DerefMut for UniqueFence { + fn deref_mut(&mut self) -> &mut FenceObject { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { &mut *self.0 } + } +} + +impl private::Sealed for UniqueFence {} +impl RawDmaFence for UniqueFence { + fn raw(&self) -> *mut bindings::dma_fence { + // SAFETY: The pointer is always valid for UniqueFence objects + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl From> for UserFence { + fn from(value: UniqueFence) -> Self { + let ptr = value.0; + core::mem::forget(value); + + UserFence(ptr) + } +} + +impl Drop for UniqueFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for UniqueFence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for UniqueFence {} + +/// A shared reference to a driver-specific fence object +pub struct UserFence(*mut FenceObject); + +impl Deref for UserFence { + type Target = FenceObject; + + fn deref(&self) -> &FenceObject { + // SAFETY: The pointer is always valid for UserFence objects + unsafe { &*self.0 } + } +} + +impl Clone for UserFence { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { + bindings::dma_fence_get(self.raw()); + Self(self.0) + } + } +} + +impl private::Sealed for UserFence {} +impl RawDmaFence for UserFence { + fn raw(&self) -> *mut bindings::dma_fence { + // SAFETY: The pointer is always valid for UserFence objects + unsafe { addr_of_mut!((*self.0).fence) } + } +} + +impl Drop for UserFence { + fn drop(&mut self) { + // SAFETY: We own a reference to this fence. + unsafe { bindings::dma_fence_put(self.raw()) }; + } +} + +// SAFETY: The API for these objects is thread safe +unsafe impl Sync for UserFence {} +// SAFETY: The API for these objects is thread safe +unsafe impl Send for UserFence {} + +/// An array of fence contexts, out of which fences can be created. +pub struct FenceContexts { + start: u64, + count: u32, + seqnos: KVec, + lock_name: &'static CStr, + lock_key: Pin<&'static LockClassKey>, +} + +impl FenceContexts { + /// Create a new set of fence contexts. + pub fn new( + count: u32, + name: &'static CStr, + key: Pin<&'static LockClassKey>, + ) -> Result { + let mut seqnos: KVec = KVec::new(); + + seqnos.reserve(count as usize, GFP_KERNEL)?; + + for _ in 0..count { + seqnos.push(Default::default(), GFP_KERNEL)?; + } + + // SAFETY: This is always safe to call + let start = unsafe { bindings::dma_fence_context_alloc(count as crate::ffi::c_uint) }; + + Ok(FenceContexts { + start, + count, + seqnos, + lock_name: name, + lock_key: key, + }) + } + + /// Create a new fence in a given context index. + pub fn new_fence(&self, context: u32, inner: T) -> Result> { + if context > self.count { + return Err(EINVAL); + } + + // SAFETY: krealloc is always safe to call like this + let p = unsafe { + bindings::krealloc_node_align( + core::ptr::null_mut(), + FenceObject::::SIZE, + 1, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + bindings::NUMA_NO_NODE, + ) as *mut FenceObject + }; + + if p.is_null() { + return Err(ENOMEM); + } + + let seqno = self.seqnos[context as usize].fetch_add(1, Ordering::Relaxed); + + // SAFETY: The pointer is valid, so pointers to members are too. + // After this, all fields are initialized. + unsafe { + addr_of_mut!((*p).inner).write(inner); + bindings::__spin_lock_init( + addr_of_mut!((*p).lock) as *mut _, + self.lock_name.as_char_ptr(), + self.lock_key.as_ptr(), + ); + bindings::dma_fence_init64( + addr_of_mut!((*p).fence), + &FenceObject::::VTABLE, + addr_of_mut!((*p).lock) as *mut _, + self.start + context as u64, + seqno, + ); + }; + + Ok(UniqueFence(p)) + } +} + +/// A DMA Fence Chain Object +/// +/// # Invariants +/// ptr is a valid pointer to a dma_fence_chain which we own. +pub struct FenceChain { + ptr: *mut bindings::dma_fence_chain, +} + +impl FenceChain { + /// Create a new DmaFenceChain object. + pub fn new() -> Result { + // SAFETY: This function is safe to call and takes no arguments. + let ptr = unsafe { bindings::dma_fence_chain_alloc() }; + + if ptr.is_null() { + Err(ENOMEM) + } else { + Ok(FenceChain { ptr }) + } + } + + /// Convert the DmaFenceChain into the underlying raw pointer. + /// + /// This assumes the caller will take ownership of the object. + pub(crate) fn into_raw(self) -> *mut bindings::dma_fence_chain { + let ptr = self.ptr; + core::mem::forget(self); + ptr + } +} + +impl Drop for FenceChain { + fn drop(&mut self) { + // SAFETY: We own this dma_fence_chain. + unsafe { bindings::dma_fence_chain_free(self.ptr) }; + } +} diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 3ce8f62a005696..a9cf304a18a324 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -6,15 +6,29 @@ use crate::{ alloc::allocator::Kmalloc, - bindings, device, drm, - drm::driver::AllocImpl, + bindings, + device, + drm::{ + self, + driver::AllocImpl, // + }, error::from_err_ptr, - error::Result, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, - types::Opaque, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; +use core::{ + alloc::Layout, + mem, + ops::Deref, + ptr::{ + self, + NonNull, // + }, }; -use core::{alloc::Layout, mem, ops::Deref, ptr, ptr::NonNull}; #[cfg(CONFIG_DRM_LEGACY)] macro_rules! drm_legacy_fields { @@ -86,7 +100,7 @@ impl Device { name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), - driver_features: drm::driver::FEAT_GEM, + driver_features: T::FEATURES, ioctls: T::IOCTLS.as_ptr(), num_ioctls: T::IOCTLS.len() as i32, fops: &Self::GEM_FOPS, @@ -184,7 +198,10 @@ impl Device { // SAFETY: // - When `release` runs it is guaranteed that there is no further access to `this`. // - `this` is valid for dropping. - unsafe { core::ptr::drop_in_place(this) }; + // unsafe { core::ptr::drop_in_place(this) }; + // HACK: data might be uninitialized so leak the DRM device instead. The expected number + // of times the asahi device gets released is once at poweroff or reboot. + let _ = core::mem::ManuallyDrop::new(this); } } diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index e09f977b5b519e..18a8a90cf52649 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -5,15 +5,26 @@ //! C header: [`include/drm/drm_drv.h`](srctree/include/drm/drm_drv.h) use crate::{ - bindings, device, devres, drm, - error::{to_result, Result}, + bindings, + device, + devres, + drm, + error::to_result, prelude::*, - sync::aref::ARef, + sync::aref::ARef, // }; -use macros::vtable; /// Driver use the GEM memory manager. This should be set for all modern drivers. -pub(crate) const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +pub const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; +/// Driver supports dedicated render nodes. +pub const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; +/// Driver supports DRM sync objects for explicit synchronization of command submission. +pub const FEAT_SYNCOBJ: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ; +/// Driver supports the timeline flavor of DRM sync objects for explicit synchronization of command +/// submission. +pub const FEAT_SYNCOBJ_TIMELINE: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ_TIMELINE; +/// Driver supports user defined GPU VA bindings for GEM objects. +pub const FEAT_GEM_GPUVA: u32 = bindings::drm_driver_feature_DRIVER_GEM_GPUVA; /// Information data for a DRM Driver. pub struct DriverInfo { @@ -111,6 +122,9 @@ pub trait Driver { /// Driver metadata const INFO: DriverInfo; + /// Feature flags + const FEATURES: u32; + /// IOCTL list. See `kernel::drm::ioctl::declare_drm_ioctls!{}`. const IOCTLS: &'static [drm::ioctl::DrmIoctlDescriptor]; } diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 8c46f8d519516a..3603e473c4e0f6 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -4,9 +4,13 @@ //! //! C header: [`include/drm/drm_file.h`](srctree/include/drm/drm_file.h) -use crate::{bindings, drm, error::Result, prelude::*, types::Opaque}; +use crate::{ + bindings, + drm, + prelude::*, + types::Opaque, // +}; use core::marker::PhantomData; -use core::pin::Pin; /// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). pub trait DriverFile { @@ -15,6 +19,9 @@ pub trait DriverFile { /// Open a new file (called when a client opens the DRM device). fn open(device: &drm::Device) -> Result>>; + + /// Get raw drm_file pointer + fn as_raw(&self) -> *mut bindings::drm_file; } /// An open DRM File. diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index d49a9ba026356c..0bcc268f8dd8a7 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -5,15 +5,71 @@ //! C header: [`include/drm/drm_gem.h`](srctree/include/drm/drm_gem.h) use crate::{ - alloc::flags::*, - bindings, drm, - drm::driver::{AllocImpl, AllocOps}, - error::{to_result, Result}, + bindings, + dma_buf, + drm::{ + self, + driver::{ + AllocImpl, + AllocOps, // + }, + }, + error::{ + from_err_ptr, + to_result, // + }, prelude::*, - sync::aref::{ARef, AlwaysRefCounted}, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, types::Opaque, }; -use core::{ops::Deref, ptr::NonNull}; +use core::{ + ops::Deref, + marker::PhantomData, + ptr::NonNull, // +}; + +#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] +pub mod shmem; + +/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. +/// +/// Since all GEM objects use the same refcounting scheme. +#[macro_export] +macro_rules! impl_aref_for_gem_obj { + ( + impl $( <$( $tparam_id:ident ),+> )? for $type:ty + $( + where + $( $bind_param:path : $bind_trait:path ),+ + )? + ) => { + // SAFETY: All GEM objects are refcounted. + unsafe impl $( <$( $tparam_id ),+> )? $crate::types::AlwaysRefCounted for $type + where + Self: IntoGEMObject, + $( $( $bind_param : $bind_trait ),+ )? + { + fn inc_ref(&self) { + // SAFETY: The existence of a shared reference guarantees that the refcount is + // non-zero. + unsafe { bindings::drm_gem_object_get(self.as_raw()) }; + } + + unsafe fn dec_ref(obj: core::ptr::NonNull) { + // SAFETY: `obj` is a valid pointer to an `Object`. + let obj = unsafe { obj.as_ref() }.as_raw(); + + // SAFETY: The safety requirements guarantee that the refcount is non-zero. + unsafe { bindings::drm_gem_object_put(obj) }; + } + } + }; +} +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] +pub(crate) use impl_aref_for_gem_obj; /// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its /// [`DriverObject`] implementation. @@ -23,12 +79,20 @@ use core::{ops::Deref, ptr::NonNull}; pub type DriverFile = drm::File<<::Driver as drm::Driver>::File>; /// GEM object functions, which must be implemented by drivers. +#[vtable] pub trait DriverObject: Sync + Send + Sized { /// Parent `Driver` for this object. type Driver: drm::Driver; + /// The data type to use for passing arguments to [`DriverObject::new`]. + type Args; + /// Create a new driver data object for a GEM object of a given size. - fn new(dev: &drm::Device, size: usize) -> impl PinInit; + fn new( + dev: &drm::Device, + size: usize, + args: Self::Args, + ) -> impl PinInit; /// Open a new handle to an existing object, associated with a File. fn open(_obj: &::Object, _file: &DriverFile) -> Result { @@ -37,6 +101,14 @@ pub trait DriverObject: Sync + Send + Sized { /// Close a handle to an existing object, associated with a File. fn close(_obj: &::Object, _file: &DriverFile) {} + + /// Optional handle for exporting a gem object. + fn export( + _obj: &::Object, + _flags: u32, + ) -> Result::Object>> { + unimplemented!() + } } /// Trait that represents a GEM object subtype @@ -86,6 +158,21 @@ extern "C" fn close_callback( T::close(obj, file); } +extern "C" fn export_callback( + raw_obj: *mut bindings::drm_gem_object, + flags: i32, +) -> *mut bindings::dma_buf { + // SAFETY: `export_callback` is specified in the AllocOps structure for `Object`, ensuring + // that `raw_obj` is contained within a `Object`. + let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; + + match T::export(obj, flags as _) { + // DRM takes a hold of the reference + Ok(buf) => buf.into_raw(), + Err(e) => e.to_ptr(), + } +} + impl IntoGEMObject for Object { fn as_raw(&self) -> *mut bindings::drm_gem_object { self.obj.get() @@ -150,6 +237,28 @@ pub trait BaseObject: IntoGEMObject { Ok(unsafe { ARef::from_raw(obj.into()) }) } + /// Export a [`DmaBuf`] for this GEM object using the DRM prime helper library. + /// + /// `flags` should be a set of flags from [`fs::file::flags`](kernel::fs::file::flags). + fn prime_export(&self, flags: u32) -> Result> { + // SAFETY: + // - `as_raw()` always returns a valid pointer to a `drm_gem_object`. + // - `drm_gem_prime_export()` returns either an error pointer, or a valid pointer to an + // initialized `dma_buf` on success. + let dma_ptr = + from_err_ptr(unsafe { bindings::drm_gem_prime_export(self.as_raw(), flags as _) })?; + + // SAFETY: + // - We checked that dma_ptr is not an error, so it must point to an initialized dma_buf + // - We used drm_gem_prime_export(), so `dma_ptr` will remain valid until a call to + // `drm_gem_prime_release()` which we don't call here. + let dma_buf = unsafe { dma_buf::DmaBuf::as_ref(dma_ptr) }; + + // INVARIANT: We used drm_gem_prime_export() to create this dma_buf, fulfilling the + // invariant that this dma_buf came from a GEM object of type `Self`. + Ok(DmaBuf(dma_buf.into(), PhantomData)) + } + /// Creates an mmap offset to map the object from userspace. fn create_mmap_offset(&self) -> Result { // SAFETY: The arguments are valid per the type invariant. @@ -158,10 +267,36 @@ pub trait BaseObject: IntoGEMObject { // SAFETY: The arguments are valid per the type invariant. Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) }) } + + /// Lock the gpuva lock + fn lock_gpuva(&self) { + unsafe { + bindings::mutex_lock(&raw mut (*self.as_raw()).gpuva.lock); + } + } + + /// Lock the gpuva lock + fn unlock_gpuva(&self) { + unsafe { + bindings::mutex_unlock(&raw mut (*self.as_raw()).gpuva.lock); + } + } } impl BaseObject for T {} +/// Crate-private base operations shared by all GEM object classes. +#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] +pub(crate) trait BaseObjectPrivate: IntoGEMObject { + /// Return a pointer to this object's dma_resv. + fn raw_dma_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `as_gem_obj()` always returns a valid pointer to the base DRM gem object + unsafe { (*self.as_raw()).resv } + } +} + +impl BaseObjectPrivate for T {} + /// A base GEM object. /// /// # Invariants @@ -181,7 +316,11 @@ impl Object { open: Some(open_callback::), close: Some(close_callback::), print_info: None, - export: None, + export: if T::HAS_EXPORT { + Some(export_callback::) + } else { + None + }, pin: None, unpin: None, get_sg_table: None, @@ -195,11 +334,11 @@ impl Object { }; /// Create a new GEM object. - pub fn new(dev: &drm::Device, size: usize) -> Result> { + pub fn new(dev: &drm::Device, size: usize, args: T::Args) -> Result> { let obj: Pin> = KBox::pin_init( try_pin_init!(Self { obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size), + data <- T::new(dev, size, args), }), GFP_KERNEL, )?; @@ -207,8 +346,17 @@ impl Object { // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS }; - // SAFETY: The arguments are all valid per the type invariants. - to_result(unsafe { bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) })?; + if let Err(err) = + // SAFETY: The arguments are all valid per the type invariants. + to_result(unsafe { + bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) + }) + { + // SAFETY: `drm_gem_object_init()` initializes the private GEM object state before + // failing, so `drm_gem_private_object_fini()` is the matching cleanup. + unsafe { bindings::drm_gem_private_object_fini(obj.obj.get()) }; + return Err(err); + } // SAFETY: We will never move out of `Self` as `ARef` is always treated as pinned. let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); @@ -252,21 +400,7 @@ impl Object { } } -// SAFETY: Instances of `Object` are always reference-counted. -unsafe impl crate::sync::aref::AlwaysRefCounted for Object { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: `obj` is a valid pointer to an `Object`. - let obj = unsafe { obj.as_ref() }; - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj.as_raw()) } - } -} +impl_aref_for_gem_obj!(impl for Object where T: DriverObject); impl super::private::Sealed for Object {} @@ -292,8 +426,53 @@ impl AllocImpl for Object { }; } +/// A [`dma_buf::DmaBuf`] which has been exported from a GEM object. +/// +/// The [`dma_buf::DmaBuf`] will be released when this type is dropped. +/// +/// # Invariants +/// +/// - `self.0` points to a valid initialized [`dma_buf::DmaBuf`] for the lifetime of this object. +/// - The GEM object from which this [`dma_buf::DmaBuf`] was exported from is guaranteed to be of +/// type `T`. +pub struct DmaBuf(NonNull, PhantomData); + +impl Deref for DmaBuf { + type Target = dma_buf::DmaBuf; + + #[inline] + fn deref(&self) -> &Self::Target { + // SAFETY: This pointer is guaranteed to be valid by our type invariants. + unsafe { self.0.as_ref() } + } +} + +impl Drop for DmaBuf { + #[inline] + fn drop(&mut self) { + // SAFETY: + // - `dma_buf::DmaBuf` is guaranteed to have an identical layout to `struct dma_buf` + // by its type invariants. + // - We hold the last reference to this `DmaBuf`, making it safe to destroy. + unsafe { bindings::drm_gem_dmabuf_release(self.0.cast().as_ptr()) } + } +} + +impl DmaBuf { + /// Leak the reference for this [`DmaBuf`] and return a raw pointer to it. + #[inline] + pub(crate) fn into_raw(self) -> *mut bindings::dma_buf { + let dma_ptr = self.as_raw(); + + core::mem::forget(self); + dma_ptr + } +} + pub(super) const fn create_fops() -> bindings::file_operations { - let mut fops: bindings::file_operations = pin_init::zeroed(); + // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` + // zeroed. + let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() }; fops.owner = core::ptr::null_mut(); fops.open = Some(bindings::drm_open); diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs new file mode 100644 index 00000000000000..b7cf3a689c3d85 --- /dev/null +++ b/rust/kernel/drm/gem/shmem.rs @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! DRM GEM shmem helper objects +//! +//! C header: [`include/linux/drm/drm_gem_shmem_helper.h`](srctree/include/drm/drm_gem_shmem_helper.h) + +// TODO: +// - There are a number of spots here that manually acquire/release the DMA reservation lock using +// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to +// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. + +use crate::{ + container_of, + drm::{ + device, + driver, + gem, + private::Sealed, // + }, + error::{ + from_err_ptr, + to_result, // + }, + iosys_map::*, + prelude::*, + scatterlist, + transmute::*, + types::{ + ARef, + Opaque, // + }, // +}; +use core::{ + mem::{ + self, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, // + }, + ptr::NonNull, +}; +use gem::{ + BaseObject, + BaseObjectPrivate, + DriverObject, + IntoGEMObject, // +}; + +/// A struct for controlling the creation of shmem-backed GEM objects. +/// +/// This is used with [`Object::new()`] to control various properties that can only be set when +/// initially creating a shmem-backed GEM object. +#[derive(Default)] +pub struct ObjectConfig<'a, T: DriverObject> { + /// Whether to set the write-combine map flag. + pub map_wc: bool, + + /// Reuse the DMA reservation from another GEM object. + /// + /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. + pub parent_resv_obj: Option<&'a Object>, +} + +/// A shmem-backed GEM object. +/// +/// # Invariants +/// +/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this +/// object. +#[repr(C)] +#[pin_data] +pub struct Object { + #[pin] + obj: Opaque, + /// Parent object that owns this object's DMA reservation object. + parent_resv_obj: Option>>, + #[pin] + inner: T, +} + +super::impl_aref_for_gem_obj!(impl for Object where T: DriverObject); + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Send for Object {} + +// SAFETY: All GEM objects are thread-safe. +unsafe impl Sync for Object {} + +impl Object { + /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. + const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { + free: Some(Self::free_callback), + open: Some(super::open_callback::), + close: Some(super::close_callback::), + print_info: Some(bindings::drm_gem_shmem_object_print_info), + export: if T::HAS_EXPORT { + Some(super::export_callback::) + } else { + None + }, + pin: Some(bindings::drm_gem_shmem_object_pin), + unpin: Some(bindings::drm_gem_shmem_object_unpin), + get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), + vmap: Some(bindings::drm_gem_shmem_object_vmap), + vunmap: Some(bindings::drm_gem_shmem_object_vunmap), + mmap: Some(bindings::drm_gem_shmem_object_mmap), + status: None, + rss: None, + #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] + // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. + vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, + evict: None, + }; + + /// Return a raw pointer to the embedded drm_gem_shmem_object. + fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { + self.obj.get() + } + + /// Create a new shmem-backed DRM object of the given size. + /// + /// Additional config options can be specified using `config`. + pub fn new( + dev: &device::Device, + size: usize, + config: ObjectConfig<'_, T>, + args: T::Args, + ) -> Result> { + let new: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + )?; + + // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. + unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; + + // SAFETY: The arguments are all valid via the type invariants. + to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; + + // SAFETY: We never move out of `self`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. + let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; + + // Start filling out values from `config` + if let Some(parent_resv) = config.parent_resv_obj { + // SAFETY: We have yet to expose the new gem object outside of this function, so it is + // safe to modify this field. + unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; + } + + // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed + // to have exclusive access - thus making this safe to hold a mutable reference to. + let shmem = unsafe { &mut *obj.as_raw_shmem() }; + shmem.set_map_wc(config.map_wc); + + Ok(obj) + } + + /// Returns the `Device` that owns this GEM object. + pub fn dev(&self) -> &device::Device { + // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. + unsafe { device::Device::from_raw((*self.as_raw()).dev) } + } + + extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { + // SAFETY: + // - DRM always passes a valid gem object here + // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that + // `obj` is contained within a drm_gem_shmem_object + let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; + + // SAFETY: + // - We're in free_callback - so this function is safe to call. + // - We won't be using the gem resources on `this` after this call. + unsafe { bindings::drm_gem_shmem_release(this) }; + + // SAFETY: + // - We verified above that `obj` is valid, which makes `this` valid + // - This function is set in AllocOps, so we know that `this` is contained within a + // `Object` + let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); + + // SAFETY: We're recovering the Kbox<> we created in gem_create_object() + let _ = unsafe { KBox::from_raw(this) }; + } + + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA + /// pages for this object. + /// + /// This will pin the object in memory. + #[inline] + pub fn sg_table(&self) -> Result<&scatterlist::SGTable> { + // SAFETY: + // - drm_gem_shmem_get_pages_sgt is thread-safe. + // - drm_gem_shmem_get_pages_sgt returns either a valid pointer to a scatterlist, or an + // error pointer. + let sgt = from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })?; + + // SAFETY: We checked above that `sgt` is not an error pointer, so it must be a valid + // pointer to a scatterlist + Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }) + } + + /// Creates (if necessary) and returns an owned reference to a scatter-gather table of DMA pages + /// for this object. + /// + /// This is the same as [`sg_table`](Self::sg_table), except that it instead returns an + /// [`shmem::SGTable`] which holds a reference to the associated gem object, instead of a + /// reference to an [`scatterlist::SGTable`]. + /// + /// This will pin the object in memory. + /// + /// [`shmem::SGTable`]: SGTable + pub fn owned_sg_table(&self) -> Result> { + Ok(SGTable { + sgt: self.sg_table()?.into(), + // INVARIANT: We take an owned refcount to `self` here, ensuring that `sgt` remains + // valid for as long as this `SGTable`. + _owner: self.into(), + }) + } + + /// Attempt to create a [`RawIoSysMap`] from the gem object. + fn raw_vmap(&self) -> Result> { + build_assert!( + mem::size_of::() > 0, + "It doesn't make sense for the mapping type to be a ZST" + ); + + let mut map: MaybeUninit = MaybeUninit::uninit(); + + // SAFETY: drm_gem_shmem_vmap can be called with the DMA reservation lock held + to_result(unsafe { + // TODO: see top of file + bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); + let ret = bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()); + bindings::dma_resv_unlock(self.raw_dma_resv()); + ret + })?; + + // SAFETY: if drm_gem_shmem_vmap did not fail, map is initialized now + Ok(unsafe { RawIoSysMap::from_raw(map.assume_init()) }) + } + + /// Unmap a [`RawIoSysMap`] from the gem object. + /// + /// # Safety + /// + /// - The caller promises that `map` came from a prior call to [`Self::raw_vmap`] on this gem + /// object. + /// - The caller promises that the memory pointed to by `map` will no longer be accesed through + /// this instance. + unsafe fn raw_vunmap(&self, map: &mut RawIoSysMap) { + let resv = self.raw_dma_resv(); + + // SAFETY: + // - This function is safe to call with the DMA reservation lock held + // - Our `ARef` is proof that the underlying gem object here is initialized and thus safe to + // dereference. + unsafe { + // TODO: see top of file + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), map.as_raw_mut()); + bindings::dma_resv_unlock(resv); + } + } + + /// Creates and returns a virtual kernel memory mapping for this object. + pub fn vmap(&self) -> Result> { + let map = self.raw_vmap()?; + + Ok(VMapRef { + // SAFETY: + // - The size of the vmap is the same as the size of the gem + // - The vmap will remain alive until this object is dropped. + map: unsafe { IoSysMapRef::new(map, self.size()) }, + owner: self, + }) + } + + /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. + pub fn owned_vmap(&self) -> Result> { + Ok(VMap { + map: self.raw_vmap()?, + owner: self.into(), + }) + } +} + +impl Deref for Object { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Object { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Sealed for Object {} + +impl gem::IntoGEMObject for Object { + fn as_raw(&self) -> *mut bindings::drm_gem_object { + // SAFETY: + // - Our immutable reference is proof that this is safe to dereference. + // - `obj` is always a valid drm_gem_shmem_object via our type invariants. + unsafe { &raw mut (*self.obj.get()).base } + } + + unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { + // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within + // `Self` + unsafe { + let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); + + &*container_of!(obj, Object, obj) + } + } +} + +impl driver::AllocImpl for Object { + type Driver = T::Driver; + + const ALLOC_OPS: driver::AllocOps = driver::AllocOps { + gem_create_object: None, + prime_handle_to_fd: None, + prime_fd_to_handle: None, + gem_prime_import: None, + gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), + dumb_create: Some(bindings::drm_gem_shmem_dumb_create), + dumb_map_offset: None, + }; +} + +/// A borrowed reference to a virtual mapping for a shmem-based GEM object in kernel address space. +pub struct VMapRef<'a, D: DriverObject, T: AsBytes + FromBytes> { + map: IoSysMapRef<'a, T>, + owner: &'a Object, +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Clone for VMapRef<'a, D, T> { + fn clone(&self) -> Self { + // SAFETY: We have a successful vmap already, so this can't fail + unsafe { self.owner.vmap().unwrap_unchecked() } + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Deref for VMapRef<'a, D, T> { + type Target = IoSysMapRef<'a, T>; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> DerefMut for VMapRef<'a, D, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> Drop for VMapRef<'a, D, T> { + fn drop(&mut self) { + // SAFETY: Our existence is proof that this map was previously created using self.owner. + unsafe { self.owner.raw_vunmap(&mut self.map) }; + } +} + +/// An owned reference to a virtual mapping for a shmem-based GEM object in kernel address space. +/// +/// # Invariants +/// +/// - The memory pointed to by `map` is at least as large as `T`. +/// - The memory pointed to by `map` remains valid at least until this object is dropped. +pub struct VMap { + map: RawIoSysMap, + owner: ARef>, +} + +impl Clone for VMap { + fn clone(&self) -> Self { + // SAFETY: We have a successful vmap already, so this can't fail + unsafe { self.owner.owned_vmap().unwrap_unchecked() } + } +} + +impl<'a, D: DriverObject, T: AsBytes + FromBytes> From> for VMap { + fn from(value: VMapRef<'a, D, T>) -> Self { + let this = Self { + map: value.map.clone(), + owner: value.owner.into(), + }; + + mem::forget(value); + this + } +} + +impl VMap { + /// Return a reference to the iosys map for this `VMap`. + pub fn get(&self) -> IoSysMapRef<'_, T> { + // SAFETY: The size of the iosys_map is equivalent to the size of the gem object. + unsafe { IoSysMapRef::new(self.map.clone(), self.owner.size()) } + } + + /// Borrows a reference to the object that owns this virtual mapping. + pub fn owner(&self) -> &Object { + &self.owner + } +} + +impl Drop for VMap { + fn drop(&mut self) { + // SAFETY: Our existence is proof that this map was previously created using self.owner + unsafe { self.owner.raw_vunmap(&mut self.map) }; + } +} + +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Send for VMap {} +/// SAFETY: `iosys_map` objects are safe to send across threads. +unsafe impl Sync for VMap {} + +/// An owned reference to a scatter-gather table of DMA address spans for a GEM shmem object. +/// +/// This object holds an owned reference to the underlying GEM shmem object, ensuring that the +/// [`scatterlist::SGTable`] referenced by this type remains valid for the lifetime of this object. +/// +/// # Invariants +/// +/// - `sgt` is kept alive by `_owner`, ensuring it remains valid for as long as `Self`. +/// - `sgt` corresponds to the owned object in `_owner`. +/// - This object is only exposed in situations where we know the underlying `SGTable` will not be +/// modified for the lifetime of this object. Thus, it is safe to send/access this type across +/// threads. +pub struct SGTable { + sgt: NonNull, + _owner: ARef>, +} + +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Send for SGTable {} +// SAFETY: This object is thread-safe via our type invariants. +unsafe impl Sync for SGTable {} + +impl Deref for SGTable { + type Target = scatterlist::SGTable; + + fn deref(&self) -> &Self::Target { + // SAFETY: Creating an immutable reference to this is safe via our type invariants. + unsafe { self.sgt.as_ref() } + } +} diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs new file mode 100644 index 00000000000000..f3939a599acc1e --- /dev/null +++ b/rust/kernel/drm/gpuvm.rs @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/drm/drm_gpuvm.h`](../../../../include/drm/drm_gpuvm.h) + +#![allow(missing_docs)] + +use crate::{ + bindings, drm, + drm::{ + device, + gem::{ + BaseObject, + IntoGEMObject, // + }, + }, + error::{ + code::{EINVAL, ENOMEM}, + from_result, to_result, Error, Result, + }, + prelude::*, + types::{ARef, AlwaysRefCounted, Opaque}, +}; + +use core::cell::UnsafeCell; +use core::marker::{PhantomData, PhantomPinned}; +use core::mem::ManuallyDrop; +use core::ops::{Deref, DerefMut, Range}; +use core::ptr::NonNull; +use pin_init; + +/// GpuVaFlags to be used for a GpuVa. +/// +/// They can be combined with the operators `|`, `&`, and `!`. +#[derive(Clone, Copy, PartialEq, Default)] +pub struct GpuVaFlags(u32); + +impl GpuVaFlags { + /// No GpuVaFlags (zero) + pub const NONE: GpuVaFlags = GpuVaFlags(0); + + /// The backing GEM is invalidated. + pub const INVALIDATED: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_INVALIDATED); + + /// The GpuVa is a sparse mapping. + pub const SPARSE: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_SPARSE); + + /// The GpuVa is a repeat mapping. + pub const REPEAT: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_REPEAT); + + /// Construct a driver-specific GpuVaFlag. + /// + /// The argument must be a flag index in the range [0..28]. + pub const fn user_flag(index: u32) -> GpuVaFlags { + let flags = bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS << index; + assert!(flags != 0); + GpuVaFlags(flags) + } + + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> u32 { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: GpuVaFlags) -> bool { + (self & flags) == flags + } +} + +impl core::ops::BitOr for GpuVaFlags { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } +} + +impl core::ops::BitAnd for GpuVaFlags { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } +} + +impl core::ops::Not for GpuVaFlags { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } +} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVm (a GPU address space). +pub trait DriverGpuVm: Sized { + /// The parent `Driver` implementation for this `DriverGpuVm`. + type Driver: drm::Driver; + type GpuVa: DriverGpuVa = (); + type GpuVmBo: DriverGpuVmBo = (); + type StepContext = (); + + fn step_map( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpMap, + ctx: &mut Self::StepContext, + ) -> Result; + fn step_unmap( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpUnMap, + ctx: &mut Self::StepContext, + ) -> Result; + fn step_remap( + self: &mut UpdatingGpuVm<'_, Self>, + op: &mut OpReMap, + vm_bo: &GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result; +} + +struct StepContext<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + ctx: &'a mut T::StepContext, +} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVa (a mapping in GPU address space). +pub trait DriverGpuVa: Sized {} + +impl DriverGpuVa for () {} + +/// Trait that must be implemented by DRM drivers to represent a DRM GpuVmBo (a connection between a BO and a VM). +pub trait DriverGpuVmBo: Sized { + fn new() -> impl PinInit; +} + +/// Provide a default implementation for trivial types +impl DriverGpuVmBo for T { + fn new() -> impl PinInit { + pin_init::default() + } +} + +/// A convenience type for the driver's GEM object. +type Object = <::Driver as drm::driver::Driver>::Object; + +#[repr(transparent)] +pub struct OpMap(bindings::drm_gpuva_op_map, PhantomData); +#[repr(transparent)] +pub struct OpUnMap(bindings::drm_gpuva_op_unmap, PhantomData); +#[repr(transparent)] +pub struct OpReMap(bindings::drm_gpuva_op_remap, PhantomData); + +impl OpMap { + pub fn addr(&self) -> u64 { + self.0.va.addr + } + pub fn range(&self) -> u64 { + self.0.va.range + } + pub fn offset(&self) -> u64 { + self.0.gem.offset + } + pub fn flags(&self) -> GpuVaFlags { + GpuVaFlags(self.0.flags) + } + pub fn object(&self) -> &Object { + let p = unsafe { as IntoGEMObject>::from_raw(self.0.gem.obj) }; + // SAFETY: The GEM object has an active reference for the lifetime of this op + &*p + } + pub fn map_and_link_va( + &mut self, + gpuvm: &mut UpdatingGpuVm<'_, T>, + gpuva: Pin>>, + gpuvmbo: &GpuVmBo, + ) -> Result<(), Pin>>> { + // SAFETY: We are handing off the GpuVa ownership and it will not be moved. + let p = KBox::leak(unsafe { Pin::into_inner_unchecked(gpuva) }); + // SAFETY: These C functions are called with the correct invariants + unsafe { + bindings::drm_gpuva_init_from_op(&mut p.gpuva, &mut self.0); + if bindings::drm_gpuva_insert(gpuvm.0.gpuvm() as *mut _, &mut p.gpuva) != 0 { + // EEXIST, return the GpuVa to the caller as an error + return Err(Pin::new_unchecked(KBox::from_raw(p))); + }; + // SAFETY: This takes a new reference to the gpuvmbo. + gpuvmbo.lock_gpuva(); + bindings::drm_gpuva_link(&mut p.gpuva, &gpuvmbo.bo as *const _ as *mut _); + gpuvmbo.unlock_gpuva(); + } + Ok(()) + } +} + +impl OpUnMap { + pub fn va(&self) -> Option<&GpuVa> { + if self.0.va.is_null() { + return None; + } + // SAFETY: Container invariant is guaranteed for ops structs created for our types. + let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + Some(unsafe { &*p }) + } + pub fn unmap_and_unlink_va(&mut self) -> Option>>> { + self.do_unmap_and_unlink_va(false) + } + pub fn unmap_and_unlink_va_defer(&mut self) -> Option>>> { + self.do_unmap_and_unlink_va(true) + } + fn do_unmap_and_unlink_va(&mut self, defer: bool) -> Option>>> { + if self.0.va.is_null() { + return None; + } + // SAFETY: Container invariant is guaranteed for ops structs created for our types. + let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; + + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + unsafe { + bindings::drm_gpuva_unmap(&mut self.0); + if defer { + bindings::drm_gpuva_unlink_defer(self.0.va); + } else { + bindings::drm_gpuva_unlink(self.0.va); + } + } + + // Unlinking/unmapping relinquishes ownership of the GpuVa object, + // so clear the pointer + self.0.va = core::ptr::null_mut(); + // SAFETY: The GpuVa object reference is valid per the op_unmap contract + Some(unsafe { Pin::new_unchecked(KBox::from_raw(p)) }) + } +} + +impl OpReMap { + pub fn prev_map(&mut self) -> Option<&mut OpMap> { + // SAFETY: The prev pointer must be valid if not-NULL per the op_remap contract + unsafe { (self.0.prev as *mut OpMap).as_mut() } + } + pub fn next_map(&mut self) -> Option<&mut OpMap> { + // SAFETY: The next pointer must be valid if not-NULL per the op_remap contract + unsafe { (self.0.next as *mut OpMap).as_mut() } + } + pub fn unmap(&mut self) -> &mut OpUnMap { + // SAFETY: The unmap pointer is always valid per the op_remap contract + unsafe { (self.0.unmap as *mut OpUnMap).as_mut().unwrap() } + } +} + +/// A base GPU VA. +#[repr(C)] +#[pin_data] +pub struct GpuVa { + #[pin] + gpuva: bindings::drm_gpuva, + #[pin] + inner: T::GpuVa, + #[pin] + _p: PhantomPinned, +} + +impl GpuVa { + pub fn new(inner: impl PinInit) -> Result>>> + where + Error: From, + { + KBox::try_pin_init( + try_pin_init!(Self { + gpuva <- pin_init::init_zeroed(), + inner <- inner, + _p: PhantomPinned + }), + GFP_KERNEL, + ) + } + + pub fn addr(&self) -> u64 { + self.gpuva.va.addr + } + pub fn range(&self) -> u64 { + self.gpuva.va.range + } + pub fn offset(&self) -> u64 { + self.gpuva.gem.offset + } + pub fn flags(&self) -> GpuVaFlags { + GpuVaFlags(self.gpuva.flags) + } +} + +/// A base GpuVm BO. +#[repr(C)] +#[pin_data] +pub struct GpuVmBo { + #[pin] + bo: bindings::drm_gpuvm_bo, + #[pin] + inner: T::GpuVmBo, + #[pin] + _p: PhantomPinned, +} + +impl GpuVmBo { + /// Return a reference to the inner driver data for this GpuVmBo + pub fn inner(&self) -> &T::GpuVmBo { + &self.inner + } + /// Lock the GpuVmBo's gem boject gpuva lock + pub fn lock_gpuva(&self) { + unsafe { + let lock = &raw mut (*self.bo.obj).gpuva.lock; + bindings::mutex_lock(lock); + } + } + /// Unlock the GpuVmBo's gem boject gpuva lock + pub fn unlock_gpuva(&self) { + unsafe { + let lock = &raw mut (*self.bo.obj).gpuva.lock; + bindings::mutex_unlock(lock); + } + } +} + +// SAFETY: DRM GpuVmBo objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for GpuVmBo { + fn inc_ref(&self) { + // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). + unsafe { bindings::drm_gpuvm_bo_get(&self.bo as *const _ as *mut _) }; + } + + unsafe fn dec_ref(mut obj: NonNull) { + // SAFETY: drm_gpuvm_bo_put() requires holding the gpuva lock, which is the dma_resv lock by default. + // The drm_gpuvm_put function satisfies the requirements for dec_ref(). + // (We do not support custom locks yet.) + unsafe { + let resv = (*obj.as_mut().bo.obj).resv; + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gpuvm_bo_put(&mut obj.as_mut().bo); + bindings::dma_resv_unlock(resv); + } + } +} + +/// A base GPU VM. +#[repr(C)] +#[pin_data] +pub struct GpuVm { + #[pin] + gpuvm: Opaque, + #[pin] + inner: UnsafeCell, + #[pin] + _p: PhantomPinned, +} + +pub(super) unsafe extern "C" fn vm_free_callback( + raw_gpuvm: *mut bindings::drm_gpuvm, +) { + // SAFETY: Container invariant is guaranteed for objects using our callback. + let p = unsafe { + crate::container_of!( + raw_gpuvm as *mut Opaque, + GpuVm, + gpuvm + ) as *mut GpuVm + }; + + // SAFETY: p is guaranteed to be valid for drm_gpuvm objects using this callback. + unsafe { drop(KBox::from_raw(p)) }; +} + +pub(super) unsafe extern "C" fn vm_bo_alloc_callback() -> *mut bindings::drm_gpuvm_bo +{ + let obj: Result>>> = KBox::try_pin_init( + try_pin_init!(GpuVmBo:: { + bo <- pin_init::default(), + inner <- T::GpuVmBo::new(), + _p: PhantomPinned + }), + GFP_KERNEL, + ); + + match obj { + Ok(obj) => + // SAFETY: The DRM core will keep this object pinned + unsafe { + let p = KBox::leak(Pin::into_inner_unchecked(obj)); + &mut p.bo + }, + Err(_) => core::ptr::null_mut(), + } +} + +pub(super) unsafe extern "C" fn vm_bo_free_callback( + raw_vm_bo: *mut bindings::drm_gpuvm_bo, +) { + // SAFETY: Container invariant is guaranteed for objects using this callback. + let p = unsafe { crate::container_of!(raw_vm_bo, GpuVmBo, bo) as *mut GpuVmBo }; + + // SAFETY: p is guaranteed to be valid for drm_gpuvm_bo objects using this callback. + unsafe { drop(KBox::from_raw(p)) }; +} + +pub(super) unsafe extern "C" fn step_map_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpMap is a transparent wrapper. + let map = unsafe { &mut *((&mut (*op).__bindgen_anon_1.map) as *mut _ as *mut OpMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_map(map, ctx.ctx)?; + Ok(0) + }) +} + +pub(super) unsafe extern "C" fn step_remap_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpReMap is a transparent wrapper. + let remap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.remap) as *mut _ as *mut OpReMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + let p_vm_bo = remap.unmap().va().unwrap().gpuva.vm_bo; + + let res = { + // SAFETY: vm_bo pointer must be valid and non-null by the step_remap invariants. + // Since we grab a ref, this reference's lifetime is until the decref. + let vm_bo_ref = unsafe { + bindings::drm_gpuvm_bo_get(p_vm_bo); + &*(crate::container_of!(p_vm_bo, GpuVmBo, bo) as *mut GpuVmBo) + }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_remap(remap, vm_bo_ref, ctx.ctx)?; + Ok(0) + }) + }; + + // SAFETY: We incremented the refcount above, and the Rust reference we took is + // no longer in scope. + unsafe { bindings::drm_gpuvm_bo_put_deferred(p_vm_bo) }; + + res +} +pub(super) unsafe extern "C" fn step_unmap_callback( + op: *mut bindings::drm_gpuva_op, + _priv: *mut core::ffi::c_void, +) -> core::ffi::c_int { + // SAFETY: We know this is a map op, and OpUnMap is a transparent wrapper. + let unmap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.unmap) as *mut _ as *mut OpUnMap) }; + // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is + // guaranteed to outlive this function. + let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; + + from_result(|| { + UpdatingGpuVm(ctx.gpuvm).step_unmap(unmap, ctx.ctx)?; + Ok(0) + }) +} + +pub(super) unsafe extern "C" fn exec_lock_gem_object( + vm_exec: *mut bindings::drm_gpuvm_exec, +) -> core::ffi::c_int { + // SAFETY: The gpuvm_exec object is valid and priv_ is a GEM object pointer + // when this callback is used + unsafe { bindings::drm_exec_lock_obj(&mut (*vm_exec).exec, (*vm_exec).extra.priv_ as *mut _) } +} + +impl GpuVm { + const OPS: bindings::drm_gpuvm_ops = bindings::drm_gpuvm_ops { + vm_free: Some(vm_free_callback::), + op_alloc: None, + op_free: None, + vm_bo_alloc: Some(vm_bo_alloc_callback::), + vm_bo_free: Some(vm_bo_free_callback::), + vm_bo_validate: None, + sm_step_map: Some(step_map_callback::), + sm_step_remap: Some(step_remap_callback::), + sm_step_unmap: Some(step_unmap_callback::), + sm_can_merge_flags: None, + }; + + fn gpuvm(&self) -> *const bindings::drm_gpuvm { + self.gpuvm.get() + } + + pub fn new( + name: &'static CStr, + flags: bindings::drm_gpuvm_flags, + dev: &device::Device, + r_obj: ARef>, + range: Range, + reserve_range: Range, + inner: impl PinInit, + ) -> Result>> + where + Error: From, + { + let obj: Pin> = KBox::try_pin_init( + try_pin_init!(Self { + // SAFETY: drm_gpuvm_init cannot fail and always initializes the member + gpuvm <- unsafe { + pin_init::pin_init_from_closure(move |slot: *mut Opaque | { + // Zero-init required by drm_gpuvm_init + *slot = Opaque::zeroed(); + bindings::drm_gpuvm_init( + Opaque::cast_into(slot), + name.as_char_ptr(), + flags, + dev.as_raw(), + r_obj.as_raw() as *const _ as *mut _, + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + &Self::OPS + ); + Ok(()) + }) + }, + // SAFETY: Just passing through to the initializer argument + inner <- unsafe { + pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell | { + inner.__pinned_init(slot as *mut _) + }) + }, + _p: PhantomPinned + }), + GFP_KERNEL, + )?; + + // SAFETY: We never move out of the object + let vm_ref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::leak( + Pin::into_inner_unchecked(obj), + ))) + }; + + Ok(vm_ref) + } + + pub fn exec_lock<'a, 'b>( + &'a self, + obj: Option<&'b Object>, + interruptible: bool, + ) -> Result> { + // Do not try to lock the object if it is internal (since it is already locked). + let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); + + let mut guard = ManuallyDrop::new(LockedGpuVm { + gpuvm: self, + // vm_exec needs to be pinned, so stick it in a Box. + vm_exec: KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }, + exec: Default::default(), + extra: match (is_ext, obj) { + (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { + fn_: Some(exec_lock_gem_object), + priv_: obj.as_raw() as *const _ as *mut _, + }, + _ => Default::default(), + }, + num_fences: 0, + }), + GFP_KERNEL, + )?, + obj, + }); + + // SAFETY: The object is valid and was initialized above + to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *guard.vm_exec) })?; + + Ok(ManuallyDrop::into_inner(guard)) + } + + /// Returns true if the given object is external to the GPUVM + /// (that is, if it does not share the DMA reservation object of the GPUVM). + pub fn is_extobj(&self, obj: &impl IntoGEMObject) -> bool { + let gem = obj.as_raw() as *const _ as *mut _; + // SAFETY: This is safe to call as long as the arguments are valid pointers. + unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) } + } + + pub fn bo_deferred_cleanup(&self) { + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) } + } + + pub fn find_bo(&self, obj: &Object) -> Option>> { + obj.lock_gpuva(); + // SAFETY: drm_gem_object.gpuva.lock was just locked. + let p = unsafe { + bindings::drm_gpuvm_bo_find(self.gpuvm() as *mut _, obj.as_raw() as *const _ as *mut _) + }; + obj.unlock_gpuva(); + if p.is_null() { + None + } else { + // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn obtain_bo(&self, obj: &Object) -> Result>> { + obj.lock_gpuva(); + // SAFETY: drm_gem_object.gpuva.lock was just locked. + let p = unsafe { + bindings::drm_gpuvm_bo_obtain_locked( + self.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + obj.unlock_gpuva(); + if p.is_null() { + Err(ENOMEM) + } else { + // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn bo_unmap(&self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { + let mut ctx = StepContext { ctx, gpuvm: self }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) + }) + } +} + +// SAFETY: DRM GpuVm objects are always reference counted and the get/put functions +// satisfy the requirements. +unsafe impl AlwaysRefCounted for GpuVm { + fn inc_ref(&self) { + // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). + unsafe { bindings::drm_gpuvm_get(&self.gpuvm as *const _ as *mut _) }; + } + + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: The drm_gpuvm_put function satisfies the requirements for dec_ref(). + unsafe { bindings::drm_gpuvm_put(Opaque::cast_into(&(*obj.as_ptr()).gpuvm)) }; + } +} + +pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + vm_exec: KBox, + obj: Option<&'b Object>, +} + +impl LockedGpuVm<'_, '_, T> { + pub fn find_bo(&mut self) -> Option>> { + let obj = self.obj?; + // SAFETY: LockedGpuVm implies the right locks are held. + let p = unsafe { + bindings::drm_gpuvm_bo_find( + self.gpuvm.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + if p.is_null() { + None + } else { + // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn obtain_bo(&mut self) -> Result>> { + let obj = self.obj.ok_or(EINVAL)?; + // SAFETY: LockedGpuVm implies the right locks are held. + let p = unsafe { + bindings::drm_gpuvm_bo_obtain_locked( + self.gpuvm.gpuvm() as *mut _, + obj.as_raw() as *const _ as *mut _, + ) + }; + if p.is_null() { + Err(ENOMEM) + } else { + // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. + let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; + // SAFETY: We checked for NULL above, and the types ensure that + // this object was created by vm_bo_alloc_callback. + Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) + } + } + + pub fn sm_map( + &mut self, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { + let obj = self.obj.ok_or(EINVAL)?; + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + + let req = bindings::drm_gpuvm_map_req { + map: bindings::drm_gpuva_op_map { + va: bindings::drm_gpuva_op_map__bindgen_ty_1 { + addr: req_addr, + range: req_range, + }, + gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { + offset: req_offset, + range: req_gem_range, + obj: obj.as_raw(), + }, + flags: flags.as_raw(), + }, + }; + + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_sm_map( + self.gpuvm.gpuvm() as *mut _, + &mut ctx as *mut _ as *mut _, + &raw const req, + ) + }) + } + + pub fn sm_unmap(&mut self, ctx: &mut T::StepContext, req_addr: u64, req_range: u64) -> Result { + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_sm_unmap( + self.gpuvm.gpuvm() as *mut _, + &mut ctx as *mut _ as *mut _, + req_addr, + req_range, + ) + }) + } + + pub fn bo_unmap(&mut self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { + let mut ctx = StepContext { + ctx, + gpuvm: self.gpuvm, + }; + // SAFETY: LockedGpuVm implies the right locks are held. + to_result(unsafe { + bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) + }) + } +} + +impl Deref for LockedGpuVm<'_, '_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The existence of this LockedGpuVm implies the lock is held, + // so this is the only reference + unsafe { &*self.gpuvm.inner.get() } + } +} + +impl DerefMut for LockedGpuVm<'_, '_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &mut *self.gpuvm.inner.get() } + } +} + +impl Drop for LockedGpuVm<'_, '_, T> { + fn drop(&mut self) { + // SAFETY: We hold the lock, so it's safe to unlock + unsafe { + bindings::drm_gpuvm_exec_unlock(&mut *self.vm_exec); + } + } +} + +pub struct UpdatingGpuVm<'a, T: DriverGpuVm>(&'a GpuVm); + +impl UpdatingGpuVm<'_, T> {} + +impl Deref for UpdatingGpuVm<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &*self.0.inner.get() } + } +} + +impl DerefMut for UpdatingGpuVm<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, + // so this is the only reference + unsafe { &mut *self.0.inner.get() } + } +} + +// SAFETY: All our trait methods take locks +unsafe impl Sync for GpuVm {} +// SAFETY: All our trait methods take locks +unsafe impl Send for GpuVm {} + +// SAFETY: All our trait methods take locks +unsafe impl Sync for GpuVmBo {} +// SAFETY: All our trait methods take locks +unsafe impl Send for GpuVmBo {} diff --git a/rust/kernel/drm/mm.rs b/rust/kernel/drm/mm.rs new file mode 100644 index 00000000000000..7b13cfd7d53095 --- /dev/null +++ b/rust/kernel/drm/mm.rs @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM MM range allocator +//! +//! C header: [`include/drm/drm_mm.h`](../../../../include/drm/drm_mm.h) + +use crate::{ + alloc::flags::*, + bindings, + error::{to_result, Result}, + sync::{new_mutex, Arc, Mutex, UniqueArc}, + types::Opaque, +}; + +use crate::init::InPlaceInit; +use crate::prelude::KBox; + +use core::{ + marker::{PhantomData, PhantomPinned}, + ops::Deref, + pin::Pin, +}; + +/// Type alias representing a DRM MM node. +pub type Node = Pin>>; + +/// Trait which must be implemented by the inner allocator state type provided by the user. +pub trait AllocInner { + /// Notification that a node was dropped from the allocator. + fn drop_object(&mut self, _start: u64, _size: u64, _color: usize, _object: &mut T) {} +} + +impl AllocInner for () {} + +/// Wrapper type for a `struct drm_mm` plus user AllocInner object. +/// +/// # Invariants +/// The `drm_mm` struct is valid and initialized. +struct MmInner, T>(Opaque, A, PhantomData); + +/// Represents a single allocated node in the MM allocator +pub struct NodeData, T> { + node: bindings::drm_mm_node, + mm: Arc>>, + valid: bool, + /// A drm_mm_node needs to be pinned because nodes reference each other in a linked list. + _pin: PhantomPinned, + inner: T, +} + +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Send> Send for NodeData {} +// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. +unsafe impl, T: Sync> Sync for NodeData {} + +/// Available MM node insertion modes +#[repr(u32)] +pub enum InsertMode { + /// Search for the smallest hole (within the search range) that fits the desired node. + /// + /// Allocates the node from the bottom of the found hole. + Best = bindings::drm_mm_insert_mode_DRM_MM_INSERT_BEST, + + /// Search for the lowest hole (address closest to 0, within the search range) that fits the + /// desired node. + /// + /// Allocates the node from the bottom of the found hole. + Low = bindings::drm_mm_insert_mode_DRM_MM_INSERT_LOW, + + /// Search for the highest hole (address closest to U64_MAX, within the search range) that fits + /// the desired node. + /// + /// Allocates the node from the top of the found hole. The specified alignment for the node is + /// applied to the base of the node (`Node.start()`). + High = bindings::drm_mm_insert_mode_DRM_MM_INSERT_HIGH, + + /// Search for the most recently evicted hole (within the search range) that fits the desired + /// node. This is appropriate for use immediately after performing an eviction scan and removing + /// the selected nodes to form a hole. + /// + /// Allocates the node from the bottom of the found hole. + Evict = bindings::drm_mm_insert_mode_DRM_MM_INSERT_EVICT, +} + +/// A clonable, interlocked reference to the allocator state. +/// +/// This is useful to perform actions on the user-supplied `AllocInner` type given just a Node, +/// without immediately taking the lock. +#[derive(Clone)] +pub struct InnerRef, T>(Arc>>); + +impl, T> InnerRef { + /// Operate on the user `AllocInner` implementation, taking the lock. + pub fn with(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.0.lock(); + cb(&mut l.1) + } +} + +impl, T> NodeData { + /// Returns the color of the node (an opaque value) + pub fn color(&self) -> usize { + self.node.color as usize + } + + /// Returns the start address of the node + pub fn start(&self) -> u64 { + self.node.start + } + + /// Returns the size of the node in bytes + pub fn size(&self) -> u64 { + self.node.size + } + + /// Operate on the user `AllocInner` implementation associated with this node's allocator. + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut l = self.mm.lock(); + cb(&mut l.1) + } + + /// Return a clonable, detached reference to the allocator inner data. + pub fn alloc_ref(&self) -> InnerRef { + InnerRef(self.mm.clone()) + } + + /// Return a mutable reference to the inner data. + pub fn inner_mut(self: Pin<&mut Self>) -> &mut T { + // SAFETY: This is okay because inner is not structural + unsafe { &mut self.get_unchecked_mut().inner } + } +} + +impl, T> Deref for NodeData { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl, T> Drop for NodeData { + fn drop(&mut self) { + if self.valid { + let mut guard = self.mm.lock(); + + // Inform the user allocator that a node is being dropped. + guard + .1 + .drop_object(self.start(), self.size(), self.color(), &mut self.inner); + // SAFETY: The MM lock is still taken, so we can safely remove the node. + unsafe { bindings::drm_mm_remove_node(&mut self.node) }; + } + } +} + +/// An instance of a DRM MM range allocator. +pub struct Allocator, T> { + mm: Arc>>, + _p: PhantomData, +} + +impl, T> Allocator { + /// Create a new range allocator for the given start and size range of addresses. + /// + /// The user may optionally provide an inner object representing allocator state, which will + /// be protected by the same lock. If not required, `()` can be used. + #[track_caller] + pub fn new(start: u64, size: u64, inner: A) -> Result> { + // SAFETY: We call `Mutex::init_lock` below. + let mm = UniqueArc::pin_init( + new_mutex!(MmInner(Opaque::uninit(), inner, PhantomData)), + GFP_KERNEL, + )?; + + // SAFETY: The Opaque instance provides a valid pointer, and it is initialized after + // this call. + unsafe { + bindings::drm_mm_init(mm.lock().0.get(), start, size); + } + + Ok(Allocator { + mm: mm.into(), + _p: PhantomData, + }) + } + + /// Insert a new node into the allocator of a given size. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node(&mut self, node: T, size: u64) -> Result> { + self.insert_node_generic(node, size, 0, 0, InsertMode::Best) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, and insertion mode. + /// + /// `node` is the user `T` type data to store into the node. + pub fn insert_node_generic( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + mode: InsertMode, + ) -> Result> { + self.insert_node_in_range(node, size, alignment, color, 0, u64::MAX, mode) + } + + /// Insert a new node into the allocator of a given size, with configurable alignment, + /// color, insertion mode, and sub-range to allocate from. + /// + /// `node` is the user `T` type data to store into the node. + #[allow(clippy::too_many_arguments)] + pub fn insert_node_in_range( + &mut self, + node: T, + size: u64, + alignment: u64, + color: usize, + start: u64, + end: u64, + mode: InsertMode, + ) -> Result> { + let mut mm_node = KBox::new( + NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + }, + GFP_KERNEL, + )?; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { + bindings::drm_mm_insert_node_in_range( + guard.0.get(), + &mut mm_node.node, + size, + alignment, + color, + start, + end, + mode as u32, + ) + })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Insert a node into the allocator at a fixed start address. + /// + /// `node` is the user `T` type data to store into the node. + pub fn reserve_node( + &mut self, + node: T, + start: u64, + size: u64, + color: usize, + ) -> Result> { + let mut mm_node = KBox::new( + NodeData { + // SAFETY: This C struct should be zero-initialized. + node: unsafe { core::mem::zeroed() }, + valid: false, + inner: node, + mm: self.mm.clone(), + _pin: PhantomPinned, + }, + GFP_KERNEL, + )?; + + mm_node.node.start = start; + mm_node.node.size = size; + mm_node.node.color = color as crate::ffi::c_ulong; + + let guard = self.mm.lock(); + // SAFETY: We hold the lock and all pointers are valid. + to_result(unsafe { bindings::drm_mm_reserve_node(guard.0.get(), &mut mm_node.node) })?; + + mm_node.valid = true; + + Ok(Pin::from(mm_node)) + } + + /// Operate on the inner user type `A`, taking the allocator lock + pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { + let mut guard = self.mm.lock(); + cb(&mut guard.1) + } +} + +impl, T> Drop for MmInner { + fn drop(&mut self) { + // SAFETY: If the MmInner is dropped then all nodes are gone (since they hold references), + // so it is safe to tear down the allocator. + unsafe { + bindings::drm_mm_takedown(self.0.get()); + } + } +} + +// SAFETY: MmInner is safely Send if the AllocInner user type is Send. +unsafe impl, T> Send for MmInner {} diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs index 1b82b6945edf25..882841415aa414 100644 --- a/rust/kernel/drm/mod.rs +++ b/rust/kernel/drm/mod.rs @@ -6,7 +6,12 @@ pub mod device; pub mod driver; pub mod file; pub mod gem; +#[cfg(CONFIG_DRM_GPUVM = "y")] +pub mod gpuvm; pub mod ioctl; +pub mod mm; +pub mod sched; +pub mod syncobj; pub use self::device::Device; pub use self::driver::Driver; diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs new file mode 100644 index 00000000000000..e2f5cd96014f93 --- /dev/null +++ b/rust/kernel/drm/sched.rs @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Scheduler +//! +//! C header: [`include/drm/gpu_scheduler.h`](../../../../include/drm/gpu_scheduler.h) + +use crate::{ + bindings, device, + dma_fence::*, + error::{to_result, Result}, + prelude::*, + sync::{Arc, UniqueArc}, + time::{self, msecs_to_jiffies}, +}; +use core::marker::PhantomData; +use core::mem::MaybeUninit; +use core::ops::{Deref, DerefMut}; +use core::ptr::{addr_of, addr_of_mut}; + +/// Scheduler status after timeout recovery +#[repr(u32)] +pub enum Status { + /// Device recovered from the timeout and can execute jobs again + Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, + /// Device is no longer available + NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, +} + +/// Scheduler priorities +#[repr(u32)] +pub enum Priority { + /// Low userspace priority + Low = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_LOW, + /// Normal userspace priority + Normal = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_NORMAL, + /// High userspace priority + High = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_HIGH, + /// Kernel priority (highest) + Kernel = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_KERNEL, +} + +/// Trait to be implemented by driver job objects. +pub trait JobImpl: Sized { + /// Called when the scheduler is considering scheduling this job next, to get another Fence + /// for this job to block on. Once it returns None, run() may be called. + fn prepare(_job: &mut Job) -> Option { + None // Equivalent to NULL function pointer + } + + /// Called to execute the job once all of the dependencies have been resolved. This may be + /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides + /// to try it again. + fn run(job: &mut Job) -> Result>; + + /// Called when a job has taken too long to execute, to trigger GPU recovery. + /// + /// This method is called in a workqueue context. + fn timed_out(job: &mut Job) -> Status; + + /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences + /// get signalled before the scheduler is torn down. + fn cancel(job: &mut Job); +} + +unsafe extern "C" fn prepare_job_cb( + sched_job: *mut bindings::drm_sched_job, + _s_entity: *mut bindings::drm_sched_entity, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + match T::prepare(unsafe { &mut *p }) { + None => core::ptr::null_mut(), + Some(fence) => fence.into_raw(), + } +} + +unsafe extern "C" fn run_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> *mut bindings::dma_fence { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + match T::run(unsafe { &mut *p }) { + Err(e) => e.to_ptr(), + Ok(None) => core::ptr::null_mut(), + Ok(Some(fence)) => fence.into_raw(), + } +} + +unsafe extern "C" fn timedout_job_cb( + sched_job: *mut bindings::drm_sched_job, +) -> bindings::drm_gpu_sched_stat { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat +} + +unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // Convert the job back to a Box and drop it + // SAFETY: All of our Jobs are created inside a box. + unsafe { drop(KBox::from_raw(p)) }; +} + +unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { + // SAFETY: All of our jobs are Job. + let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + + // SAFETY: All of our jobs are Job. + T::cancel(unsafe { &mut *p }); + + let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; + fence.set_error(ECANCELED); + let _ = fence.signal(); +} + +/// A DRM scheduler job. +pub struct Job { + job: bindings::drm_sched_job, + inner: T, +} + +impl Deref for Job { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl DerefMut for Job { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl Drop for Job { + fn drop(&mut self) { + // SAFETY: At this point the job has either been submitted and this is being called from + // `free_job_cb` above, or it hasn't and it is safe to call `drm_sched_job_cleanup`. + unsafe { bindings::drm_sched_job_cleanup(&mut self.job) }; + } +} + +/// A pending DRM scheduler job (not yet armed) +pub struct PendingJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> PendingJob<'a, T> { + /// Add a fence as a dependency to the job + pub fn add_dependency(&mut self, fence: Fence) -> Result { + // SAFETY: C call with correct arguments + to_result(unsafe { + bindings::drm_sched_job_add_dependency(&mut self.0.job, fence.into_raw()) + }) + } + + /// Arm the job to make it ready for execution + pub fn arm(mut self) -> ArmedJob<'a, T> { + // SAFETY: C call with correct arguments + unsafe { bindings::drm_sched_job_arm(&mut self.0.job) }; + ArmedJob(self.0, PhantomData) + } +} + +impl<'a, T: JobImpl> Deref for PendingJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for PendingJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// An armed DRM scheduler job (not yet submitted) +pub struct ArmedJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); + +impl<'a, T: JobImpl> ArmedJob<'a, T> { + /// Returns the job fences + pub fn fences(&mut self) -> JobFences<'_> { + // SAFETY: s_fence is always a valid drm_sched_fence pointer + JobFences(unsafe { &mut *self.0.job.s_fence }) + } + + /// Push the job for execution into the scheduler + pub fn push(self) { + // After this point, the job is submitted and owned by the scheduler + let ptr = match self { + ArmedJob(job, _) => KBox::>::into_raw(job), + }; + + // SAFETY: We are passing in ownership of a valid Box raw pointer. + unsafe { bindings::drm_sched_entity_push_job(addr_of_mut!((*ptr).job)) }; + } +} +impl<'a, T: JobImpl> Deref for ArmedJob<'a, T> { + type Target = Job; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a, T: JobImpl> DerefMut for ArmedJob<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +/// Reference to the bundle of fences attached to a DRM scheduler job +pub struct JobFences<'a>(&'a mut bindings::drm_sched_fence); + +impl<'a> JobFences<'a> { + /// Returns a new reference to the job scheduled fence. + pub fn scheduled(&mut self) -> Fence { + // SAFETY: self.0.scheduled is always a valid fence + unsafe { Fence::get_raw(&mut self.0.scheduled) } + } + + /// Returns a new reference to the job finished fence. + pub fn finished(&mut self) -> Fence { + // SAFETY: self.0.finished is always a valid fence + unsafe { Fence::get_raw(&mut self.0.finished) } + } +} + +struct EntityInner { + entity: bindings::drm_sched_entity, + // TODO: Allow users to share guilty flag between entities + sched: Arc>, + guilty: bindings::atomic_t, + _p: PhantomData, +} + +impl Drop for EntityInner { + fn drop(&mut self) { + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for EntityInner {} +// SAFETY: TODO +unsafe impl Send for EntityInner {} + +/// A DRM scheduler entity. +pub struct Entity(Pin>>); + +impl Entity { + /// Create a new scheduler entity. + pub fn new(sched: &Scheduler, priority: Priority) -> Result { + let mut entity: KBox>> = + KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + + let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + + // SAFETY: The Box is allocated above and valid. + unsafe { + bindings::drm_sched_entity_init( + addr_of_mut!((*entity.as_mut_ptr()).entity), + priority as _, + &mut sched_ptr, + 1, + addr_of_mut!((*entity.as_mut_ptr()).guilty), + ) + }; + + // SAFETY: The Box is allocated above and valid. + unsafe { addr_of_mut!((*entity.as_mut_ptr()).sched).write(sched.0.clone()) }; + + // SAFETY: entity is now initialized. + Ok(Self(Pin::from(unsafe { entity.assume_init() }))) + } + + /// Create a new job on this entity. + /// + /// The entity must outlive the pending job until it transitions into the submitted state, + /// after which the scheduler owns it. Since jobs must be submitted in creation order, + /// this requires a mutable reference to the entity, ensuring that only one new job can be + /// in flight at once. + pub fn new_job(&mut self, credits: u32, inner: T) -> Result> { + let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + + // SAFETY: We hold a reference to the entity (which is a valid pointer), + // and the job object was just allocated above. + to_result(unsafe { + bindings::drm_sched_job_init( + addr_of_mut!((*job.as_mut_ptr()).job), + &self.0.as_ref().get_ref().entity as *const _ as *mut _, + credits, + core::ptr::null_mut(), + 0, + ) + })?; + + // SAFETY: The Box pointer is valid, and this initializes the inner member. + unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + + // SAFETY: All fields of the Job are now initialized. + Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) + } +} + +/// DRM scheduler inner data +pub struct SchedulerInner { + sched: bindings::drm_gpu_scheduler, + _p: PhantomData, +} + +impl Drop for SchedulerInner { + fn drop(&mut self) { + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of + // freeing all in-progress jobs. + unsafe { bindings::drm_sched_stop(&mut self.sched, core::ptr::null_mut()) }; + unsafe { bindings::drm_sched_fini(&mut self.sched) }; + } +} + +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + +/// A DRM Scheduler +pub struct Scheduler(Arc>); + +impl Scheduler { + const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { + prepare_job: Some(prepare_job_cb::), + run_job: Some(run_job_cb::), + timedout_job: Some(timedout_job_cb::), + free_job: Some(free_job_cb::), + cancel_job: Some(cancel_job_cb::), + }; + /// Creates a new DRM Scheduler object + // TODO: Shared timeout workqueues & scores + pub fn new( + device: &device::Device, + num_rqs: u32, + credit_limit: u32, + hang_limit: u32, + timeout_ms: time::Msecs, + name: &'static CStr, + ) -> Result> { + let mut sched: UniqueArc>> = + UniqueArc::new_uninit(GFP_KERNEL)?; + + // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation + // TODO: allocate sched zzeroed instead + unsafe { + (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); + }; + + let init_ops = bindings::drm_sched_init_args { + ops: &Self::OPS, + submit_wq: core::ptr::null_mut(), + timeout_wq: core::ptr::null_mut(), + num_rqs, + credit_limit, + hang_limit, + timeout: msecs_to_jiffies(timeout_ms).try_into()?, + score: core::ptr::null_mut(), + name: name.as_char_ptr(), + dev: device.as_raw(), + }; + + // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. + // `device` is valid by its type invarants + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + addr_of!(init_ops), + ) + })?; + + // SAFETY: All fields of SchedulerInner are now initialized. + Ok(Scheduler(unsafe { sched.assume_init() }.into())) + } +} diff --git a/rust/kernel/drm/syncobj.rs b/rust/kernel/drm/syncobj.rs new file mode 100644 index 00000000000000..a022e08223588b --- /dev/null +++ b/rust/kernel/drm/syncobj.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DRM Sync Objects +//! +//! C header: [`include/drm/drm_syncobj.h`](../../../../include/drm/drm_syncobj.h) + +use crate::{bindings, dma_fence::*, drm, error::Result, prelude::*}; + +/// A DRM Sync Object +/// +/// # Invariants +/// ptr is a valid pointer to a drm_syncobj and we own a reference to it. +pub struct SyncObj { + ptr: *mut bindings::drm_syncobj, +} + +impl SyncObj { + /// Looks up a sync object by its handle for a given `File`. + pub fn lookup_handle( + file: &drm::File, + handle: u32, + ) -> Result { + // SAFETY: The arguments are all valid per the type invariants. + let ptr = unsafe { bindings::drm_syncobj_find(file.as_raw() as *mut _, handle) }; + + if ptr.is_null() { + Err(ENOENT) + } else { + Ok(SyncObj { ptr }) + } + } + + /// Returns the DMA fence associated with this sync object, if any. + pub fn fence_get(&self) -> Option { + // SAFETY: self.ptr is always valid + let fence = unsafe { bindings::drm_syncobj_fence_get(self.ptr) }; + if fence.is_null() { + None + } else { + // SAFETY: The pointer is non-NULL and drm_syncobj_fence_get acquired an + // additional reference. + Some(unsafe { Fence::from_raw(fence) }) + } + } + + /// Replaces the DMA fence with a new one, or removes it if fence is None. + pub fn replace_fence(&self, fence: Option<&Fence>) { + // SAFETY: All arguments should be valid per the respective type invariants. + unsafe { + bindings::drm_syncobj_replace_fence( + self.ptr, + fence.map_or(core::ptr::null_mut(), |a| a.raw()), + ) + }; + } + + /// Adds a new timeline point to the syncobj. + pub fn add_point(&self, chain: FenceChain, fence: &Fence, point: u64) { + // SAFETY: All arguments should be valid per the respective type invariants. + // This takes over the FenceChain ownership. + unsafe { bindings::drm_syncobj_add_point(self.ptr, chain.into_raw(), fence.raw(), point) }; + } +} + +impl Drop for SyncObj { + fn drop(&mut self) { + // SAFETY: We own a reference to this syncobj. + unsafe { bindings::drm_syncobj_put(self.ptr) }; + } +} + +impl Clone for SyncObj { + fn clone(&self) -> Self { + // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. + unsafe { bindings::drm_syncobj_get(self.ptr) }; + SyncObj { ptr: self.ptr } + } +} + +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Sync for SyncObj {} +// SAFETY: drm_syncobj operations are internally locked. +unsafe impl Send for SyncObj {} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs index 258b12afdcba35..bc116561f6c09c 100644 --- a/rust/kernel/error.rs +++ b/rust/kernel/error.rs @@ -66,8 +66,11 @@ pub mod code { declare_err!(EPIPE, "Broken pipe."); declare_err!(EDOM, "Math argument out of domain of func."); declare_err!(ERANGE, "Math result not representable."); + declare_err!(ENOSYS, "Invalid system call number."); + declare_err!(ENODATA, "No data available."); declare_err!(EOVERFLOW, "Value too large for defined data type."); declare_err!(ETIMEDOUT, "Connection timed out."); + declare_err!(ECANCELED, "Operation Canceled."); declare_err!(ERESTARTSYS, "Restart the system call."); declare_err!(ERESTARTNOINTR, "System call was interrupted by a signal and will be restarted."); declare_err!(ERESTARTNOHAND, "Restart if no handler."); diff --git a/rust/kernel/iio/common/aop_sensors.rs b/rust/kernel/iio/common/aop_sensors.rs new file mode 100644 index 00000000000000..fe0d8af5606555 --- /dev/null +++ b/rust/kernel/iio/common/aop_sensors.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple AOP sensors common code +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::marker::{PhantomData, PhantomPinned}; +use core::ptr; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kernel::{ + bindings, device, + prelude::*, + soc::apple::aop::FakehidListener, + sync::Arc, + types::{ARef, ForeignOwnable}, + ThisModule, // +}; + +/// TODO: add documentation +pub trait MessageProcessor { + /// TODO: add documentation + fn process(&self, message: &[u8]) -> u32; +} + +/// TODO: add documentation +pub struct AopSensorData { + dev: ARef, + ty: u32, + value: AtomicU32, + msg_proc: T, +} + +impl AopSensorData { + /// TODO: add documentation + pub fn new(dev: ARef, ty: u32, msg_proc: T) -> Result>> { + Ok(Arc::new( + AopSensorData { + dev, + ty, + value: AtomicU32::new(0), + msg_proc, + }, + GFP_KERNEL, + )?) + } +} + +impl FakehidListener for AopSensorData { + fn process_fakehid_report(&self, data: &[u8]) -> Result<()> { + self.value + .store(self.msg_proc.process(data), Ordering::Relaxed); + Ok(()) + } +} + +unsafe extern "C" fn aop_read_raw( + dev: *mut bindings::iio_dev, + chan: *const bindings::iio_chan_spec, + val: *mut i32, + _: *mut i32, + mask: isize, +) -> i32 { + let data = unsafe { Arc::>::borrow((*dev).priv_.cast()) }; + let ty = unsafe { (*chan).type_ }; + if mask != bindings::BINDINGS_IIO_CHAN_INFO_PROCESSED as isize + && mask != bindings::BINDINGS_IIO_CHAN_INFO_RAW as isize + { + return EINVAL.to_errno(); + } + if data.ty != ty { + return EINVAL.to_errno(); + } + let value = data.value.load(Ordering::Relaxed); + unsafe { + *val = value as i32; + } + bindings::IIO_VAL_INT as i32 +} + +struct IIOSpec { + spec: [bindings::iio_chan_spec; 1], + vtable: bindings::iio_info, + _p: PhantomPinned, +} + +/// TODO: add documentation +pub struct IIORegistration { + dev: *mut bindings::iio_dev, + spec: Pin>, + registered: bool, + _p: PhantomData>, +} + +impl IIORegistration { + /// TODO: add documentation + pub fn new( + data: Arc>, + name: &'static CStr, + ty: u32, + info_mask: usize, + module: &ThisModule, + ) -> Result { + let spec = KBox::pin( + IIOSpec { + spec: [bindings::iio_chan_spec { + type_: ty, + __bindgen_anon_1: bindings::iio_chan_spec__bindgen_ty_1 { + scan_type: bindings::iio_scan_type { + sign: b'u' as _, + realbits: 32, + storagebits: 32, + ..Default::default() + }, + }, + info_mask_separate: info_mask, + ..Default::default() + }], + vtable: bindings::iio_info { + read_raw: Some(aop_read_raw::), + ..Default::default() + }, + _p: PhantomPinned, + }, + GFP_KERNEL, + )?; + let mut this = IIORegistration { + dev: ptr::null_mut(), + spec, + registered: false, + _p: PhantomData, + }; + this.dev = unsafe { bindings::iio_device_alloc(data.dev.as_raw(), 0) }; + unsafe { + (*this.dev).priv_ = data.clone().into_foreign().cast(); + (*this.dev).name = name.as_ptr() as _; + // spec is now pinned + (*this.dev).channels = this.spec.spec.as_ptr(); + (*this.dev).num_channels = this.spec.spec.len() as i32; + (*this.dev).info = &this.spec.vtable; + } + let ret = unsafe { bindings::__iio_device_register(this.dev, module.as_ptr()) }; + if ret < 0 { + dev_err!(data.dev, "Unable to register iio sensor"); + return Err(Error::from_errno(ret)); + } + this.registered = true; + Ok(this) + } +} + +impl Drop for IIORegistration { + fn drop(&mut self) { + if self.dev != ptr::null_mut() { + unsafe { + if self.registered { + bindings::iio_device_unregister(self.dev); + } + Arc::>::from_foreign((*self.dev).priv_.cast()); + bindings::iio_device_free(self.dev); + } + } + } +} + +unsafe impl Send for IIORegistration {} +unsafe impl Sync for IIORegistration {} diff --git a/rust/kernel/iio/common/mod.rs b/rust/kernel/iio/common/mod.rs new file mode 100644 index 00000000000000..b789e9bf44c9bf --- /dev/null +++ b/rust/kernel/iio/common/mod.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! IIO common modules + +#[cfg(any(CONFIG_IIO_AOP_SENSOR_LAS, CONFIG_IIO_AOP_SENSOR_ALS,))] +pub mod aop_sensors; diff --git a/rust/kernel/iio/mod.rs b/rust/kernel/iio/mod.rs new file mode 100644 index 00000000000000..b0cb308f0b454c --- /dev/null +++ b/rust/kernel/iio/mod.rs @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0 or MIT + +//! Industrial IO drivers + +pub mod common; diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index e5fba6bf6db072..3955f9b58b8877 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -320,6 +320,17 @@ const fn offset_valid(offset: usize, size: usize) -> bool { } } +/// Checks whether a memcpy of `length` bytes at the given `offset` +/// is valid within this region. +#[inline] +const fn length_valid(offset: usize, length: usize, size: usize) -> bool { + if let Some(end) = offset.checked_add(length) { + end <= size + } else { + false + } +} + /// Marker trait indicating that an I/O backend supports operations of a certain type. /// /// Different I/O backends can implement this trait to expose only the operations they support. @@ -367,6 +378,67 @@ pub trait Io { self.addr().checked_add(offset).ok_or(EINVAL) } + /// Copy memory block from an i/o memory by filling the specified buffer with it. + /// + /// # Examples + /// ``` + /// use kernel::io::mem::IoMem; + /// use kernel::io::mem::Resource; + /// + /// fn test(device: &Device, res: Resource) -> Result { + /// // Create an i/o memory block of at least 100 bytes. + /// let devres_mem = IoMem::<100>::new(res, device)?; + /// // aquire access to memory block + /// let mem = devres_mem.try_access()?; + /// + /// let mut buffer: [u8; 32] = [0; 32]; + /// + /// // Memcpy 16 bytes from an offset 10 of i/o memory block into the buffer. + /// mem.try_memcpy_fromio(&mut buffer[..16], 10)?; + /// + /// Ok(()) + /// } + /// ``` + fn try_memcpy_fromio(&self, buffer: &mut [u8], offset: usize) -> Result { + if buffer.len() == 0 || !length_valid(offset, buffer.len(), self.maxsize()) { + return Err(EINVAL); + } + let addr = self.io_addr::(offset)?; + + // SAFETY: + // - The type invariants guarantee that `adr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `length_valid`. + unsafe { + bindings::memcpy_fromio( + buffer.as_mut_ptr() as *mut _, + addr as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + + /// Copy memory block to i/o memory from the specified buffer. + fn try_memcpy_toio(&self, offset: usize, buffer: &[u8]) -> Result { + if buffer.len() == 0 || !length_valid(offset, buffer.len(), self.maxsize()) { + return Err(EINVAL); + } + // no need to check since offset + buffer.len() - 1 is valid + let addr = self.io_addr::(offset)?; + + // SAFETY: + // - The type invariants guarantee that `adr` is a valid pointer. + // - The bounds of `buffer` are checked with a call to `length_valid`. + unsafe { + bindings::memcpy_toio( + addr as *mut _, + buffer.as_ptr() as *const _, + buffer.len() as _, + ) + }; + Ok(()) + } + /// Fallible 8-bit read with runtime bounds check. #[inline(always)] fn try_read8(&self, _offset: usize) -> Result diff --git a/rust/kernel/io/mem.rs b/rust/kernel/io/mem.rs index 620022cff4013b..5bd3aae7192232 100644 --- a/rust/kernel/io/mem.rs +++ b/rust/kernel/io/mem.rs @@ -3,6 +3,7 @@ //! Generic memory-mapped IO. use core::ops::Deref; +use core::ptr::NonNull; use crate::{ device::{ @@ -10,6 +11,7 @@ use crate::{ Device, // }, devres::Devres, + impl_flags, io::{ self, resource::{ @@ -294,3 +296,105 @@ impl Deref for IoMem { unsafe { Mmio::from_raw(&self.io) } } } + +impl_flags!( + /// Flags to be used when remapping memory. + #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)] + pub struct MemFlags(usize); + + /// Enum mirroring the C MEMREMAP_* eum values + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum MemFlag { + /// Matches the default mapping for System RAM on the architecture. + /// + /// This is usually a read-allocate write-back cache. Moreover, if this flag is specified and + /// the requested remap region is RAM, memremap() will bypass establishing a new mapping and + /// instead return a pointer into the direct map. + WB = bindings::MEMREMAP_WB as usize, + + /// Establish a mapping whereby writes either bypass the cache or are written through to memory + /// and never exist in a cache-dirty state with respect to program visibility. + /// + /// Attempts to map System RAM with this mapping type will fail. + WT = bindings::MEMREMAP_WT as usize, + + /// Establish a writecombine mapping, whereby writes may be coalesced together (e.g. in the + /// CPU's write buffers), but is otherwise uncached. + /// + /// Attempts to map System RAM with this mapping type will fail. + WC = bindings::MEMREMAP_WC as usize, + // Note: Skipping MEMREMAP_ENC/DEC since they are under-documented and have zero + // users outside of arch/x86. + } +); + +/// Represents a non-MMIO memory block. This is like [`IoMem`], but for cases where it is known +/// that the resource being mapped does not have I/O side effects. +// Invariants: +// `ptr` is a non-null and valid address of at least `usize` bytes and returned by a `memremap` +// call. +// ``` +pub struct Mem { + ptr: NonNull, + size: usize, +} + +impl Mem { + /// Tries to create a new instance of a memory block from a Resource. + /// + /// The resource described by `res` is mapped into the CPU's address space so that it can be + /// accessed directly. It is also consumed by this function so that it can't be mapped again + /// to a different address. + /// + /// If multiple caching flags are specified, the different mapping types will be attempted in + /// the order [`MemFlag::WB`], [`MemFlag::WT`], [`MemFlag::WC`]. + /// + /// # Flags + /// + /// * [`MemFlag::WB`]: Matches the default mapping for System RAM on the architecture. + /// This is usually a read-allocate write-back cache. Moreover, if this flag is specified and + /// the requested remap region is RAM, memremap() will bypass establishing a new mapping and + /// instead return a pointer into the direct map. + /// + /// * [`MemFlag::WT`]: Establish a mapping whereby writes either bypass the cache or are written + /// through to memory and never exist in a cache-dirty state with respect to program visibility. + /// Attempts to map System RAM with this mapping type will fail. + /// * [`MemFlag::WC`]: Establish a writecombine mapping, whereby writes may be coalesced together + /// (e.g. in the CPU's write buffers), but is otherwise uncached. Attempts to map System RAM with + /// this mapping type will fail. + /// + /// # Safety + /// + /// Callers must ensure that either (a) the resulting interface cannot be used to initiate DMA + /// operations, or (b) that DMA operations initiated via the returned interface use DMA handles + /// allocated through the `dma` module. + pub unsafe fn try_new(res: Resource, flags: MemFlags) -> Result { + let size: usize = res.size().try_into()?; + + let addr = unsafe { bindings::memremap(res.start(), size, flags.into()) }; + let ptr = NonNull::new(addr).ok_or(ENOMEM)?; + // INVARIANT: `ptr` is non-null and was returned by `memremap`, so it is valid. + Ok(Self { ptr, size }) + } + + /// Returns the base address of the memory mapping as a raw pointer. + /// + /// It is up to the caller to use this pointer safely, depending on the requirements of the + /// hardware backing this memory block. + pub fn ptr(&self) -> *mut u8 { + self.ptr.cast().as_ptr() + } + + /// Returns the size of this mapped memory block. + pub fn size(&self) -> usize { + self.size + } +} + +impl Drop for Mem { + fn drop(&mut self) { + // SAFETY: By the type invariant, `self.ptr` is a value returned by a previous successful + // call to `memremap`. + unsafe { bindings::memunmap(self.ptr.as_ptr()) }; + } +} diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index b7ac9faf141d2e..2c7fe290b51440 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -79,6 +79,18 @@ unsafe impl Sync for Region {} pub struct Resource(Opaque); impl Resource { + /// Create a new zeroed [`Resource`] + pub(crate) fn zeroed() -> Self { + Resource { + 0: Opaque::::zeroed(), + } + } + + /// Gets the raw pointer to the wrapped `bindings::resource`. + pub(crate) fn as_raw(&self) -> *mut bindings::resource { + self.0.get() + } + /// Creates a reference to a [`Resource`] from a valid pointer. /// /// # Safety diff --git a/rust/kernel/iosys_map.rs b/rust/kernel/iosys_map.rs new file mode 100644 index 00000000000000..039137c051a2f1 --- /dev/null +++ b/rust/kernel/iosys_map.rs @@ -0,0 +1,647 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! IO-agnostic memory mapping interfaces. +//! +//! This crate provides bindings for the `struct iosys_map` type, which provides a common interface +//! for memory mappings which can reside within coherent memory, or within IO memory. +//! +//! C header: [`include/linux/iosys-map.h`](srctree/include/linux/pci.h) + +use crate::{ + prelude::*, + transmute::{AsBytes, FromBytes}, +}; +use bindings; +use core::{ + marker::PhantomData, + mem::{self, MaybeUninit}, + ops::{Deref, DerefMut, Range}, +}; + +/// Raw unsized representation of a `struct iosys_map`. +/// +/// This struct is a transparent wrapper around `struct iosys_map`. The C API does not provide the +/// size of the mapping by default, and thus this type also does not include the size of the +/// mapping. As such, it cannot be used for actually accessing the underlying data pointed to by the +/// mapping. +/// +/// With the exception of kernel crates which may provide their own wrappers around `RawIoSysMap`, +/// users will typically not interact with this type directly. +pub struct RawIoSysMap(bindings::iosys_map, PhantomData); + +impl RawIoSysMap { + /// Convert from a raw `bindings::iosys_map`. + #[allow(unused)] + #[inline] + pub(crate) fn from_raw(val: bindings::iosys_map) -> Self { + Self(val, PhantomData) + } + + /// Convert from a `RawIoSysMap` to a raw `bindings::iosys_map` ref. + #[inline] + pub(crate) fn as_raw(&self) -> &bindings::iosys_map { + &self.0 + } + + /// Convert from a `RawIoSysMap` to a raw mutable `bindings::iosys_map` ref. + #[inline] + pub(crate) fn as_raw_mut(&mut self) -> &mut bindings::iosys_map { + &mut self.0 + } + + /// Returns whether the mapping is within IO memory space or not. + #[inline] + pub fn is_iomem(&self) -> bool { + self.0.is_iomem + } + + /// Returns the size of a single item in this mapping. + pub const fn item_size(&self) -> usize { + mem::size_of::() + } + + /// Returns a mutable address to the memory pointed to by this iosys map. + /// + /// Note that this address is not guaranteed to reside in system memory, and may reside in IO + /// memory. + #[inline] + pub fn as_mut_ptr(&self) -> *mut T { + if self.is_iomem() { + // SAFETY: We confirmed above that this iosys map is contained within iomem, so it's + // safe to read vaddr_iomem + unsafe { self.0.__bindgen_anon_1.vaddr_iomem } + } else { + // SAFETY: We confirmed above that this iosys map is not contaned within iomem, so it's + // safe to read vaddr. + unsafe { self.0.__bindgen_anon_1.vaddr } + } + .cast() + } + + /// Returns an immutable address to the memory pointed to by this iosys map. + /// + /// Note that this address is not guaranteed to reside in system memory, and may reside in IO + /// memory. + #[inline] + pub fn as_ptr(&self) -> *const T { + self.as_mut_ptr().cast_const() + } +} + +// SAFETY: As we make no guarantees about the validity of the mapping, there's no issue with sending +// this type between threads. +unsafe impl Send for RawIoSysMap {} + +impl Clone for RawIoSysMap { + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } +} + +/// A sized version of a [`RawIoSysMap`]. +/// +/// Since this type includes the size of the [`RawIoSysMap`], it can be used for accessing the +/// underlying data pointed to by it. +/// +/// # Invariants +/// +/// - The iosys mapping referenced by this type is guaranteed to be of at least `size` bytes in +/// size +/// - The iosys mapping referenced by this type is valid for the lifetime `'a`. +#[derive(Clone)] +pub struct IoSysMapRef<'a, T: AsBytes + FromBytes> { + map: RawIoSysMap, + size: usize, + _p: PhantomData<&'a T>, +} + +impl<'a, T: AsBytes + FromBytes> IoSysMapRef<'a, T> { + /// Create a new [`IoSysMapRef`] from a [`RawIoSysMap`]. + /// + /// # Safety + /// + /// - The caller guarantees that the mapping referenced by `map` is of at least `size` bytes in + /// size. + /// - The caller guarantees that the mapping referenced by `map` remains valid for the lifetime + /// of `'a`. + #[allow(unused)] + pub(crate) unsafe fn new(map: RawIoSysMap, size: usize) -> IoSysMapRef<'a, T> { + // INVARIANT: Our safety contract fulfills the type invariants of `IoSysMapRef`. + IoSysMapRef { + map, + size, + _p: PhantomData, + } + } + + /// Return the size of the `IoSysMapRef`. + #[inline] + pub fn size(&self) -> usize { + self.size + } + + /// Writes `src` to the region starting from `offset`. + /// + /// `offset` is in units of `T`, not the number of bytes. + /// + /// This function can return the following errors: + /// + /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. + /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// + /// # Examples + /// + /// ``` + /// use kernel::iosys_map::*; + /// + /// # fn test() -> Result { + /// # let mut map = tests::VecIoSysMap::new(&[0; 3])?; + /// # { + /// # let mut map = map.get(); + /// map.write(&[1, 2, 3], 0)?; // (now [1, 2, 3]) + /// map.write(&[4], 2)?; // (now [1, 2, 4]) + /// # } + /// # + /// # map.assert_eq(&[1, 2, 4]); + /// # + /// # Ok::<(), Error>(()) } + /// # assert!(test().is_ok()); + /// ``` + pub fn write(&mut self, src: &[T], offset: usize) -> Result { + let range = self.compute_range(offset, src.len())?; + + // SAFETY: + // - The address pointed to by this iosys_map is guaranteed to be valid via IoSysMapRef's + // type invariants. + // - We checked that this range of memory is within bounds above + unsafe { + bindings::iosys_map_memcpy_to( + self.as_raw_mut(), + range.start, + src.as_ptr().cast(), + range.len(), + ) + }; + + Ok(()) + } + + /// Memset the region starting from `offset`. + /// + /// `offset` and `len` are in units of `T`, not the number of bytes. + /// + /// This function can return the following errors: + /// + /// * [`EOVERFLOW`] if calculating the length of the slice results in an overflow. + /// * [`EINVAL`] if the slice would go out of bounds of the memory region. + /// + /// # Examples + /// + /// ``` + /// use kernel::iosys_map::*; + /// + /// # fn test() -> Result { + /// # let mut map = tests::VecIoSysMap::new(&[0u8; 3])?; + /// # { + /// # let mut map = map.get(); + /// map.memset(7)?; // (now [7, 7, 7]) + /// # } + /// # + /// # map.assert_eq(&[7, 7, 7]); + /// # + /// # Ok::<(), Error>(()) } + /// # assert!(test().is_ok()); + /// ``` + pub fn memset(&mut self, value: i32) { + // SAFETY: + // - The address pointed to by this iosys_map is guaranteed to be valid via IoSysMapRef's + // type invariants. + unsafe { bindings::iosys_map_memset(self.as_raw_mut(), 0, value, self.size()) }; + } + + /// Attempt to compute the offset of an item within the iosys map using its index. + /// + /// Returns an error if an overflow occurs. + /// + /// # Safety + /// + /// This function checks for overflows, but it explicitly does not check if the offset goes out + /// of bounds. It is the caller's responsibility to check for this before using the returned + /// offset with the iosys_map API. + unsafe fn item_from_index(&self, idx: usize) -> Result { + self.item_size().checked_mul(idx).ok_or(EOVERFLOW) + } + + /// Compute the range within this mapping a specific data type at a given offset would occupy. + /// + /// This function returns the computed range if it doesn't overflow, but does not check whether + /// or not the range is within the bounds of the allocated region pointed to by this iosys + /// mapping. + /// + /// On success, the range returned by this function is guaranteed: + /// + /// * To be a valid range of memory within the virtual mapping for this gem object. + /// * To be properly aligned to [`RawIoSysMap::item_size()`]. + fn compute_range(&self, offset: usize, count: usize) -> Result> { + // SAFETY: If the offset is out of bounds, we'll catch this via overflow checks or when + // checking range_end. + let offset = unsafe { self.item_from_index(offset)? }; + let range_size = count.checked_mul(self.item_size()).ok_or(EOVERFLOW)?; + let range_end = offset.checked_add(range_size).ok_or(EOVERFLOW)?; + + if range_end > self.size { + return Err(EINVAL); + } + + // INVARIANT: Since `offset` and `count` are both in units of `T`, we're guaranteed that the + // range returned here is properly aligned to `T`. + Ok(offset..range_end) + } + + /// Common helper to compute the memory address of an item within the iosys mapping. + /// + /// Public but hidden, since it should only be used from [`iosys_map_read`] and + /// [`iosys_map_write`]. + #[doc(hidden)] + pub fn ptr_from_index(&self, offset: usize) -> Result<*mut T> { + // SAFETY: We check if the resulting offset goes out of bounds below. + let offset = unsafe { self.item_from_index(offset)? }; + + if offset.checked_add(self.item_size()).ok_or(EOVERFLOW)? > self.size() { + return Err(EINVAL); + } + + // SAFETY: We confirmed that `offset` + the item size does not go out of bounds above. + Ok(unsafe { self.as_mut_ptr().byte_add(offset) }) + } + + // TODO: + // This function is currently needed for making the iosys_map_read!() and iosys_map_write!() + // macros work due to a combination of a few limitations: + // + // * The current C API for iosys_map requires that we use offsets for reading/writing + // iosys_maps. + // * Calculating the offset of a field within a struct requires that we either: + // * Use field projection for calculating the offset of the field. We don't have this yet. + // * Explicitly specify the type of the struct, which would be cumbersome to require in the + // read/write macros. + // * Provide a typed pointer (or other reference) to the struct in question, allowing the + // use of &raw const and &raw mut. + // * Keep in mind: we can't simply cast the offset of an item in the iosys map into a typed + // pointer to fulfill the third option. While having invalid memory addresses as pointers + // is ok, adding an offset to a pointer in rust requires that the resulting memory address + // is within the same allocation. Since an invalid pointer has no allocation, we can't + // make that guarantee. + // + // So, until we have field projection the way we workaround this: + // + // * Calculate the offset (self.item_from_index()) of the struct within the iosys map + // * Calculate the memory address of the struct using the offset from the last step + // (self.ptr_from_index()). + // * Use that memory address with &raw const/&raw mut in order to calculate the memory address + // of the desired field, ensuring it remains in the same allocation (happens within the + // macros). + // * Convert the address from the last step back into an offset within the iosys map + // (offset_from_ptr()). + // + // Once we do get field projection, this silly code should be removed. + // + /// Convert a pointer to an item within the iosys map back into an offset. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to data within the iosys map. + unsafe fn offset_from_ptr(&self, ptr: *const F) -> usize { + // SAFETY: `ptr` always points to data within the memory pointed to by the iosys map, + // meaning it is within the same memory allocation. + // + // Additionally, since `ptr` is within the iosys mapping, the offset here will always be + // positive and safe to cast to a usize. + // (TODO: replace this with byte_offset_from_unsigned once it's available in the kernel) + unsafe { ptr.byte_offset_from(self.as_ptr()) as usize } + } + + /// Reads the value of `field` and ensures that its type is [`FromBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`iosys_map_read`] macro which ensures that the `field` + /// pointer is validated beforehand. + /// + /// Public but hidden since it should only be used from the [`iosys_map_read`] macro. + #[doc(hidden)] + pub unsafe fn field_read(&self, field: *const F) -> F { + let mut field_val = MaybeUninit::::uninit(); + + // SAFETY: `field` is guaranteed valid via our safety contract. + let offset = unsafe { self.offset_from_ptr(field) }; + + // SAFETY: Since we verified `field` is valid above, `offset_from_ptr` will always return a + // valid offset within the iosys map. + unsafe { + bindings::iosys_map_memcpy_from( + field_val.as_mut_ptr().cast(), + self.as_raw(), + offset, + mem::size_of::(), + ) + } + + // SAFETY: We just initialized `field_val` above. + unsafe { field_val.assume_init() } + } + + /// Writes the value of `field` and ensures that its type is [`AsBytes`]. + /// + /// # Safety + /// + /// This must be called from the [`iosys_map_write`] macro which ensures that the `field` + /// pointers validated beforehand. + /// + /// Public but hidden since it should only be used from the [`iosys_map_write`] macro. + #[doc(hidden)] + pub unsafe fn field_write(&mut self, field: *mut F, val: F) { + // SAFETY: `field` is guaranteed valid via our safety contract. + let offset = unsafe { self.offset_from_ptr(field) }; + + // SAFETY: `offset_from_ptr` always returns a valid offset within the iosys map. + unsafe { + bindings::iosys_map_memcpy_to( + self.as_raw_mut(), + offset, + core::ptr::from_ref(&val).cast(), + mem::size_of::(), + ) + } + } +} + +impl<'a, T: AsBytes + FromBytes> Deref for IoSysMapRef<'a, T> { + type Target = RawIoSysMap; + + fn deref(&self) -> &Self::Target { + &self.map + } +} + +impl<'a, T: AsBytes + FromBytes> DerefMut for IoSysMapRef<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.map + } +} + +/// Reads from a field of an item from an iosys map ref. +/// +/// # Examples +/// +/// ``` +/// use kernel::{iosys_map::*, transmute::*}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// struct MyStruct { a: u32, b: u16 } +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl FromBytes for MyStruct {}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl AsBytes for MyStruct {}; +/// +/// # fn test() -> Result { +/// # let mut map = tests::VecIoSysMap::new(&[MyStruct { a: 42, b: 2 }; 3])?; +/// # let map = map.get(); +/// let whole = kernel::iosys_map_read!(map[2])?; +/// assert_eq!(whole, MyStruct { a: 42, b: 2 }); +/// +/// let field = kernel::iosys_map_read!(map[1].b)?; +/// assert_eq!(field, 2); +/// # Ok::<(), Error>(()) } +/// # assert!(test().is_ok()); +/// ``` +#[macro_export] +macro_rules! iosys_map_read { + ($map:expr, $idx:expr, $($field:tt)*) => {{ + (|| -> ::core::result::Result<_, $crate::error::Error> { + let map = &$map; + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + + // SAFETY: `ptr_from_index()` ensures that `item` is always a valid (although + // potentially not dereferenceable, which is fine here) pointer to within the iosys + // mapping. + unsafe { + let ptr_field = &raw const (*item) $($field)*; + ::core::result::Result::Ok( + $crate::iosys_map::IoSysMapRef::field_read(map, ptr_field) + ) + } + })() + }}; + ($map:ident [ $idx: expr ] $($field:tt)* ) => { + $crate::iosys_map_read!($map, $idx, $($field)*) + }; + ($($map:ident).* [ $idx:expr ] $($field:tt)* ) => { + $crate::iosys_map_read!($($map).*, $idx, $($field)*) + }; +} + +/// Writes to a field of an item from an iosys map ref. +/// +/// # Examples +/// +/// ``` +/// use kernel::{iosys_map::*, transmute::*}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// struct MyStruct { a: u32, b: u16 }; +/// +/// // SAFETY: All bit patterns are acceptable values for `MyStruct`. +/// unsafe impl FromBytes for MyStruct {}; +/// // SAFETY: Instances of `MyStruct` have no uninitialized portions. +/// unsafe impl AsBytes for MyStruct {}; +/// +/// # fn test() -> Result { +/// # let mut map = tests::VecIoSysMap::new(&[MyStruct { a: 42, b: 2 }; 3])?; +/// # let mut map = map.get(); +/// kernel::iosys_map_write!(map[2].b = 1337)?; +/// # assert_eq!(kernel::iosys_map_read!(map[2].b)?, 1337); +/// +/// kernel::iosys_map_write!(map[1] = MyStruct { a: 10, b: 20 })?; +/// # assert_eq!(kernel::iosys_map_read!(map[1])?, MyStruct { a: 10, b: 20 }); +/// # Ok::<(), Error>(()) } +/// # assert!(test().is_ok()); +/// ``` +#[macro_export] +macro_rules! iosys_map_write { + ($map:ident [ $idx:expr ] $($field:tt)*) => {{ + $crate::iosys_map_write!($map, $idx, $($field)*) + }}; + ($($map:ident).* [ $idx:expr ] $($field:tt)* ) => {{ + $crate::iosys_map_write!($($map).*, $idx, $($field)*) + }}; + ($map:expr, $idx:expr, = $val:expr) => { + (|| -> ::core::result::Result<_, $crate::error::Error> { + // (expand these outside of the unsafe block (clippy::macro-metavars-in-unsafe) + let map = &mut $map; + let val = $val; + + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + // SAFETY: `item_from_index` ensures that `item` is always a valid item. + unsafe { $crate::iosys_map::IoSysMapRef::field_write(map, item, val) }; + ::core::result::Result::Ok(()) + })() + }; + ($map:expr, $idx:expr, $(.$field:ident)* = $val:expr) => { + (|| -> ::core::result::Result<_, $crate::error::Error> { + // (expand these outside of the unsafe block (clippy::macro-metavars-in-unsafe) + let map = &mut $map; + let val = $val; + + let item = $crate::iosys_map::IoSysMapRef::ptr_from_index(map, $idx)?; + + // SAFETY: `ptr_from_index()` ensures that `item` is always a valid (although + // potentially not dereferenceable, which is fine here) pointer to within the iosys + // mapping. + unsafe { + let ptr_field = &raw mut (*item) $(.$field)*; + $crate::iosys_map::IoSysMapRef::field_write(map, ptr_field, val) + }; + ::core::result::Result::Ok(()) + })() + }; +} + +#[doc(hidden)] +#[kunit_tests(rust_iosys_map)] +pub mod tests { + use super::*; + + /// A helper struct for managed IoSysMapRef structs which point to a [`Vec`]. + pub struct VecIoSysMap { + map: RawIoSysMap, + vec: KVec, + } + + impl VecIoSysMap { + pub fn new(src: &[T]) -> Result { + let mut vec = KVec::::new(); + + vec.extend_from_slice(src, GFP_KERNEL)?; + + let map = RawIoSysMap( + bindings::iosys_map { + is_iomem: false, + __bindgen_anon_1: bindings::iosys_map__bindgen_ty_1 { + vaddr: vec.as_mut_ptr().cast(), + }, + }, + PhantomData, + ); + + Ok(Self { map, vec }) + } + + pub fn get(&mut self) -> IoSysMapRef<'_, T> { + // SAFETY: + // * `map` points to `vec`, so the size of `map` is the size of the `vec`. + unsafe { IoSysMapRef::new(self.map.clone(), self.vec.len() * self.map.item_size()) } + } + + /// Assert whether or not the contents of this struct match src. + pub fn assert_eq(&self, src: &[T]) { + assert_eq!(*self.vec.as_ref(), *src) + } + } + + #[test] + fn basic() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + map.get().write(&[1, 2, 3], 0)?; + map.assert_eq(&[1, 2, 3]); + + map.get().write(&[42], 1)?; + map.assert_eq(&[1, 42, 3]); + + Ok(()) + } + + #[test] + fn oob_accesses() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + assert!(map.get().write(&[1, 2, 3, 69], 0).is_err()); + assert!(map.get().write(&[1, 2, 3], 69).is_err()); + map.assert_eq(&[0; 3]); + + Ok(()) + } + + #[test] + fn overflows() -> Result { + let mut map = VecIoSysMap::new(&[0; 3])?; + + assert!(map.get().write(&[1], usize::MAX).is_err()); + map.assert_eq(&[0; 3]); + + Ok(()) + } + + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + struct TestStruct { + a: u32, + b: u64, + } + + // SAFETY: All bit patterns are acceptable values for `TestStruct`. + unsafe impl FromBytes for TestStruct {} + // SAFETY: Instances of `TestStruct` have no uninitialized portions. + unsafe impl AsBytes for TestStruct {} + + #[test] + fn basic_macro() -> Result { + let mut expected = [TestStruct { a: 1, b: 2 }; 5]; + let mut map = VecIoSysMap::new(&expected)?; + + { + let mut map_ref = map.get(); + + iosys_map_write!(map_ref[3].a = u32::MAX)?; + expected[3].a = u32::MAX; + + assert_eq!(iosys_map_read!(map_ref[3].a)?, u32::MAX); + assert_eq!( + iosys_map_read!(map_ref[3])?, + TestStruct { a: u32::MAX, b: 2 } + ); + } + + // Compare the entire array, so that we catch any mis-sized writes. + map.assert_eq(&expected); + + Ok(()) + } + + #[test] + fn macro_oob_accesses() -> Result { + let mut map = VecIoSysMap::new(&[TestStruct { a: 1, b: 2 }; 3])?; + let mut map = map.get(); + + assert!(iosys_map_read!(map[5].b).is_err()); + assert!(iosys_map_read!(map[1000]).is_err()); + assert!(iosys_map_write!(map[6969].a = 999).is_err()); + assert!(iosys_map_write!(map[243] = TestStruct { a: 99, b: 22 }).is_err()); + + Ok(()) + } + + #[test] + fn macro_overflows() -> Result { + let mut map = VecIoSysMap::new(&[TestStruct { a: 1, b: 2 }; 3])?; + let mut map = map.get(); + + assert!(iosys_map_read!(map[usize::MAX]).is_err()); + assert!(iosys_map_read!(map[usize::MAX].b).is_err()); + assert!(iosys_map_write!(map[usize::MAX] = TestStruct { a: 1, b: 1 }).is_err()); + assert!(iosys_map_write!(map[usize::MAX].b = 1).is_err()); + + Ok(()) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index d93292d47420f1..11f11421d7657e 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -16,6 +16,21 @@ // Please see https://github.com/Rust-for-Linux/linux/issues/2 for details on // the unstable features in use. // +// ============ start asahi downstream features =========== +#![feature(associated_type_defaults)] +// +#![feature(cfg_version)] +// +// Stable since Rust 1.87.0. +#![feature(ptr_sub_ptr)] +// +#![feature(sized_type_properties)] +// +#![feature(slice_range)] +// +#![cfg_attr(CONFIG_RUSTC_HAS_COERCE_POINTEE, feature(pin_coerce_unsized_trait))] +// ============ end asahi dowanstream features ============ +// // Stable since Rust 1.79.0. #![feature(generic_nonzero)] #![feature(inline_const)] @@ -37,6 +52,7 @@ #![feature(const_option)] #![feature(const_ptr_write)] #![feature(const_refs_to_cell)] +#![feature(const_refs_to_static)] // // Stable since Rust 1.84.0. #![feature(strict_provenance)] @@ -69,6 +85,7 @@ extern crate self as kernel; pub use ffi; pub mod acpi; +pub mod addr; pub mod alloc; #[cfg(CONFIG_AUXILIARY_BUS)] pub mod auxiliary; @@ -88,10 +105,15 @@ pub mod cpufreq; pub mod cpumask; pub mod cred; pub mod debugfs; +#[cfg(CONFIG_DEV_COREDUMP)] +pub mod devcoredump; pub mod device; pub mod device_id; pub mod devres; pub mod dma; +pub mod dma_buf; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] +pub mod dma_fence; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; @@ -104,12 +126,15 @@ pub mod fs; #[cfg(CONFIG_I2C = "y")] pub mod i2c; pub mod id_pool; +#[cfg(CONFIG_IIO)] +pub mod iio; #[doc(hidden)] pub mod impl_flags; pub mod init; pub mod io; pub mod ioctl; pub mod iommu; +pub mod iosys_map; pub mod iov; pub mod irq; pub mod jump_label; @@ -146,7 +171,6 @@ pub mod security; pub mod seq_file; pub mod sizes; pub mod slice; -#[cfg(CONFIG_SOC_BUS)] pub mod soc; mod static_assert; #[doc(hidden)] diff --git a/rust/kernel/of.rs b/rust/kernel/of.rs index 58b20c367f993f..80c604ee831452 100644 --- a/rust/kernel/of.rs +++ b/rust/kernel/of.rs @@ -8,6 +8,17 @@ use crate::{ prelude::*, }; +// Note: Most OF functions turn into inline dummies with CONFIG_OF(_*) disabled. +// We have to either add config conditionals to helpers.c or here; let's do it +// here for now. In the future, once bindgen can auto-generate static inline +// helpers, this can go away if desired. + +use core::marker::PhantomData; +use core::num::NonZeroU32; + +use crate::error::to_result; +use crate::io::resource::Resource; + /// IdTable type for OF drivers. pub type IdTable = &'static dyn kernel::device_id::IdTable; @@ -50,6 +61,518 @@ impl DeviceId { } } +/// Type alias for an OF phandle +pub type PHandle = bindings::phandle; + +/// An OF device tree node. +/// +/// # Invariants +/// +/// `raw_node` points to a valid OF node, and we hold a reference to it. +pub struct Node { + raw_node: *mut bindings::device_node, +} + +#[allow(dead_code)] +impl Node { + /// Creates a `Node` from a raw C pointer. The pointer must be owned (the caller + /// gives up its reference). If the pointer is NULL, returns None. + pub(crate) unsafe fn from_raw(raw_node: *mut bindings::device_node) -> Option { + if raw_node.is_null() { + None + } else { + // INVARIANT: `raw_node` is valid per the above contract, and non-null per the + // above check. + Some(Node { raw_node }) + } + } + + /// Creates a `Node` from a raw C pointer. The pointer must be borrowed (the caller + /// retains its reference, which must be valid for the duration of the call). If the + /// pointer is NULL, returns None. + pub(crate) unsafe fn get_from_raw(raw_node: *mut bindings::device_node) -> Option { + // SAFETY: `raw_node` is valid or NULL per the above contract. `of_node_get` can handle + // NULL. + unsafe { + #[cfg(CONFIG_OF_DYNAMIC)] + bindings::of_node_get(raw_node); + Node::from_raw(raw_node) + } + } + + /// Returns a reference to the underlying C `device_node` structure. + pub(crate) fn node(&self) -> &bindings::device_node { + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { &*self.raw_node } + } + + /// Returns the name of the node. + pub(crate) fn name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().name) } + } + + /// Returns the phandle for this node. + pub(crate) fn phandle(&self) -> PHandle { + self.node().phandle + } + + /// Returns the full name (with address) for this node. + pub(crate) fn full_name(&self) -> &CStr { + // SAFETY: The lifetime of the `CStr` is the same as the lifetime of this `Node`. + unsafe { CStr::from_char_ptr(self.node().full_name) } + } + + /// Returns `true` if the node is the root node. + pub(crate) fn is_root(&self) -> bool { + #[cfg(not(CONFIG_OF))] + { + false + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant + unsafe { + bindings::of_node_is_root(self.raw_node) + } + } + + /// Returns the parent node, if any. + pub(crate) fn parent(&self) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and `of_get_parent()` takes a + // new reference to the parent (or returns NULL). + unsafe { + Node::from_raw(bindings::of_get_parent(self.raw_node)) + } + } + + /// Returns an iterator over the node's children. + // TODO: use type alias for return type once type_alias_impl_trait is stable + pub fn children( + &self, + ) -> NodeIterator<'_, impl Fn(*mut bindings::device_node) -> *mut bindings::device_node + '_> + { + #[cfg(not(CONFIG_OF))] + { + NodeIterator::new(|_prev| core::ptr::null_mut()) + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant, and the lifetime of the `NodeIterator` + // does not exceed the lifetime of the `Node` so it can borrow its reference. + NodeIterator::new(|prev| unsafe { bindings::of_get_next_child(self.raw_node, prev) }) + } + + /// Find a child by its name and return it, or None if not found. + #[allow(unused_variables)] + pub(crate) fn get_child_by_name(&self, name: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + Node::from_raw(bindings::of_get_child_by_name( + self.raw_node, + name.as_char_ptr(), + )) + } + } + + /// Checks whether the node is compatible with the given compatible string. + /// + /// Returns `None` if there is no match, or `Some` if there is, with the value + /// representing as match score (higher values for more specific compatible matches). + #[allow(unused_variables)] + pub(crate) fn is_compatible(&self, compatible: &CStr) -> Option { + #[cfg(not(CONFIG_OF))] + let ret = 0; + #[cfg(CONFIG_OF)] + let ret = + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { bindings::of_device_is_compatible(self.raw_node, compatible.as_char_ptr()) }; + + NonZeroU32::new(ret.try_into().ok()?) + } + + /// Parse a phandle property and return the Node referenced at a given index, if any. + /// + /// Used only for phandle properties with no arguments. + #[allow(unused_variables)] + pub fn parse_phandle(&self, name: &CStr, index: usize) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. `of_parse_phandle` returns an + // owned reference. + unsafe { + Node::from_raw(bindings::of_parse_phandle( + self.raw_node, + name.as_char_ptr(), + index.try_into().ok()?, + )) + } + } + + #[allow(unused_variables)] + /// Get a reserved memory region as a resource + pub fn reserved_mem_region_to_resource_byname(&self, name: &CStr) -> Result { + #[cfg(not(CONFIG_OF))] + { + Err(ENOENT) + } + #[cfg(CONFIG_OF)] + { + let res = Resource::zeroed(); + // SAFETY: This function is safe to call as long as the arguments are valid pointers. + let ret = unsafe { + bindings::of_reserved_mem_region_to_resource_byname( + self.raw_node, + name.as_char_ptr(), + res.as_raw(), + ) + }; + to_result(ret)?; + + Ok(res) + } + } + + #[allow(unused_variables)] + /// Look up a node property by name, returning a `Property` object if found. + pub(crate) fn find_property(&self, propname: &CStr) -> Option> { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: `raw_node` is valid per the type invariant. The property structure + // returned borrows the reference to the owning node, and so has the same + // lifetime. + unsafe { + Property::from_raw(bindings::of_find_property( + self.raw_node, + propname.as_char_ptr(), + core::ptr::null_mut(), + )) + } + } + + /// Look up a mandatory node property by name, and decode it into a value type. + /// + /// Returns `Err(ENOENT)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_property<'a, T: TryFrom>>(&'a self, propname: &CStr) -> Result + where + crate::error::Error: From<>>::Error>, + { + Ok(self.find_property(propname).ok_or(ENOENT)?.try_into()?) + } + + /// Look up an optional node property by name, and decode it into a value type. + /// + /// Returns `Ok(None)` if the property is not found. + /// + /// The type `T` must implement `TryFrom>`. + pub fn get_opt_property<'a, T: TryFrom>>( + &'a self, + propname: &CStr, + ) -> Result> + where + crate::error::Error: From<>>::Error>, + { + self.find_property(propname) + .map_or(Ok(None), |p| Ok(Some(p.try_into()?))) + } +} + +/// A property attached to a device tree `Node`. +/// +/// # Invariants +/// +/// `raw` must be valid and point to a property that outlives the lifetime of this object. +#[derive(Copy, Clone)] +pub struct Property<'a> { + raw: *mut bindings::property, + _p: PhantomData<&'a Node>, +} + +impl<'a> Property<'a> { + #[cfg(CONFIG_OF)] + /// Create a `Property` object from a raw C pointer. Returns `None` if NULL. + /// + /// The passed pointer must be valid and outlive the lifetime argument, or NULL. + unsafe fn from_raw(raw: *mut bindings::property) -> Option> { + if raw.is_null() { + None + } else { + Some(Property { + raw, + _p: PhantomData, + }) + } + } + + /// Returns the name of the property as a `CStr`. + pub fn name(&self) -> &CStr { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the `CStr` does not + // outlive it. + unsafe { CStr::from_char_ptr((*self.raw).name) } + } + + /// Returns the name of the property as a `&[u8]`. + pub fn value(&self) -> &[u8] { + // SAFETY: `raw` is valid per the type invariant, and the lifetime of the slice does not + // outlive it. + unsafe { core::slice::from_raw_parts((*self.raw).value as *const u8, self.len()) } + } + + /// Returns the length of the property in bytes. + pub fn len(&self) -> usize { + // SAFETY: `raw` is valid per the type invariant. + unsafe { (*self.raw).length.try_into().unwrap() } + } + + /// Returns true if the property is empty (zero-length), which typically represents boolean true. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// A trait that represents a value decodable from a property with a fixed unit size. +/// +/// This allows us to auto-derive property decode implementations for `Vec`. +pub trait PropertyUnit: Sized { + /// The size in bytes of a single data unit. + const UNIT_SIZE: usize; + + /// Decode this data unit from a byte slice. The passed slice will have a length of `UNIT_SIZE`. + fn from_bytes(data: &[u8]) -> Result; +} + +// This doesn't work... +// impl<'a, T: PropertyUnit> TryFrom> for T { +// type Error = Error; +// +// fn try_from(p: Property<'_>) -> core::result::Result { +// if p.value().len() != T::UNIT_SIZE { +// Err(EINVAL) +// } else { +// Ok(T::from_bytes(p.value())?) +// } +// } +// } + +impl<'a, T: PropertyUnit> TryFrom> for KVec { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result, Self::Error> { + if p.len() % T::UNIT_SIZE != 0 { + return Err(EINVAL); + } + + let mut v = Vec::new(); + let val = p.value(); + for off in (0..p.len()).step_by(T::UNIT_SIZE) { + v.push(T::from_bytes(&val[off..off + T::UNIT_SIZE])?, GFP_KERNEL)?; + } + Ok(v) + } +} + +macro_rules! prop_int_type ( + ($type:ty) => { + impl<'a> TryFrom> for $type { + type Error = Error; + + fn try_from(p: Property<'_>) -> core::result::Result<$type, Self::Error> { + Ok(<$type>::from_be_bytes(p.value().try_into().or(Err(EINVAL))?)) + } + } + + impl PropertyUnit for $type { + const UNIT_SIZE: usize = <$type>::BITS as usize / 8; + + fn from_bytes(data: &[u8]) -> Result { + Ok(<$type>::from_be_bytes(data.try_into().or(Err(EINVAL))?)) + } + } + } +); + +prop_int_type!(u8); +prop_int_type!(u16); +prop_int_type!(u32); +prop_int_type!(u64); +prop_int_type!(i8); +prop_int_type!(i16); +prop_int_type!(i32); +prop_int_type!(i64); + +/// An iterator across a collection of Node objects. +/// +/// # Invariants +/// +/// `cur` must be NULL or a valid node owned reference. If NULL, it represents either the first +/// or last position of the iterator. +/// +/// If `done` is true, `cur` must be NULL. +/// +/// fn_next must be a callback that iterates from one node to the next, and it must not capture +/// values that exceed the lifetime of the iterator. It must return owned references and also +/// take owned references. +pub struct NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + cur: *mut bindings::device_node, + done: bool, + fn_next: T, + _p: PhantomData<&'a T>, +} + +impl<'a, T> NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn new(next: T) -> NodeIterator<'a, T> { + // INVARIANT: `cur` is initialized to NULL to represent the initial state. + NodeIterator { + cur: core::ptr::null_mut(), + done: false, + fn_next: next, + _p: PhantomData, + } + } +} + +impl<'a, T> Iterator for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + type Item = Node; + + fn next(&mut self) -> Option { + if self.done { + None + } else { + // INVARIANT: if the new `cur` is NULL, then the iterator has reached its end and we + // set `done` to `true`. + self.cur = (self.fn_next)(self.cur); + self.done = self.cur.is_null(); + // SAFETY: `fn_next` must return an owned reference per the iterator contract. + // The iterator itself is considered to own this reference, so we take another one. + unsafe { Node::get_from_raw(self.cur) } + } + } +} + +// Drop impl to ensure we drop the current node being iterated on, if any. +impl<'a, T> Drop for NodeIterator<'a, T> +where + T: Fn(*mut bindings::device_node) -> *mut bindings::device_node, +{ + fn drop(&mut self) { + // SAFETY: `cur` is valid or NULL, and `of_node_put()` can handle NULL. + #[cfg(CONFIG_OF_DYNAMIC)] + unsafe { + bindings::of_node_put(self.cur) + }; + } +} + +/// Returns the root node of the OF device tree (if any). +pub fn root() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_root is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_root) + } +} + +/// Returns the /chosen node of the OF device tree (if any). +pub fn chosen() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_chosen is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_chosen) + } +} + +/// Returns the /aliases node of the OF device tree (if any). +pub fn aliases() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_aliases is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_aliases) + } +} + +/// Returns the system stdout node of the OF device tree (if any). +pub fn stdout() -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_stdout is always valid or NULL + unsafe { + Node::get_from_raw(bindings::of_stdout) + } +} + +#[allow(unused_variables)] +/// Looks up a node in the device tree by phandle. +pub fn find_node_by_phandle(handle: PHandle) -> Option { + #[cfg(not(CONFIG_OF))] + { + None + } + #[cfg(CONFIG_OF)] + // SAFETY: bindings::of_find_node_by_phandle always returns a valid pointer or NULL + unsafe { + #[allow(dead_code)] + Node::from_raw(bindings::of_find_node_by_phandle(handle)) + } +} + +impl Clone for Node { + fn clone(&self) -> Node { + // SAFETY: `raw_node` is valid and non-NULL per the type invariant, + // so this can never return None. + unsafe { Node::get_from_raw(self.raw_node).unwrap() } + } +} + +impl Drop for Node { + fn drop(&mut self) { + #[cfg(CONFIG_OF_DYNAMIC)] + // SAFETY: `raw_node` is valid per the type invariant. + unsafe { + bindings::of_node_put(self.raw_node) + }; + } +} + /// Create an OF `IdTable` with an "alias" for modpost. #[macro_export] macro_rules! of_device_table { diff --git a/rust/kernel/page.rs b/rust/kernel/page.rs index adecb200c65495..675f5de536cdb1 100644 --- a/rust/kernel/page.rs +++ b/rust/kernel/page.rs @@ -3,10 +3,12 @@ //! Kernel page allocation and management. use crate::{ + addr::*, alloc::{AllocError, Flags}, bindings, error::code::*, error::Result, + types::{Opaque, Ownable, Owned}, uaccess::UserSliceReader, }; use core::{ @@ -99,7 +101,7 @@ pub const fn page_align(addr: usize) -> Option { /// /// [`VBox`]: kernel::alloc::VBox /// [`Vmalloc`]: kernel::alloc::allocator::Vmalloc -pub struct BorrowedPage<'a>(ManuallyDrop, PhantomData<&'a Page>); +pub struct BorrowedPage<'a>(ManuallyDrop>, PhantomData<&'a Page>); impl<'a> BorrowedPage<'a> { /// Constructs a [`BorrowedPage`] from a raw pointer to a `struct page`. @@ -109,7 +111,7 @@ impl<'a> BorrowedPage<'a> { /// - `ptr` must point to a valid `bindings::page`. /// - `ptr` must remain valid for the entire lifetime `'a`. pub unsafe fn from_raw(ptr: NonNull) -> Self { - let page = Page { page: ptr }; + let page = unsafe { Page::from_phys(bindings::page_to_phys(ptr.as_ptr())) }; // INVARIANT: The safety requirements guarantee that `ptr` is valid for the entire lifetime // `'a`. @@ -142,8 +144,9 @@ pub trait AsPageIter { /// # Invariants /// /// The pointer is valid, and has ownership over the page. +#[repr(transparent)] pub struct Page { - page: NonNull, + page: Opaque, } // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across @@ -177,19 +180,20 @@ impl Page { /// # Ok::<(), kernel::alloc::AllocError>(()) /// ``` #[inline] - pub fn alloc_page(flags: Flags) -> Result { + pub fn alloc_page(flags: Flags) -> Result, AllocError> { // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it // is always safe to call this method. let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; let page = NonNull::new(page).ok_or(AllocError)?; - // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly - // allocated page. We transfer that ownership to the new `Page` object. - Ok(Self { page }) + // SAFETY: We just successfully allocated a page, so we now have ownership of the newly + // allocated page. We transfer that ownership to the new `Owned` object. + // Since `Page` is transparent, we can cast the pointer directly. + Ok(unsafe { Owned::from_raw(page.cast()) }) } /// Returns a raw pointer to the page. pub fn as_ptr(&self) -> *mut bindings::page { - self.page.as_ptr() + Opaque::cast_into(&self.page) } /// Get the node id containing this page. @@ -214,7 +218,7 @@ impl Page { /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). - fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { + pub fn with_page_mapped(&self, f: impl FnOnce(*mut u8) -> T) -> T { // SAFETY: `page` is valid due to the type invariants on `Page`. let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; @@ -255,7 +259,7 @@ impl Page { /// different addresses. However, even if the addresses are different, the underlying memory is /// still the same for these purposes (e.g., it's still a data race if they both write to the /// same underlying byte at the same time). - fn with_pointer_into_page( + pub fn with_pointer_into_page( &self, off: usize, len: usize, @@ -362,12 +366,77 @@ impl Page { reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) }) } + + /// Returns the physical address of this page. + pub fn phys(&self) -> PhysicalAddr { + // SAFETY: `page` is valid due to the type invariants on `Page`. + unsafe { bindings::page_to_phys(self.as_ptr()) } + } + + /// Converts a Rust-owned Page into its physical address. + /// The caller is responsible for calling `from_phys()` to avoid + /// leaking memory. + pub fn into_phys(this: Owned) -> PhysicalAddr { + ManuallyDrop::new(this).phys() + } + + /// Converts a physical address to a Rust-owned Page. + /// + /// SAFETY: + /// The caller must ensure that the physical address was previously returned + /// by a call to `Page::into_phys()`, and that the physical address is no + /// longer used after this call, nor is `from_phys()` called again on it. + pub unsafe fn from_phys(phys: PhysicalAddr) -> Owned { + // SAFETY: By the safety requirements, the physical address must be valid and + // have come from `into_phys()`, so phys_to_page() cannot fail and + // must return the original struct page pointer. + unsafe { Owned::from_raw(NonNull::new_unchecked(bindings::phys_to_page(phys)).cast()) } + } + + /// Borrows a Page from a physical address, without taking over ownership. + /// + /// If the physical address does not have a `struct page` entry or is not + /// part of the System RAM region, returns None. + /// + /// SAFETY: + /// The caller must ensure that the physical address, if it is backed by a + /// `struct page`, remains available for the duration of the borrowed + /// lifetime. + pub unsafe fn borrow_phys(phys: &PhysicalAddr) -> Option<&Self> { + // SAFETY: This is always safe, as it is just arithmetic + let pfn = unsafe { bindings::phys_to_pfn(*phys) }; + // SAFETY: This function is safe to call with any pfn + if !unsafe { bindings::pfn_valid(pfn) && bindings::page_is_ram(pfn) != 0 } { + None + } else { + // SAFETY: We have just checked that the pfn is valid above, so it must + // have a corresponding struct page. By the safety requirements, we can + // return a borrowed reference to it. + Some(unsafe { &*(bindings::pfn_to_page(pfn) as *mut Self as *const Self) }) + } + } + + /// Borrows a Page from a physical address, without taking over ownership + /// nor checking for validity. + /// + /// SAFETY: + /// The caller must ensure that the physical address is backed by a + /// `struct page` and corresponds to System RAM. + pub unsafe fn borrow_phys_unchecked(phys: &PhysicalAddr) -> &Self { + // SAFETY: This is always safe, as it is just arithmetic + let pfn = unsafe { bindings::phys_to_pfn(*phys) }; + // SAFETY: The caller guarantees that the pfn is valid. By the safety + // requirements, we can return a borrowed reference to it. + unsafe { &*(bindings::pfn_to_page(pfn) as *mut Self as *const Self) } + } } -impl Drop for Page { +// SAFETY: See below. +unsafe impl Ownable for Page { #[inline] - fn drop(&mut self) { + unsafe fn release(this: NonNull) { // SAFETY: By the type invariants, we have ownership of the page and can free it. - unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; + // Since Page is transparent, we can cast the raw pointer directly. + unsafe { bindings::__free_pages(this.cast().as_ptr(), 0) }; } } diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index 8917d4ee499fb3..0e47321591dc3a 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -299,6 +299,20 @@ impl Device { // returned by `platform_get_resource`. Some(unsafe { Resource::from_raw(resource) }) } + + /// excute closure while the device is bound + pub fn while_bound_with(&self, f: F) -> Result + where + F: FnOnce(&Device) -> Result, + { + let _guard = self.as_ref().lock(); + if unsafe { !bindings::device_is_bound(self.as_ref().as_raw()) } { + return Err(ENODEV); + } + let ptr: *const Self = self; + let ptr = ptr.cast::>(); + f(unsafe { &*ptr }) + } } impl Device { diff --git a/rust/kernel/soc/apple/aop.rs b/rust/kernel/soc/apple/aop.rs new file mode 100644 index 00000000000000..da46ce0bcb027d --- /dev/null +++ b/rust/kernel/soc/apple/aop.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Common code for AOP endpoint drivers + +use kernel::{prelude::*, sync::Arc}; + +/// Representation of an "EPIC" service. +#[derive(Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct EPICService { + /// Channel id + pub channel: u32, + /// RTKit endpoint + pub endpoint: u8, +} + +/// Listener for the "HID" events sent by aop +pub trait FakehidListener { + /// Process the event. + fn process_fakehid_report(&self, data: &[u8]) -> Result<()>; +} + +/// AOP communications manager. +pub trait AOP: Send + Sync { + /// Calls a method on a specified service + fn epic_call(&self, svc: &EPICService, subtype: u16, msg_bytes: &[u8]) -> Result; + /// Just like epic_call, but also returns a value + fn epic_call_ret( + &self, + svc: &EPICService, + subtype: u16, + msg_bytes: &[u8], + ret_len: usize, + ) -> Result<(u32, KVec)>; + + /// Adds the listener for the specified service + fn add_fakehid_listener( + &self, + svc: EPICService, + listener: Arc, + ) -> Result<()>; + /// Remove the listener for the specified service + fn remove_fakehid_listener(&self, svc: &EPICService) -> bool; + /// Internal method to detach the device. + fn remove(&self); +} + +/// Converts a text representation of a FourCC to u32 +pub const fn from_fourcc(b: &[u8]) -> u32 { + b[3] as u32 | (b[2] as u32) << 8 | (b[1] as u32) << 16 | (b[0] as u32) << 24 +} diff --git a/rust/kernel/soc/apple/mailbox.rs b/rust/kernel/soc/apple/mailbox.rs new file mode 100644 index 00000000000000..f477ddc0afb00e --- /dev/null +++ b/rust/kernel/soc/apple/mailbox.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple ASC Mailbox. +//! +//! C header: [`include/linux/soc/apple/mailbox.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + bindings, + device, + error::{ + from_err_ptr, + to_result, // + }, + prelude::*, + str::CStrExt, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // +}; + +use core::marker::PhantomData; + +/// 96-bit message. What it means is up to the upper layer +pub type Message = bindings::apple_mbox_msg; + +/// Mailbox receive callback +pub trait MailCallback { + /// Callback context + type Data: ForeignOwnable + Send + Sync; + + /// The actual callback. Called in an interrupt context. + fn recv_message(data: ::Borrowed<'_>, msg: Message); +} + +/// Wrapper over `struct apple_mbox *` +#[repr(transparent)] +pub struct Mailbox { + mbox: *mut bindings::apple_mbox, + _p: PhantomData, +} + +extern "C" fn mailbox_rx_callback( + _mbox: *mut bindings::apple_mbox, + msg: Message, + cookie: *mut crate::ffi::c_void, +) { + // SAFETY: cookie came from a call to `into_foreign` + T::recv_message(unsafe { T::Data::borrow(cookie.cast()) }, msg); +} + +impl Mailbox { + /// Creates a mailbox for the specified name. + pub fn new_byname( + dev: &device::Device, + mbox_name: &'static CStr, + data: T::Data, + ) -> Result> { + let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr.cast()) }; + }); + // SAFETY: Just calling the c function, all values are valid. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get_byname( + dev.as_raw(), + mbox_name.as_char_ptr(), + ))? + }; + // SAFETY: mbox is a valid pointer + unsafe { + (*mbox).cookie = ptr; + (*mbox).rx = Some(mailbox_rx_callback::); + to_result(bindings::apple_mbox_start(mbox))?; + } + guard.dismiss(); + Ok(Mailbox { + mbox, + _p: PhantomData, + }) + } + /// Sends the specified message + pub fn send(&self, msg: Message, atomic: bool) -> Result<()> { + // SAFETY: Calling the c function, `mbox` is a valid pointer + to_result(unsafe { bindings::apple_mbox_send(self.mbox, msg, atomic) }) + } +} + +impl Drop for Mailbox { + fn drop(&mut self) { + // SAFETY: mbox is a valid pointer + unsafe { bindings::apple_mbox_stop(self.mbox) }; + // SAFETY: `cookie` came from `into_foreign` + unsafe { T::Data::from_foreign((*self.mbox).cookie.cast()) }; + } +} + +unsafe impl Sync for Mailbox +where + T: MailCallback, + T::Data: Sync, +{ +} + +unsafe impl Send for Mailbox +where + T: MailCallback, + T::Data: Send, +{ +} diff --git a/rust/kernel/soc/apple/mod.rs b/rust/kernel/soc/apple/mod.rs new file mode 100644 index 00000000000000..e77eba782a5867 --- /dev/null +++ b/rust/kernel/soc/apple/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Apple SoC drivers + +#[cfg(CONFIG_RUST_APPLE_RTKIT = "y")] +pub mod rtkit; + +#[cfg(any(CONFIG_APPLE_AOP = "y", CONFIG_APPLE_AOP = "m"))] +pub mod aop; + +#[cfg(CONFIG_RUST_APPLE_MAILBOX = "y")] +pub mod mailbox; diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs new file mode 100644 index 00000000000000..a5bfcfb8fa7334 --- /dev/null +++ b/rust/kernel/soc/apple/rtkit.rs @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Support for Apple RTKit coprocessors. +//! +//! C header: [`include/linux/soc/apple/rtkit.h`](../../../../include/linux/gpio/driver.h) + +use crate::{ + alloc::flags::*, + bindings, + device, + error::{ + from_err_ptr, + from_result, + to_result, // + }, + iosys_map::IoSysMapRef, + prelude::*, + str::CStrExt, + types::{ + ForeignOwnable, + ScopeGuard, // + }, // +}; + +use core::marker::PhantomData; +use core::ptr; +use macros::vtable; + +/// Trait to represent allocatable buffers for the RTKit core. +/// +/// Users must implement this trait for their own representation of those allocations. +pub trait Buffer { + /// Returns the IOVA (virtual address) of the buffer from RTKit's point of view, or an error if + /// unavailable. + fn iova(&self) -> Result; + + /// Returns a mutable byte slice of the buffer contents, or an + /// error if unavailable. + fn buf(&mut self) -> Result>; +} + +/// Callback operations for an RTKit client. +#[vtable] +pub trait Operations { + /// Arbitrary user context type. + type Data: ForeignOwnable + Send + Sync; + + /// Type representing an allocated buffer for RTKit. + type Buffer: Buffer; + + /// Called when RTKit crashes. + fn crashed(_data: ::Borrowed<'_>, _crashlog: Option<&[u8]>) {} + + /// Called when a message was received on a non-system endpoint. Called in non-IRQ context. + fn recv_message( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) { + } + + /// Called in IRQ context when a message was received on a non-system endpoint. + /// + /// Must return `true` if the message is handled, or `false` to process it in + /// the handling thread. + fn recv_message_early( + _data: ::Borrowed<'_>, + _endpoint: u8, + _message: u64, + ) -> bool { + false + } + + /// Allocate a buffer for use by RTKit. + fn shmem_alloc( + _data: ::Borrowed<'_>, + _size: usize, + ) -> Result { + Err(EINVAL) + } + + /// Map an existing buffer used by RTKit at a device-specified virtual address. + fn shmem_map( + _data: ::Borrowed<'_>, + _iova: usize, + _size: usize, + ) -> Result { + Err(EINVAL) + } +} + +/// Represents `struct apple_rtkit *`. +/// +/// # Invariants +/// +/// The rtk pointer is valid. +/// The data pointer is a valid pointer from T::Data::into_foreign(). +pub struct RtKit { + rtk: *mut bindings::apple_rtkit, + data: *mut core::ffi::c_void, + _p: PhantomData, +} + +unsafe extern "C" fn crashed_callback( + cookie: *mut core::ffi::c_void, + crashlog: *const core::ffi::c_void, + crashlog_size: usize, +) { + let crashlog = if !crashlog.is_null() && crashlog_size > 0 { + // SAFETY: The crashlog is either missing or a byte buffer of the specified size + Some(unsafe { core::slice::from_raw_parts(crashlog as *const u8, crashlog_size) }) + } else { + None + }; + // SAFETY: cookie is always a T::Data in this API + T::crashed(unsafe { T::Data::borrow(cookie.cast()) }, crashlog); +} + +unsafe extern "C" fn recv_message_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) { + // SAFETY: cookie is always a T::Data in this API + T::recv_message(unsafe { T::Data::borrow(cookie.cast()) }, endpoint, message); +} + +unsafe extern "C" fn recv_message_early_callback( + cookie: *mut core::ffi::c_void, + endpoint: u8, + message: u64, +) -> bool { + // SAFETY: cookie is always a T::Data in this API + T::recv_message_early(unsafe { T::Data::borrow(cookie.cast()) }, endpoint, message) +} + +unsafe extern "C" fn shmem_setup_callback( + cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) -> core::ffi::c_int { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + + from_result(|| { + let mut buf = if bfr_mut.iova != 0 { + bfr_mut.is_mapped = true; + T::shmem_map( + // SAFETY: `cookie` came from a previous call to `into_foreign`. + unsafe { T::Data::borrow(cookie.cast()) }, + bfr_mut.iova as usize, + bfr_mut.size, + )? + } else { + bfr_mut.is_mapped = false; + // SAFETY: `cookie` came from a previous call to `into_foreign`. + T::shmem_alloc(unsafe { T::Data::borrow(cookie.cast()) }, bfr_mut.size)? + }; + + let iova = buf.iova()?; + let iosys_map = buf.buf()?; + + if iosys_map.size() < bfr_mut.size { + return Err(ENOMEM); + } + + bfr_mut.iova = iova as u64; + bfr_mut.buffer = iosys_map.as_mut_ptr() as *mut _; + + // Now box the returned buffer type and stash it in the private pointer of the + // `apple_rtkit_shmem` struct for safekeeping. + let boxed = KBox::new(buf, GFP_KERNEL)?; + bfr_mut.private = KBox::into_raw(boxed) as *mut _; + Ok(0) + }) +} + +unsafe extern "C" fn shmem_destroy_callback( + _cookie: *mut core::ffi::c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) { + // SAFETY: `bfr` is a valid buffer + let bfr_mut = unsafe { &mut *bfr }; + if !bfr_mut.private.is_null() { + // SAFETY: Per shmem_setup_callback, this has to be a pointer to a Buffer if it is set. + unsafe { + core::mem::drop(KBox::from_raw(bfr_mut.private as *mut T::Buffer)); + } + bfr_mut.private = core::ptr::null_mut(); + } +} + +impl RtKit { + const VTABLE: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + crashed: Some(crashed_callback::), + recv_message: Some(recv_message_callback::), + recv_message_early: Some(recv_message_early_callback::), + shmem_setup: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_setup_callback::) + } else { + None + }, + shmem_destroy: if T::HAS_SHMEM_ALLOC || T::HAS_SHMEM_MAP { + Some(shmem_destroy_callback::) + } else { + None + }, + }; + + /// Creates a new RTKit client for a given device and optional mailbox name or index. + pub fn new( + dev: &device::Device, + mbox_name: Option<&'static CStr>, + mbox_idx: usize, + data: T::Data, + ) -> Result { + let ptr: *mut crate::ffi::c_void = data.into_foreign().cast(); + let guard = ScopeGuard::new(|| { + // SAFETY: `ptr` came from a previous call to `into_foreign`. + unsafe { T::Data::from_foreign(ptr.cast()) }; + }); + // SAFETY: `dev` is valid by its type invarants and otherwise his just + // calls the C init function. + let rtk = unsafe { + from_err_ptr(bindings::apple_rtkit_init( + dev.as_raw(), + ptr, + match mbox_name { + Some(s) => s.as_char_ptr(), + None => ptr::null(), + }, + mbox_idx.try_into()?, + &Self::VTABLE, + )) + }?; + + guard.dismiss(); + // INVARIANT: `rtk` and `data` are valid here. + Ok(Self { + rtk, + data: ptr, + _p: PhantomData, + }) + } + + /// Boots (wakes up) the RTKit coprocessor. + pub fn wake(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) + } + + /// Waits for the RTKit coprocessor to finish booting. + pub fn boot(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_boot(self.rtk) }) + } + + /// Starts a non-system endpoint. + pub fn start_endpoint(self: Pin<&mut Self>, endpoint: u8) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_start_ep(self.rtk, endpoint) }) + } + + /// Sends a message to a given endpoint. + pub fn send_message(self: Pin<&mut Self>, endpoint: u8, message: u64) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { + bindings::apple_rtkit_send_message(self.rtk, endpoint, message, ptr::null_mut(), false) + }) + } + + /// Checks if an endpoint is present + pub fn has_endpoint(self: Pin<&mut Self>, endpoint: u8) -> bool { + unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } + } +} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Sync for RtKit {} + +// SAFETY: `RtKit` operations require a mutable reference +unsafe impl Send for RtKit {} + +impl Drop for RtKit { + fn drop(&mut self) { + // SAFETY: The pointer is valid by the type invariant. + unsafe { bindings::apple_rtkit_free(self.rtk) }; + + // Free context data. + // + // SAFETY: This matches the call to `into_foreign` from `new` in the success case. + unsafe { T::Data::from_foreign(self.data.cast()) }; + } +} diff --git a/rust/kernel/soc.rs b/rust/kernel/soc/bus.rs similarity index 100% rename from rust/kernel/soc.rs rename to rust/kernel/soc/bus.rs diff --git a/rust/kernel/soc/mod.rs b/rust/kernel/soc/mod.rs new file mode 100644 index 00000000000000..117e533612f5d3 --- /dev/null +++ b/rust/kernel/soc/mod.rs @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! SoC drivers + +pub mod apple; +#[cfg(CONFIG_SOC_BUS)] +pub mod bus; diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs index 4aebeacb961a29..296b25e83bbb95 100644 --- a/rust/kernel/sync/atomic.rs +++ b/rust/kernel/sync/atomic.rs @@ -204,10 +204,7 @@ impl Atomic { /// // no data race. /// unsafe { Atomic::from_ptr(foo_a_ptr) }.store(2, Release); /// ``` - pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self - where - T: Sync, - { + pub unsafe fn from_ptr<'a>(ptr: *mut T) -> &'a Self { // CAST: `T` and `Atomic` have the same size, alignment and bit validity. // SAFETY: Per function safety requirement, `ptr` is a valid pointer and the object will // live long enough. It's safe to return a `&Atomic` because function safety requirement diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 10b6b5e9b024f2..7ef06813ff88ce 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -8,6 +8,7 @@ use super::LockClassKey; use crate::{ str::{CStr, CStrExt as _}, + try_pin_init, types::{NotThreadSafe, Opaque, ScopeGuard}, }; use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; @@ -143,6 +144,31 @@ impl Lock { }), }) } + + /// Constructs a new lock initialiser taking an initialiser. + pub fn pin_init( + t: impl PinInit, + name: &'static CStr, + key: &'static LockClassKey, + ) -> impl PinInit + where + E: core::convert::From, + { + try_pin_init!(Self { + // SAFETY: We are just forwarding the initialization across a + // cast away from UnsafeCell, so the pin_init_from_closure and + // __pinned_init() requirements are in sync. + data <- unsafe { pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell| { + t.__pinned_init(slot as *mut T) + })}, + _pin: PhantomPinned, + // SAFETY: `slot` is valid while the closure is called and both `name` and `key` have + // static lifetimes so they live indefinitely. + state <- Opaque::ffi_init(|slot| unsafe { + B::init(slot, name.as_char_ptr(), key.as_ptr()) + }), + }? E) + } } impl Lock<(), B> { @@ -286,8 +312,8 @@ impl core::ops::Deref for Guard<'_, T, B> { } impl core::ops::DerefMut for Guard<'_, T, B> -where - T: Unpin, +// where +// T: Unpin, { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index cda0203efefb9f..5e4bd9d78d6d6f 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -17,6 +17,19 @@ macro_rules! new_mutex { } pub use new_mutex; +/// Creates a [`Mutex`] initialiser with the given name and a newly-created lock class, +/// given an initialiser for the inner type. +/// +/// It uses the name if one is given, otherwise it generates one based on the file name and line +/// number. +#[macro_export] +macro_rules! new_mutex_pinned { + ($inner:expr $(, $name:literal)? $(,)?) => { + $crate::sync::Mutex::pin_init( + $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} + /// A mutual exclusion primitive. /// /// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs index 9c5e7dbf16323f..4081f73f29f85c 100644 --- a/rust/kernel/types.rs +++ b/rust/kernel/types.rs @@ -6,8 +6,9 @@ use crate::ffi::c_void; use core::{ cell::UnsafeCell, marker::{PhantomData, PhantomPinned}, - mem::MaybeUninit, + mem::{ManuallyDrop, MaybeUninit}, ops::{Deref, DerefMut}, + ptr::NonNull, }; use pin_init::{PinInit, Wrapper, Zeroable}; @@ -422,6 +423,114 @@ impl Wrapper for Opaque { } } +/// Types that may be owned by Rust code or borrowed, but have a lifetime managed by C code. +/// +/// It allows such types to define their own custom destructor function to be called when +/// a Rust-owned reference is dropped. +/// +/// This is usually implemented by wrappers to existing structures on the C side of the code. +/// +/// # Safety +/// +/// Implementers must ensure that any objects borrowed directly stay alive for the duration +/// of the borrow lifetime, and that any objects deemed owned by Rust stay alive while +/// that owned reference exists, until the [`Ownable::release()`] function is called. +pub unsafe trait Ownable { + /// Releases the object (frees it or returns it to foreign ownership). + /// + /// # Safety + /// + /// Callers must ensure that the object is no longer referenced after this call. + unsafe fn release(this: NonNull); +} + +/// A subtrait of Ownable that asserts that an Owned Rust reference is not only unique +/// within Rust, but also follows the same rules in kernel C code. That is, the kernel +/// will never mutate the contents of the object while Rust owns it. +/// +/// When this type is implemented for an Ownable type, it allows Owned to be dereferenced +/// into a &mut T. + +/// # Safety +/// +/// Implementers must ensure that the kernel never mutates the underlying type while +/// Rust owns it. +pub unsafe trait OwnableMut: Ownable {} + +/// An owned reference to an ownable kernel object. +/// +/// The object is automatically freed or released when an instance of [`Owned`] is +/// dropped. +/// +/// # Invariants +/// +/// The pointer stored in `ptr` is non-null and valid for the lifetime of the [`Owned`] instance. +pub struct Owned { + ptr: NonNull, + _p: PhantomData, +} + +// SAFETY: It is safe to send `Owned` to another thread when the underlying `T` is `Send` because +// it effectively means sharing `&mut T` (which is safe because `T` is `Send`). +unsafe impl Send for Owned {} + +// SAFETY: It is safe to send `&Owned` to another thread when the underlying `T` is `Sync` +// because it effectively means sharing `&T` (which is safe because `T` is `Sync`). +unsafe impl Sync for Owned {} + +impl Owned { + /// Creates a new instance of [`Owned`]. + /// + /// It takes over ownership of the underlying object. + /// + /// # Safety + /// + /// Callers must ensure that the underlying object is acquired and can be considered owned by + /// Rust. + pub unsafe fn from_raw(ptr: NonNull) -> Self { + // INVARIANT: The safety requirements guarantee that the new instance now owns the + // reference. + Self { + ptr, + _p: PhantomData, + } + } + + /// Consumes the `Owned`, returning a raw pointer. + /// + /// This function does not actually relinquish ownership of the object. + /// After calling this function, the caller is responsible for ownership previously managed + /// by the `Owned`. + pub fn into_raw(me: Self) -> NonNull { + ManuallyDrop::new(me).ptr + } +} + +impl Deref for Owned { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: The type invariants guarantee that the object is valid. + unsafe { self.ptr.as_ref() } + } +} + +impl DerefMut for Owned { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: The type invariants guarantee that the object is valid, + // and that we can safely return a mutable reference to it. + unsafe { self.ptr.as_mut() } + } +} + +impl Drop for Owned { + fn drop(&mut self) { + // SAFETY: The type invariants guarantee that the `Owned` owns the object we're about to + // release. + unsafe { T::release(self.ptr) }; + } +} + /// Zero-sized type to mark types not [`Send`]. /// /// Add this type as a field to your struct if your type should not be sent to a different task. @@ -442,3 +551,86 @@ pub type NotThreadSafe = PhantomData<*mut ()>; /// [`NotThreadSafe`]: type@NotThreadSafe #[allow(non_upper_case_globals)] pub const NotThreadSafe: NotThreadSafe = PhantomData; + +/// Helper macro to declare a bitfield style type. The type will automatically +/// gain boolean operator implementations, as well as the `as_raw()` and `contains()` +/// methods, Debug, Copy, Clone, and PartialEq implementations. +/// +/// Optionally, a default value can be specified with `= value` syntax, which +/// will add a Default trait implementation. +/// +/// # Examples +/// +/// ``` +/// declare_flags_type! { +/// /// Flags to be used for foo. +/// pub struct FooFlags(u32); +/// } +/// +/// declare_flags_type! { +/// /// Flags to be used for bar. +/// pub struct BarFlags(u32) = 0; +/// } +/// ``` +macro_rules! declare_flags_type ( + ( + $(#[$outer:meta])* + $v:vis struct $t:ident ( $base:ty ); + $($rest:tt)* + ) => { + $(#[$outer])* + #[derive(Debug, Clone, Copy, PartialEq)] + $v struct $t($base); + + impl $t { + /// Get the raw representation of this flag. + pub(crate) fn as_raw(self) -> $base { + self.0 + } + + /// Check whether `flags` is contained in `self`. + pub fn contains(self, flags: Self) -> bool { + (self & flags) == flags + } + } + + impl core::ops::BitOr for $t { + type Output = Self; + fn bitor(self, rhs: Self) -> Self::Output { + Self(self.0 | rhs.0) + } + } + + impl core::ops::BitAnd for $t { + type Output = Self; + fn bitand(self, rhs: Self) -> Self::Output { + Self(self.0 & rhs.0) + } + } + + impl core::ops::Not for $t { + type Output = Self; + fn not(self) -> Self::Output { + Self(!self.0) + } + } + }; + ( + $(#[$outer:meta])* + $v:vis struct $t:ident ( $base:ty ) = $default:expr; + $($rest:tt)* + ) => { + declare_flags_type! { + $(#[$outer])* + $v struct $t ($base); + $($rest)* + } + impl Default for $t { + fn default() -> Self { + Self($default) + } + } + }; +); + +pub(crate) use declare_flags_type; diff --git a/rust/kernel/xarray.rs b/rust/kernel/xarray.rs index a49d6db2884588..e1f7e2d9b629f0 100644 --- a/rust/kernel/xarray.rs +++ b/rust/kernel/xarray.rs @@ -5,17 +5,20 @@ //! C header: [`include/linux/xarray.h`](srctree/include/linux/xarray.h) use crate::{ - alloc, bindings, build_assert, - error::{Error, Result}, - ffi::c_void, - types::{ForeignOwnable, NotThreadSafe, Opaque}, + alloc, + prelude::*, + types::{ForeignOwnable, NotThreadSafe, Opaque, ScopeGuard}, +}; +use core::{ + fmt, iter, + marker::PhantomData, + mem, ops, + ptr::{null_mut, NonNull}, }; -use core::{iter, marker::PhantomData, pin::Pin, ptr::NonNull}; -use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// An array which efficiently maps sparse integer indices to owned objects. /// -/// This is similar to a [`crate::alloc::kvec::Vec>`], but more efficient when there are +/// This is similar to a [`Vec>`], but more efficient when there are /// holes in the index space, and can be efficiently grown. /// /// # Invariants @@ -26,10 +29,11 @@ use pin_init::{pin_data, pin_init, pinned_drop, PinInit}; /// # Examples /// /// ```rust -/// use kernel::alloc::KBox; -/// use kernel::xarray::{AllocKind, XArray}; +/// # use kernel::alloc::KBox; +/// # use kernel::xarray::XArray; +/// # use pin_init::stack_pin_init; /// -/// let xa = KBox::pin_init(XArray::new(AllocKind::Alloc1), GFP_KERNEL)?; +/// stack_pin_init!(let xa = XArray::new(Default::default())); /// /// let dead = KBox::new(0xdead, GFP_KERNEL)?; /// let beef = KBox::new(0xbeef, GFP_KERNEL)?; @@ -77,8 +81,10 @@ impl PinnedDrop for XArray { } /// Flags passed to [`XArray::new`] to configure the array's allocation tracking behavior. +#[derive(Default)] pub enum AllocKind { /// Consider the first element to be at index 0. + #[default] Alloc, /// Consider the first element to be at index 1. Alloc1, @@ -105,19 +111,69 @@ impl XArray { fn iter(&self) -> impl Iterator> + '_ { let mut index = 0; - // SAFETY: `self.xa` is always valid by the type invariant. - iter::once(unsafe { - bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) - }) - .chain(iter::from_fn(move || { + core::iter::Iterator::chain( // SAFETY: `self.xa` is always valid by the type invariant. - Some(unsafe { - bindings::xa_find_after(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) - }) - })) + iter::once(unsafe { + bindings::xa_find(self.xa.get(), &mut index, usize::MAX, bindings::XA_PRESENT) + }), + iter::from_fn(move || { + // SAFETY: `self.xa` is always valid by the type invariant. + Some(unsafe { + bindings::xa_find_after( + self.xa.get(), + &mut index, + usize::MAX, + bindings::XA_PRESENT, + ) + }) + }), + ) .map_while(|ptr| NonNull::new(ptr.cast())) } + /// Looks up and returns a reference to the lowest entry in the array between index and max, + /// returning a tuple of its index and a `Guard` if one exists. + /// + /// This guard blocks all other actions on the `XArray`. Callers are expected to drop the + /// `Guard` eagerly to avoid blocking other users, such as by taking a clone of the value. + pub fn find(&self, index: usize, max: usize) -> Option<(usize, ValueGuard<'_, T>)> { + let mut index: usize = index; + + // SAFETY: `self.xa` is always valid by the type invariant. + unsafe { bindings::xa_lock(self.xa.get()) }; + + // SAFETY: `self.xa` is always valid by the type invariant. + let guard = ScopeGuard::new(|| unsafe { bindings::xa_unlock(self.xa.get()) }); + + // SAFETY: `self.xa` is always valid by the type invariant. + let p = unsafe { bindings::xa_find(self.xa.get(), &mut index, max, bindings::XA_PRESENT) }; + + NonNull::new(p as *mut T).map(|ptr| { + guard.dismiss(); + ( + index, + ValueGuard { + xa: self, + ptr, + _not_send: NotThreadSafe, + }, + ) + }) + } + + fn with_guard(&self, guard: Option<&mut Guard<'_, T>>, f: F) -> U + where + F: FnOnce(&mut Guard<'_, T>) -> U, + { + match guard { + None => f(&mut self.lock()), + Some(guard) => { + assert_eq!(guard.xa.xa.get(), self.xa.get()); + f(guard) + } + } + } + /// Attempts to lock the [`XArray`] for exclusive access. pub fn try_lock(&self) -> Option> { // SAFETY: `self.xa` is always valid by the type invariant. @@ -141,6 +197,12 @@ impl XArray { _not_send: NotThreadSafe, } } + + /// Removes and returns the element at the given index. + pub fn remove(&self, index: usize) -> Option { + let mut guard = self.lock(); + guard.remove(index) + } } /// A lock guard. @@ -161,9 +223,41 @@ impl Drop for Guard<'_, T> { } } +/// A lock guard. +/// +/// The lock is unlocked when the guard goes out of scope. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct ValueGuard<'a, T: ForeignOwnable> { + xa: &'a XArray, + ptr: NonNull, + _not_send: NotThreadSafe, +} + +impl<'a, T: ForeignOwnable> ValueGuard<'a, T> { + /// Borrow the underlying value wrapped by the `Guard`. + /// + /// Returns a `T::Borrowed` type for the owned `ForeignOwnable` type. + pub fn borrow(&self) -> T::Borrowed<'_> { + // SAFETY: The value is owned by the `XArray`, the lifetime it is borrowed for must not + // outlive the `XArray` itself, nor the Guard that holds the lock ensuring the value + // remains in the `XArray`. + unsafe { T::borrow(self.ptr.as_ptr() as _) } + } +} + +impl Drop for ValueGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: + // - `self.xa.xa` is always valid by the type invariant. + // - The caller holds the lock, so it is safe to unlock it. + unsafe { bindings::xa_unlock(self.xa.xa.get()) }; + } +} + /// The error returned by [`store`](Guard::store). /// /// Contains the underlying error and the value that was not stored. +#[derive(Debug)] pub struct StoreError { /// The error that occurred. pub error: Error, @@ -177,6 +271,11 @@ impl From> for Error { } } +fn to_usize(i: u32) -> usize { + i.try_into() + .unwrap_or_else(|_| build_error!("cannot convert u32 to usize")) +} + impl<'a, T: ForeignOwnable> Guard<'a, T> { fn load(&self, index: usize, f: F) -> Option where @@ -211,7 +310,7 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { // - The caller holds the lock. let ptr = unsafe { bindings::__xa_erase(self.xa.xa.get(), index) }.cast(); // SAFETY: - // - `ptr` is either NULL or came from `T::into_foreign`. + // - `ptr` is either `NULL` or came from `T::into_foreign`. // - `&mut self` guarantees that the lifetimes of [`T::Borrowed`] and [`T::BorrowedMut`] // borrowed from `self` have ended. unsafe { T::try_from_foreign(ptr) } @@ -259,13 +358,272 @@ impl<'a, T: ForeignOwnable> Guard<'a, T> { }) } else { let old = old.cast(); - // SAFETY: `ptr` is either NULL or came from `T::into_foreign`. + // SAFETY: `ptr` is either `NULL` or came from `T::into_foreign`. // // NB: `XA_ZERO_ENTRY` is never returned by functions belonging to the Normal XArray // API; such entries present as `NULL`. Ok(unsafe { T::try_from_foreign(old) }) } } + + /// Stores an element at the given index if no entry is present. + /// + /// May drop the lock if needed to allocate memory, and then reacquire it afterwards. + /// + /// On failure, returns the element which was attempted to be stored. + pub fn insert( + &mut self, + index: usize, + value: T, + gfp: alloc::Flags, + ) -> Result<(), StoreError> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let ptr = value.into_foreign(); + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` came from `T::into_foreign`. + match unsafe { bindings::__xa_insert(self.xa.xa.get(), index, ptr.cast(), gfp.as_raw()) } { + 0 => Ok(()), + errno => { + // SAFETY: `ptr` came from `T::into_foreign` and `__xa_insert` does not take + // ownership of the value on error. + let value = unsafe { T::from_foreign(ptr) }; + Err(StoreError { + value, + error: Error::from_errno(errno), + }) + } + } + } + + /// Wrapper around `__xa_alloc`. + /// + /// On success, takes ownership of pointers passed in `op`. + /// + /// On failure, ownership returns to the caller. + /// + /// # Safety + /// + /// `ptr` must be `NULL` or have come from a previous call to `T::into_foreign`. + unsafe fn alloc( + &mut self, + limit: impl ops::RangeBounds, + ptr: *mut c_void, + gfp: alloc::Flags, + ) -> Result { + // NB: `xa_limit::{max,min}` are inclusive. + let limit = bindings::xa_limit { + max: match limit.end_bound() { + ops::Bound::Included(&end) => end, + ops::Bound::Excluded(&end) => end - 1, + ops::Bound::Unbounded => u32::MAX, + }, + min: match limit.start_bound() { + ops::Bound::Included(&start) => start, + ops::Bound::Excluded(&start) => start + 1, + ops::Bound::Unbounded => 0, + }, + }; + + let mut index = u32::MAX; + + // SAFETY: + // - `self.xa` is always valid by the type invariant. + // - `self.xa` was initialized with `XA_FLAGS_ALLOC` or `XA_FLAGS_ALLOC1`. + // + // INVARIANT: `ptr` is either `NULL` or came from `T::into_foreign`. + match unsafe { + bindings::__xa_alloc( + self.xa.xa.get(), + &mut index, + ptr.cast(), + limit, + gfp.as_raw(), + ) + } { + 0 => Ok(to_usize(index)), + errno => Err(Error::from_errno(errno)), + } + } + + /// Allocates an entry somewhere in the array. + /// + /// On success, returns the index at which the entry was stored. + /// + /// On failure, returns the entry which was attempted to be stored. + pub fn insert_limit( + &mut self, + limit: impl ops::RangeBounds, + value: T, + gfp: alloc::Flags, + ) -> Result> { + build_assert!( + T::FOREIGN_ALIGN >= 4, + "pointers stored in XArray must be 4-byte aligned" + ); + let ptr = value.into_foreign(); + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { self.alloc(limit, ptr, gfp) }.map_err(|error| { + // SAFETY: `ptr` came from `T::into_foreign` and `self.alloc` does not take ownership of + // the value on error. + let value = unsafe { T::from_foreign(ptr) }; + StoreError { value, error } + }) + } + + /// Reserves an entry in the array. + pub fn reserve(&mut self, index: usize, gfp: alloc::Flags) -> Result> { + // NB: `__xa_insert` internally coerces `NULL` to `XA_ZERO_ENTRY` on ingress. + let ptr = null_mut(); + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` is `NULL`. + match unsafe { bindings::__xa_insert(self.xa.xa.get(), index, ptr, gfp.as_raw()) } { + 0 => Ok(Reservation { xa: self.xa, index }), + errno => Err(Error::from_errno(errno)), + } + } + + /// Reserves an entry somewhere in the array. + pub fn reserve_limit( + &mut self, + limit: impl ops::RangeBounds, + gfp: alloc::Flags, + ) -> Result> { + // NB: `__xa_alloc` internally coerces `NULL` to `XA_ZERO_ENTRY` on ingress. + let ptr = null_mut(); + // SAFETY: `ptr` is `NULL`. + unsafe { self.alloc(limit, ptr, gfp) }.map(|index| Reservation { xa: self.xa, index }) + } +} + +/// A reserved slot in an array. +/// +/// The slot is released when the reservation goes out of scope. +/// +/// Note that the array lock *must not* be held when the reservation is filled or dropped as this +/// will lead to deadlock. [`Reservation::fill_locked`] and [`Reservation::release_locked`] can be +/// used in context where the array lock is held. +#[must_use = "the reservation is released immediately when the reservation is unused"] +pub struct Reservation<'a, T: ForeignOwnable> { + xa: &'a XArray, + index: usize, +} + +impl fmt::Debug for Reservation<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Reservation") + .field("index", &self.index()) + .finish() + } +} + +impl Reservation<'_, T> { + /// Returns the index of the reservation. + pub fn index(&self) -> usize { + self.index + } + + /// Replaces the reserved entry with the given entry. + /// + /// # Safety + /// + /// `ptr` must be `NULL` or have come from a previous call to `T::into_foreign`. + unsafe fn replace(guard: &mut Guard<'_, T>, index: usize, ptr: *mut c_void) -> Result { + // SAFETY: `xa_zero_entry` wraps `XA_ZERO_ENTRY` which is always safe to use. + let old = unsafe { bindings::xa_zero_entry() }; + + // NB: `__xa_cmpxchg_raw` is used over `__xa_cmpxchg` because the latter coerces + // `XA_ZERO_ENTRY` to `NULL` on egress, which would prevent us from determining whether a + // replacement was made. + // + // SAFETY: `self.xa` is always valid by the type invariant. + // + // INVARIANT: `ptr` is either `NULL` or came from `T::into_foreign` and `old` is + // `XA_ZERO_ENTRY`. + let ret = + unsafe { bindings::__xa_cmpxchg_raw(guard.xa.xa.get(), index, old, ptr.cast(), 0) }; + + // SAFETY: `__xa_cmpxchg_raw` returns the old entry at this index on success or `xa_err` if + // an error happened. + match unsafe { bindings::xa_err(ret) } { + 0 => { + if ret == old { + Ok(()) + } else { + Err(EBUSY) + } + } + errno => Err(Error::from_errno(errno)), + } + } + + fn fill_inner(&self, guard: Option<&mut Guard<'_, T>>, value: T) -> Result<(), StoreError> { + let Self { xa, index } = self; + let index = *index; + + let ptr = value.into_foreign(); + xa.with_guard(guard, |guard| { + // SAFETY: `ptr` came from `T::into_foreign`. + unsafe { Self::replace(guard, index, ptr) } + }) + .map_err(|error| { + // SAFETY: `ptr` came from `T::into_foreign` and `Self::replace` does not take ownership + // of the value on error. + let value = unsafe { T::from_foreign(ptr) }; + StoreError { value, error } + }) + } + + /// Fills the reservation. + pub fn fill(self, value: T) -> Result<(), StoreError> { + let result = self.fill_inner(None, value); + mem::forget(self); + result + } + + /// Fills the reservation without acquiring the array lock. + /// + /// # Panics + /// + /// Panics if the passed guard locks a different array. + pub fn fill_locked(self, guard: &mut Guard<'_, T>, value: T) -> Result<(), StoreError> { + let result = self.fill_inner(Some(guard), value); + mem::forget(self); + result + } + + fn release_inner(&self, guard: Option<&mut Guard<'_, T>>) -> Result { + let Self { xa, index } = self; + let index = *index; + + xa.with_guard(guard, |guard| { + let ptr = null_mut(); + // SAFETY: `ptr` is `NULL`. + unsafe { Self::replace(guard, index, ptr) } + }) + } + + /// Releases the reservation without acquiring the array lock. + /// + /// # Panics + /// + /// Panics if the passed guard locks a different array. + pub fn release_locked(self, guard: &mut Guard<'_, T>) -> Result { + let result = self.release_inner(Some(guard)); + mem::forget(self); + result + } +} + +impl Drop for Reservation<'_, T> { + fn drop(&mut self) { + // NB: Errors here are possible since `Guard::store` does not honor reservations. + let _: Result = self.release_inner(None); + } } // SAFETY: `XArray` has no shared mutable state so it is `Send` iff `T` is `Send`. @@ -274,3 +632,133 @@ unsafe impl Send for XArray {} // SAFETY: `XArray` serialises the interior mutability it provides so it is `Sync` iff `T` is // `Send`. unsafe impl Sync for XArray {} + +#[macros::kunit_tests(rust_xarray_kunit)] +mod tests { + use super::*; + use pin_init::stack_pin_init; + + fn new_kbox(value: T) -> Result> { + KBox::new(value, GFP_KERNEL).map_err(Into::into) + } + + #[test] + fn test_alloc_kind_alloc() -> Result { + test_alloc_kind(AllocKind::Alloc, 0) + } + + #[test] + fn test_alloc_kind_alloc1() -> Result { + test_alloc_kind(AllocKind::Alloc1, 1) + } + + fn test_alloc_kind(kind: AllocKind, expected_index: usize) -> Result { + stack_pin_init!(let xa = XArray::new(kind)); + let mut guard = xa.lock(); + + let reservation = guard.reserve_limit(.., GFP_KERNEL)?; + assert_eq!(reservation.index(), expected_index); + reservation.release_locked(&mut guard)?; + + let insertion = guard.insert_limit(.., new_kbox(0x1337)?, GFP_KERNEL); + assert!(insertion.is_ok()); + let insertion_index = insertion.unwrap(); + assert_eq!(insertion_index, expected_index); + + Ok(()) + } + + const IDX: usize = 0x1337; + + fn insert(guard: &mut Guard<'_, T>, value: T) -> Result<(), StoreError> { + guard.insert(IDX, value, GFP_KERNEL) + } + + fn reserve<'a, T: ForeignOwnable>(guard: &mut Guard<'a, T>) -> Result> { + guard.reserve(IDX, GFP_KERNEL) + } + + #[track_caller] + fn check_not_vacant<'a>(guard: &mut Guard<'a, KBox>) -> Result { + // Insertion fails. + { + let beef = new_kbox(0xbeef)?; + let ret = insert(guard, beef); + assert!(ret.is_err()); + let StoreError { error, value } = ret.unwrap_err(); + assert_eq!(error, EBUSY); + assert_eq!(*value, 0xbeef); + } + + // Reservation fails. + { + let ret = reserve(guard); + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err(), EBUSY); + } + + Ok(()) + } + + #[test] + fn test_insert_and_reserve_interaction() -> Result { + stack_pin_init!(let xa = XArray::new(Default::default())); + let mut guard = xa.lock(); + + // Vacant. + assert_eq!(guard.get(IDX), None); + + // Reservation succeeds. + let reservation = { + let ret = reserve(&mut guard); + assert!(ret.is_ok()); + ret.unwrap() + }; + + // Reserved presents as vacant. + assert_eq!(guard.get(IDX), None); + + check_not_vacant(&mut guard)?; + + // Release reservation. + { + let ret = reservation.release_locked(&mut guard); + assert!(ret.is_ok()); + let () = ret.unwrap(); + } + + // Vacant again. + assert_eq!(guard.get(IDX), None); + + // Insert succeeds. + { + let dead = new_kbox(0xdead)?; + let ret = insert(&mut guard, dead); + assert!(ret.is_ok()); + let () = ret.unwrap(); + } + + check_not_vacant(&mut guard)?; + + // Remove. + assert_eq!(guard.remove(IDX).as_deref(), Some(&0xdead)); + + // Reserve and fill. + { + let beef = new_kbox(0xbeef)?; + let ret = reserve(&mut guard); + assert!(ret.is_ok()); + let reservation = ret.unwrap(); + let ret = reservation.fill_locked(&mut guard, beef); + assert!(ret.is_ok()); + let () = ret.unwrap(); + }; + + check_not_vacant(&mut guard)?; + + // Remove. + assert_eq!(guard.remove(IDX).as_deref(), Some(&0xbeef)); + + Ok(()) + } +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 0c36194d997168..62b73b5071f9b0 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -18,6 +18,7 @@ mod helpers; mod kunit; mod module; mod paste; +mod versions; mod vtable; use proc_macro::TokenStream; @@ -137,6 +138,12 @@ pub fn module(input: TokenStream) -> TokenStream { .into() } +/// Declares multiple variants of a structure or impl code +#[proc_macro_attribute] +pub fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + versions::versions(attr, item) +} + /// Declares or implements a vtable trait. /// /// Linux's use of pure vtables is very close to Rust traits, but they differ diff --git a/rust/macros/versions.rs b/rust/macros/versions.rs new file mode 100644 index 00000000000000..b13a5d55c0e17b --- /dev/null +++ b/rust/macros/versions.rs @@ -0,0 +1,341 @@ +use proc_macro::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream, TokenTree}; + +//use crate::helpers::expect_punct; + +fn expect_group(it: &mut impl Iterator) -> Group { + if let Some(TokenTree::Group(group)) = it.next() { + group + } else { + panic!("Expected Group") + } +} + +fn expect_punct(it: &mut impl Iterator) -> String { + if let Some(TokenTree::Punct(punct)) = it.next() { + punct.to_string() + } else { + panic!("Expected Group") + } +} + +fn drop_until_punct(it: &mut impl Iterator, delimiter: &str, is_struct: bool) { + let mut depth: isize = 0; + let mut colons: isize = 0; + for token in it.by_ref() { + if let TokenTree::Punct(punct) = token { + match punct.as_char() { + ':' => { + colons += 1; + } + '<' => { + if depth > 0 || colons == 2 || is_struct { + depth += 1; + } + colons = 0; + } + '>' => { + if depth > 0 { + depth -= 1; + } + colons = 0; + } + _ => { + colons = 0; + if depth == 0 && delimiter.contains(&punct.to_string()) { + break; + } + } + } + } + } +} + +fn drop_until_braces(it: &mut impl Iterator) { + let mut depth: isize = 0; + let mut colons: isize = 0; + for token in it.by_ref() { + match token { + TokenTree::Punct(punct) => match punct.as_char() { + ':' => { + colons += 1; + } + '<' => { + if depth > 0 || colons == 2 { + depth += 1; + } + colons = 0; + } + '>' => { + if depth > 0 { + depth -= 1; + } + colons = 0; + } + _ => colons = 0, + }, + TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => { + if depth == 0 { + break; + } + } + _ => (), + } + } +} + +struct VersionConfig { + fields: &'static [&'static str], + enums: &'static [&'static [&'static str]], + versions: &'static [&'static [&'static str]], +} + +static AGX_VERSIONS: VersionConfig = VersionConfig { + fields: &["G", "V"], + enums: &[ + &["G13", "G14", "G14X"], + &["V12_3", "V12_4", "V13_0B4", "V13_2", "V13_3", "V13_5"], + ], + versions: &[ + &["G13", "V12_3"], + &["G14", "V12_4"], + &["G13", "V13_5"], + &["G14", "V13_5"], + &["G14X", "V13_5"], + ], +}; + +fn check_version( + config: &VersionConfig, + ver: &[usize], + it: &mut impl Iterator, +) -> bool { + let first = it.next().unwrap(); + let val: bool = match &first { + TokenTree::Group(group) => check_version(config, ver, &mut group.stream().into_iter()), + TokenTree::Ident(ident) => { + let key = config + .fields + .iter() + .position(|&r| r == ident.to_string()) + .unwrap_or_else(|| panic!("Unknown field {}", ident)); + let mut operator = expect_punct(it); + let mut rhs_token = it.next().unwrap(); + if let TokenTree::Punct(punct) = &rhs_token { + operator.extend(std::iter::once(punct.as_char())); + rhs_token = it.next().unwrap(); + } + let rhs_name = if let TokenTree::Ident(ident) = &rhs_token { + ident.to_string() + } else { + panic!("Unexpected token {}", ident) + }; + + let rhs = config.enums[key] + .iter() + .position(|&r| r == rhs_name) + .unwrap_or_else(|| panic!("Unknown value for {}:{}", ident, rhs_name)); + let lhs = ver[key]; + + match operator.as_str() { + "==" => lhs == rhs, + "!=" => lhs != rhs, + ">" => lhs > rhs, + ">=" => lhs >= rhs, + "<" => lhs < rhs, + "<=" => lhs <= rhs, + _ => panic!("Unknown operator {}", operator), + } + } + _ => { + panic!("Unknown token {}", first) + } + }; + + let boolop = it.next(); + match boolop { + Some(TokenTree::Punct(punct)) => { + let right = expect_punct(it); + if right != punct.to_string() { + panic!("Unexpected op {}{}", punct, right); + } + match punct.as_char() { + '&' => val && check_version(config, ver, it), + '|' => val || check_version(config, ver, it), + _ => panic!("Unexpected op {}{}", right, right), + } + } + Some(a) => panic!("Unexpected op {}", a), + None => val, + } +} + +fn filter_versions( + config: &VersionConfig, + tag: &str, + ver: &[usize], + tree: impl IntoIterator, + is_struct: bool, +) -> Vec { + let mut out = Vec::::new(); + let mut it = tree.into_iter(); + + while let Some(token) = it.next() { + let mut tail: Option = None; + match &token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + let group = expect_group(&mut it); + let mut grp_it = group.stream().into_iter(); + let attr = grp_it.next().unwrap(); + match attr { + TokenTree::Ident(ident) if ident.to_string() == "ver" => { + if check_version(config, ver, &mut grp_it) { + } else if is_struct { + drop_until_punct(&mut it, ",", true); + } else { + let first = it.next().unwrap(); + match &first { + TokenTree::Ident(ident) + if ["while", "for", "loop", "if", "match", "unsafe", "fn"] + .contains(&ident.to_string().as_str()) => + { + drop_until_braces(&mut it); + } + TokenTree::Group(_) => (), + _ => { + drop_until_punct(&mut it, ",;", false); + } + } + } + } + _ => { + out.push(token.clone()); + out.push(TokenTree::Group(group.clone())); + } + } + continue; + } + TokenTree::Punct(punct) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Punct(punct)) if punct.to_string() == ":" => { + let next = it.next(); + match next { + Some(TokenTree::Ident(idtag)) if idtag.to_string() == "ver" => { + let ident = match out.pop() { + Some(TokenTree::Ident(ident)) => ident, + a => panic!("$ver not following ident: {:?}", a), + }; + let name = ident.to_string() + tag; + let new_ident = Ident::new(name.as_str(), ident.span()); + out.push(TokenTree::Ident(new_ident)); + continue; + } + Some(a) => { + out.push(token.clone()); + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + out.push(token.clone()); + } + } + } + Some(a) => { + out.push(token.clone()); + tail = Some(a); + } + None => { + out.push(token.clone()); + continue; + } + } + } + _ => { + tail = Some(token); + } + } + match &tail { + Some(TokenTree::Group(group)) => { + let new_body = + filter_versions(config, tag, ver, group.stream().into_iter(), is_struct); + let mut stream = TokenStream::new(); + stream.extend(new_body); + let mut filtered_group = Group::new(group.delimiter(), stream); + filtered_group.set_span(group.span()); + out.push(TokenTree::Group(filtered_group)); + } + Some(token) => { + out.push(token.clone()); + } + None => {} + } + } + + out +} + +pub(crate) fn versions(attr: TokenStream, item: TokenStream) -> TokenStream { + let config = match attr.to_string().as_str() { + "AGX" => &AGX_VERSIONS, + _ => panic!("Unknown version group {}", attr), + }; + + let mut it = item.into_iter(); + let mut out = TokenStream::new(); + let mut body: Vec = Vec::new(); + let mut is_struct = false; + + while let Some(token) = it.next() { + match token { + TokenTree::Punct(punct) if punct.to_string() == "#" => { + body.push(TokenTree::Punct(punct)); + body.push(it.next().unwrap()); + } + TokenTree::Ident(ident) + if ["struct", "enum", "union", "const", "type"] + .contains(&ident.to_string().as_str()) => + { + is_struct = ident.to_string() != "const"; + body.push(TokenTree::Ident(ident)); + body.push(it.next().unwrap()); + // This isn't valid syntax in a struct definition, so add it for the user + body.push(TokenTree::Punct(Punct::new(':', Spacing::Joint))); + body.push(TokenTree::Punct(Punct::new(':', Spacing::Alone))); + body.push(TokenTree::Ident(Ident::new("ver", Span::call_site()))); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "impl" => { + body.push(TokenTree::Ident(ident)); + break; + } + TokenTree::Ident(ident) if ident.to_string() == "fn" => { + body.push(TokenTree::Ident(ident)); + break; + } + _ => { + body.push(token); + } + } + } + + body.extend(it); + + for ver in config.versions { + let tag = ver.join(""); + let mut ver_num = Vec::::new(); + for (i, comp) in ver.iter().enumerate() { + let idx = config.enums[i].iter().position(|&r| r == *comp).unwrap(); + ver_num.push(idx); + } + out.extend(filter_versions( + config, + &tag, + &ver_num, + body.clone(), + is_struct, + )); + } + + out +} diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs index 2fe918f4d82aa1..bda2ae923c7846 100644 --- a/rust/pin-init/internal/src/init.rs +++ b/rust/pin-init/internal/src/init.rs @@ -243,22 +243,6 @@ fn init_fields( }); // Again span for better diagnostics let write = quote_spanned!(ident.span()=> ::core::ptr::write); - // NOTE: the field accessor ensures that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` below has the same requirement. - let accessor = if pinned { - let project_ident = format_ident!("__project_{ident}"); - quote! { - // SAFETY: TODO - unsafe { #data.#project_ident(&mut (*#slot).#ident) } - } - } else { - quote! { - // SAFETY: TODO - unsafe { &mut (*#slot).#ident } - } - }; quote! { #(#attrs)* { @@ -266,51 +250,31 @@ fn init_fields( // SAFETY: TODO unsafe { #write(::core::ptr::addr_of_mut!((*#slot).#ident), #value_ident) }; } - #(#cfgs)* - #[allow(unused_variables)] - let #ident = #accessor; } } InitializerKind::Init { ident, value, .. } => { // Again span for better diagnostics let init = format_ident!("init", span = value.span()); - // NOTE: the field accessor ensures that the initialized field is properly aligned. - // Unaligned fields will cause the compiler to emit E0793. We do not support - // unaligned fields since `Init::__init` requires an aligned pointer; the call to - // `ptr::write` below has the same requirement. - let (value_init, accessor) = if pinned { - let project_ident = format_ident!("__project_{ident}"); - ( - quote! { - // SAFETY: - // - `slot` is valid, because we are inside of an initializer closure, we - // return when an error/panic occurs. - // - We also use `#data` to require the correct trait (`Init` or `PinInit`) - // for `#ident`. - unsafe { #data.#ident(::core::ptr::addr_of_mut!((*#slot).#ident), #init)? }; - }, - quote! { - // SAFETY: TODO - unsafe { #data.#project_ident(&mut (*#slot).#ident) } - }, - ) + let value_init = if pinned { + quote! { + // SAFETY: + // - `slot` is valid, because we are inside of an initializer closure, we + // return when an error/panic occurs. + // - We also use `#data` to require the correct trait (`Init` or `PinInit`) + // for `#ident`. + unsafe { #data.#ident(::core::ptr::addr_of_mut!((*#slot).#ident), #init)? }; + } } else { - ( - quote! { - // SAFETY: `slot` is valid, because we are inside of an initializer - // closure, we return when an error/panic occurs. - unsafe { - ::pin_init::Init::__init( - #init, - ::core::ptr::addr_of_mut!((*#slot).#ident), - )? - }; - }, - quote! { - // SAFETY: TODO - unsafe { &mut (*#slot).#ident } - }, - ) + quote! { + // SAFETY: `slot` is valid, because we are inside of an initializer + // closure, we return when an error/panic occurs. + unsafe { + ::pin_init::Init::__init( + #init, + ::core::ptr::addr_of_mut!((*#slot).#ident), + )? + }; + } }; quote! { #(#attrs)* @@ -318,9 +282,6 @@ fn init_fields( let #init = #value; #value_init } - #(#cfgs)* - #[allow(unused_variables)] - let #ident = #accessor; } } InitializerKind::Code { block: value, .. } => quote! { @@ -333,18 +294,41 @@ fn init_fields( if let Some(ident) = kind.ident() { // `mixed_site` ensures that the guard is not accessible to the user-controlled code. let guard = format_ident!("__{ident}_guard", span = Span::mixed_site()); + + // NOTE: The reference is derived from the guard so that it only lives as long as the + // guard does and cannot escape the scope. If it's created via `&mut (*#slot).#ident` + // like the unaligned field guard, it will become effectively `'static`. + let accessor = if pinned { + let project_ident = format_ident!("__project_{ident}"); + quote! { + // SAFETY: the initialization is pinned. + unsafe { #data.#project_ident(#guard.let_binding()) } + } + } else { + quote! { + #guard.let_binding() + } + }; + res.extend(quote! { #(#cfgs)* - // Create the drop guard: + // Create the drop guard. // - // We rely on macro hygiene to make it impossible for users to access this local - // variable. - // SAFETY: We forget the guard later when initialization has succeeded. - let #guard = unsafe { + // SAFETY: + // - `&raw mut (*slot).#ident` is valid. + // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned. + // - `(*slot).#ident` has been initialized above. + // - We only need the ownership to the pointee back when initialization has + // succeeded, where we `forget` the guard. + let mut #guard = unsafe { ::pin_init::__internal::DropGuard::new( ::core::ptr::addr_of_mut!((*slot).#ident) ) }; + + #(#cfgs)* + #[allow(unused_variables)] + let #ident = #accessor; }); guards.push(guard); guard_attrs.push(cfgs); @@ -361,49 +345,49 @@ fn init_fields( } } -/// Generate the check for ensuring that every field has been initialized. +/// Generate the check for ensuring that every field has been initialized and aligned. fn make_field_check( fields: &Punctuated, init_kind: InitKind, path: &Path, ) -> TokenStream { - let field_attrs = fields + let field_attrs: Vec<_> = fields .iter() - .filter_map(|f| f.kind.ident().map(|_| &f.attrs)); - let field_name = fields.iter().filter_map(|f| f.kind.ident()); - match init_kind { - InitKind::Normal => quote! { - // We use unreachable code to ensure that all fields have been mentioned exactly once, - // this struct initializer will still be type-checked and complain with a very natural - // error message if a field is forgotten/mentioned more than once. - #[allow(unreachable_code, clippy::diverging_sub_expression)] - // SAFETY: this code is never executed. - let _ = || unsafe { - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: ::core::panic!(), - )* - }) - }; - }, - InitKind::Zeroing => quote! { - // We use unreachable code to ensure that all fields have been mentioned at most once. - // Since the user specified `..Zeroable::zeroed()` at the end, all missing fields will - // be zeroed. This struct initializer will still be type-checked and complain with a - // very natural error message if a field is mentioned more than once, or doesn't exist. - #[allow(unreachable_code, clippy::diverging_sub_expression, unused_assignments)] - // SAFETY: this code is never executed. - let _ = || unsafe { - ::core::ptr::write(slot, #path { - #( - #(#field_attrs)* - #field_name: ::core::panic!(), - )* - ..::core::mem::zeroed() - }) - }; - }, + .filter_map(|f| f.kind.ident().map(|_| &f.attrs)) + .collect(); + let field_name: Vec<_> = fields.iter().filter_map(|f| f.kind.ident()).collect(); + let zeroing_trailer = match init_kind { + InitKind::Normal => None, + InitKind::Zeroing => Some(quote! { + ..::core::mem::zeroed() + }), + }; + quote! { + #[allow(unreachable_code, clippy::diverging_sub_expression)] + // We use unreachable code to perform field checks. They're still checked by the compiler. + // SAFETY: this code is never executed. + let _ = || unsafe { + // Create references to ensure that the initialized field is properly aligned. + // Unaligned fields will cause the compiler to emit E0793. We do not support + // unaligned fields since `Init::__init` requires an aligned pointer; the call to + // `ptr::write` for value-initialization case has the same requirement. + #( + #(#field_attrs)* + let _ = &(*slot).#field_name; + )* + + // If the zeroing trailer is not present, this checks that all fields have been + // mentioned exactly once. If the zeroing trailer is present, all missing fields will be + // zeroed, so this checks that all fields have been mentioned at most once. The use of + // struct initializer will still generate very natural error messages for any misuse. + ::core::ptr::write(slot, #path { + #( + #(#field_attrs)* + #field_name: ::core::panic!(), + )* + #zeroing_trailer + }) + }; } } diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs index 90adbdc1893bbf..5720a621aed74b 100644 --- a/rust/pin-init/src/__internal.rs +++ b/rust/pin-init/src/__internal.rs @@ -238,32 +238,42 @@ fn stack_init_reuse() { /// When a value of this type is dropped, it drops a `T`. /// /// Can be forgotten to prevent the drop. +/// +/// # Invariants +/// +/// - `ptr` is valid and properly aligned. +/// - `*ptr` is initialized and owned by this guard. pub struct DropGuard { ptr: *mut T, } impl DropGuard { - /// Creates a new [`DropGuard`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped. + /// Creates a drop guard and transfer the ownership of the pointer content. /// - /// # Safety + /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`]. /// - /// `ptr` must be a valid pointer. + /// # Safety /// - /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`: - /// - has not been dropped, - /// - is not accessible by any other means, - /// - will not be dropped by any other means. + /// - `ptr` is valid and properly aligned. + /// - `*ptr` is initialized, and the ownership is transferred to this guard. #[inline] pub unsafe fn new(ptr: *mut T) -> Self { + // INVARIANT: By safety requirement. Self { ptr } } + + /// Create a let binding for accessor use. + #[inline] + pub fn let_binding(&mut self) -> &mut T { + // SAFETY: Per type invariant. + unsafe { &mut *self.ptr } + } } impl Drop for DropGuard { #[inline] fn drop(&mut self) { - // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function - // ensuring that this operation is safe. + // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard. unsafe { ptr::drop_in_place(self.ptr) } } } diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs index fe4c85ae3f02a8..4b33df02c39433 100644 --- a/rust/pin-init/src/lib.rs +++ b/rust/pin-init/src/lib.rs @@ -1468,6 +1468,21 @@ pub unsafe trait PinnedDrop: __internal::HasPinData { fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); } +/// Create a new default T. +/// +/// The returned initializer will use Default::default to initialize the `slot`. +#[inline] +pub fn default() -> impl Init { + // SAFETY: Because `T: Default`, T cannot require pinning and + // we can just move the data into the slot. + unsafe { + init_from_closure(|slot: *mut T| { + *slot = Default::default(); + Ok(()) + }) + } +} + /// Marker trait for types that can be initialized by writing just zeroes. /// /// # Safety diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8daba..512ac0aea08fde 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -7,10 +7,13 @@ */ #include +#include #include #include #include #include +#include +#include #include #include #include diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 3652b85be54594..0cd32c007d1159 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -323,6 +323,9 @@ $(obj)/%.lst: $(obj)/%.c FORCE # the unstable features in use. rust_allowed_features := asm_const,asm_goto,arbitrary_self_types,lint_reasons,offset_of_nested,raw_ref_op,slice_ptr_len,strict_provenance,used_with_arg +# additional rust features used by the downstream asahi kernel +rust_allowed_features := $(rust_allowed_features),ptr_sub_ptr + # `--out-dir` is required to avoid temporaries being created by `rustc` in the # current working directory, which may be not accessible in the out-of-tree # modules case. diff --git a/scripts/check-uapi.sh b/scripts/check-uapi.sh index 955581735cb3c3..9fa45cbdecc257 100755 --- a/scripts/check-uapi.sh +++ b/scripts/check-uapi.sh @@ -178,8 +178,11 @@ do_compile() { local -r inc_dir="$1" local -r header="$2" local -r out="$3" - printf "int main(void) { return 0; }\n" | \ - "$CC" -c \ + printf "int f(void) { return 0; }\n" | \ + "$CC" \ + -shared \ + -nostdlib \ + -fPIC \ -o "$out" \ -x c \ -O0 \ diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index e56374662ff797..27a43a4d9c4384 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -641,6 +641,7 @@ sub hash_show_words { Reviewed-by:| Reported-by:| Suggested-by:| + Assisted-by:| To:| Cc: )}; @@ -3105,6 +3106,15 @@ sub process { } } + # Assisted-by uses AGENT_NAME:MODEL_VERSION format, not email + if ($sign_off =~ /^Assisted-by:/i) { + if ($email !~ /^\S+:\S+/) { + WARN("BAD_SIGN_OFF", + "Assisted-by expects 'AGENT_NAME:MODEL_VERSION [TOOL1] [TOOL2]' format\n" . $herecurr); + } + next; + } + my ($email_name, $name_comment, $email_address, $comment) = parse_email($email); my $suggested_email = format_email(($email_name, $name_comment, $email_address, $comment)); if ($suggested_email eq "") { diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c index 5b25aa06041613..ce449824c80a06 100644 --- a/scripts/dtc/data.c +++ b/scripts/dtc/data.c @@ -184,6 +184,33 @@ struct data data_append_integer(struct data d, uint64_t value, int bits) } } +struct data data_append_float(struct data d, double value, int bits) +{ + float f32; + uint32_t u32; + double f64; + uint64_t u64; + fdt32_t value_32; + fdt64_t value_64; + + switch (bits) { + case 32: + f32 = value; + memcpy(&u32, &f32, sizeof(u32)); + value_32 = cpu_to_fdt32(u32); + return data_append_data(d, &value_32, 4); + + case 64: + f64 = value; + memcpy(&u64, &f64, sizeof(u64)); + value_64 = cpu_to_fdt64(u64); + return data_append_data(d, &value_64, 8); + + default: + die("Invalid literal size (%d)\n", bits); + } +} + struct data data_append_re(struct data d, uint64_t address, uint64_t size) { struct fdt_reserve_entry re; diff --git a/scripts/dtc/dtc-lexer.l b/scripts/dtc/dtc-lexer.l index 15d585c8079802..f625e35eaf0240 100644 --- a/scripts/dtc/dtc-lexer.l +++ b/scripts/dtc/dtc-lexer.l @@ -39,8 +39,6 @@ extern bool treesource_error; #define DPRINT(fmt, ...) do { } while (0) #endif -static int dts_version = 1; - #define BEGIN_DEFAULT() DPRINT("\n"); \ BEGIN(V1); \ @@ -101,7 +99,6 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); <*>"/dts-v1/" { DPRINT("Keyword: /dts-v1/\n"); - dts_version = 1; BEGIN_DEFAULT(); return DT_V1; } @@ -151,6 +148,28 @@ static void PRINTF(1, 2) lexical_error(const char *fmt, ...); return DT_LABEL; } +[-+]?(([0-9]+\.[0-9]*)|([0-9]*\.[0-9]+))(e[-+]?[0-9]+)?f? { + char *e; + DPRINT("Floating-point Literal: '%s'\n", yytext); + + errno = 0; + yylval.floating = strtod(yytext, &e); + + if (*e && (*e != 'f' || e[1])) { + lexical_error("Bad floating-point literal '%s'", + yytext); + } + + if (errno == ERANGE) + lexical_error("Floating-point literal '%s' out of range", + yytext); + else + /* ERANGE is the only strtod error triggerable + * by strings matching the pattern */ + assert(errno == 0); + return DT_FP_LITERAL; + } + {LABEL} { /* Missed includes or macro definitions while * preprocessing can lead to unexpected identifiers in diff --git a/scripts/dtc/dtc-parser.y b/scripts/dtc/dtc-parser.y index 4d5eece5262434..225a6b41b14fcf 100644 --- a/scripts/dtc/dtc-parser.y +++ b/scripts/dtc/dtc-parser.y @@ -48,6 +48,7 @@ static bool is_ref_relative(const char *ref) struct node *nodelist; struct reserve_info *re; uint64_t integer; + double floating; unsigned int flags; } @@ -61,6 +62,7 @@ static bool is_ref_relative(const char *ref) %token DT_OMIT_NO_REF %token DT_PROPNODENAME %token DT_LITERAL +%token DT_FP_LITERAL %token DT_CHAR_LITERAL %token DT_BYTE %token DT_STRING @@ -86,6 +88,7 @@ static bool is_ref_relative(const char *ref) %type subnode %type subnodes +%type floating_prim %type integer_prim %type integer_unary %type integer_mul @@ -395,6 +398,15 @@ arrayprefix: $$.data = data_add_marker(empty_data, TYPE_UINT32, NULL); $$.bits = 32; } + | arrayprefix floating_prim + { + if ($1.bits < 32) { + ERROR(&@2, "Floating-point values must be" + " 32-bit or 64-bit"); + } + + $$.data = data_append_float($1.data, $2, $1.bits); + } | arrayprefix integer_prim { if ($1.bits < 64) { @@ -439,6 +451,10 @@ arrayprefix: } ; +floating_prim: + DT_FP_LITERAL + ; + integer_prim: DT_LITERAL | DT_CHAR_LITERAL diff --git a/scripts/dtc/dtc.h b/scripts/dtc/dtc.h index 7231200e5d02cc..6f29281e12305d 100644 --- a/scripts/dtc/dtc.h +++ b/scripts/dtc/dtc.h @@ -177,6 +177,7 @@ struct data data_insert_at_marker(struct data d, struct marker *m, struct data data_merge(struct data d1, struct data d2); struct data data_append_cell(struct data d, cell_t word); struct data data_append_integer(struct data d, uint64_t word, int bits); +struct data data_append_float(struct data d, double value, int bits); struct data data_append_re(struct data d, uint64_t address, uint64_t size); struct data data_append_addr(struct data d, uint64_t addr); struct data data_append_byte(struct data d, uint8_t byte); diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index d4308b7261838b..943ff1228b487e 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -298,7 +298,7 @@ def invoke(self, arg, from_tty): if p == "-bpf": monitor_bpf = True else: - p.append(os.path.abspath(os.path.expanduser(p))) + self.module_paths.append(os.path.abspath(os.path.expanduser(p))) self.module_paths.append(os.getcwd()) if self.breakpoint is not None: diff --git a/scripts/gdb/linux/timerlist.py b/scripts/gdb/linux/timerlist.py index ccc24d30de8063..9fb3436a217cc1 100644 --- a/scripts/gdb/linux/timerlist.py +++ b/scripts/gdb/linux/timerlist.py @@ -20,7 +20,7 @@ def ktime_get(): We can't read the hardware timer itself to add any nanoseconds that need to be added since we last stored the time in the timekeeper. But this is probably good enough for debug purposes.""" - tk_core = gdb.parse_and_eval("&tk_core") + tk_core = gdb.parse_and_eval("&timekeeper_data[TIMEKEEPER_CORE]") return tk_core['timekeeper']['tkr_mono']['base'] diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index f9b545104f21f3..92543d4f91b56f 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -188,11 +188,24 @@ def append_crate_with_generated( append_crate_with_generated("uapi", ["core", "ffi", "pin_init"]) append_crate_with_generated("kernel", ["core", "macros", "build_error", "pin_init", "ffi", "bindings", "uapi"]) + scripts = srctree / "scripts" + makefile = (scripts / "Makefile").read_text() + for path in scripts.glob("*.rs"): + name = path.stem + if f"{name}-rust" not in makefile: + continue + append_crate( + name, + path, + ["std"], + ) + def is_root_crate(build_file, target): try: - return f"{target}.o" in open(build_file).read() + contents = build_file.read_text() except FileNotFoundError: return False + return f"{target}.o" in contents # Then, the rest outside of `rust/`. # @@ -203,7 +216,7 @@ def is_root_crate(build_file, target): for folder in extra_dirs: for path in folder.rglob("*.rs"): logging.info("Checking %s", path) - name = path.name.replace(".rs", "") + name = path.stem # Skip those that are not crate roots. if not is_root_crate(path.parent / "Makefile", name) and \ diff --git a/scripts/module.lds.S b/scripts/module.lds.S index 054ef99e828867..7b9f4341434373 100644 --- a/scripts/module.lds.S +++ b/scripts/module.lds.S @@ -32,30 +32,32 @@ SECTIONS { __jump_table 0 : ALIGN(8) { KEEP(*(__jump_table)) } __ex_table 0 : ALIGN(4) { KEEP(*(__ex_table)) } - __patchable_function_entries : { *(__patchable_function_entries) } + __patchable_function_entries 0 : { *(__patchable_function_entries) } .init.klp_funcs 0 : ALIGN(8) { KEEP(*(.init.klp_funcs)) } .init.klp_objects 0 : ALIGN(8) { KEEP(*(.init.klp_objects)) } #ifdef CONFIG_ARCH_USES_CFI_TRAPS - __kcfi_traps : { KEEP(*(.kcfi_traps)) } + __kcfi_traps 0 : { KEEP(*(.kcfi_traps)) } #endif - .text : { +#ifndef CONFIG_ARCH_WANTS_MODULES_TEXT_SECTIONS + .text 0 : { *(.text .text.[0-9a-zA-Z_]*) } +#endif - .bss : { + .bss 0 : { *(.bss .bss.[0-9a-zA-Z_]*) *(.bss..L*) } - .data : { + .data 0 : { *(.data .data.[0-9a-zA-Z_]*) *(.data..L*) } - .rodata : { + .rodata 0 : { *(.rodata .rodata.[0-9a-zA-Z_]*) *(.rodata..L*) } diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 3627ca227e5a5d..ba1defc616524f 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -139,7 +139,13 @@ install_kernel_headers () { pdir=debian/$1 version=${1#linux-headers-} - CC="${DEB_HOST_GNU_TYPE}-gcc" "${srctree}/scripts/package/install-extmod-build" "${pdir}/usr/src/linux-headers-${version}" + # Override $CC only for cross-compiles, to not unnecessarily rebuild + # scripts/ including plugins, which may lead to a full kernel rebuild. + if [ -n "${CROSS_COMPILE}" ]; then + CC="${DEB_HOST_GNU_TYPE}-gcc" "${srctree}/scripts/package/install-extmod-build" "${pdir}/usr/src/linux-headers-${version}" + else + "${srctree}/scripts/package/install-extmod-build" "${pdir}/usr/src/linux-headers-${version}" + fi mkdir -p $pdir/lib/modules/$version/ ln -s /usr/src/linux-headers-$version $pdir/lib/modules/$version/build diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c index c1d42fc72fdb4b..d3af2d10fc22c5 100644 --- a/security/apparmor/lsm.c +++ b/security/apparmor/lsm.c @@ -822,25 +822,23 @@ static int apparmor_getprocattr(struct task_struct *task, const char *name, char **value) { int error = -ENOENT; - /* released below */ - const struct cred *cred = get_task_cred(task); - struct aa_task_ctx *ctx = task_ctx(current); struct aa_label *label = NULL; + rcu_read_lock(); if (strcmp(name, "current") == 0) - label = aa_get_newest_label(cred_label(cred)); - else if (strcmp(name, "prev") == 0 && ctx->previous) - label = aa_get_newest_label(ctx->previous); - else if (strcmp(name, "exec") == 0 && ctx->onexec) - label = aa_get_newest_label(ctx->onexec); + label = aa_get_newest_cred_label(__task_cred(task)); + else if (strcmp(name, "prev") == 0 && task_ctx(task)->previous) + label = aa_get_newest_label(task_ctx(task)->previous); + else if (strcmp(name, "exec") == 0 && task_ctx(task)->onexec) + label = aa_get_newest_label(task_ctx(task)->onexec); else error = -EINVAL; + rcu_read_unlock(); if (label) error = aa_getprocattr(label, value, true); aa_put_label(label); - put_cred(cred); return error; } diff --git a/security/apparmor/path.c b/security/apparmor/path.c index 65a0ca5cc1bdda..2494e810153841 100644 --- a/security/apparmor/path.c +++ b/security/apparmor/path.c @@ -164,14 +164,16 @@ static int d_namespace_path(const struct path *path, char *buf, char **name, } out: - /* Append "/" to directory paths, except for root "/" which - * already ends in a slash. + /* Append "/" to directory paths and reterminate string, except for + * root "/" which already ends in a slash. */ if (!error && isdir) { bool is_root = (*name)[0] == '/' && (*name)[1] == '\0'; - if (!is_root) + if (!is_root) { buf[aa_g_path_max - 2] = '/'; + buf[aa_g_path_max - 1] = '\0'; + } } return error; diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c index aff61643415dec..85c433e39c008a 100644 --- a/security/integrity/ima/ima_crypto.c +++ b/security/integrity/ima/ima_crypto.c @@ -832,7 +832,7 @@ static int ima_calc_boot_aggregate_tfm(char *digest, u16 alg_id, } } if (!rc) - crypto_shash_final(shash, digest); + rc = crypto_shash_final(shash, digest); return rc; } diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c index 012a58959ff022..f3c461ad7062b8 100644 --- a/security/integrity/ima/ima_fs.c +++ b/security/integrity/ima/ima_fs.c @@ -404,16 +404,24 @@ static int __init create_securityfs_measurement_lists(void) char file_name[NAME_MAX + 1]; struct dentry *dentry; - sprintf(file_name, "ascii_runtime_measurements_%s", - hash_algo_name[algo]); + if (algo == HASH_ALGO__LAST) + sprintf(file_name, "ascii_runtime_measurements_tpm_alg_%x", + ima_tpm_chip->allocated_banks[i].alg_id); + else + sprintf(file_name, "ascii_runtime_measurements_%s", + hash_algo_name[algo]); dentry = securityfs_create_file(file_name, S_IRUSR | S_IRGRP, ima_dir, (void *)(uintptr_t)i, &ima_ascii_measurements_ops); if (IS_ERR(dentry)) return PTR_ERR(dentry); - sprintf(file_name, "binary_runtime_measurements_%s", - hash_algo_name[algo]); + if (algo == HASH_ALGO__LAST) + sprintf(file_name, "binary_runtime_measurements_tpm_alg_%x", + ima_tpm_chip->allocated_banks[i].alg_id); + else + sprintf(file_name, "binary_runtime_measurements_%s", + hash_algo_name[algo]); dentry = securityfs_create_file(file_name, S_IRUSR | S_IRGRP, ima_dir, (void *)(uintptr_t)i, &ima_measurements_ops); diff --git a/security/landlock/cred.c b/security/landlock/cred.c index 0cb3edde4d18ab..cc419de75cd6bd 100644 --- a/security/landlock/cred.c +++ b/security/landlock/cred.c @@ -22,10 +22,8 @@ static void hook_cred_transfer(struct cred *const new, const struct landlock_cred_security *const old_llcred = landlock_cred(old); - if (old_llcred->domain) { - landlock_get_ruleset(old_llcred->domain); - *landlock_cred(new) = *old_llcred; - } + landlock_get_ruleset(old_llcred->domain); + *landlock_cred(new) = *old_llcred; } static int hook_cred_prepare(struct cred *const new, diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c index 0d66a68677b700..a0bb664e0d315b 100644 --- a/security/landlock/syscalls.c +++ b/security/landlock/syscalls.c @@ -512,10 +512,13 @@ SYSCALL_DEFINE2(landlock_restrict_self, const int, ruleset_fd, const __u32, /* * It is allowed to set LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF with - * -1 as ruleset_fd, but no other flag must be set. + * -1 as ruleset_fd, optionally combined with + * LANDLOCK_RESTRICT_SELF_TSYNC to propagate this configuration to all + * threads. No other flag must be set. */ if (!(ruleset_fd == -1 && - flags == LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF)) { + (flags & ~LANDLOCK_RESTRICT_SELF_TSYNC) == + LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF)) { /* Gets and checks the ruleset. */ ruleset = get_ruleset_from_fd(ruleset_fd, FMODE_CAN_READ); if (IS_ERR(ruleset)) @@ -537,9 +540,10 @@ SYSCALL_DEFINE2(landlock_restrict_self, const int, ruleset_fd, const __u32, /* * The only case when a ruleset may not be set is if - * LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF is set and ruleset_fd is -1. - * We could optimize this case by not calling commit_creds() if this flag - * was already set, but it is not worth the complexity. + * LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF is set (optionally with + * LANDLOCK_RESTRICT_SELF_TSYNC) and ruleset_fd is -1. We could + * optimize this case by not calling commit_creds() if this flag was + * already set, but it is not worth the complexity. */ if (ruleset) { /* diff --git a/security/lsm.h b/security/lsm.h index db77cc83e1582b..32f808ad4335a8 100644 --- a/security/lsm.h +++ b/security/lsm.h @@ -29,6 +29,7 @@ extern struct lsm_blob_sizes blob_sizes; /* LSM blob caches */ extern struct kmem_cache *lsm_file_cache; +extern struct kmem_cache *lsm_backing_file_cache; extern struct kmem_cache *lsm_inode_cache; /* LSM blob allocators */ diff --git a/security/lsm_init.c b/security/lsm_init.c index 573e2a7250c416..7c0fd17f1601f2 100644 --- a/security/lsm_init.c +++ b/security/lsm_init.c @@ -293,6 +293,8 @@ static void __init lsm_prepare(struct lsm_info *lsm) blobs = lsm->blobs; lsm_blob_size_update(&blobs->lbs_cred, &blob_sizes.lbs_cred); lsm_blob_size_update(&blobs->lbs_file, &blob_sizes.lbs_file); + lsm_blob_size_update(&blobs->lbs_backing_file, + &blob_sizes.lbs_backing_file); lsm_blob_size_update(&blobs->lbs_ib, &blob_sizes.lbs_ib); /* inode blob gets an rcu_head in addition to LSM blobs. */ if (blobs->lbs_inode && blob_sizes.lbs_inode == 0) @@ -441,6 +443,8 @@ int __init security_init(void) if (lsm_debug) { lsm_pr("blob(cred) size %d\n", blob_sizes.lbs_cred); lsm_pr("blob(file) size %d\n", blob_sizes.lbs_file); + lsm_pr("blob(backing_file) size %d\n", + blob_sizes.lbs_backing_file); lsm_pr("blob(ib) size %d\n", blob_sizes.lbs_ib); lsm_pr("blob(inode) size %d\n", blob_sizes.lbs_inode); lsm_pr("blob(ipc) size %d\n", blob_sizes.lbs_ipc); @@ -462,6 +466,11 @@ int __init security_init(void) lsm_file_cache = kmem_cache_create("lsm_file_cache", blob_sizes.lbs_file, 0, SLAB_PANIC, NULL); + if (blob_sizes.lbs_backing_file) + lsm_backing_file_cache = kmem_cache_create( + "lsm_backing_file_cache", + blob_sizes.lbs_backing_file, + 0, SLAB_PANIC, NULL); if (blob_sizes.lbs_inode) lsm_inode_cache = kmem_cache_create("lsm_inode_cache", blob_sizes.lbs_inode, 0, diff --git a/security/security.c b/security/security.c index a26c1474e2e499..048560ef6a1a40 100644 --- a/security/security.c +++ b/security/security.c @@ -82,6 +82,7 @@ const struct lsm_id *lsm_idlist[MAX_LSM_COUNT]; struct lsm_blob_sizes blob_sizes; struct kmem_cache *lsm_file_cache; +struct kmem_cache *lsm_backing_file_cache; struct kmem_cache *lsm_inode_cache; #define SECURITY_HOOK_ACTIVE_KEY(HOOK, IDX) security_hook_active_##HOOK##_##IDX @@ -173,6 +174,30 @@ static int lsm_file_alloc(struct file *file) return 0; } +/** + * lsm_backing_file_alloc - allocate a composite backing file blob + * @backing_file: the backing file + * + * Allocate the backing file blob for all the modules. + * + * Returns 0, or -ENOMEM if memory can't be allocated. + */ +static int lsm_backing_file_alloc(struct file *backing_file) +{ + void *blob; + + if (!lsm_backing_file_cache) { + backing_file_set_security(backing_file, NULL); + return 0; + } + + blob = kmem_cache_zalloc(lsm_backing_file_cache, GFP_KERNEL); + backing_file_set_security(backing_file, blob); + if (!blob) + return -ENOMEM; + return 0; +} + /** * lsm_blob_alloc - allocate a composite blob * @dest: the destination for the blob @@ -2418,6 +2443,57 @@ void security_file_free(struct file *file) } } +/** + * security_backing_file_alloc() - Allocate and setup a backing file blob + * @backing_file: the backing file + * @user_file: the associated user visible file + * + * Allocate a backing file LSM blob and perform any necessary initialization of + * the LSM blob. There will be some operations where the LSM will not have + * access to @user_file after this point, so any important state associated + * with @user_file that is important to the LSM should be captured in the + * backing file's LSM blob. + * + * LSM's should avoid taking a reference to @user_file in this hook as it will + * result in problems later when the system attempts to drop/put the file + * references due to a circular dependency. + * + * Return: Return 0 if the hook is successful, negative values otherwise. + */ +int security_backing_file_alloc(struct file *backing_file, + const struct file *user_file) +{ + int rc; + + rc = lsm_backing_file_alloc(backing_file); + if (rc) + return rc; + rc = call_int_hook(backing_file_alloc, backing_file, user_file); + if (unlikely(rc)) + security_backing_file_free(backing_file); + + return rc; +} + +/** + * security_backing_file_free() - Free a backing file blob + * @backing_file: the backing file + * + * Free any LSM state associate with a backing file's LSM blob, including the + * blob itself. + */ +void security_backing_file_free(struct file *backing_file) +{ + void *blob = backing_file_security(backing_file); + + call_void_hook(backing_file_free, backing_file); + + if (blob) { + backing_file_set_security(backing_file, NULL); + kmem_cache_free(lsm_backing_file_cache, blob); + } +} + /** * security_file_ioctl() - Check if an ioctl is allowed * @file: associated file @@ -2506,6 +2582,32 @@ int security_mmap_file(struct file *file, unsigned long prot, flags); } +/** + * security_mmap_backing_file - Check if mmap'ing a backing file is allowed + * @vma: the vm_area_struct for the mmap'd region + * @backing_file: the backing file being mmap'd + * @user_file: the user file being mmap'd + * + * Check permissions for a mmap operation on a stacked filesystem. This hook + * is called after the security_mmap_file() and is responsible for authorizing + * the mmap on @backing_file. It is important to note that the mmap operation + * on @user_file has already been authorized and the @vma->vm_file has been + * set to @backing_file. + * + * Return: Returns 0 if permission is granted. + */ +int security_mmap_backing_file(struct vm_area_struct *vma, + struct file *backing_file, + struct file *user_file) +{ + /* recommended by the stackable filesystem devs */ + if (WARN_ON_ONCE(!(backing_file->f_mode & FMODE_BACKING))) + return -EIO; + + return call_int_hook(mmap_backing_file, vma, backing_file, user_file); +} +EXPORT_SYMBOL_GPL(security_mmap_backing_file); + /** * security_mmap_addr() - Check if mmap'ing an address is allowed * @addr: address diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index d8224ea113d1ac..6c154a4d94b9de 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -1745,49 +1745,72 @@ static inline int file_path_has_perm(const struct cred *cred, static int bpf_fd_pass(const struct file *file, u32 sid); #endif -/* Check whether a task can use an open file descriptor to - access an inode in a given way. Check access to the - descriptor itself, and then use dentry_has_perm to - check a particular permission to the file. - Access to the descriptor is implicitly granted if it - has the same SID as the process. If av is zero, then - access to the file is not checked, e.g. for cases - where only the descriptor is affected like seek. */ -static int file_has_perm(const struct cred *cred, - struct file *file, - u32 av) +static int __file_has_perm(const struct cred *cred, const struct file *file, + u32 av, bool bf_user_file) + { - struct file_security_struct *fsec = selinux_file(file); - struct inode *inode = file_inode(file); struct common_audit_data ad; - u32 sid = cred_sid(cred); + struct inode *inode; + u32 ssid = cred_sid(cred); + u32 tsid_fd; int rc; - ad.type = LSM_AUDIT_DATA_FILE; - ad.u.file = file; + if (bf_user_file) { + struct backing_file_security_struct *bfsec; + const struct path *path; - if (sid != fsec->sid) { - rc = avc_has_perm(sid, fsec->sid, - SECCLASS_FD, - FD__USE, - &ad); + if (WARN_ON(!(file->f_mode & FMODE_BACKING))) + return -EIO; + + bfsec = selinux_backing_file(file); + path = backing_file_user_path(file); + tsid_fd = bfsec->uf_sid; + inode = d_inode(path->dentry); + + ad.type = LSM_AUDIT_DATA_PATH; + ad.u.path = *path; + } else { + struct file_security_struct *fsec = selinux_file(file); + + tsid_fd = fsec->sid; + inode = file_inode(file); + + ad.type = LSM_AUDIT_DATA_FILE; + ad.u.file = file; + } + + if (ssid != tsid_fd) { + rc = avc_has_perm(ssid, tsid_fd, SECCLASS_FD, FD__USE, &ad); if (rc) - goto out; + return rc; } #ifdef CONFIG_BPF_SYSCALL - rc = bpf_fd_pass(file, cred_sid(cred)); + /* regardless of backing vs user file, use the underlying file here */ + rc = bpf_fd_pass(file, ssid); if (rc) return rc; #endif /* av is zero if only checking access to the descriptor. */ - rc = 0; if (av) - rc = inode_has_perm(cred, inode, av, &ad); + return inode_has_perm(cred, inode, av, &ad); -out: - return rc; + return 0; +} + +/* Check whether a task can use an open file descriptor to + access an inode in a given way. Check access to the + descriptor itself, and then use dentry_has_perm to + check a particular permission to the file. + Access to the descriptor is implicitly granted if it + has the same SID as the process. If av is zero, then + access to the file is not checked, e.g. for cases + where only the descriptor is affected like seek. */ +static inline int file_has_perm(const struct cred *cred, + const struct file *file, u32 av) +{ + return __file_has_perm(cred, file, av, false); } /* @@ -2942,7 +2965,7 @@ static int selinux_inode_init_security(struct inode *inode, struct inode *dir, { const struct cred_security_struct *crsec = selinux_cred(current_cred()); struct superblock_security_struct *sbsec; - struct xattr *xattr = lsm_get_xattr_slot(xattrs, xattr_count); + struct xattr *xattr; u32 newsid, clen; u16 newsclass; int rc; @@ -2968,6 +2991,7 @@ static int selinux_inode_init_security(struct inode *inode, struct inode *dir, !(sbsec->flags & SBLABEL_MNT)) return -EOPNOTSUPP; + xattr = lsm_get_xattr_slot(xattrs, xattr_count); if (xattr) { rc = security_sid_to_context_force(newsid, &context, &clen); @@ -3184,15 +3208,13 @@ static inline int task_avdcache_search(struct task_security_struct *tsec, * @tsec: the task's security state * @isec: the inode associated with the cache entry * @avd: the AVD to cache - * @audited: the permission audit bitmask to cache * - * Update the AVD cache in @tsec with the @avdc and @audited info associated + * Update the AVD cache in @tsec with the @avd info associated * with @isec. */ static inline void task_avdcache_update(struct task_security_struct *tsec, struct inode_security_struct *isec, - struct av_decision *avd, - u32 audited) + struct av_decision *avd) { int spot; @@ -3204,9 +3226,7 @@ static inline void task_avdcache_update(struct task_security_struct *tsec, spot = (tsec->avdcache.dir_spot + 1) & (TSEC_AVDC_DIR_SIZE - 1); tsec->avdcache.dir_spot = spot; tsec->avdcache.dir[spot].isid = isec->sid; - tsec->avdcache.dir[spot].audited = audited; - tsec->avdcache.dir[spot].allowed = avd->allowed; - tsec->avdcache.dir[spot].permissive = avd->flags & AVD_FLAGS_PERMISSIVE; + tsec->avdcache.dir[spot].avd = *avd; tsec->avdcache.permissive_neveraudit = (avd->flags == (AVD_FLAGS_PERMISSIVE|AVD_FLAGS_NEVERAUDIT)); } @@ -3227,6 +3247,7 @@ static int selinux_inode_permission(struct inode *inode, int requested) struct task_security_struct *tsec; struct inode_security_struct *isec; struct avdc_entry *avdc; + struct av_decision avd, *avdp = &avd; int rc, rc2; u32 audited, denied; @@ -3248,23 +3269,21 @@ static int selinux_inode_permission(struct inode *inode, int requested) rc = task_avdcache_search(tsec, isec, &avdc); if (likely(!rc)) { /* Cache hit. */ - audited = perms & avdc->audited; - denied = perms & ~avdc->allowed; - if (unlikely(denied && enforcing_enabled() && - !avdc->permissive)) + avdp = &avdc->avd; + denied = perms & ~avdp->allowed; + if (unlikely(denied) && enforcing_enabled() && + !(avdp->flags & AVD_FLAGS_PERMISSIVE)) rc = -EACCES; } else { - struct av_decision avd; - /* Cache miss. */ rc = avc_has_perm_noaudit(sid, isec->sid, isec->sclass, - perms, 0, &avd); - audited = avc_audit_required(perms, &avd, rc, - (requested & MAY_ACCESS) ? FILE__AUDIT_ACCESS : 0, - &denied); - task_avdcache_update(tsec, isec, &avd, audited); + perms, 0, avdp); + task_avdcache_update(tsec, isec, avdp); } + audited = avc_audit_required(perms, avdp, rc, + (requested & MAY_ACCESS) ? + FILE__AUDIT_ACCESS : 0, &denied); if (likely(!audited)) return rc; @@ -3825,6 +3844,17 @@ static int selinux_file_alloc_security(struct file *file) return 0; } +static int selinux_backing_file_alloc(struct file *backing_file, + const struct file *user_file) +{ + struct backing_file_security_struct *bfsec; + + bfsec = selinux_backing_file(backing_file); + bfsec->uf_sid = selinux_file(user_file)->sid; + + return 0; +} + /* * Check whether a task has the ioctl permission and cmd * operation to an inode. @@ -3942,42 +3972,55 @@ static int selinux_file_ioctl_compat(struct file *file, unsigned int cmd, static int default_noexec __ro_after_init; -static int file_map_prot_check(struct file *file, unsigned long prot, int shared) +static int __file_map_prot_check(const struct cred *cred, + const struct file *file, unsigned long prot, + bool shared, bool bf_user_file) { - const struct cred *cred = current_cred(); - u32 sid = cred_sid(cred); - int rc = 0; + struct inode *inode = NULL; + bool prot_exec = prot & PROT_EXEC; + bool prot_write = prot & PROT_WRITE; + + if (file) { + if (bf_user_file) + inode = d_inode(backing_file_user_path(file)->dentry); + else + inode = file_inode(file); + } + + if (default_noexec && prot_exec && + (!file || IS_PRIVATE(inode) || (!shared && prot_write))) { + int rc; + u32 sid = cred_sid(cred); - if (default_noexec && - (prot & PROT_EXEC) && (!file || IS_PRIVATE(file_inode(file)) || - (!shared && (prot & PROT_WRITE)))) { /* - * We are making executable an anonymous mapping or a - * private file mapping that will also be writable. - * This has an additional check. + * We are making executable an anonymous mapping or a private + * file mapping that will also be writable. */ - rc = avc_has_perm(sid, sid, SECCLASS_PROCESS, - PROCESS__EXECMEM, NULL); + rc = avc_has_perm(sid, sid, SECCLASS_PROCESS, PROCESS__EXECMEM, + NULL); if (rc) - goto error; + return rc; } if (file) { - /* read access is always possible with a mapping */ + /* "read" always possible, "write" only if shared */ u32 av = FILE__READ; - - /* write access only matters if the mapping is shared */ - if (shared && (prot & PROT_WRITE)) + if (shared && prot_write) av |= FILE__WRITE; - - if (prot & PROT_EXEC) + if (prot_exec) av |= FILE__EXECUTE; - return file_has_perm(cred, file, av); + return __file_has_perm(cred, file, av, bf_user_file); } -error: - return rc; + return 0; +} + +static inline int file_map_prot_check(const struct cred *cred, + const struct file *file, + unsigned long prot, bool shared) +{ + return __file_map_prot_check(cred, file, prot, shared, false); } static int selinux_mmap_addr(unsigned long addr) @@ -3993,36 +4036,80 @@ static int selinux_mmap_addr(unsigned long addr) return rc; } -static int selinux_mmap_file(struct file *file, - unsigned long reqprot __always_unused, - unsigned long prot, unsigned long flags) +static int selinux_mmap_file_common(const struct cred *cred, struct file *file, + unsigned long prot, bool shared) { - struct common_audit_data ad; - int rc; - if (file) { + int rc; + struct common_audit_data ad; + ad.type = LSM_AUDIT_DATA_FILE; ad.u.file = file; - rc = inode_has_perm(current_cred(), file_inode(file), - FILE__MAP, &ad); + rc = inode_has_perm(cred, file_inode(file), FILE__MAP, &ad); if (rc) return rc; } - return file_map_prot_check(file, prot, - (flags & MAP_TYPE) == MAP_SHARED); + return file_map_prot_check(cred, file, prot, shared); +} + +static int selinux_mmap_file(struct file *file, + unsigned long reqprot __always_unused, + unsigned long prot, unsigned long flags) +{ + return selinux_mmap_file_common(current_cred(), file, prot, + (flags & MAP_TYPE) == MAP_SHARED); +} + +/** + * selinux_mmap_backing_file - Check mmap permissions on a backing file + * @vma: memory region + * @backing_file: stacked filesystem backing file + * @user_file: user visible file + * + * This is called after selinux_mmap_file() on stacked filesystems, and it + * is this function's responsibility to verify access to @backing_file and + * setup the SELinux state for possible later use in the mprotect() code path. + * + * By the time this function is called, mmap() access to @user_file has already + * been authorized and @vma->vm_file has been set to point to @backing_file. + * + * Return zero on success, negative values otherwise. + */ +static int selinux_mmap_backing_file(struct vm_area_struct *vma, + struct file *backing_file, + struct file *user_file __always_unused) +{ + unsigned long prot = 0; + + /* translate vma->vm_flags perms into PROT perms */ + if (vma->vm_flags & VM_READ) + prot |= PROT_READ; + if (vma->vm_flags & VM_WRITE) + prot |= PROT_WRITE; + if (vma->vm_flags & VM_EXEC) + prot |= PROT_EXEC; + + return selinux_mmap_file_common(backing_file->f_cred, backing_file, + prot, vma->vm_flags & VM_SHARED); } static int selinux_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot __always_unused, unsigned long prot) { + int rc; const struct cred *cred = current_cred(); u32 sid = cred_sid(cred); + const struct file *file = vma->vm_file; + bool backing_file; + bool shared = vma->vm_flags & VM_SHARED; + + /* check if we need to trigger the "backing files are awful" mode */ + backing_file = file && (file->f_mode & FMODE_BACKING); if (default_noexec && (prot & PROT_EXEC) && !(vma->vm_flags & VM_EXEC)) { - int rc = 0; /* * We don't use the vma_is_initial_heap() helper as it has * a history of problems and is currently broken on systems @@ -4036,11 +4123,15 @@ static int selinux_file_mprotect(struct vm_area_struct *vma, vma->vm_end <= vma->vm_mm->brk) { rc = avc_has_perm(sid, sid, SECCLASS_PROCESS, PROCESS__EXECHEAP, NULL); - } else if (!vma->vm_file && (vma_is_initial_stack(vma) || + if (rc) + return rc; + } else if (!file && (vma_is_initial_stack(vma) || vma_is_stack_for_current(vma))) { rc = avc_has_perm(sid, sid, SECCLASS_PROCESS, PROCESS__EXECSTACK, NULL); - } else if (vma->vm_file && vma->anon_vma) { + if (rc) + return rc; + } else if (file && vma->anon_vma) { /* * We are making executable a file mapping that has * had some COW done. Since pages might have been @@ -4048,13 +4139,29 @@ static int selinux_file_mprotect(struct vm_area_struct *vma, * modified content. This typically should only * occur for text relocations. */ - rc = file_has_perm(cred, vma->vm_file, FILE__EXECMOD); + rc = __file_has_perm(cred, file, FILE__EXECMOD, + backing_file); + if (rc) + return rc; + if (backing_file) { + rc = file_has_perm(file->f_cred, file, + FILE__EXECMOD); + if (rc) + return rc; + } } + } + + rc = __file_map_prot_check(cred, file, prot, shared, backing_file); + if (rc) + return rc; + if (backing_file) { + rc = file_map_prot_check(file->f_cred, file, prot, shared); if (rc) return rc; } - return file_map_prot_check(vma->vm_file, prot, vma->vm_flags&VM_SHARED); + return 0; } static int selinux_file_lock(struct file *file, unsigned int cmd) @@ -4808,7 +4915,7 @@ static bool sock_skip_has_perm(u32 sid) static int sock_has_perm(struct sock *sk, u32 perms) { - struct sk_security_struct *sksec = sk->sk_security; + struct sk_security_struct *sksec = selinux_sock(sk); struct common_audit_data ad; struct lsm_network_audit net; @@ -6115,7 +6222,7 @@ static unsigned int selinux_ip_postroute(void *priv, static int nlmsg_sock_has_extended_perms(struct sock *sk, u32 perms, u16 nlmsg_type) { - struct sk_security_struct *sksec = sk->sk_security; + struct sk_security_struct *sksec = selinux_sock(sk); struct common_audit_data ad; u8 driver; u8 xperm; @@ -7393,6 +7500,7 @@ struct lsm_blob_sizes selinux_blob_sizes __ro_after_init = { .lbs_cred = sizeof(struct cred_security_struct), .lbs_task = sizeof(struct task_security_struct), .lbs_file = sizeof(struct file_security_struct), + .lbs_backing_file = sizeof(struct backing_file_security_struct), .lbs_inode = sizeof(struct inode_security_struct), .lbs_ipc = sizeof(struct ipc_security_struct), .lbs_key = sizeof(struct key_security_struct), @@ -7498,9 +7606,11 @@ static struct security_hook_list selinux_hooks[] __ro_after_init = { LSM_HOOK_INIT(file_permission, selinux_file_permission), LSM_HOOK_INIT(file_alloc_security, selinux_file_alloc_security), + LSM_HOOK_INIT(backing_file_alloc, selinux_backing_file_alloc), LSM_HOOK_INIT(file_ioctl, selinux_file_ioctl), LSM_HOOK_INIT(file_ioctl_compat, selinux_file_ioctl_compat), LSM_HOOK_INIT(mmap_file, selinux_mmap_file), + LSM_HOOK_INIT(mmap_backing_file, selinux_mmap_backing_file), LSM_HOOK_INIT(mmap_addr, selinux_mmap_addr), LSM_HOOK_INIT(file_mprotect, selinux_file_mprotect), LSM_HOOK_INIT(file_lock, selinux_file_lock), diff --git a/security/selinux/include/objsec.h b/security/selinux/include/objsec.h index 5bddd28ea5cb89..3c0a16ec978b01 100644 --- a/security/selinux/include/objsec.h +++ b/security/selinux/include/objsec.h @@ -32,9 +32,7 @@ struct avdc_entry { u32 isid; /* inode SID */ - u32 allowed; /* allowed permission bitmask */ - u32 audited; /* audited permission bitmask */ - bool permissive; /* AVC permissive flag */ + struct av_decision avd; /* av decision */ }; struct cred_security_struct { @@ -88,6 +86,10 @@ struct file_security_struct { u32 pseqno; /* Policy seqno at the time of file open */ }; +struct backing_file_security_struct { + u32 uf_sid; /* associated user file fsec->sid */ +}; + struct superblock_security_struct { u32 sid; /* SID of file system superblock */ u32 def_sid; /* default SID for labeling */ @@ -195,6 +197,13 @@ static inline struct file_security_struct *selinux_file(const struct file *file) return file->f_security + selinux_blob_sizes.lbs_file; } +static inline struct backing_file_security_struct * +selinux_backing_file(const struct file *backing_file) +{ + void *blob = backing_file_security(backing_file); + return blob + selinux_blob_sizes.lbs_backing_file; +} + static inline struct inode_security_struct * selinux_inode(const struct inode *inode) { diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index d1f16d7f684de3..0babb89921816a 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -312,8 +312,6 @@ int security_context_to_sid_default(const char *scontext, u32 scontext_len, int security_context_to_sid_force(const char *scontext, u32 scontext_len, u32 *sid); -int security_get_user_sids(u32 fromsid, const char *username, u32 **sids, u32 *nel); - int security_port_sid(u8 protocol, u16 port, u32 *out_sid); int security_ib_pkey_sid(u64 subnet_prefix, u16 pkey_num, u32 *out_sid); diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c index 3245cc531555b6..35aa25b03852ce 100644 --- a/security/selinux/selinuxfs.c +++ b/security/selinux/selinuxfs.c @@ -76,7 +76,6 @@ struct selinux_fs_info { int *bool_pending_values; struct dentry *class_dir; unsigned long last_class_ino; - bool policy_opened; unsigned long last_ino; struct super_block *sb; }; @@ -272,35 +271,13 @@ static ssize_t sel_write_disable(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char *page; - ssize_t length; - int new_value; - - if (count >= PAGE_SIZE) - return -ENOMEM; - - /* No partial writes. */ - if (*ppos != 0) - return -EINVAL; - - page = memdup_user_nul(buf, count); - if (IS_ERR(page)) - return PTR_ERR(page); - - if (sscanf(page, "%d", &new_value) != 1) { - length = -EINVAL; - goto out; - } - length = count; - - if (new_value) { - pr_err("SELinux: https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-runtime-disable\n"); - pr_err("SELinux: Runtime disable is not supported, use selinux=0 on the kernel cmdline.\n"); - } - -out: - kfree(page); - return length; + /* + * Setting disable is no longer supported, see + * https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-runtime-disable + */ + pr_err_once("SELinux: %s (%d) wrote to disable. This is no longer supported.\n", + current->comm, current->pid); + return count; } static const struct file_operations sel_disable_ops = { @@ -362,44 +339,31 @@ struct policy_load_memory { static int sel_open_policy(struct inode *inode, struct file *filp) { - struct selinux_fs_info *fsi = inode->i_sb->s_fs_info; struct policy_load_memory *plm = NULL; int rc; - BUG_ON(filp->private_data); - - mutex_lock(&selinux_state.policy_mutex); - rc = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__READ_POLICY, NULL); if (rc) - goto err; - - rc = -EBUSY; - if (fsi->policy_opened) - goto err; + return rc; - rc = -ENOMEM; plm = kzalloc_obj(*plm); if (!plm) - goto err; + return -ENOMEM; + mutex_lock(&selinux_state.policy_mutex); rc = security_read_policy(&plm->data, &plm->len); if (rc) goto err; - if ((size_t)i_size_read(inode) != plm->len) { inode_lock(inode); i_size_write(inode, plm->len); inode_unlock(inode); } - - fsi->policy_opened = 1; + mutex_unlock(&selinux_state.policy_mutex); filp->private_data = plm; - mutex_unlock(&selinux_state.policy_mutex); - return 0; err: mutex_unlock(&selinux_state.policy_mutex); @@ -412,13 +376,8 @@ static int sel_open_policy(struct inode *inode, struct file *filp) static int sel_release_policy(struct inode *inode, struct file *filp) { - struct selinux_fs_info *fsi = inode->i_sb->s_fs_info; struct policy_load_memory *plm = filp->private_data; - BUG_ON(!plm); - - fsi->policy_opened = 0; - vfree(plm->data); kfree(plm); @@ -594,34 +553,31 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, if (!count) return -EINVAL; - mutex_lock(&selinux_state.policy_mutex); - length = avc_has_perm(current_sid(), SECINITSID_SECURITY, SECCLASS_SECURITY, SECURITY__LOAD_POLICY, NULL); if (length) - goto out; + return length; data = vmalloc(count); - if (!data) { - length = -ENOMEM; - goto out; - } + if (!data) + return -ENOMEM; if (copy_from_user(data, buf, count) != 0) { length = -EFAULT; goto out; } + mutex_lock(&selinux_state.policy_mutex); length = security_load_policy(data, count, &load_state); if (length) { pr_warn_ratelimited("SELinux: failed to load policy\n"); - goto out; + goto out_unlock; } fsi = file_inode(file)->i_sb->s_fs_info; length = sel_make_policy_nodes(fsi, load_state.policy); if (length) { pr_warn_ratelimited("SELinux: failed to initialize selinuxfs\n"); selinux_policy_cancel(&load_state); - goto out; + goto out_unlock; } selinux_policy_commit(&load_state); @@ -631,8 +587,9 @@ static ssize_t sel_write_load(struct file *file, const char __user *buf, from_kuid(&init_user_ns, audit_get_loginuid(current)), audit_get_sessionid(current)); -out: +out_unlock: mutex_unlock(&selinux_state.policy_mutex); +out: vfree(data); return length; } @@ -689,46 +646,13 @@ static ssize_t sel_read_checkreqprot(struct file *filp, char __user *buf, static ssize_t sel_write_checkreqprot(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char *page; - ssize_t length; - unsigned int new_value; - - length = avc_has_perm(current_sid(), SECINITSID_SECURITY, - SECCLASS_SECURITY, SECURITY__SETCHECKREQPROT, - NULL); - if (length) - return length; - - if (count >= PAGE_SIZE) - return -ENOMEM; - - /* No partial writes. */ - if (*ppos != 0) - return -EINVAL; - - page = memdup_user_nul(buf, count); - if (IS_ERR(page)) - return PTR_ERR(page); - - if (sscanf(page, "%u", &new_value) != 1) { - length = -EINVAL; - goto out; - } - length = count; - - if (new_value) { - char comm[sizeof(current->comm)]; - - strscpy(comm, current->comm); - pr_err("SELinux: %s (%d) set checkreqprot to 1. This is no longer supported.\n", - comm, current->pid); - } - - selinux_ima_measure_state(); - -out: - kfree(page); - return length; + /* + * Setting checkreqprot is no longer supported, see + * https://github.com/SELinuxProject/selinux-kernel/wiki/DEPRECATE-checkreqprot + */ + pr_err_once("SELinux: %s (%d) wrote to checkreqprot. This is no longer supported.\n", + current->comm, current->pid); + return count; } static const struct file_operations sel_checkreqprot_ops = { .read = sel_read_checkreqprot, @@ -1073,69 +997,11 @@ static ssize_t sel_write_relabel(struct file *file, char *buf, size_t size) static ssize_t sel_write_user(struct file *file, char *buf, size_t size) { - char *con = NULL, *user = NULL, *ptr; - u32 sid, *sids = NULL; - ssize_t length; - char *newcon; - int rc; - u32 i, len, nsids; - - pr_warn_ratelimited("SELinux: %s (%d) wrote to /sys/fs/selinux/user!" - " This will not be supported in the future; please update your" - " userspace.\n", current->comm, current->pid); - ssleep(5); - - length = avc_has_perm(current_sid(), SECINITSID_SECURITY, - SECCLASS_SECURITY, SECURITY__COMPUTE_USER, - NULL); - if (length) - goto out; - - length = -ENOMEM; - con = kzalloc(size + 1, GFP_KERNEL); - if (!con) - goto out; - - length = -ENOMEM; - user = kzalloc(size + 1, GFP_KERNEL); - if (!user) - goto out; - - length = -EINVAL; - if (sscanf(buf, "%s %s", con, user) != 2) - goto out; - - length = security_context_str_to_sid(con, &sid, GFP_KERNEL); - if (length) - goto out; - - length = security_get_user_sids(sid, user, &sids, &nsids); - if (length) - goto out; - - length = sprintf(buf, "%u", nsids) + 1; - ptr = buf + length; - for (i = 0; i < nsids; i++) { - rc = security_sid_to_context(sids[i], &newcon, &len); - if (rc) { - length = rc; - goto out; - } - if ((length + len) >= SIMPLE_TRANSACTION_LIMIT) { - kfree(newcon); - length = -ERANGE; - goto out; - } - memcpy(ptr, newcon, len); - kfree(newcon); - ptr += len; - length += len; - } -out: - kfree(sids); - kfree(user); - kfree(con); - return length; + pr_err_once("SELinux: %s (%d) wrote to user. This is no longer supported.\n", + current->comm, current->pid); + buf[0] = '0'; + buf[1] = 0; + return 2; } static ssize_t sel_write_member(struct file *file, char *buf, size_t size) diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index e8e7ccbd1e4485..143021c5e326d8 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -2746,131 +2746,6 @@ int security_node_sid(u16 domain, return rc; } -#define SIDS_NEL 25 - -/** - * security_get_user_sids - Obtain reachable SIDs for a user. - * @fromsid: starting SID - * @username: username - * @sids: array of reachable SIDs for user - * @nel: number of elements in @sids - * - * Generate the set of SIDs for legal security contexts - * for a given user that can be reached by @fromsid. - * Set *@sids to point to a dynamically allocated - * array containing the set of SIDs. Set *@nel to the - * number of elements in the array. - */ - -int security_get_user_sids(u32 fromsid, - const char *username, - u32 **sids, - u32 *nel) -{ - struct selinux_policy *policy; - struct policydb *policydb; - struct sidtab *sidtab; - struct context *fromcon, usercon; - u32 *mysids = NULL, *mysids2, sid; - u32 i, j, mynel, maxnel = SIDS_NEL; - struct user_datum *user; - struct role_datum *role; - struct ebitmap_node *rnode, *tnode; - int rc; - - *sids = NULL; - *nel = 0; - - if (!selinux_initialized()) - return 0; - - mysids = kcalloc(maxnel, sizeof(*mysids), GFP_KERNEL); - if (!mysids) - return -ENOMEM; - -retry: - mynel = 0; - rcu_read_lock(); - policy = rcu_dereference(selinux_state.policy); - policydb = &policy->policydb; - sidtab = policy->sidtab; - - context_init(&usercon); - - rc = -EINVAL; - fromcon = sidtab_search(sidtab, fromsid); - if (!fromcon) - goto out_unlock; - - rc = -EINVAL; - user = symtab_search(&policydb->p_users, username); - if (!user) - goto out_unlock; - - usercon.user = user->value; - - ebitmap_for_each_positive_bit(&user->roles, rnode, i) { - role = policydb->role_val_to_struct[i]; - usercon.role = i + 1; - ebitmap_for_each_positive_bit(&role->types, tnode, j) { - usercon.type = j + 1; - - if (mls_setup_user_range(policydb, fromcon, user, - &usercon)) - continue; - - rc = sidtab_context_to_sid(sidtab, &usercon, &sid); - if (rc == -ESTALE) { - rcu_read_unlock(); - goto retry; - } - if (rc) - goto out_unlock; - if (mynel < maxnel) { - mysids[mynel++] = sid; - } else { - rc = -ENOMEM; - maxnel += SIDS_NEL; - mysids2 = kcalloc(maxnel, sizeof(*mysids2), GFP_ATOMIC); - if (!mysids2) - goto out_unlock; - memcpy(mysids2, mysids, mynel * sizeof(*mysids2)); - kfree(mysids); - mysids = mysids2; - mysids[mynel++] = sid; - } - } - } - rc = 0; -out_unlock: - rcu_read_unlock(); - if (rc || !mynel) { - kfree(mysids); - return rc; - } - - rc = -ENOMEM; - mysids2 = kcalloc(mynel, sizeof(*mysids2), GFP_KERNEL); - if (!mysids2) { - kfree(mysids); - return rc; - } - for (i = 0, j = 0; i < mynel; i++) { - struct av_decision dummy_avd; - rc = avc_has_perm_noaudit(fromsid, mysids[i], - SECCLASS_PROCESS, /* kernel value */ - PROCESS__TRANSITION, AVC_STRICT, - &dummy_avd); - if (!rc) - mysids2[j++] = mysids[i]; - cond_resched(); - } - kfree(mysids); - *sids = mysids2; - *nel = j; - return 0; -} - /** * __security_genfs_sid - Helper to obtain a SID for a file in a filesystem * @policy: policy diff --git a/sound/aoa/soundbus/i2sbus/core.c b/sound/aoa/soundbus/i2sbus/core.c index f974b96e98cd48..833c44c0a9506b 100644 --- a/sound/aoa/soundbus/i2sbus/core.c +++ b/sound/aoa/soundbus/i2sbus/core.c @@ -84,6 +84,7 @@ static void i2sbus_release_dev(struct device *dev) for (i = aoa_resource_i2smmio; i <= aoa_resource_rxdbdma; i++) free_irq(i2sdev->interrupts[i], i2sdev); i2sbus_control_remove_dev(i2sdev->control, i2sdev); + of_node_put(i2sdev->sound.ofdev.dev.of_node); mutex_destroy(&i2sdev->lock); kfree(i2sdev); } @@ -147,7 +148,6 @@ static int i2sbus_get_and_fixup_rsrc(struct device_node *np, int index, } /* Returns 1 if added, 0 for otherwise; don't return a negative value! */ -/* FIXME: look at device node refcounting */ static int i2sbus_add_dev(struct macio_dev *macio, struct i2sbus_control *control, struct device_node *np) @@ -178,8 +178,9 @@ static int i2sbus_add_dev(struct macio_dev *macio, i = 0; for_each_child_of_node(np, child) { if (of_node_name_eq(child, "sound")) { + of_node_put(sound); i++; - sound = child; + sound = of_node_get(child); } } if (i == 1) { @@ -205,6 +206,7 @@ static int i2sbus_add_dev(struct macio_dev *macio, } } } + of_node_put(sound); /* for the time being, until we can handle non-layout-id * things in some fabric, refuse to attach if there is no * layout-id property or we haven't been forced to attach. @@ -219,7 +221,7 @@ static int i2sbus_add_dev(struct macio_dev *macio, mutex_init(&dev->lock); spin_lock_init(&dev->low_lock); dev->sound.ofdev.archdata.dma_mask = macio->ofdev.archdata.dma_mask; - dev->sound.ofdev.dev.of_node = np; + dev->sound.ofdev.dev.of_node = of_node_get(np); dev->sound.ofdev.dev.dma_mask = &dev->sound.ofdev.archdata.dma_mask; dev->sound.ofdev.dev.parent = &macio->ofdev.dev; dev->sound.ofdev.dev.release = i2sbus_release_dev; @@ -327,6 +329,7 @@ static int i2sbus_add_dev(struct macio_dev *macio, for (i=0;i<3;i++) release_and_free_resource(dev->allocated_resource[i]); mutex_destroy(&dev->lock); + of_node_put(dev->sound.ofdev.dev.of_node); kfree(dev); return 0; } @@ -405,6 +408,9 @@ static int i2sbus_resume(struct macio_dev* dev) int err, ret = 0; list_for_each_entry(i2sdev, &control->list, item) { + if (list_empty(&i2sdev->sound.codec_list)) + continue; + /* reset i2s bus format etc. */ i2sbus_pcm_prepare_both(i2sdev); diff --git a/sound/aoa/soundbus/i2sbus/pcm.c b/sound/aoa/soundbus/i2sbus/pcm.c index aff99003d83376..63004ece94f933 100644 --- a/sound/aoa/soundbus/i2sbus/pcm.c +++ b/sound/aoa/soundbus/i2sbus/pcm.c @@ -165,17 +165,16 @@ static int i2sbus_pcm_open(struct i2sbus_dev *i2sdev, int in) * currently in use (if any). */ hw->rate_min = 5512; hw->rate_max = 192000; - /* if the other stream is active, then we can only - * support what it is currently using. - * FIXME: I lied. This comment is wrong. We can support - * anything that works with the same serial format, ie. - * when recording 24 bit sound we can well play 16 bit - * sound at the same time iff using the same transfer mode. + /* If the other stream is already prepared, keep this stream + * on the same duplex format and rate. + * + * i2sbus_pcm_prepare() still programs one shared transport + * configuration for both directions, so mixed duplex formats + * are not supported here. */ if (other->active) { - /* FIXME: is this guaranteed by the alsa api? */ hw->formats &= pcm_format_to_bits(i2sdev->format); - /* see above, restrict rates to the one we already have */ + /* Restrict rates to the one already in use. */ hw->rate_min = i2sdev->rate; hw->rate_max = i2sdev->rate; } @@ -283,6 +282,23 @@ void i2sbus_wait_for_stop_both(struct i2sbus_dev *i2sdev) } #endif +static void i2sbus_pcm_clear_active(struct i2sbus_dev *i2sdev, int in) +{ + struct pcm_info *pi; + + guard(mutex)(&i2sdev->lock); + + get_pcm_info(i2sdev, in, &pi, NULL); + pi->active = 0; +} + +static inline int i2sbus_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, int in) +{ + i2sbus_pcm_clear_active(snd_pcm_substream_chip(substream), in); + return 0; +} + static inline int i2sbus_hw_free(struct snd_pcm_substream *substream, int in) { struct i2sbus_dev *i2sdev = snd_pcm_substream_chip(substream); @@ -291,14 +307,27 @@ static inline int i2sbus_hw_free(struct snd_pcm_substream *substream, int in) get_pcm_info(i2sdev, in, &pi, NULL); if (pi->dbdma_ring.stopping) i2sbus_wait_for_stop(i2sdev, pi); + i2sbus_pcm_clear_active(i2sdev, in); return 0; } +static int i2sbus_playback_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + return i2sbus_hw_params(substream, params, 0); +} + static int i2sbus_playback_hw_free(struct snd_pcm_substream *substream) { return i2sbus_hw_free(substream, 0); } +static int i2sbus_record_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + return i2sbus_hw_params(substream, params, 1); +} + static int i2sbus_record_hw_free(struct snd_pcm_substream *substream) { return i2sbus_hw_free(substream, 1); @@ -335,7 +364,6 @@ static int i2sbus_pcm_prepare(struct i2sbus_dev *i2sdev, int in) return -EINVAL; runtime = pi->substream->runtime; - pi->active = 1; if (other->active && ((i2sdev->format != runtime->format) || (i2sdev->rate != runtime->rate))) @@ -383,6 +411,9 @@ static int i2sbus_pcm_prepare(struct i2sbus_dev *i2sdev, int in) /* set stop command */ command->command = cpu_to_le16(DBDMA_STOP); + cii = list_first_entry(&i2sdev->sound.codec_list, + struct codec_info_item, list); + /* ok, let's set the serial format and stuff */ switch (runtime->format) { /* 16 bit formats */ @@ -390,13 +421,7 @@ static int i2sbus_pcm_prepare(struct i2sbus_dev *i2sdev, int in) case SNDRV_PCM_FORMAT_U16_BE: /* FIXME: if we add different bus factors we need to * do more here!! */ - bi.bus_factor = 0; - list_for_each_entry(cii, &i2sdev->sound.codec_list, list) { - bi.bus_factor = cii->codec->bus_factor; - break; - } - if (!bi.bus_factor) - return -ENODEV; + bi.bus_factor = cii->codec->bus_factor; input_16bit = 1; break; case SNDRV_PCM_FORMAT_S32_BE: @@ -410,10 +435,7 @@ static int i2sbus_pcm_prepare(struct i2sbus_dev *i2sdev, int in) return -EINVAL; } /* we assume all sysclocks are the same! */ - list_for_each_entry(cii, &i2sdev->sound.codec_list, list) { - bi.sysclock_factor = cii->codec->sysclock_factor; - break; - } + bi.sysclock_factor = cii->codec->sysclock_factor; if (clock_and_divisors(bi.sysclock_factor, bi.bus_factor, @@ -450,9 +472,11 @@ static int i2sbus_pcm_prepare(struct i2sbus_dev *i2sdev, int in) /* early exit if already programmed correctly */ /* not locking these is fine since we touch them only in this function */ - if (in_le32(&i2sdev->intfregs->serial_format) == sfr - && in_le32(&i2sdev->intfregs->data_word_sizes) == dws) + if (in_le32(&i2sdev->intfregs->serial_format) == sfr && + in_le32(&i2sdev->intfregs->data_word_sizes) == dws) { + pi->active = 1; return 0; + } /* let's notify the codecs about clocks going away. * For now we only do mastering on the i2s cell... */ @@ -490,6 +514,7 @@ static int i2sbus_pcm_prepare(struct i2sbus_dev *i2sdev, int in) if (cii->codec->switch_clock) cii->codec->switch_clock(cii, CLOCK_SWITCH_SLAVE); + pi->active = 1; return 0; } @@ -734,6 +759,7 @@ static snd_pcm_uframes_t i2sbus_playback_pointer(struct snd_pcm_substream static const struct snd_pcm_ops i2sbus_playback_ops = { .open = i2sbus_playback_open, .close = i2sbus_playback_close, + .hw_params = i2sbus_playback_hw_params, .hw_free = i2sbus_playback_hw_free, .prepare = i2sbus_playback_prepare, .trigger = i2sbus_playback_trigger, @@ -802,6 +828,7 @@ static snd_pcm_uframes_t i2sbus_record_pointer(struct snd_pcm_substream static const struct snd_pcm_ops i2sbus_record_ops = { .open = i2sbus_record_open, .close = i2sbus_record_close, + .hw_params = i2sbus_record_hw_params, .hw_free = i2sbus_record_hw_free, .prepare = i2sbus_record_prepare, .trigger = i2sbus_record_trigger, diff --git a/sound/core/compress_offload.c b/sound/core/compress_offload.c index fdba6e4b25fdc8..5a0308eb4e31de 100644 --- a/sound/core/compress_offload.c +++ b/sound/core/compress_offload.c @@ -41,13 +41,6 @@ #define COMPR_CODEC_CAPS_OVERFLOW #endif -/* TODO: - * - add substream support for multiple devices in case of - * SND_DYNAMIC_MINORS is not used - * - Multiple node representation - * driver should be able to register multiple nodes - */ - struct snd_compr_file { unsigned long caps; struct snd_compr_stream stream; diff --git a/sound/core/control.c b/sound/core/control.c index 934e84e9383806..dc287fadc954d8 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -123,10 +123,12 @@ static int snd_ctl_release(struct inode *inode, struct file *file) scoped_guard(rwsem_write, &card->controls_rwsem) { list_for_each_entry(control, &card->controls, list) for (idx = 0; idx < control->count; idx++) - if (control->vd[idx].owner == ctl) + if (control->vd[idx].owner == ctl) { control->vd[idx].owner = NULL; + if (control->unlock) + control->unlock(control); + } } - snd_fasync_free(ctl->fasync); snd_ctl_empty_read_queue(ctl); put_pid(ctl->pid); @@ -303,6 +305,8 @@ struct snd_kcontrol *snd_ctl_new1(const struct snd_kcontrol_new *ncontrol, kctl->info = ncontrol->info; kctl->get = ncontrol->get; kctl->put = ncontrol->put; + kctl->lock = ncontrol->lock; + kctl->unlock = ncontrol->unlock; kctl->tlv.p = ncontrol->tlv.p; kctl->private_value = ncontrol->private_value; @@ -1359,6 +1363,12 @@ static int snd_ctl_elem_lock(struct snd_ctl_file *file, vd = &kctl->vd[snd_ctl_get_ioff(kctl, &id)]; if (vd->owner) return -EBUSY; + + if (kctl->lock) { + int err = kctl->lock(kctl, file); + if (err < 0) + return err; + } vd->owner = file; return 0; } @@ -1383,6 +1393,8 @@ static int snd_ctl_elem_unlock(struct snd_ctl_file *file, if (vd->owner != file) return -EPERM; vd->owner = NULL; + if (kctl->unlock) + kctl->unlock(kctl); return 0; } @@ -1574,6 +1586,10 @@ static int snd_ctl_elem_init_enum_names(struct user_element *ue) /* check that there are enough valid names */ p = names; for (i = 0; i < ue->info.value.enumerated.items; ++i) { + if (buf_len == 0) { + kvfree(names); + return -EINVAL; + } name_len = strnlen(p, buf_len); if (name_len == 0 || name_len >= 64 || name_len == buf_len) { kvfree(names); diff --git a/sound/core/misc.c b/sound/core/misc.c index 88d9e1f9a6e9ff..833124c8e4fa83 100644 --- a/sound/core/misc.c +++ b/sound/core/misc.c @@ -100,14 +100,18 @@ static LIST_HEAD(snd_fasync_list); static void snd_fasync_work_fn(struct work_struct *work) { struct snd_fasync *fasync; + int signal, poll; spin_lock_irq(&snd_fasync_lock); while (!list_empty(&snd_fasync_list)) { fasync = list_first_entry(&snd_fasync_list, struct snd_fasync, list); list_del_init(&fasync->list); + if (!fasync->on) + continue; + signal = fasync->signal; + poll = fasync->poll; spin_unlock_irq(&snd_fasync_lock); - if (fasync->on) - kill_fasync(&fasync->fasync, fasync->signal, fasync->poll); + kill_fasync(&fasync->fasync, signal, poll); spin_lock_irq(&snd_fasync_lock); } spin_unlock_irq(&snd_fasync_lock); @@ -144,9 +148,11 @@ EXPORT_SYMBOL_GPL(snd_fasync_helper); void snd_kill_fasync(struct snd_fasync *fasync, int signal, int poll) { - if (!fasync || !fasync->on) + if (!fasync) return; guard(spinlock_irqsave)(&snd_fasync_lock); + if (!fasync->on) + return; fasync->signal = signal; fasync->poll = poll; list_move(&fasync->list, &snd_fasync_list); @@ -158,7 +164,12 @@ void snd_fasync_free(struct snd_fasync *fasync) { if (!fasync) return; - fasync->on = 0; + + scoped_guard(spinlock_irq, &snd_fasync_lock) { + fasync->on = 0; + list_del_init(&fasync->list); + } + flush_work(&snd_fasync_work); kfree(fasync); } diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c index d4fd4dfc7fc312..6af26ec2ecfd59 100644 --- a/sound/core/oss/pcm_oss.c +++ b/sound/core/oss/pcm_oss.c @@ -2149,10 +2149,16 @@ static int snd_pcm_oss_get_trigger(struct snd_pcm_oss_file *pcm_oss_file) psubstream = pcm_oss_file->streams[SNDRV_PCM_STREAM_PLAYBACK]; csubstream = pcm_oss_file->streams[SNDRV_PCM_STREAM_CAPTURE]; - if (psubstream && psubstream->runtime && psubstream->runtime->oss.trigger) - result |= PCM_ENABLE_OUTPUT; - if (csubstream && csubstream->runtime && csubstream->runtime->oss.trigger) - result |= PCM_ENABLE_INPUT; + if (psubstream && psubstream->runtime) { + guard(mutex)(&psubstream->runtime->oss.params_lock); + if (psubstream->runtime->oss.trigger) + result |= PCM_ENABLE_OUTPUT; + } + if (csubstream && csubstream->runtime) { + guard(mutex)(&csubstream->runtime->oss.params_lock); + if (csubstream->runtime->oss.trigger) + result |= PCM_ENABLE_INPUT; + } return result; } @@ -2826,6 +2832,17 @@ static int snd_pcm_oss_capture_ready(struct snd_pcm_substream *substream) runtime->oss.period_frames; } +static bool need_input_retrigger(struct snd_pcm_runtime *runtime) +{ + bool ret; + + guard(mutex)(&runtime->oss.params_lock); + ret = runtime->oss.trigger; + if (ret) + runtime->oss.trigger = 0; + return ret; +} + static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait) { struct snd_pcm_oss_file *pcm_oss_file; @@ -2858,11 +2875,11 @@ static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait) snd_pcm_oss_capture_ready(csubstream)) mask |= EPOLLIN | EPOLLRDNORM; } - if (ostate != SNDRV_PCM_STATE_RUNNING && runtime->oss.trigger) { + if (ostate != SNDRV_PCM_STATE_RUNNING && + need_input_retrigger(runtime)) { struct snd_pcm_oss_file ofile; memset(&ofile, 0, sizeof(ofile)); ofile.streams[SNDRV_PCM_STREAM_CAPTURE] = pcm_oss_file->streams[SNDRV_PCM_STREAM_CAPTURE]; - runtime->oss.trigger = 0; snd_pcm_oss_set_trigger(&ofile, PCM_ENABLE_INPUT); } } diff --git a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c index 1306b04be171ae..554a4df9db7557 100644 --- a/sound/core/pcm_dmaengine.c +++ b/sound/core/pcm_dmaengine.c @@ -22,6 +22,8 @@ struct dmaengine_pcm_runtime_data { struct dma_chan *dma_chan; dma_cookie_t cookie; + struct work_struct complete_wq; /* for nonatomic PCM */ + struct snd_pcm_substream *substream; unsigned int pos; }; @@ -147,6 +149,21 @@ static void dmaengine_pcm_dma_complete(void *arg) snd_pcm_period_elapsed(substream); } +static void dmaengine_pcm_dma_complete_nonatomic(struct work_struct *wq) +{ + struct dmaengine_pcm_runtime_data *prtd = \ + container_of(wq, struct dmaengine_pcm_runtime_data, complete_wq); + struct snd_pcm_substream *substream = prtd->substream; + dmaengine_pcm_dma_complete(substream); +} + +static void dmaengine_pcm_dma_complete_nonatomic_callback(void *arg) +{ + struct snd_pcm_substream *substream = arg; + struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); + schedule_work(&prtd->complete_wq); +} + static int dmaengine_pcm_prepare_and_submit(struct snd_pcm_substream *substream) { struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream); @@ -169,7 +186,11 @@ static int dmaengine_pcm_prepare_and_submit(struct snd_pcm_substream *substream) if (!desc) return -ENOMEM; - desc->callback = dmaengine_pcm_dma_complete; + if (substream->pcm->nonatomic) + desc->callback = dmaengine_pcm_dma_complete_nonatomic_callback; + else + desc->callback = dmaengine_pcm_dma_complete; + desc->callback_param = substream; prtd->cookie = dmaengine_submit(desc); @@ -322,6 +343,10 @@ int snd_dmaengine_pcm_open(struct snd_pcm_substream *substream, if (!prtd) return -ENOMEM; + if (substream->pcm->nonatomic) + INIT_WORK(&prtd->complete_wq, dmaengine_pcm_dma_complete_nonatomic); + + prtd->substream = substream; prtd->dma_chan = chan; substream->runtime->private_data = prtd; @@ -355,7 +380,14 @@ static void __snd_dmaengine_pcm_close(struct snd_pcm_substream *substream, if (status == DMA_PAUSED) dmaengine_terminate_async(prtd->dma_chan); + /* + * The PCM might have been closed while suspended, which would + * skip the STOP trigger. Make sure we terminate. + */ + dmaengine_terminate_async(prtd->dma_chan); dmaengine_synchronize(prtd->dma_chan); + if (substream->pcm->nonatomic) + flush_work(&prtd->complete_wq); if (release_channel) dma_release_channel(prtd->dma_chan); kfree(prtd); diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c index 09c421cd9319e7..ae80091e2ff825 100644 --- a/sound/core/pcm_lib.c +++ b/sound/core/pcm_lib.c @@ -1149,6 +1149,43 @@ static int snd_interval_step(struct snd_interval *i, unsigned int step) return changed; } +/** + * snd_interval_rate_bits - refine the rate interval from a rate bitmask + * @i: the rate interval to refine + * @mask: the rate bitmask + * + * Refines the interval value, assumed to be the sample rate, according to + * a bitmask of available rates (an ORed combination of SNDRV_PCM_RATE_*). + * + * Return: Positive if the value is changed, zero if it's not changed, or a + * negative error code. + */ +int snd_interval_rate_bits(struct snd_interval *i, unsigned int mask) +{ + unsigned int k; + struct snd_interval mask_range; + + if (!mask) + return -EINVAL; + + snd_interval_any(&mask_range); + mask_range.min = UINT_MAX; + mask_range.max = 0; + for (k = 0; k < snd_pcm_known_rates.count; k++) { + unsigned int rate = snd_pcm_known_rates.list[k]; + if (!(mask & (1 << k))) + continue; + + if (rate > mask_range.max) + mask_range.max = rate; + + if (rate < mask_range.min) + mask_range.min = rate; + } + return snd_interval_refine(i, &mask_range); +} +EXPORT_SYMBOL(snd_interval_rate_bits); + /* Info constraints helpers */ /** diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 5a64453da7283a..28f2304fd3ccc1 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -944,8 +944,9 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream) goto unlock; result = do_hw_free(substream); snd_pcm_set_state(substream, SNDRV_PCM_STATE_OPEN); - cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); - unlock: + if (cpu_latency_qos_request_active(&substream->latency_pm_qos_req)) + cpu_latency_qos_remove_request(&substream->latency_pm_qos_req); +unlock: snd_pcm_buffer_access_unlock(runtime); return result; } @@ -2470,6 +2471,7 @@ const struct snd_pcm_hw_constraint_list snd_pcm_known_rates = { .count = ARRAY_SIZE(rates), .list = rates, }; +EXPORT_SYMBOL_GPL(snd_pcm_known_rates); static int snd_pcm_hw_rule_rate(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) diff --git a/sound/core/seq/oss/seq_oss_rw.c b/sound/core/seq/oss/seq_oss_rw.c index 8a142fd54a1907..307ef98c44c7b5 100644 --- a/sound/core/seq/oss/seq_oss_rw.c +++ b/sound/core/seq/oss/seq_oss_rw.c @@ -101,9 +101,9 @@ snd_seq_oss_write(struct seq_oss_devinfo *dp, const char __user *buf, int count, break; } fmt = (*(unsigned short *)rec.c) & 0xffff; - /* FIXME the return value isn't correct */ - return snd_seq_oss_synth_load_patch(dp, rec.s.dev, - fmt, buf, 0, count); + err = snd_seq_oss_synth_load_patch(dp, rec.s.dev, + fmt, buf, 0, count); + return err < 0 ? err : count; } if (ev_is_long(&rec)) { /* extended code */ diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 75a7a2af9d8c96..5719637575a911 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -1253,7 +1253,7 @@ static int snd_seq_ioctl_set_client_info(struct snd_seq_client *client, if (client->user_pversion >= SNDRV_PROTOCOL_VERSION(1, 0, 3)) client->midi_version = client_info->midi_version; memcpy(client->event_filter, client_info->event_filter, 32); - client->group_filter = client_info->group_filter; + client->group_filter = client_info->group_filter & SND_SEQ_GROUP_FILTER_MASK; /* notify the change */ snd_seq_system_client_ev_client_change(client->number); diff --git a/sound/core/seq/seq_clientmgr.h b/sound/core/seq/seq_clientmgr.h index ece02c58db702d..feea8bb7d9870a 100644 --- a/sound/core/seq/seq_clientmgr.h +++ b/sound/core/seq/seq_clientmgr.h @@ -14,6 +14,9 @@ /* client manager */ +#define SND_SEQ_GROUP_FILTER_MASK GENMASK(SNDRV_UMP_MAX_GROUPS, 0) +#define SND_SEQ_GROUP_FILTER_GROUPS GENMASK(SNDRV_UMP_MAX_GROUPS, 1) + struct snd_seq_user_client { struct file *file; /* file struct of client */ /* ... */ @@ -40,7 +43,7 @@ struct snd_seq_client { int number; /* client number */ unsigned int filter; /* filter flags */ DECLARE_BITMAP(event_filter, 256); - unsigned short group_filter; + unsigned int group_filter; snd_use_lock_t use_lock; int event_lost; /* ports */ diff --git a/sound/core/seq/seq_ump_client.c b/sound/core/seq/seq_ump_client.c index fdc76f23e03f48..9079ccfdc8666d 100644 --- a/sound/core/seq/seq_ump_client.c +++ b/sound/core/seq/seq_ump_client.c @@ -369,7 +369,7 @@ static void setup_client_group_filter(struct seq_ump_client *client) cptr = snd_seq_kernel_client_get(client->seq_client); if (!cptr) return; - filter = ~(1U << 0); /* always allow groupless messages */ + filter = SND_SEQ_GROUP_FILTER_GROUPS; /* always allow groupless messages */ for (p = 0; p < SNDRV_UMP_MAX_GROUPS; p++) { if (client->ump->groups[p].active) filter &= ~(1U << (p + 1)); diff --git a/sound/core/sound.c b/sound/core/sound.c index 93436db24710b6..8d05fe0d263b23 100644 --- a/sound/core/sound.c +++ b/sound/core/sound.c @@ -216,9 +216,16 @@ static int snd_find_free_minor(int type, struct snd_card *card, int dev) case SNDRV_DEVICE_TYPE_RAWMIDI: case SNDRV_DEVICE_TYPE_PCM_PLAYBACK: case SNDRV_DEVICE_TYPE_PCM_CAPTURE: + if (snd_BUG_ON(!card)) + return -EINVAL; + minor = SNDRV_MINOR(card->number, type + dev); + break; case SNDRV_DEVICE_TYPE_COMPRESS: if (snd_BUG_ON(!card)) return -EINVAL; + if (dev < 0 || + dev >= SNDRV_MINOR_HWDEP - SNDRV_MINOR_COMPRESS) + return -EINVAL; minor = SNDRV_MINOR(card->number, type + dev); break; default: diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index aa0d2fcb1a180c..a37a1695f51c7d 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -99,6 +99,9 @@ struct loopback_ops { struct loopback_cable { spinlock_t lock; struct loopback_pcm *streams[2]; + /* in-flight peer stops running outside cable->lock */ + atomic_t stop_count; + wait_queue_head_t stop_wait; struct snd_pcm_hardware hw; /* flags */ unsigned int valid; @@ -366,8 +369,11 @@ static int loopback_check_format(struct loopback_cable *cable, int stream) return 0; if (stream == SNDRV_PCM_STREAM_CAPTURE) return -EIO; - else if (cruntime->state == SNDRV_PCM_STATE_RUNNING) + else if (cruntime->state == SNDRV_PCM_STATE_RUNNING) { + /* close must not free the peer runtime below */ + atomic_inc(&cable->stop_count); stop_capture = true; + } } setup = get_setup(dpcm_play); @@ -396,8 +402,11 @@ static int loopback_check_format(struct loopback_cable *cable, int stream) } } - if (stop_capture) + if (stop_capture) { snd_pcm_stop(dpcm_capt->substream, SNDRV_PCM_STATE_DRAINING); + if (atomic_dec_and_test(&cable->stop_count)) + wake_up(&cable->stop_wait); + } return 0; } @@ -1049,23 +1058,29 @@ static void free_cable(struct snd_pcm_substream *substream) struct loopback *loopback = substream->private_data; int dev = get_cable_index(substream); struct loopback_cable *cable; + struct loopback_pcm *dpcm; + bool other_alive; cable = loopback->cables[substream->number][dev]; if (!cable) return; - if (cable->streams[!substream->stream]) { - /* other stream is still alive */ - guard(spinlock_irq)(&cable->lock); - cable->streams[substream->stream] = NULL; - } else { - struct loopback_pcm *dpcm = substream->runtime->private_data; - if (cable->ops && cable->ops->close_cable && dpcm) - cable->ops->close_cable(dpcm); - /* free the cable */ - loopback->cables[substream->number][dev] = NULL; - kfree(cable); + scoped_guard(spinlock_irq, &cable->lock) { + cable->streams[substream->stream] = NULL; + other_alive = cable->streams[!substream->stream]; } + + /* Pair with the stop_count increment in loopback_check_format(). */ + wait_event(cable->stop_wait, !atomic_read(&cable->stop_count)); + if (other_alive) + return; + + dpcm = substream->runtime->private_data; + if (cable->ops && cable->ops->close_cable && dpcm) + cable->ops->close_cable(dpcm); + /* free the cable */ + loopback->cables[substream->number][dev] = NULL; + kfree(cable); } static int loopback_jiffies_timer_open(struct loopback_pcm *dpcm) @@ -1260,6 +1275,8 @@ static int loopback_open(struct snd_pcm_substream *substream) goto unlock; } spin_lock_init(&cable->lock); + atomic_set(&cable->stop_count, 0); + init_waitqueue_head(&cable->stop_wait); cable->hw = loopback_pcm_hardware; if (loopback->timer_source) cable->ops = &loopback_snd_timer_ops; diff --git a/sound/drivers/pcmtest.c b/sound/drivers/pcmtest.c index 768bb698adfb42..fe31ff1e5b3c3c 100644 --- a/sound/drivers/pcmtest.c +++ b/sound/drivers/pcmtest.c @@ -754,13 +754,24 @@ static int __init mod_init(void) err = init_debug_files(buf_allocated); if (err) - return err; + goto err_free_patterns; err = platform_device_register(&pcmtst_pdev); - if (err) - return err; + if (err) { + platform_device_put(&pcmtst_pdev); + goto err_clear_debug; + } err = platform_driver_register(&pcmtst_pdrv); - if (err) + if (err) { platform_device_unregister(&pcmtst_pdev); + goto err_clear_debug; + } + + return 0; + +err_clear_debug: + clear_debug_files(); +err_free_patterns: + free_pattern_buffers(); return err; } diff --git a/sound/firewire/fireworks/fireworks_command.c b/sound/firewire/fireworks/fireworks_command.c index 2b595ee0bc357b..05550f36fac551 100644 --- a/sound/firewire/fireworks/fireworks_command.c +++ b/sound/firewire/fireworks/fireworks_command.c @@ -151,10 +151,13 @@ efw_transaction(struct snd_efw *efw, unsigned int category, (be32_to_cpu(header->category) != category) || (be32_to_cpu(header->command) != command) || (be32_to_cpu(header->status) != EFR_STATUS_OK)) { + u32 st = be32_to_cpu(header->status); + dev_err(&efw->unit->device, "EFW command failed [%u/%u]: %s\n", be32_to_cpu(header->category), be32_to_cpu(header->command), - efr_status_names[be32_to_cpu(header->status)]); + st < ARRAY_SIZE(efr_status_names) ? + efr_status_names[st] : "unknown"); err = -EIO; goto end; } diff --git a/sound/firewire/tascam/tascam-hwdep.c b/sound/firewire/tascam/tascam-hwdep.c index 867b4ea1096e13..6270263e7bf48b 100644 --- a/sound/firewire/tascam/tascam-hwdep.c +++ b/sound/firewire/tascam/tascam-hwdep.c @@ -73,6 +73,7 @@ static long tscm_hwdep_read_queue(struct snd_tscm *tscm, char __user *buf, length = rounddown(remained, sizeof(*entries)); if (length == 0) break; + tail_pos = head_pos + length / sizeof(*entries); spin_unlock_irq(&tscm->lock); if (copy_to_user(pos, &entries[head_pos], length)) diff --git a/sound/hda/codecs/cmedia.c b/sound/hda/codecs/cmedia.c index e6e12c01339f60..88dd80d987d414 100644 --- a/sound/hda/codecs/cmedia.c +++ b/sound/hda/codecs/cmedia.c @@ -39,13 +39,6 @@ static int cmedia_probe(struct hda_codec *codec, const struct hda_device_id *id) spec->out_vol_mask = (1ULL << 0x10); } - err = snd_hda_parse_pin_defcfg(codec, cfg, NULL, 0); - if (err < 0) - goto error; - err = snd_hda_gen_parse_auto_config(codec, cfg); - if (err < 0) - goto error; - err = snd_hda_parse_pin_defcfg(codec, cfg, NULL, 0); if (err < 0) goto error; diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index aa726eb323ebe2..a7689f9ef9675b 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -1183,6 +1183,7 @@ static void add_cx5051_fake_mutes(struct hda_codec *codec) static int cx_probe(struct hda_codec *codec, const struct hda_device_id *id) { struct conexant_spec *spec; + struct hda_jack_callback *callback; int err; codec_info(codec, "%s: BIOS auto-probing.\n", codec->core.chip_name); @@ -1198,7 +1199,12 @@ static int cx_probe(struct hda_codec *codec, const struct hda_device_id *id) case 0x14f11f86: case 0x14f11f87: spec->is_cx11880_sn6140 = true; - snd_hda_jack_detect_enable_callback(codec, 0x19, cx_update_headset_mic_vref); + callback = snd_hda_jack_detect_enable_callback(codec, 0x19, + cx_update_headset_mic_vref); + if (IS_ERR(callback)) { + err = PTR_ERR(callback); + goto error; + } break; } diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 45f9d648738857..8f7d8337b4bc6f 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -2296,9 +2296,9 @@ static void alc_fixup_headset_mode_alc255_no_hp_mic(struct hda_codec *codec, struct alc_spec *spec = codec->spec; spec->parse_flags |= HDA_PINCFG_HEADSET_MIC; alc255_set_default_jack_type(codec); - } - else + } else { alc_fixup_headset_mode(codec, fix, action); + } } static void alc288_update_headset_jack_cb(struct hda_codec *codec, @@ -3397,6 +3397,19 @@ static void alc256_fixup_mic_no_presence_and_resume(struct hda_codec *codec, } } +static void alc256_fixup_xiaomi_pro15_resume(struct hda_codec *codec, + const struct hda_fixup *fix, + int action) +{ + /* + * On the Xiaomi Mi Laptop Pro 15 (TM1905, SSID 1d72:1905) the ALC256 + * codec sets coefficient 0x10 bit 9 to 1 after S3 resume, silencing + * the internal speaker. Bluetooth and HDMI audio are unaffected. + * Clear the bit so the speaker keeps working across suspend cycles. + */ + alc_update_coef_idx(codec, 0x10, 1<<9, 0); +} + static void alc256_decrease_headphone_amp_val(struct hda_codec *codec, const struct hda_fixup *fix, int action) { @@ -3661,22 +3674,11 @@ static void alc287_alc1318_playback_pcm_hook(struct hda_pcm_stream *hinfo, struct snd_pcm_substream *substream, int action) { - static const struct coef_fw dis_coefs[] = { - WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), - WRITE_COEF(0x28, 0x0004), WRITE_COEF(0x29, 0xb023), - }; /* Disable AMP silence detection */ - static const struct coef_fw en_coefs[] = { - WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), - WRITE_COEF(0x28, 0x0084), WRITE_COEF(0x29, 0xb023), - }; /* Enable AMP silence detection */ - switch (action) { case HDA_GEN_PCM_ACT_OPEN: - alc_process_coef_fw(codec, dis_coefs); alc_write_coefex_idx(codec, 0x5a, 0x00, 0x954f); /* write gpio3 to high */ break; case HDA_GEN_PCM_ACT_CLOSE: - alc_process_coef_fw(codec, en_coefs); alc_write_coefex_idx(codec, 0x5a, 0x00, 0x554f); /* write gpio3 as default value */ break; } @@ -3699,10 +3701,15 @@ static void alc287_fixup_lenovo_thinkpad_with_alc1318(struct hda_codec *codec, WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC301), WRITE_COEF(0x28, 0x0001), WRITE_COEF(0x29, 0xb023), }; + static const struct coef_fw dis_coefs[] = { + WRITE_COEF(0x24, 0x0013), WRITE_COEF(0x25, 0x0000), WRITE_COEF(0x26, 0xC203), + WRITE_COEF(0x28, 0x0004), WRITE_COEF(0x29, 0xb023), + }; /* Disable AMP silence detection */ if (action != HDA_FIXUP_ACT_PRE_PROBE) return; alc_update_coef_idx(codec, 0x10, 1<<11, 1<<11); + alc_process_coef_fw(codec, dis_coefs); alc_process_coef_fw(codec, coefs); spec->power_hook = alc287_s4_power_gpio3_default; spec->gen.pcm_playback_hook = alc287_alc1318_playback_pcm_hook; @@ -4054,6 +4061,7 @@ enum { ALC256_FIXUP_SYSTEM76_MIC_NO_PRESENCE, ALC233_FIXUP_NO_AUDIO_JACK, ALC256_FIXUP_MIC_NO_PRESENCE_AND_RESUME, + ALC256_FIXUP_XIAOMI_PRO15_RESUME, ALC285_FIXUP_LEGION_Y9000X_SPEAKERS, ALC285_FIXUP_LEGION_Y9000X_AUTOMUTE, ALC287_FIXUP_LEGION_16ACHG6, @@ -6241,6 +6249,10 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC269_FIXUP_HEADSET_MODE_NO_HP_MIC }, + [ALC256_FIXUP_XIAOMI_PRO15_RESUME] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc256_fixup_xiaomi_pro15_resume, + }, [ALC287_FIXUP_LEGION_16ACHG6] = { .type = HDA_FIXUP_FUNC, .v.func = alc287_fixup_legion_16achg6_speakers, @@ -6639,10 +6651,10 @@ static const struct hda_fixup alc269_fixups[] = { .type = HDA_FIXUP_FUNC, .v.func = alc288_fixup_surface_swap_dacs, }, - [ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY] = { - .type = HDA_FIXUP_FUNC, - .v.func = alc233_fixup_lenovo_gpio2_mic_hotkey, - }, + [ALC233_FIXUP_LENOVO_GPIO2_MIC_HOTKEY] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc233_fixup_lenovo_gpio2_mic_hotkey, + }, [ALC245_FIXUP_BASS_HP_DAC] = { .type = HDA_FIXUP_FUNC, /* Borrow the DAC routing selected for those Thinkpads */ @@ -7134,6 +7146,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8ca4, "HP ZBook Fury", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8ca7, "HP ZBook Fury", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8caf, "HP Elite mt645 G8 Mobile Thin Client", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), + SND_PCI_QUIRK(0x103c, 0x8cbc, "HP Pavilion Laptop 16-ag0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x8cbd, "HP Pavilion Aero Laptop 13-bg0xxx", ALC245_FIXUP_HP_X360_MUTE_LEDS), SND_PCI_QUIRK(0x103c, 0x8cdd, "HP Spectre", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX), SND_PCI_QUIRK(0x103c, 0x8cde, "HP OmniBook Ultra Flip Laptop 14t", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX), @@ -7153,6 +7166,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8d90, "HP EliteBook 16 G12", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8d91, "HP ZBook Firefly 14 G12", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8d92, "HP ZBook Firefly 16 G12", ALC285_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8dcd, "HP Victus 15-fa2xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8d9b, "HP 17 Turbine OmniBook 7 UMA", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8d9c, "HP 17 Turbine OmniBook 7 DIS", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8d9d, "HP 17 Turbine OmniBook X UMA", ALC287_FIXUP_CS35L41_I2C_2), @@ -7434,6 +7448,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc870, "Samsung Galaxy Book2 Pro (NP950XED)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc872, "Samsung Galaxy Book2 Pro (NP950XEE)", ALC298_FIXUP_SAMSUNG_AMP_V2_2_AMPS), SND_PCI_QUIRK(0x144d, 0xc886, "Samsung Galaxy Book3 Pro (NP964XFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x144d, 0xc902, "Samsung Galaxy Book5 360 (NP750QHA)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc1ca, "Samsung Galaxy Book3 Pro 360 (NP960QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1cb, "Samsung Galaxy Book3 Pro 360 (NP965QFG)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x144d, 0xc1cc, "Samsung Galaxy Book3 Ultra (NT960XFH)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), @@ -7605,6 +7620,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3801, "Lenovo Yoga9 14IAP7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), HDA_CODEC_QUIRK(0x17aa, 0x3802, "DuetITL 2021", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3802, "Lenovo Yoga Pro 9 14IRP8", ALC287_FIXUP_TAS2781_I2C), + SND_PCI_QUIRK(0x17aa, 0x3811, "Legion S7 15IMH05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3813, "Legion 7i 15IMHG05", ALC287_FIXUP_LEGION_15IMHG05_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3818, "Lenovo C940 / Yoga Duet 7", ALC298_FIXUP_LENOVO_C940_DUET7), SND_PCI_QUIRK(0x17aa, 0x3819, "Lenovo 13s Gen2 ITL", ALC287_FIXUP_13S_GEN2_SPEAKERS), @@ -7749,6 +7765,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1d72, 0x1901, "RedmiBook 14", ALC256_FIXUP_ASUS_HEADSET_MIC), + SND_PCI_QUIRK(0x1d72, 0x1905, "Xiaomi Mi Laptop Pro 15", ALC256_FIXUP_XIAOMI_PRO15_RESUME), SND_PCI_QUIRK(0x1d72, 0x1945, "Redmi G", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1947, "RedmiBook Air", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1e39, 0xca14, "MEDION NM14LNL", ALC233_FIXUP_MEDION_MTL_SPK), diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c index 1ace4beef50857..4c8d01799931c8 100644 --- a/sound/hda/codecs/side-codecs/cs35l56_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c @@ -180,11 +180,15 @@ static int cs35l56_hda_mixer_get(struct snd_kcontrol *kcontrol, { struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol); unsigned int reg_val; - int i; + int i, ret; cs35l56_hda_wait_dsp_ready(cs35l56); - regmap_read(cs35l56->base.regmap, kcontrol->private_value, ®_val); + ret = regmap_read(cs35l56->base.regmap, kcontrol->private_value, + ®_val); + if (ret) + return ret; + reg_val &= CS35L56_ASP_TXn_SRC_MASK; for (i = 0; i < CS35L56_NUM_INPUT_SRC; ++i) { @@ -203,15 +207,20 @@ static int cs35l56_hda_mixer_put(struct snd_kcontrol *kcontrol, struct cs35l56_hda *cs35l56 = snd_kcontrol_chip(kcontrol); unsigned int item = ucontrol->value.enumerated.item[0]; bool changed; + int ret; if (item >= CS35L56_NUM_INPUT_SRC) return -EINVAL; cs35l56_hda_wait_dsp_ready(cs35l56); - regmap_update_bits_check(cs35l56->base.regmap, kcontrol->private_value, - CS35L56_INPUT_MASK, cs35l56_tx_input_values[item], - &changed); + ret = regmap_update_bits_check(cs35l56->base.regmap, + kcontrol->private_value, + CS35L56_INPUT_MASK, + cs35l56_tx_input_values[item], + &changed); + if (ret) + return ret; return changed; } @@ -967,6 +976,7 @@ static int cs35l56_hda_system_resume(struct device *dev) static int cs35l56_hda_fixup_yoga9(struct cs35l56_hda *cs35l56, int *bus_addr) { /* The cirrus,dev-index property has the wrong values */ + cs35l56->num_amps = 2; switch (*bus_addr) { case 0x30: cs35l56->index = 1; @@ -1016,7 +1026,6 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id) char hid_string[8]; struct acpi_device *adev; const char *property, *sub; - size_t nval; int i, ret; /* @@ -1052,13 +1061,14 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id) ret = -EINVAL; goto err; } - nval = ret; + cs35l56->num_amps = ret; - ret = device_property_read_u32_array(cs35l56->base.dev, property, values, nval); + ret = device_property_read_u32_array(cs35l56->base.dev, property, values, + cs35l56->num_amps); if (ret) goto err; - for (i = 0; i < nval; i++) { + for (i = 0; i < cs35l56->num_amps; i++) { if (values[i] == id) { cs35l56->index = i; break; @@ -1081,7 +1091,8 @@ static int cs35l56_hda_read_acpi(struct cs35l56_hda *cs35l56, int hid, int id) "Read ACPI _SUB failed(%ld): fallback to generic firmware\n", PTR_ERR(sub)); } else { - ret = cirrus_scodec_get_speaker_id(cs35l56->base.dev, cs35l56->index, nval, -1); + ret = cirrus_scodec_get_speaker_id(cs35l56->base.dev, cs35l56->index, + cs35l56->num_amps, -1); if (ret == -ENOENT) { cs35l56->system_name = sub; } else if (ret >= 0) { diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.h b/sound/hda/codecs/side-codecs/cs35l56_hda.h index cb4b5e7356a358..3705af7c186b37 100644 --- a/sound/hda/codecs/side-codecs/cs35l56_hda.h +++ b/sound/hda/codecs/side-codecs/cs35l56_hda.h @@ -26,6 +26,7 @@ struct cs35l56_hda { struct work_struct dsp_work; int index; + int num_amps; const char *system_name; const char *amp_name; diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index f860e0eb7602a7..6c736b17c98317 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -132,10 +132,18 @@ static int tasdevice_spi_dev_update_bits(struct tasdevice_priv *tas_priv, int ret, val; /* - * In our TAS2781 SPI mode, read/write was masked in last bit of - * address, it cause regmap_update_bits() not work as expected. + * In TAS2781 SPI mode, when accessing non-book-zero or page numbers + * greater than 1 in book 0, an additional byte must be read. The + * first byte in such cases is a dummy byte and should be ignored. */ - ret = tasdevice_dev_read(tas_priv, chn, reg, &val); + if ((TASDEVICE_BOOK_ID(reg) > 0) || (TASDEVICE_PAGE_ID(reg) > 1)) { + unsigned char buf[2]; + + ret = tasdevice_dev_bulk_read(tas_priv, chn, reg, buf, 2); + val = buf[1]; + } else { + ret = tasdevice_dev_read(tas_priv, chn, reg, &val); + } if (ret < 0) { dev_err(tas_priv->dev, "%s, E=%d\n", __func__, ret); return ret; diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c index 09b1329bb8f351..5123df32ad89f5 100644 --- a/sound/hda/common/codec.c +++ b/sound/hda/common/codec.c @@ -2529,7 +2529,10 @@ EXPORT_SYMBOL_GPL(snd_hda_spdif_ctls_assign); static int spdif_share_sw_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { - struct hda_multi_out *mout = snd_kcontrol_chip(kcontrol); + struct hda_codec *codec = snd_kcontrol_chip(kcontrol); + struct hda_multi_out *mout = (void *)kcontrol->private_value; + + guard(mutex)(&codec->spdif_mutex); ucontrol->value.integer.value[0] = mout->share_spdif; return 0; } @@ -2537,9 +2540,15 @@ static int spdif_share_sw_get(struct snd_kcontrol *kcontrol, static int spdif_share_sw_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { - struct hda_multi_out *mout = snd_kcontrol_chip(kcontrol); - mout->share_spdif = !!ucontrol->value.integer.value[0]; - return 0; + struct hda_codec *codec = snd_kcontrol_chip(kcontrol); + struct hda_multi_out *mout = (void *)kcontrol->private_value; + bool val = !!ucontrol->value.integer.value[0]; + int change; + + guard(mutex)(&codec->spdif_mutex); + change = mout->share_spdif != val; + mout->share_spdif = val; + return change; } static const struct snd_kcontrol_new spdif_share_sw = { @@ -2550,6 +2559,14 @@ static const struct snd_kcontrol_new spdif_share_sw = { .put = spdif_share_sw_put, }; +static void notify_spdif_share_sw(struct hda_codec *codec, + struct hda_multi_out *mout) +{ + if (mout->share_spdif_kctl) + snd_ctl_notify_one(codec->card, SNDRV_CTL_EVENT_MASK_VALUE, + mout->share_spdif_kctl, 0); +} + /** * snd_hda_create_spdif_share_sw - create Default PCM switch * @codec: the HDA codec @@ -2559,15 +2576,24 @@ int snd_hda_create_spdif_share_sw(struct hda_codec *codec, struct hda_multi_out *mout) { struct snd_kcontrol *kctl; + int err; if (!mout->dig_out_nid) return 0; - kctl = snd_ctl_new1(&spdif_share_sw, mout); + kctl = snd_ctl_new1(&spdif_share_sw, codec); if (!kctl) return -ENOMEM; - /* ATTENTION: here mout is passed as private_data, instead of codec */ - return snd_hda_ctl_add(codec, mout->dig_out_nid, kctl); + /* snd_ctl_new1() stores @codec in private_data; stash @mout in + * private_value for the share-switch callbacks and cache the + * assigned control for forced-disable notifications. + */ + kctl->private_value = (unsigned long)mout; + err = snd_hda_ctl_add(codec, mout->dig_out_nid, kctl); + if (err < 0) + return err; + mout->share_spdif_kctl = kctl; + return 0; } EXPORT_SYMBOL_GPL(snd_hda_create_spdif_share_sw); @@ -3701,6 +3727,8 @@ int snd_hda_multi_out_analog_open(struct hda_codec *codec, struct hda_pcm_stream *hinfo) { struct snd_pcm_runtime *runtime = substream->runtime; + bool notify_share_sw = false; + runtime->hw.channels_max = mout->max_channels; if (mout->dig_out_nid) { if (!mout->analog_rates) { @@ -3729,10 +3757,12 @@ int snd_hda_multi_out_analog_open(struct hda_codec *codec, hinfo->maxbps = mout->spdif_maxbps; } else { mout->share_spdif = 0; - /* FIXME: need notify? */ + notify_share_sw = true; } } } + if (notify_share_sw) + notify_spdif_share_sw(codec, mout); return snd_pcm_hw_constraint_step(substream->runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, 2); } diff --git a/sound/hda/common/hda_local.h b/sound/hda/common/hda_local.h index ab423f1cef5492..98b2c4acebc273 100644 --- a/sound/hda/common/hda_local.h +++ b/sound/hda/common/hda_local.h @@ -221,6 +221,7 @@ struct hda_multi_out { unsigned int spdif_rates; unsigned int spdif_maxbps; u64 spdif_formats; + struct snd_kcontrol *share_spdif_kctl; /* cached shared SPDIF switch */ }; int snd_hda_create_spdif_share_sw(struct hda_codec *codec, diff --git a/sound/isa/sc6000.c b/sound/isa/sc6000.c index 6d618cc2ba4577..9949e06403f61a 100644 --- a/sound/isa/sc6000.c +++ b/sound/isa/sc6000.c @@ -100,6 +100,15 @@ MODULE_PARM_DESC(joystick, "Enable gameport."); #define PFX "sc6000: " #define DRV_NAME "SC-6000" +struct snd_sc6000 { + char __iomem *vport; + char __iomem *vmss_port; + u8 mss_config; + u8 config; + u8 hw_cfg[2]; + bool old_dsp; +}; + /* hardware dependent functions */ /* @@ -267,7 +276,7 @@ static int sc6000_dsp_reset(char __iomem *vport) /* detection and initialization */ static int sc6000_hw_cfg_write(struct device *devptr, - char __iomem *vport, const int *cfg) + char __iomem *vport, const u8 *cfg) { if (sc6000_write(devptr, vport, COMMAND_6C) < 0) { dev_warn(devptr, "CMD 0x%x: failed!\n", COMMAND_6C); @@ -353,8 +362,7 @@ static int sc6000_init_mss(struct device *devptr, return 0; } -static void sc6000_hw_cfg_encode(struct device *devptr, - char __iomem *vport, int *cfg, +static void sc6000_hw_cfg_encode(struct device *devptr, u8 *cfg, long xport, long xmpu, long xmss_port, int joystick) { @@ -376,27 +384,83 @@ static void sc6000_hw_cfg_encode(struct device *devptr, dev_dbg(devptr, "hw cfg %x, %x\n", cfg[0], cfg[1]); } -static int sc6000_init_board(struct device *devptr, - char __iomem *vport, - char __iomem *vmss_port, int dev) +static void sc6000_prepare_board(struct device *devptr, + struct snd_sc6000 *sc6000, + unsigned int dev, int xirq, int xdma) +{ + sc6000->mss_config = sc6000_irq_to_softcfg(xirq) | + sc6000_dma_to_softcfg(xdma); + sc6000->config = sc6000->mss_config | + sc6000_mpu_irq_to_softcfg(mpu_irq[dev]); + sc6000_hw_cfg_encode(devptr, sc6000->hw_cfg, port[dev], mpu_port[dev], + mss_port[dev], joystick[dev]); +} + +static void sc6000_detect_old_dsp(struct device *devptr, + struct snd_sc6000 *sc6000) +{ + sc6000_write(devptr, sc6000->vport, COMMAND_5C); + sc6000->old_dsp = sc6000_read(sc6000->vport) < 0; +} + +static int sc6000_program_board(struct device *devptr, + struct snd_sc6000 *sc6000) +{ + int err; + + if (!sc6000->old_dsp) { + if (sc6000_hw_cfg_write(devptr, sc6000->vport, + sc6000->hw_cfg) < 0) { + dev_err(devptr, "sc6000_hw_cfg_write: failed!\n"); + return -EIO; + } + } + + err = sc6000_setup_board(devptr, sc6000->vport, sc6000->config); + if (err < 0) { + dev_err(devptr, "sc6000_setup_board: failed!\n"); + return -ENODEV; + } + + sc6000_dsp_reset(sc6000->vport); + + if (!sc6000->old_dsp) { + sc6000_write(devptr, sc6000->vport, COMMAND_60); + sc6000_write(devptr, sc6000->vport, 0x02); + sc6000_dsp_reset(sc6000->vport); + } + + err = sc6000_setup_board(devptr, sc6000->vport, sc6000->config); + if (err < 0) { + dev_err(devptr, "sc6000_setup_board: failed!\n"); + return -ENODEV; + } + + err = sc6000_init_mss(devptr, sc6000->vport, sc6000->config, + sc6000->vmss_port, sc6000->mss_config); + if (err < 0) { + dev_err(devptr, "Cannot initialize Microsoft Sound System mode.\n"); + return -ENODEV; + } + + return 0; +} + +static int sc6000_init_board(struct device *devptr, struct snd_sc6000 *sc6000) { char answer[15]; char version[2]; - int mss_config = sc6000_irq_to_softcfg(irq[dev]) | - sc6000_dma_to_softcfg(dma[dev]); - int config = mss_config | - sc6000_mpu_irq_to_softcfg(mpu_irq[dev]); int err; - int old = 0; - err = sc6000_dsp_reset(vport); + err = sc6000_dsp_reset(sc6000->vport); if (err < 0) { dev_err(devptr, "sc6000_dsp_reset: failed!\n"); return err; } memset(answer, 0, sizeof(answer)); - err = sc6000_dsp_get_answer(devptr, vport, GET_DSP_COPYRIGHT, answer, 15); + err = sc6000_dsp_get_answer(devptr, sc6000->vport, GET_DSP_COPYRIGHT, + answer, 15); if (err <= 0) { dev_err(devptr, "sc6000_dsp_copyright: failed!\n"); return -ENODEV; @@ -408,54 +472,17 @@ static int sc6000_init_board(struct device *devptr, if (strncmp("SC-6000", answer, 7)) dev_warn(devptr, "Warning: non SC-6000 audio card!\n"); - if (sc6000_dsp_get_answer(devptr, vport, GET_DSP_VERSION, version, 2) < 2) { + if (sc6000_dsp_get_answer(devptr, sc6000->vport, + GET_DSP_VERSION, version, 2) < 2) { dev_err(devptr, "sc6000_dsp_version: failed!\n"); return -ENODEV; } dev_info(devptr, "Detected model: %s, DSP version %d.%d\n", answer, version[0], version[1]); - /* set configuration */ - sc6000_write(devptr, vport, COMMAND_5C); - if (sc6000_read(vport) < 0) - old = 1; - - if (!old) { - int cfg[2]; - sc6000_hw_cfg_encode(devptr, - vport, &cfg[0], port[dev], mpu_port[dev], - mss_port[dev], joystick[dev]); - if (sc6000_hw_cfg_write(devptr, vport, cfg) < 0) { - dev_err(devptr, "sc6000_hw_cfg_write: failed!\n"); - return -EIO; - } - } - err = sc6000_setup_board(devptr, vport, config); - if (err < 0) { - dev_err(devptr, "sc6000_setup_board: failed!\n"); - return -ENODEV; - } - - sc6000_dsp_reset(vport); - - if (!old) { - sc6000_write(devptr, vport, COMMAND_60); - sc6000_write(devptr, vport, 0x02); - sc6000_dsp_reset(vport); - } + sc6000_detect_old_dsp(devptr, sc6000); - err = sc6000_setup_board(devptr, vport, config); - if (err < 0) { - dev_err(devptr, "sc6000_setup_board: failed!\n"); - return -ENODEV; - } - err = sc6000_init_mss(devptr, vport, config, vmss_port, mss_config); - if (err < 0) { - dev_err(devptr, "Cannot initialize Microsoft Sound System mode.\n"); - return -ENODEV; - } - - return 0; + return sc6000_program_board(devptr, sc6000); } static int snd_sc6000_mixer(struct snd_wss *chip) @@ -538,10 +565,10 @@ static int snd_sc6000_match(struct device *devptr, unsigned int dev) static void snd_sc6000_free(struct snd_card *card) { - char __iomem *vport = (char __force __iomem *)card->private_data; + struct snd_sc6000 *sc6000 = card->private_data; - if (vport) - sc6000_setup_board(card->dev, vport, 0); + if (sc6000->vport) + sc6000_setup_board(card->dev, sc6000->vport, 0); } static int __snd_sc6000_probe(struct device *devptr, unsigned int dev) @@ -552,15 +579,17 @@ static int __snd_sc6000_probe(struct device *devptr, unsigned int dev) int xirq = irq[dev]; int xdma = dma[dev]; struct snd_card *card; + struct snd_sc6000 *sc6000; struct snd_wss *chip; struct snd_opl3 *opl3; char __iomem *vport; char __iomem *vmss_port; err = snd_devm_card_new(devptr, index[dev], id[dev], THIS_MODULE, - 0, &card); + sizeof(*sc6000), &card); if (err < 0) return err; + sc6000 = card->private_data; if (xirq == SNDRV_AUTO_IRQ) { xirq = snd_legacy_find_free_irq(possible_irqs); @@ -587,7 +616,7 @@ static int __snd_sc6000_probe(struct device *devptr, unsigned int dev) dev_err(devptr, "I/O port cannot be iomapped.\n"); return -EBUSY; } - card->private_data = (void __force *)vport; + sc6000->vport = vport; /* to make it marked as used */ if (!devm_request_region(devptr, mss_port[dev], 4, DRV_NAME)) { @@ -600,12 +629,15 @@ static int __snd_sc6000_probe(struct device *devptr, unsigned int dev) dev_err(devptr, "MSS port I/O cannot be iomapped.\n"); return -EBUSY; } + sc6000->vmss_port = vmss_port; dev_dbg(devptr, "Initializing BASE[0x%lx] IRQ[%d] DMA[%d] MIRQ[%d]\n", port[dev], xirq, xdma, mpu_irq[dev] == SNDRV_AUTO_IRQ ? 0 : mpu_irq[dev]); - err = sc6000_init_board(devptr, vport, vmss_port, dev); + sc6000_prepare_board(devptr, sc6000, dev, xirq, xdma); + + err = sc6000_init_board(devptr, sc6000); if (err < 0) return err; card->private_free = snd_sc6000_free; diff --git a/sound/pci/ctxfi/ctatc.c b/sound/pci/ctxfi/ctatc.c index da2667cb2489ed..206b4baa6d9291 100644 --- a/sound/pci/ctxfi/ctatc.c +++ b/sound/pci/ctxfi/ctatc.c @@ -794,7 +794,8 @@ static int spdif_passthru_playback_get_resources(struct ct_atc *atc, struct src *src; int err; int n_amixer = apcm->substream->runtime->channels, i; - unsigned int pitch, rsr = atc->pll_rate; + unsigned int pitch; + unsigned int rsr = atc->pll_rate ? atc->pll_rate : atc->rsr; /* first release old resources */ atc_pcm_release_resources(atc, apcm); diff --git a/sound/pci/ctxfi/ctvmem.h b/sound/pci/ctxfi/ctvmem.h index da54cbcdb0be4d..43a0065b40c354 100644 --- a/sound/pci/ctxfi/ctvmem.h +++ b/sound/pci/ctxfi/ctvmem.h @@ -15,7 +15,7 @@ #ifndef CTVMEM_H #define CTVMEM_H -#define CT_PTP_NUM 4 /* num of device page table pages */ +#define CT_PTP_NUM 1 /* num of device page table pages */ #include #include diff --git a/sound/soc/amd/acp-pcm-dma.c b/sound/soc/amd/acp-pcm-dma.c index c76a4bcc964592..6ad70aa0ea8376 100644 --- a/sound/soc/amd/acp-pcm-dma.c +++ b/sound/soc/amd/acp-pcm-dma.c @@ -1252,7 +1252,7 @@ static const struct snd_soc_component_driver acp_asoc_platform = { .pointer = acp_dma_pointer, .delay = acp_dma_delay, .prepare = acp_dma_prepare, - .pcm_construct = acp_dma_new, + .pcm_new = acp_dma_new, }; static int acp_audio_probe(struct platform_device *pdev) diff --git a/sound/soc/amd/acp/acp-legacy-mach.c b/sound/soc/amd/acp/acp-legacy-mach.c index a7a551366a4097..235d6cc83fa989 100644 --- a/sound/soc/amd/acp/acp-legacy-mach.c +++ b/sound/soc/amd/acp/acp-legacy-mach.c @@ -174,7 +174,7 @@ static int acp_asoc_probe(struct platform_device *pdev) acp_card_drvdata->acp_rev = mach->mach_params.subsystem_rev; dmi_id = dmi_first_match(acp_quirk_table); - if (dmi_id && dmi_id->driver_data) + if (dmi_id && dmi_id->driver_data == (void *)QUIRK_TDM_MODE_ENABLE) acp_card_drvdata->tdm_mode = dmi_id->driver_data; ret = acp_legacy_dai_links_create(card); diff --git a/sound/soc/amd/acp/acp-mach-common.c b/sound/soc/amd/acp/acp-mach-common.c index 09f6c9a2c04104..ef784cca13f2b8 100644 --- a/sound/soc/amd/acp/acp-mach-common.c +++ b/sound/soc/amd/acp/acp-mach-common.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "../../codecs/rt5682.h" #include "../../codecs/rt1019.h" @@ -37,15 +38,21 @@ #define NAU8821_FREQ_OUT 12288000 #define MAX98388_CODEC_DAI "max98388-aif1" -#define TDM_MODE_ENABLE 1 - const struct dmi_system_id acp_quirk_table[] = { { /* Google skyrim proto-0 */ .matches = { DMI_EXACT_MATCH(DMI_PRODUCT_FAMILY, "Google_Skyrim"), }, - .driver_data = (void *)TDM_MODE_ENABLE, + .driver_data = (void *)QUIRK_TDM_MODE_ENABLE, + }, + { + /* Valve Steam Deck OLED */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Valve"), + DMI_MATCH(DMI_PRODUCT_NAME, "Galileo"), + }, + .driver_data = (void *)QUIRK_REMAP_DMIC_BT, }, {} }; @@ -1401,6 +1408,7 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card) struct snd_soc_dai_link *links; struct device *dev = card->dev; struct acp_card_drvdata *drv_data = card->drvdata; + const struct dmi_system_id *dmi_id = dmi_first_match(acp_quirk_table); int i = 0, num_links = 0; if (drv_data->hs_cpu_id) @@ -1572,6 +1580,9 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card) links[i].codecs = &snd_soc_dummy_dlc; links[i].num_codecs = 1; } + + if (dmi_id && dmi_id->driver_data == (void *)QUIRK_REMAP_DMIC_BT) + links[i].id = DMIC_BE_ID; i++; } @@ -1587,6 +1598,11 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card) links[i].capture_only = 1; links[i].nonatomic = true; links[i].no_pcm = 1; + + if (dmi_id && dmi_id->driver_data == (void *)QUIRK_REMAP_DMIC_BT) { + links[i].id = BT_BE_ID; + dev_dbg(dev, "quirk REMAP_DMIC_BT enabled\n"); + } } card->dai_link = links; diff --git a/sound/soc/amd/acp/acp-mach.h b/sound/soc/amd/acp/acp-mach.h index f94c30c20f20b6..7177d3fd961927 100644 --- a/sound/soc/amd/acp/acp-mach.h +++ b/sound/soc/amd/acp/acp-mach.h @@ -26,6 +26,10 @@ #define acp_get_drvdata(card) ((struct acp_card_drvdata *)(card)->drvdata) +/* List of DMI quirks - check acp-mach-common.c for usage. */ +#define QUIRK_TDM_MODE_ENABLE 1 +#define QUIRK_REMAP_DMIC_BT 2 + enum be_id { HEADSET_BE_ID = 0, AMP_BE_ID, diff --git a/sound/soc/amd/acp/acp-platform.c b/sound/soc/amd/acp/acp-platform.c index 88613569fd64f1..6b1e18b31c1c69 100644 --- a/sound/soc/amd/acp/acp-platform.c +++ b/sound/soc/amd/acp/acp-platform.c @@ -321,7 +321,7 @@ static const struct snd_soc_component_driver acp_pcm_component = { .close = acp_dma_close, .hw_params = acp_dma_hw_params, .pointer = acp_dma_pointer, - .pcm_construct = acp_dma_new, + .pcm_new = acp_dma_new, .legacy_dai_naming = 1, }; diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index cd7b1acc7216d0..a9c8d9545281e5 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -551,11 +551,11 @@ static int mc_probe(struct platform_device *pdev) " cfg-amp:%d", amp_num); if (!card->components) return -ENOMEM; - if (mach->mach_params.dmic_num) { + if (soc_sdw_quirk & ASOC_SDW_ACP_DMIC) { card->components = devm_kasprintf(card->dev, GFP_KERNEL, "%s mic:dmic cfg-mics:%d", card->components, - mach->mach_params.dmic_num); + 1); if (!card->components) return -ENOMEM; } diff --git a/sound/soc/amd/acp/acp-sof-mach.c b/sound/soc/amd/acp/acp-sof-mach.c index 6215e31eceddf8..36ecef7013b9cb 100644 --- a/sound/soc/amd/acp/acp-sof-mach.c +++ b/sound/soc/amd/acp/acp-sof-mach.c @@ -110,7 +110,7 @@ static int acp_sof_probe(struct platform_device *pdev) acp_card_drvdata = card->drvdata; dmi_id = dmi_first_match(acp_quirk_table); - if (dmi_id && dmi_id->driver_data) + if (dmi_id && dmi_id->driver_data == (void *)QUIRK_TDM_MODE_ENABLE) acp_card_drvdata->tdm_mode = dmi_id->driver_data; acp_card_drvdata->acp_rev = mach->mach_params.subsystem_rev; diff --git a/sound/soc/amd/ps/ps-pdm-dma.c b/sound/soc/amd/ps/ps-pdm-dma.c index 7c529fc6ba9976..04c014349347ef 100644 --- a/sound/soc/amd/ps/ps-pdm-dma.c +++ b/sound/soc/amd/ps/ps-pdm-dma.c @@ -351,7 +351,8 @@ static const struct snd_soc_component_driver acp63_pdm_component = { .close = acp63_pdm_dma_close, .hw_params = acp63_pdm_dma_hw_params, .pointer = acp63_pdm_dma_pointer, - .pcm_construct = acp63_pdm_dma_new, + .pcm_new = acp63_pdm_dma_new, + .use_dai_pcm_id = true, }; static int acp63_pdm_audio_probe(struct platform_device *pdev) diff --git a/sound/soc/amd/ps/ps-sdw-dma.c b/sound/soc/amd/ps/ps-sdw-dma.c index 366d7c4bb07e94..f27ebbd213798b 100644 --- a/sound/soc/amd/ps/ps-sdw-dma.c +++ b/sound/soc/amd/ps/ps-sdw-dma.c @@ -634,7 +634,7 @@ static const struct snd_soc_component_driver acp63_sdw_component = { .hw_params = acp63_sdw_dma_hw_params, .trigger = acp63_sdw_dma_trigger, .pointer = acp63_sdw_dma_pointer, - .pcm_construct = acp63_sdw_dma_new, + .pcm_new = acp63_sdw_dma_new, .use_dai_pcm_id = true, }; diff --git a/sound/soc/amd/raven/acp3x-pcm-dma.c b/sound/soc/amd/raven/acp3x-pcm-dma.c index 4529404ebd9351..37ea5c572eb948 100644 --- a/sound/soc/amd/raven/acp3x-pcm-dma.c +++ b/sound/soc/amd/raven/acp3x-pcm-dma.c @@ -363,7 +363,7 @@ static const struct snd_soc_component_driver acp3x_i2s_component = { .close = acp3x_dma_close, .hw_params = acp3x_dma_hw_params, .pointer = acp3x_dma_pointer, - .pcm_construct = acp3x_dma_new, + .pcm_new = acp3x_dma_new, }; static int acp3x_audio_probe(struct platform_device *pdev) diff --git a/sound/soc/amd/renoir/acp3x-pdm-dma.c b/sound/soc/amd/renoir/acp3x-pdm-dma.c index e832c7c4b96fac..e60e3821703ccb 100644 --- a/sound/soc/amd/renoir/acp3x-pdm-dma.c +++ b/sound/soc/amd/renoir/acp3x-pdm-dma.c @@ -376,7 +376,7 @@ static const struct snd_soc_component_driver acp_pdm_component = { .close = acp_pdm_dma_close, .hw_params = acp_pdm_dma_hw_params, .pointer = acp_pdm_dma_pointer, - .pcm_construct = acp_pdm_dma_new, + .pcm_new = acp_pdm_dma_new, .legacy_dai_naming = 1, }; diff --git a/sound/soc/amd/vangogh/acp5x-pcm-dma.c b/sound/soc/amd/vangogh/acp5x-pcm-dma.c index 6ce82cd8859b8d..831e30e9b04263 100644 --- a/sound/soc/amd/vangogh/acp5x-pcm-dma.c +++ b/sound/soc/amd/vangogh/acp5x-pcm-dma.c @@ -357,7 +357,7 @@ static const struct snd_soc_component_driver acp5x_i2s_component = { .close = acp5x_dma_close, .hw_params = acp5x_dma_hw_params, .pointer = acp5x_dma_pointer, - .pcm_construct = acp5x_dma_new, + .pcm_new = acp5x_dma_new, }; static int acp5x_audio_probe(struct platform_device *pdev) diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index aa620093318254..2f7a51d7eb1156 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -52,6 +52,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "HP Laptop 15-fc0xxx"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "OMEN Gaming Laptop 16-ap0xxx"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -654,6 +661,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "8EE4"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_BOARD_NAME, "8E35"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/amd/yc/acp6x-pdm-dma.c b/sound/soc/amd/yc/acp6x-pdm-dma.c index 1c8aad84991644..710db721ffa480 100644 --- a/sound/soc/amd/yc/acp6x-pdm-dma.c +++ b/sound/soc/amd/yc/acp6x-pdm-dma.c @@ -346,7 +346,7 @@ static const struct snd_soc_component_driver acp6x_pdm_component = { .close = acp6x_pdm_dma_close, .hw_params = acp6x_pdm_dma_hw_params, .pointer = acp6x_pdm_dma_pointer, - .pcm_construct = acp6x_pdm_dma_new, + .pcm_new = acp6x_pdm_dma_new, .legacy_dai_naming = 1, }; diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig index d8dc2f1ccc83e0..d47b189e0d611d 100644 --- a/sound/soc/apple/Kconfig +++ b/sound/soc/apple/Kconfig @@ -1,5 +1,16 @@ menu "Apple" +config SND_SOC_APPLE_AOP_AUDIO + tristate "AOP audio driver" + depends on ARCH_APPLE || COMPILE_TEST + depends on RUST + select APPLE_AOP + select SND_DMAENGINE_PCM + help + This option enables an ASoC driver for sound devices connected to the AOP + co-processor on ARM Macs. This includes the built-in microphone on those + machines. + config SND_SOC_APPLE_MCA tristate "Apple Silicon MCA driver" depends on ARCH_APPLE || COMPILE_TEST @@ -8,4 +19,21 @@ config SND_SOC_APPLE_MCA This option enables an ASoC platform driver for MCA peripherals found on Apple Silicon SoCs. +config SND_SOC_APPLE_MACAUDIO + tristate "Sound support for Apple Silicon Macs" + depends on ARCH_APPLE || COMPILE_TEST + select SND_SOC_APPLE_MCA + select SND_SIMPLE_CARD_UTILS + select APPLE_ADMAC if DMADEVICES + select COMMON_CLK_APPLE_NCO if COMMON_CLK + select SND_SOC_TAS2764 if I2C + select SND_SOC_TAS2770 if I2C + select SND_SOC_CS42L83 if I2C + select SND_SOC_CS42L84 if I2C + select REGULATOR_FIXED_VOLTAGE if REGULATOR + help + This option enables an ASoC machine-level driver for Apple Silicon Macs + and it also enables the required SoC and codec drivers for overall + sound support on these machines. + endmenu diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile index 1eb8fbef60c617..7d4901f407014d 100644 --- a/sound/soc/apple/Makefile +++ b/sound/soc/apple/Makefile @@ -1,3 +1,10 @@ +snd-soc-aop-y := aop_audio.o +obj-$(CONFIG_SND_SOC_APPLE_AOP_AUDIO) += snd-soc-aop.o + snd-soc-apple-mca-y := mca.o obj-$(CONFIG_SND_SOC_APPLE_MCA) += snd-soc-apple-mca.o + +snd-soc-macaudio-objs := macaudio.o + +obj-$(CONFIG_SND_SOC_APPLE_MACAUDIO) += snd-soc-macaudio.o diff --git a/sound/soc/apple/aop_audio.rs b/sound/soc/apple/aop_audio.rs new file mode 100644 index 00000000000000..10b57a4518ab03 --- /dev/null +++ b/sound/soc/apple/aop_audio.rs @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#![recursion_limit = "2048"] + +//! Apple AOP audio driver +//! +//! Copyright (C) The Asahi Linux Contributors + +use core::fmt; +use core::sync::atomic::{AtomicU32, Ordering}; +use core::{mem, ptr, slice}; + +use kernel::{ + bindings, c_str, device, + device::property::FwNode, + device::Core, + error::from_err_ptr, + module_platform_driver, of, platform, + prelude::*, + soc::apple::aop::{from_fourcc, EPICService, AOP}, + str::CString, + sync::Arc, + types::{ARef, ForeignOwnable}, +}; + +use pin_init::Zeroable; + +/// An unaligned u32 type. +/// +/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible +/// with `#[derive(Debug)]` and atomics. +#[derive(Copy, Clone, Default)] +#[repr(C, packed(1))] +pub(crate) struct U32(pub(crate) u32); + +// SAFETY: U32 is zeroable just like u32 +unsafe impl Zeroable for U32 {} + +impl fmt::Debug for U32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let v = self.0; + f.write_fmt(format_args!("{:#x}", v)) + } +} + +const EPIC_SUBTYPE_WRAPPED_CALL: u16 = 0x20; +const CALLTYPE_AUDIO_ATTACH_DEVICE: u32 = 0xc3000002; +const CALLTYPE_AUDIO_SET_PROP: u32 = 0xc3000005; +const PDM_NUM_COEFFS: usize = 120; +const DECIMATION_RATIOS: [u8; 3] = [0xf, 5, 2]; +const COEFFICIENTS: [u8; PDM_NUM_COEFFS * mem::size_of::()] = [ + 0x88, 0x03, 0x00, 0x00, 0x82, 0x08, 0x00, 0x00, 0x51, 0x12, 0x00, 0x00, 0x0a, 0x23, 0x00, 0x00, + 0xce, 0x3d, 0x00, 0x00, 0x97, 0x66, 0x00, 0x00, 0x43, 0xa2, 0x00, 0x00, 0x9c, 0xf6, 0x00, 0x00, + 0x53, 0x6a, 0x01, 0x00, 0xe6, 0x04, 0x02, 0x00, 0x7e, 0xce, 0x02, 0x00, 0xae, 0xcf, 0x03, 0x00, + 0x2e, 0x11, 0x05, 0x00, 0x7d, 0x9b, 0x06, 0x00, 0x75, 0x76, 0x08, 0x00, 0xd8, 0xa8, 0x0a, 0x00, + 0xd2, 0x37, 0x0d, 0x00, 0x82, 0x26, 0x10, 0x00, 0x86, 0x75, 0x13, 0x00, 0x97, 0x22, 0x17, 0x00, + 0x39, 0x28, 0x1b, 0x00, 0x89, 0x7d, 0x1f, 0x00, 0x2e, 0x16, 0x24, 0x00, 0x69, 0xe2, 0x28, 0x00, + 0x56, 0xcf, 0x2d, 0x00, 0x51, 0xc7, 0x32, 0x00, 0x80, 0xb2, 0x37, 0x00, 0x87, 0x77, 0x3c, 0x00, + 0x4c, 0xfc, 0x40, 0x00, 0xd9, 0x26, 0x45, 0x00, 0x47, 0xde, 0x48, 0x00, 0xa0, 0x0b, 0x4c, 0x00, + 0xc1, 0x9a, 0x4e, 0x00, 0x1f, 0x7b, 0x50, 0x00, 0x68, 0xa0, 0x51, 0x00, 0x06, 0x03, 0x52, 0x00, + 0x4a, 0x25, 0x00, 0x00, 0x4c, 0xaf, 0x00, 0x00, 0xc0, 0x07, 0x02, 0x00, 0x45, 0x99, 0x04, 0x00, + 0x9a, 0x84, 0x08, 0x00, 0x7d, 0x38, 0x0d, 0x00, 0x5f, 0x1a, 0x11, 0x00, 0xd9, 0x81, 0x11, 0x00, + 0x80, 0x44, 0x0b, 0x00, 0x8e, 0xe5, 0xfb, 0xff, 0xca, 0x32, 0xe3, 0xff, 0x52, 0xc7, 0xc4, 0xff, + 0xa6, 0xbc, 0xa8, 0xff, 0x83, 0xe6, 0x9a, 0xff, 0xb8, 0x5b, 0xa8, 0xff, 0x6b, 0xae, 0xdb, 0xff, + 0xe7, 0xd8, 0x38, 0x00, 0x24, 0x42, 0xba, 0x00, 0x33, 0x20, 0x50, 0x01, 0x6e, 0xdc, 0xe2, 0x01, + 0x42, 0x23, 0x58, 0x02, 0x2c, 0x50, 0x99, 0x02, 0xcf, 0xfa, 0xff, 0xff, 0x53, 0x0a, 0xff, 0xff, + 0x66, 0x23, 0xfb, 0xff, 0xa0, 0x3e, 0xf4, 0xff, 0xe6, 0x68, 0xf0, 0xff, 0xb8, 0x35, 0xf7, 0xff, + 0x56, 0xec, 0x04, 0x00, 0x37, 0xa3, 0x09, 0x00, 0x00, 0xd4, 0xfe, 0xff, 0x78, 0xa3, 0xf5, 0xff, + 0x03, 0xbf, 0xfe, 0xff, 0x84, 0xd5, 0x0b, 0x00, 0xbe, 0x0b, 0x04, 0x00, 0x52, 0x54, 0xf2, 0xff, + 0x6d, 0x3f, 0xf8, 0xff, 0xc5, 0x7f, 0x0f, 0x00, 0xe6, 0x9e, 0x0c, 0x00, 0x79, 0x03, 0xef, 0xff, + 0xd5, 0x33, 0xed, 0xff, 0xec, 0xd1, 0x11, 0x00, 0x7d, 0x69, 0x1a, 0x00, 0xd6, 0x55, 0xee, 0xff, + 0x88, 0x66, 0xdc, 0xff, 0x57, 0x26, 0x10, 0x00, 0xc7, 0x8d, 0x2e, 0x00, 0x82, 0x2e, 0xf3, 0xff, + 0x63, 0x69, 0xc4, 0xff, 0xcd, 0x08, 0x07, 0x00, 0x35, 0x34, 0x4b, 0x00, 0xaf, 0x21, 0x02, 0x00, + 0x83, 0xb6, 0xa1, 0xff, 0xe2, 0xd5, 0xef, 0xff, 0x94, 0x9b, 0x76, 0x00, 0xf3, 0xd7, 0x25, 0x00, + 0xff, 0xfc, 0x67, 0xff, 0xe3, 0xac, 0xb6, 0xff, 0x52, 0x1b, 0xcc, 0x00, 0x3c, 0x8a, 0x8b, 0x00, + 0x9f, 0x0c, 0xcd, 0xfe, 0x5c, 0x68, 0xcc, 0xfe, 0x4d, 0xc5, 0x98, 0x02, 0x82, 0xcf, 0xfb, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +]; +const FILTER_LENGTHS: u32 = 0x542c47; +const AUDIO_DEV_PDM0: u32 = from_fourcc(b"pdm0"); +const AUDIO_DEV_LPAI: u32 = from_fourcc(b"lpai"); +const AUDIO_DEV_HPAI: u32 = from_fourcc(b"hpai"); +const POWER_STATE_OFF: u32 = from_fourcc(b"idle"); +const POWER_STATE_IDLE: u32 = from_fourcc(b"pw1 "); +const POWER_STATE_ON: u32 = from_fourcc(b"pwrd"); + +#[repr(C, packed)] +#[derive(Clone, Copy, Default)] +struct AudioAttachDevice { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: u64, + _zero2: u64, + _pad0: u32, + len: u64, + dev_id: u32, + _pad1: u32, +} + +impl AudioAttachDevice { + fn new(dev_id: u32) -> AudioAttachDevice { + AudioAttachDevice { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_ATTACH_DEVICE, + dev_id, + len: 0x2c, + ..AudioAttachDevice::default() + } + } +} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default)] +struct LpaiChannelConfig { + unk1: u32, + unk2: u32, + unk3: u32, + unk4: u32, +} + +#[repr(C, packed(4))] +#[derive(Debug, Copy, Clone)] +struct PDMConfig { + bytes_per_sample: u32, + clock_source: u32, + pdm_frequency: u32, + pdmc_frequency: u32, + slow_clock_speed: u32, + fast_clock_speed: u32, + channel_polarity_select: u32, + channel_phase_select: u32, + unk1: u32, + unk2: u16, + ratio1: u8, + ratio2: u8, + ratio3: u8, + _pad0: u8, + filter_lengths: U32, + coeff_bulk: U32, + coeffs: [u8; PDM_NUM_COEFFS * mem::size_of::()], + unk3: U32, + mic_turn_on_time_ms: U32, + _zero0: [u8; 8], + _zero1: [u8; 8], + unk4: U32, + mic_settle_time_ms: U32, + _zero2: [u8; 69], // ????? + _pad_extra: u8, // extra padding to increase the struct size to multiple of mem::size_of::() +} +// PDMConfig is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::() == 36 + 14 + (120 * 4) + 32 + 69 + 1); + +unsafe impl Zeroable for PDMConfig {} + +#[repr(C, packed(4))] +#[derive(Debug, Copy, Clone)] +struct DecimatorConfig { + latency: u32, + ratio1: u8, + ratio2: u8, + ratio3: u8, + _pad0: u8, + filter_lengths: u32, + coeff_bulk: u32, + coeffs: [u8; PDM_NUM_COEFFS * mem::size_of::()], +} +// DecimatorConfig is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::() == 16 + (120 * 4)); + +unsafe impl Zeroable for DecimatorConfig {} + +#[repr(C, packed(4))] +#[derive(Clone, Copy, Default, Debug)] +struct PowerSetting { + dev_id: u32, + cookie: u32, + _unk0: u32, + _zero0: [u8; 8], + target_pstate: u32, + unk1: u32, + _zero1: [u8; 20], +} + +impl PowerSetting { + fn new(dev_id: u32, cookie: u32, target_pstate: u32, unk1: u32) -> PowerSetting { + PowerSetting { + dev_id, + cookie, + target_pstate, + unk1, + ..PowerSetting::default() + } + } +} + +#[repr(C)] +#[derive(Clone, Copy, Default, Debug)] +struct AudioSetDeviceProp { + _zero0: u32, + unk0: u32, + calltype: u32, + _zero1: [u8; 8], + _zero2: [u8; 8], + _pad0: u32, + len: u32, + _pad1: u32, + dev_id: u32, + modifier: u32, + len2: u32, + data: T, +} +// AudioSetDeviceProp is intended to use `#[repr(C, packed)]` but this +// conflicts ith pin_init. Instead just ensure that it has the same size as if +// it where packed. +static_assert!(mem::size_of::>() == 52 + mem::size_of::()); +static_assert!( + mem::size_of::>() == 52 + mem::size_of::() +); +static_assert!( + mem::size_of::>() + == 52 + mem::size_of::() +); +static_assert!( + mem::size_of::>() == 52 + mem::size_of::() +); + +impl AudioSetDeviceProp { + fn new(dev_id: u32, modifier: u32, data: T) -> AudioSetDeviceProp { + AudioSetDeviceProp { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_SET_PROP, + dev_id, + modifier, + len: mem::size_of::() as u32 + 0x30, + len2: mem::size_of::() as u32, + data, + ..AudioSetDeviceProp::default() + } + } +} + +unsafe impl Zeroable for AudioSetDeviceProp {} + +impl AudioSetDeviceProp { + fn try_init( + dev_id: u32, + modifier: u32, + data: impl Init, + ) -> impl Init, Error> + where + Error: From, + { + try_init!( + AudioSetDeviceProp { + unk0: 0xFFFFFFFF, + calltype: CALLTYPE_AUDIO_SET_PROP, + dev_id, + modifier, + len: mem::size_of::() as u32 + 0x30, + len2: mem::size_of::() as u32, + data <- data, + ..Zeroable::init_zeroed() + } + ) + } +} + +struct SndSocAopData { + dev: ARef, + adata: Arc, + service: EPICService, + pstate_cookie: AtomicU32, + fwnode: ARef, +} + +impl SndSocAopData { + fn new( + dev: ARef, + adata: Arc, + service: EPICService, + fwnode: ARef, + ) -> Result> { + Ok(Arc::new( + SndSocAopData { + dev, + adata, + service, + fwnode, + pstate_cookie: AtomicU32::new(1), + }, + GFP_KERNEL, + )?) + } + fn set_pdm_config(&self) -> Result<()> { + let pdm_cfg = init!(PDMConfig { + bytes_per_sample: 2, + clock_source: 0x706c6c20, // 'pll ' + pdm_frequency: 2400000, + pdmc_frequency: 24000000, + slow_clock_speed: 24000000, + fast_clock_speed: 24000000, + channel_polarity_select: 256, + channel_phase_select: 0, + unk1: 0xf7600, + unk2: 0, + ratio1: DECIMATION_RATIOS[0], + ratio2: DECIMATION_RATIOS[1], + ratio3: DECIMATION_RATIOS[2], + filter_lengths: U32(FILTER_LENGTHS), + coeff_bulk: U32(PDM_NUM_COEFFS as u32), + coeffs: COEFFICIENTS, + unk3: U32(1), + mic_turn_on_time_ms: U32(20), + unk4: U32(1), + mic_settle_time_ms: U32(50), + ..Zeroable::init_zeroed() + }); + let set_prop = AudioSetDeviceProp::::try_init(AUDIO_DEV_PDM0, 200, pdm_cfg); + let msg = KBox::try_init(set_prop, GFP_KERNEL)?; + let ret = self.epic_wrapped_call(msg.as_ref())?; + if ret != 0 { + dev_err!(self.dev, "Unable to set pdm config, return code {}", ret); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_decimator_config(&self) -> Result<()> { + let pdm_cfg = init!(DecimatorConfig { + latency: 15, + ratio1: DECIMATION_RATIOS[0], + ratio2: DECIMATION_RATIOS[1], + ratio3: DECIMATION_RATIOS[2], + filter_lengths: FILTER_LENGTHS, + coeff_bulk: PDM_NUM_COEFFS as u32, + coeffs: COEFFICIENTS, + ..Zeroable::init_zeroed() + }); + let set_prop = + AudioSetDeviceProp::::try_init(AUDIO_DEV_PDM0, 210, pdm_cfg); + let msg = KBox::try_init(set_prop, GFP_KERNEL)?; + let ret = self.epic_wrapped_call(msg.as_ref())?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set decimator config, return code {}", + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_lpai_channel_cfg(&self) -> Result<()> { + let cfg = LpaiChannelConfig { + unk1: 7, + unk2: 7, + unk3: 1, + unk4: 7, + }; + let msg = AudioSetDeviceProp::new(AUDIO_DEV_LPAI, 301, cfg); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set lpai channel config, return code {}", + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn audio_attach_device(&self, dev_id: u32) -> Result<()> { + let msg = AudioAttachDevice::new(dev_id); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to attach device {:?}, return code {}", + dev_id, + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn set_audio_power(&self, pstate: u32, unk1: u32) -> Result<()> { + let set_pstate = PowerSetting::new( + AUDIO_DEV_HPAI, + self.pstate_cookie.fetch_add(1, Ordering::Relaxed), + pstate, + unk1, + ); + let msg = AudioSetDeviceProp::new(AUDIO_DEV_HPAI, 202, set_pstate); + let ret = self.epic_wrapped_call(&msg)?; + if ret != 0 { + dev_err!( + self.dev, + "Unable to set power state {:?}, return code {}", + pstate, + ret + ); + return Err(EIO); + } else { + Ok(()) + } + } + fn epic_wrapped_call(&self, data: &T) -> Result { + let msg_bytes = + unsafe { slice::from_raw_parts(data as *const T as *const u8, mem::size_of::()) }; + self.adata + .epic_call(&self.service, EPIC_SUBTYPE_WRAPPED_CALL, msg_bytes) + } + fn request_dma_channel(&self) -> Result<*mut bindings::dma_chan> { + let res = unsafe { + from_err_ptr(bindings::dma_request_chan( + self.dev.as_raw(), + c_str!("dma").as_ptr() as _, + )) + }; + if res.is_err() { + dev_err!(self.dev, "Unable to get dma channel"); + } + res + } +} + +#[repr(transparent)] +struct SndSocAopDriver(*mut bindings::snd_card); + +fn copy_str(target: &mut [u8], source: &[u8]) { + target[..source.len()].copy_from_slice(source) +} + +unsafe fn dmaengine_slave_config( + chan: *mut bindings::dma_chan, + config: *mut bindings::dma_slave_config, +) -> i32 { + unsafe { + match (*(*chan).device).device_config { + Some(dc) => dc(chan, config), + None => ENOSYS.to_errno(), + } + } +} + +unsafe extern "C" fn aop_hw_params( + substream: *mut bindings::snd_pcm_substream, + params: *mut bindings::snd_pcm_hw_params, +) -> i32 { + let chan = unsafe { bindings::snd_dmaengine_pcm_get_chan(substream) }; + let mut slave_config = bindings::dma_slave_config::default(); + let ret = + unsafe { bindings::snd_hwparams_to_dma_slave_config(substream, params, &mut slave_config) }; + if ret < 0 { + return ret; + } + slave_config.src_port_window_size = 4; + unsafe { dmaengine_slave_config(chan, &mut slave_config) } +} + +unsafe extern "C" fn aop_pcm_open(substream: *mut bindings::snd_pcm_substream) -> i32 { + let data = unsafe { Arc::::borrow((*substream).private_data.cast()) }; + if let Err(e) = data.set_audio_power(POWER_STATE_IDLE, 0) { + dev_err!(data.dev, "Unable to enter 'pw1 ' state"); + return e.to_errno(); + } + let mut hwparams = bindings::snd_pcm_hardware { + info: bindings::SNDRV_PCM_INFO_MMAP + | bindings::SNDRV_PCM_INFO_MMAP_VALID + | bindings::SNDRV_PCM_INFO_INTERLEAVED, + formats: bindings::BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE, + subformats: 0, + rates: bindings::SNDRV_PCM_RATE_48000, + rate_min: 48000, + rate_max: 48000, + channels_min: 3, + channels_max: 3, + periods_min: 2, + buffer_bytes_max: usize::MAX, + period_bytes_max: 0x4000, + periods_max: u32::MAX, + period_bytes_min: 256, + fifo_size: 16, + }; + let dma_chan = match data.request_dma_channel() { + Ok(dc) => dc, + Err(e) => return e.to_errno(), + }; + + if unsafe { (*substream).dma_buffer.dev.type_ == bindings::SNDRV_DMA_TYPE_UNKNOWN as _ } { + let ret = unsafe { + bindings::snd_pcm_set_managed_buffer( + substream, + bindings::SNDRV_DMA_TYPE_DEV_IRAM as i32, + (*(*dma_chan).device).dev, + 0, + 0, + ) + }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate dma buffers"); + unsafe { + bindings::dma_release_channel(dma_chan); + } + return ret; + } + } + + let ret = unsafe { + let mut dai_data = bindings::snd_dmaengine_dai_dma_data::default(); + bindings::snd_dmaengine_pcm_refine_runtime_hwparams( + substream, + &mut dai_data, + &mut hwparams, + dma_chan, + ) + }; + if ret != 0 { + dev_err!(data.dev, "Unable to refine hwparams"); + return ret; + } + if let Err(e) = data.set_audio_power(POWER_STATE_ON, 1) { + dev_err!(data.dev, "Unable to power mic on"); + return e.to_errno(); + } + unsafe { + (*(*substream).runtime).hw = hwparams; + bindings::snd_dmaengine_pcm_open(substream, dma_chan) + } +} + +unsafe extern "C" fn aop_pcm_prepare(_: *mut bindings::snd_pcm_substream) -> i32 { + 0 +} + +unsafe extern "C" fn aop_pcm_close(substream: *mut bindings::snd_pcm_substream) -> i32 { + let data = unsafe { Arc::::borrow((*substream).private_data.cast()) }; + if let Err(e) = data.set_audio_power(POWER_STATE_IDLE, 1) { + dev_err!(data.dev, "Unable to power mic off"); + return e.to_errno(); + } + let ret = unsafe { bindings::snd_dmaengine_pcm_close_release_chan(substream) }; + if ret != 0 { + dev_err!(data.dev, "Unable to close channel"); + return ret; + } + if let Err(e) = data.set_audio_power(POWER_STATE_OFF, 0) { + dev_err!(data.dev, "Unable to enter 'idle' power state"); + return e.to_errno(); + } + 0 +} + +unsafe extern "C" fn aop_pcm_free_private(pcm: *mut bindings::snd_pcm) { + unsafe { + Arc::::from_foreign((*pcm).private_data.cast()); + } +} + +impl SndSocAopDriver { + const VTABLE: bindings::snd_pcm_ops = bindings::snd_pcm_ops { + open: Some(aop_pcm_open), + close: Some(aop_pcm_close), + prepare: Some(aop_pcm_prepare), + trigger: Some(bindings::snd_dmaengine_pcm_trigger), + pointer: Some(bindings::snd_dmaengine_pcm_pointer), + ioctl: None, + hw_params: Some(aop_hw_params), + hw_free: None, + sync_stop: None, + get_time_info: None, + fill_silence: None, + copy: None, + page: None, + mmap: None, + ack: None, + }; + fn new(data: Arc) -> Result { + let mut this = SndSocAopDriver(ptr::null_mut()); + let ret = unsafe { + bindings::snd_card_new( + data.dev.as_raw(), + -1, + ptr::null(), + THIS_MODULE.as_ptr(), + 0, + &mut this.0, + ) + }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate sound card"); + return Err(Error::from_errno(ret)); + } + let chassis = data + .fwnode + .property_read::(c_str!("apple,chassis-name")) + .required_by(&data.dev)?; + let machine_kind = data + .fwnode + .property_read::(c_str!("apple,machine-kind")) + .required_by(&data.dev)?; + unsafe { + copy_str(&mut (*this.0).driver, c"aop_audio".to_bytes_with_nul()); + } + let id_str = CString::try_from_fmt(fmt!("Apple{}HPAI", *chassis))?; + unsafe { + copy_str(&mut (*this.0).id, id_str.to_bytes_with_nul()); + } + let shortname = CString::try_from_fmt(fmt!("{} {} HPAI", *machine_kind, *chassis))?; + unsafe { + copy_str(&mut (*this.0).shortname, shortname.to_bytes_with_nul()); + } + let longname = CString::try_from_fmt(fmt!( + "{} {} High-Power Audio Interface", + *machine_kind, + *chassis + ))?; + unsafe { + copy_str(&mut (*this.0).longname, longname.to_bytes_with_nul()); + } + + let mut pcm = ptr::null_mut(); + let ret = + unsafe { bindings::snd_pcm_new(this.0, longname.as_ptr() as _, 0, 0, 1, &mut pcm) }; + if ret < 0 { + dev_err!(data.dev, "Unable to allocate PCM device"); + return Err(Error::from_errno(ret)); + } + + unsafe { + bindings::snd_pcm_set_ops( + pcm, + bindings::SNDRV_PCM_STREAM_CAPTURE as i32, + &Self::VTABLE, + ); + } + + unsafe { + (*pcm).private_data = data.clone().into_foreign() as _; + (*pcm).private_free = Some(aop_pcm_free_private); + (*pcm).info_flags = 0; + copy_str(&mut (*pcm).name, c"aop_audio".to_bytes_with_nul()); + } + + let ret = unsafe { bindings::snd_card_register(this.0) }; + if ret < 0 { + dev_err!(data.dev, "Unable to register sound card"); + return Err(Error::from_errno(ret)); + } + Ok(this) + } +} + +impl Drop for SndSocAopDriver { + fn drop(&mut self) { + if self.0 != ptr::null_mut() { + unsafe { + bindings::snd_card_free(self.0); + } + } + } +} + +unsafe impl Send for SndSocAopDriver {} +unsafe impl Sync for SndSocAopDriver {} + +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c_str!("apple,aop-audio")), ())] +); + +impl platform::Driver for SndSocAopDriver { + type IdInfo = (); + + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + fn probe(pdev: &platform::Device, _info: Option<&()>) -> impl PinInit { + let dev = ARef::::from(pdev.as_ref()); + let parent = pdev.as_ref().parent().unwrap(); + // SAFETY: our parent is AOP, and AopDriver is repr(transparent) for Arc + let adata_ptr = unsafe { Pin::>>::borrow(parent.get_drvdata()) }; + let adata = (&*adata_ptr).clone(); + // SAFETY: AOP sets the platform data correctly + let svc = unsafe { *((*dev.as_raw()).platform_data as *const EPICService) }; + let parent_fwnode = parent.fwnode().ok_or(ENOENT)?; + let fwnode = parent_fwnode + .get_child_by_name(c_str!("audio")) + .ok_or(EIO)?; + let audio = *module_parameters::mic_check_123.value() != 0; + if !audio && parent_fwnode.property_present(c_str!("apple,no-beamforming")) { + return Err(ENODEV); + } + let data = SndSocAopData::new(dev, adata, svc, fwnode)?; + for dev in [AUDIO_DEV_PDM0, AUDIO_DEV_HPAI, AUDIO_DEV_LPAI] { + data.audio_attach_device(dev)?; + } + data.set_lpai_channel_cfg()?; + data.set_pdm_config()?; + data.set_decimator_config()?; + Ok(Self::new(data)?) + } +} + +module_platform_driver! { + type: SndSocAopDriver, + name: "snd_soc_apple_aop", + description: "AOP microphone capture driver", + license: "Dual MIT/GPL", + alias: ["platform:snd_soc_apple_aop"], + params: { + mic_check_123: u8 { + default: 0, + description: "Enable mics without user space handling", + }, + }, +} diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c new file mode 100644 index 00000000000000..1ec66a461754ed --- /dev/null +++ b/sound/soc/apple/macaudio.c @@ -0,0 +1,1686 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ASoC machine driver for Apple Silicon Macs + * + * Copyright (C) The Asahi Linux Contributors + * + * Based on sound/soc/qcom/{sc7180.c|common.c} + * Copyright (c) 2018, Linaro Limited. + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + * + * The platform driver has independent frontend and backend DAIs with the + * option of routing backends to any of the frontends. The platform + * driver configures the routing based on DPCM couplings in ASoC runtime + * structures, which in turn are determined from DAPM paths by ASoC. But the + * platform driver doesn't supply relevant DAPM paths and leaves that up for + * the machine driver to fill in. The filled-in virtual topology can be + * anything as long as any backend isn't connected to more than one frontend + * at any given time. (The limitation is due to the unsupported case of + * reparenting of live BEs.) + */ + +/* #define DEBUG */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRIVER_NAME "snd-soc-macaudio" + +/* + * CPU side is bit and frame clock provider + * I2S has both clocks inverted + */ +#define MACAUDIO_DAI_FMT (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBC_CFC | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF) +#define MACAUDIO_JACK_MASK (SND_JACK_HEADSET | SND_JACK_HEADPHONE) +#define MACAUDIO_SLOTWIDTH 32 +/* + * Maximum BCLK frequency + * + * Codec maximums: + * CS42L42 26.0 MHz + * TAS2770 27.1 MHz + * TAS2764 24.576 MHz + */ +#define MACAUDIO_MAX_BCLK_FREQ 24576000 + +#define SPEAKER_MAGIC_VALUE (s32)0xdec1be15 +/* milliseconds */ +#define SPEAKER_LOCK_TIMEOUT 250 + +enum macaudio_amp_type { + AMP_NONE, + AMP_TAS5770, + AMP_SN012776, + AMP_SSM3515, +}; + +enum macaudio_spkr_config { + SPKR_NONE, /* No speakers */ + SPKR_1W, /* 1 woofer / ch */ + SPKR_2W, /* 2 woofers / ch */ + SPKR_1W1T, /* 1 woofer + 1 tweeter / ch */ + SPKR_2W1T, /* 2 woofers + 1 tweeter / ch */ +}; + +struct macaudio_platform_cfg { + bool enable_speakers; + enum macaudio_amp_type amp; + enum macaudio_spkr_config speakers; + bool stereo; + int amp_gain; + int safe_vol; +}; + +static const char *volume_control_names[] = { + [AMP_TAS5770] = "* Speaker Playback Volume", + [AMP_SN012776] = "* Speaker Volume", + [AMP_SSM3515] = "* DAC Playback Volume", +}; + +#define SN012776_0DB 201 +#define SN012776_DB(x) (SN012776_0DB + 2 * (x)) +/* Same as SN012776 */ +#define TAS5770_0DB SN012776_0DB +#define TAS5770_DB(x) SN012776_DB(x) + +#define SSM3515_0DB (255 - 64) /* +24dB max, steps of 3/8 dB */ +#define SSM3515_DB(x) (SSM3515_0DB + (8 * (x) / 3)) + +struct macaudio_snd_data { + struct snd_soc_card card; + struct snd_soc_jack jack; + int jack_plugin_state; + + const struct macaudio_platform_cfg *cfg; + bool has_speakers; + bool has_sense; + bool has_safety; + unsigned int max_channels; + + struct macaudio_link_props { + /* frontend props */ + unsigned int bclk_ratio; + bool is_sense; + + /* backend props */ + bool is_speakers; + bool is_headphones; + unsigned int tdm_mask; + } *link_props; + + int speaker_sample_rate; + struct snd_kcontrol *speaker_sample_rate_kctl; + + struct mutex volume_lock_mutex; + bool speaker_volume_unlocked; + bool speaker_volume_was_locked; + struct snd_kcontrol *speaker_lock_kctl; + struct snd_ctl_file *speaker_lock_owner; + u64 bes_active; + bool speaker_lock_timeout_enabled; + ktime_t speaker_lock_timeout; + ktime_t speaker_lock_remain; + struct delayed_work lock_timeout_work; + struct work_struct lock_update_work; + +}; + +static int please_blow_up_my_speakers; +module_param(please_blow_up_my_speakers, int, 0644); +MODULE_PARM_DESC(please_blow_up_my_speakers, "Allow unsafe or untested operating configurations"); + +SND_SOC_DAILINK_DEFS(primary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); // platform (filled at runtime) + +SND_SOC_DAILINK_DEFS(secondary, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-1")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +SND_SOC_DAILINK_DEFS(sense, + DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-2")), // CPU + DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC + DAILINK_COMP_ARRAY(COMP_EMPTY())); + +static struct snd_soc_dai_link macaudio_fe_links[] = { + { + .name = "Primary", + .stream_name = "Primary", + .dynamic = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + SND_SOC_DAILINK_REG(primary), + }, + { + .name = "Secondary", + .stream_name = "Secondary", + .dynamic = 1, + .dpcm_merged_rate = 1, + .dpcm_merged_chan = 1, + .dpcm_merged_format = 1, + .dai_fmt = MACAUDIO_DAI_FMT, + .playback_only = 1, + SND_SOC_DAILINK_REG(secondary), + }, + { + .name = "Speaker Sense", + .stream_name = "Speaker Sense", + .capture_only = 1, + .dynamic = 1, + .dai_fmt = (SND_SOC_DAIFMT_I2S | \ + SND_SOC_DAIFMT_CBP_CFP | \ + SND_SOC_DAIFMT_GATED | \ + SND_SOC_DAIFMT_IB_IF), + SND_SOC_DAILINK_REG(sense), + }, +}; + +static struct macaudio_link_props macaudio_fe_link_props[] = { + { + /* + * Primary FE + * + * The bclk ratio at 64 for the primary frontend is important + * to ensure that the headphones codec's idea of left and right + * in a stereo stream over I2S fits in nicely with everyone else's. + * (This is until the headphones codec's driver supports + * set_tdm_slot.) + * + * The low bclk ratio precludes transmitting more than two + * channels over I2S, but that's okay since there is the secondary + * FE for speaker arrays anyway. + */ + .bclk_ratio = 64, + }, + { + /* + * Secondary FE + * + * Here we want frames plenty long to be able to drive all + * those fancy speaker arrays. + */ + .bclk_ratio = 256, + }, + { + .is_sense = 1, + } +}; + +static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) +{ + int ret, max; + const char *name = volume_control_names[ma->cfg->amp]; + + if (!name) { + WARN_ON_ONCE(1); + return; + } + + switch (ma->cfg->amp) { + case AMP_NONE: + WARN_ON_ONCE(1); + return; + case AMP_TAS5770: + if (unlock) + max = TAS5770_0DB; + else + max = 1; //TAS5770_DB(ma->cfg->safe_vol); + break; + case AMP_SN012776: + if (unlock) + max = SN012776_0DB; + else + max = 1; //SN012776_DB(ma->cfg->safe_vol); + break; + case AMP_SSM3515: + if (unlock) + max = SSM3515_0DB; + else + max = SSM3515_DB(ma->cfg->safe_vol); + break; + } + + ret = snd_soc_limit_volume(&ma->card, name, max); + if (ret < 0) + dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", + unlock ? "un" : "", name, ret); +} + +static void macaudio_vlimit_update(struct macaudio_snd_data *ma) +{ + int i; + bool unlock = true; + struct snd_kcontrol *kctl; + const char *reason; + + /* Do nothing if there is no safety configured */ + if (!ma->has_safety) + return; + + /* Check that someone is holding the main lock */ + if (!ma->speaker_lock_owner) { + reason = "Main control not locked"; + unlock = false; + } + + /* Check that the control has been pinged within the timeout */ + if (ma->speaker_lock_remain <= 0) { + reason = "Lock timeout"; + unlock = false; + } + + /* Check that *every* limited control is locked by the same owner */ + list_for_each_entry(kctl, &ma->card.snd_card->controls, list) { + if(!snd_soc_control_matches(kctl, volume_control_names[ma->cfg->amp])) + continue; + + for (i = 0; i < kctl->count; i++) { + if (kctl->vd[i].owner != ma->speaker_lock_owner) { + reason = "Not all child controls locked by the same process"; + unlock = false; + } + } + } + + + if (unlock != ma->speaker_volume_unlocked) { + if (unlock) { + dev_info(ma->card.dev, "Speaker volumes unlocked\n"); + } else { + dev_info(ma->card.dev, "Speaker volumes locked: %s\n", reason); + ma->speaker_volume_was_locked = true; + } + + macaudio_vlimit_unlock(ma, unlock); + ma->speaker_volume_unlocked = unlock; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_lock_kctl->id); + } +} + +static void macaudio_vlimit_enable_timeout(struct macaudio_snd_data *ma) +{ + mutex_lock(&ma->volume_lock_mutex); + + if (ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + if (ma->speaker_lock_remain > 0) { + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + dev_dbg(ma->card.dev, "Enabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + } + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = true; + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_disable_timeout(struct macaudio_snd_data *ma) +{ + ktime_t now; + + mutex_lock(&ma->volume_lock_mutex); + + if (!ma->speaker_lock_timeout_enabled) { + mutex_unlock(&ma->volume_lock_mutex); + return; + } + + now = ktime_get(); + + cancel_delayed_work(&ma->lock_timeout_work); + + if (ktime_after(now, ma->speaker_lock_timeout)) + ma->speaker_lock_remain = 0; + else if (ma->speaker_lock_remain > 0) + ma->speaker_lock_remain = ktime_sub(ma->speaker_lock_timeout, now); + + dev_dbg(ma->card.dev, "Disabling volume limit timeout: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + + macaudio_vlimit_update(ma); + + ma->speaker_lock_timeout_enabled = false; + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_timeout_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(to_delayed_work(wrk), + struct macaudio_snd_data, lock_timeout_work); + + mutex_lock(&ma->volume_lock_mutex); + + ma->speaker_lock_remain = 0; + macaudio_vlimit_update(ma); + + mutex_unlock(&ma->volume_lock_mutex); +} + +static void macaudio_vlimit_update_work(struct work_struct *wrk) +{ + struct macaudio_snd_data *ma = container_of(wrk, + struct macaudio_snd_data, lock_update_work); + + if (ma->bes_active) + macaudio_vlimit_enable_timeout(ma); + else + macaudio_vlimit_disable_timeout(ma); +} + +static int macaudio_copy_link(struct device *dev, struct snd_soc_dai_link *target, + struct snd_soc_dai_link *source) +{ + memcpy(target, source, sizeof(struct snd_soc_dai_link)); + + target->cpus = devm_kmemdup(dev, target->cpus, + sizeof(*target->cpus) * target->num_cpus, + GFP_KERNEL); + target->codecs = devm_kmemdup(dev, target->codecs, + sizeof(*target->codecs) * target->num_codecs, + GFP_KERNEL); + target->platforms = devm_kmemdup(dev, target->platforms, + sizeof(*target->platforms) * target->num_platforms, + GFP_KERNEL); + + if (!target->cpus || !target->codecs || !target->platforms) + return -ENOMEM; + + return 0; +} + +static int macaudio_parse_of_component(struct device_node *node, int index, + struct snd_soc_dai_link_component *comp) +{ + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_args(node, "sound-dai", "#sound-dai-cells", + index, &args); + if (ret) + return ret; + comp->of_node = args.np; + return snd_soc_get_dai_name(&args, &comp->dai_name); +} + +/* + * Parse one DPCM backend from the devicetree. This means taking one + * of the CPU DAIs and combining it with one or more CODEC DAIs. + */ +static int macaudio_parse_of_be_dai_link(struct macaudio_snd_data *ma, + struct snd_soc_dai_link *link, + int be_index, int ncodecs_per_be, + struct device_node *cpu, + struct device_node *codec) +{ + struct snd_soc_dai_link_component *comp; + struct device *dev = ma->card.dev; + int codec_base = be_index * ncodecs_per_be; + int ret, i; + + link->no_pcm = 1; + + link->dai_fmt = MACAUDIO_DAI_FMT; + + link->num_codecs = ncodecs_per_be; + link->codecs = devm_kcalloc(dev, ncodecs_per_be, + sizeof(*comp), GFP_KERNEL); + link->num_cpus = 1; + link->cpus = devm_kzalloc(dev, sizeof(*comp), GFP_KERNEL); + + if (!link->codecs || !link->cpus) + return -ENOMEM; + + link->num_platforms = 0; + + for_each_link_codecs(link, i, comp) { + ret = macaudio_parse_of_component(codec, codec_base + i, comp); + if (ret) + return dev_err_probe(ma->card.dev, ret, "parsing CODEC DAI of link '%s' at %pOF\n", + link->name, codec); + } + + ret = macaudio_parse_of_component(cpu, be_index, link->cpus); + if (ret) + return dev_err_probe(ma->card.dev, ret, "parsing CPU DAI of link '%s' at %pOF\n", + link->name, codec); + + link->name = link->cpus[0].dai_name; + + return 0; +} + +static int macaudio_parse_of(struct macaudio_snd_data *ma) +{ + struct device_node *codec = NULL; + struct device_node *cpu = NULL; + struct device_node *np = NULL; + struct device_node *platform = NULL; + struct snd_soc_dai_link *link = NULL; + struct snd_soc_card *card = &ma->card; + struct device *dev = card->dev; + struct macaudio_link_props *link_props; + int ret, num_links, i; + + ret = snd_soc_of_parse_card_name(card, "model"); + if (ret) { + dev_err_probe(dev, ret, "parsing card name\n"); + return ret; + } + /* + * Set long_name to prevent snd_soc_set_dmi_name() from setting one from + * make believe data u-boot provides in its SMBIOS emulation. + */ + card->long_name = card->name; + + /* Populate links, start with the fixed number of FE links */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Now add together the (dynamic) number of BE links */ + for_each_available_child_of_node(dev->of_node, np) { + int num_cpus; + + cpu = of_get_child_by_name(np, "cpu"); + if (!cpu) { + ret = dev_err_probe(dev, -EINVAL, + "missing CPU DAI node at %pOF\n", np); + goto err_free; + } + + num_cpus = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + + if (num_cpus <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", cpu); + goto err_free; + } + of_node_put(cpu); + cpu = NULL; + + /* Each CPU specified counts as one BE link */ + num_links += num_cpus; + } + + /* Allocate the DAI link array */ + card->dai_link = devm_kcalloc(dev, num_links, sizeof(*link), GFP_KERNEL); + ma->link_props = devm_kcalloc(dev, num_links, sizeof(*ma->link_props), GFP_KERNEL); + if (!card->dai_link || !ma->link_props) + return -ENOMEM; + + link = card->dai_link; + link_props = ma->link_props; + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + ret = macaudio_copy_link(dev, link, &macaudio_fe_links[i]); + if (ret) + goto err_free; + + memcpy(link_props, &macaudio_fe_link_props[i], sizeof(struct macaudio_link_props)); + link++; link_props++; + } + + for (i = 0; i < num_links; i++) + card->dai_link[i].id = i; + + /* We might disable the speakers, so count again */ + num_links = ARRAY_SIZE(macaudio_fe_links); + + /* Fill in the BEs */ + for_each_available_child_of_node(dev->of_node, np) { + const char *link_name; + bool speakers; + int be_index, num_codecs, num_bes, ncodecs_per_cpu, nchannels; + unsigned int left_mask, right_mask; + + ret = of_property_read_string(np, "link-name", &link_name); + if (ret) { + dev_err_probe(card->dev, ret, "missing link name\n"); + goto err_free; + } + + dev_dbg(ma->card.dev, "parsing link '%s'\n", link_name); + + speakers = !strcmp(link_name, "Speaker") + || !strcmp(link_name, "Speakers"); + if (speakers) { + if (!ma->cfg->enable_speakers && !please_blow_up_my_speakers) { + dev_err(card->dev, "driver can't assure safety on this model, disabling speakers\n"); + continue; + } + ma->has_speakers = 1; + if (ma->cfg->amp != AMP_SSM3515 && ma->cfg->safe_vol != 0) + ma->has_sense = 1; + } + + cpu = of_get_child_by_name(np, "cpu"); + codec = of_get_child_by_name(np, "codec"); + + if (!codec || !cpu) { + ret = dev_err_probe(dev, -EINVAL, + "missing DAI specifications for '%s'\n", link_name); + goto err_free; + } + + num_bes = of_count_phandle_with_args(cpu, "sound-dai", + "#sound-dai-cells"); + if (num_bes <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", cpu); + goto err_free; + } + + num_codecs = of_count_phandle_with_args(codec, "sound-dai", + "#sound-dai-cells"); + if (num_codecs <= 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "missing sound-dai property at %pOF\n", codec); + goto err_free; + } + + dev_dbg(ma->card.dev, "link '%s': %d CPUs %d CODECs\n", + link_name, num_bes, num_codecs); + + if (num_codecs % num_bes != 0) { + ret = dev_err_probe(card->dev, -EINVAL, + "bad combination of CODEC (%d) and CPU (%d) number at %pOF\n", + num_codecs, num_bes, np); + goto err_free; + } + + /* + * Now parse the cpu/codec lists into a number of DPCM backend links. + * In each link there will be one DAI from the cpu list paired with + * an evenly distributed number of DAIs from the codec list. (As is + * the binding semantics.) + */ + ncodecs_per_cpu = num_codecs / num_bes; + nchannels = num_codecs * (speakers ? 1 : 2); + + /* Save the max number of channels on the platform */ + if (nchannels > ma->max_channels) + ma->max_channels = nchannels; + + /* + * If there is a single speaker, assign two channels to it, because + * it can do downmix. + */ + if (nchannels < 2) + nchannels = 2; + + left_mask = 0; + for (i = 0; i < nchannels; i += 2) + left_mask = left_mask << 2 | 1; + right_mask = left_mask << 1; + + for (be_index = 0; be_index < num_bes; be_index++) { + /* + * Set initial link name to be overwritten by a BE-specific + * name later so that we can use at least use the provisional + * name in error messages. + */ + link->name = link_name; + + ret = macaudio_parse_of_be_dai_link(ma, link, be_index, + ncodecs_per_cpu, cpu, codec); + if (ret) + goto err_free; + + link_props->is_speakers = speakers; + link_props->is_headphones = !speakers; + + if (num_bes == 2) + /* This sound peripheral is split between left and right BE */ + link_props->tdm_mask = be_index ? right_mask : left_mask; + else + /* One BE covers all of the peripheral */ + link_props->tdm_mask = left_mask | right_mask; + + /* Steal platform OF reference for use in FE links later */ + platform = link->cpus->of_node; + + link++; link_props++; + } + + of_node_put(codec); + of_node_put(cpu); + cpu = codec = NULL; + + num_links += num_bes; + } + + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) + card->dai_link[i].platforms->of_node = platform; + + /* Skip the speaker sense PCM link if this amp has no sense (or no speakers) */ + if (!ma->has_sense) { + for (i = 0; i < ARRAY_SIZE(macaudio_fe_links); i++) { + if (ma->link_props[i].is_sense) { + memmove(&card->dai_link[i], &card->dai_link[i + 1], + (num_links - i - 1) * sizeof (struct snd_soc_dai_link)); + num_links--; + break; + } + } + } + + card->num_links = num_links; + + return 0; + +err_free: + of_node_put(codec); + of_node_put(cpu); + of_node_put(np); + + if (!card->dai_link) + return ret; + + for (i = 0; i < num_links; i++) { + /* + * TODO: If we don't go through this path are the references + * freed inside ASoC? + */ + snd_soc_of_put_dai_link_codecs(&card->dai_link[i]); + snd_soc_of_put_dai_link_cpus(&card->dai_link[i]); + } + + return ret; +} + +static int macaudio_get_runtime_bclk_ratio(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct snd_soc_dpcm *dpcm; + + /* + * If this is a FE, look it up in link_props directly. + * If this is a BE, look it up in the respective FE. + */ + if (!rtd->dai_link->no_pcm) + return ma->link_props[rtd->dai_link->id].bclk_ratio; + + for_each_dpcm_fe(rtd, substream->stream, dpcm) { + int fe_id = dpcm->fe->dai_link->id; + + return ma->link_props[fe_id].bclk_ratio; + } + + return 0; +} + +static int macaudio_dpcm_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); + struct snd_interval *rate = hw_param_interval(params, + SNDRV_PCM_HW_PARAM_RATE); + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (props->is_sense) { + rate->min = rate->max = cpu_dai->symmetric_rate; + return 0; + } + + /* Speakers BE */ + if (props->is_speakers) { + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) { + /* Sense PCM: keep the existing BE rate (0 if not already running) */ + rate->min = rate->max = cpu_dai->symmetric_rate; + + return 0; + } else { + /* + * Set the sense PCM rate control to inform userspace of the + * new sample rate. + */ + ma->speaker_sample_rate = params_rate(params); + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); + } + } + + if (bclk_ratio) { + struct snd_soc_dai *dai; + int mclk = params_rate(params) * bclk_ratio; + + for_each_rtd_codec_dais(rtd, i, dai) { + snd_soc_dai_set_sysclk(dai, 0, mclk, SND_SOC_CLOCK_IN); + snd_soc_dai_set_bclk_ratio(dai, bclk_ratio); + } + + snd_soc_dai_set_sysclk(cpu_dai, 0, mclk, SND_SOC_CLOCK_OUT); + snd_soc_dai_set_bclk_ratio(cpu_dai, bclk_ratio); + } + + return 0; +} + +static int macaudio_fe_startup(struct snd_pcm_substream *substream) +{ + + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int max_rate, ret; + + if (props->is_sense) { + /* + * Sense stream will not return data while playback is inactive, + * so do not time out. + */ + substream->wait_time = MAX_SCHEDULE_TIMEOUT; + return 0; + } + + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_CHANNELS, + 0, ma->max_channels); + if (ret < 0) + return ret; + + max_rate = MACAUDIO_MAX_BCLK_FREQ / props->bclk_ratio; + ret = snd_pcm_hw_constraint_minmax(substream->runtime, + SNDRV_PCM_HW_PARAM_RATE, + 0, max_rate); + if (ret < 0) + return ret; + + return 0; +} + +static int macaudio_fe_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *be; + struct snd_soc_dpcm *dpcm; + + be = NULL; + for_each_dpcm_be(rtd, substream->stream, dpcm) { + be = dpcm->be; + break; + } + + if (!be) { + dev_err(rtd->dev, "opening PCM device '%s' with no audio route configured by the user\n", + rtd->dai_link->name); + return -EINVAL; + } + + return macaudio_dpcm_hw_params(substream, params); +} + + +static void macaudio_dpcm_shutdown(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *dai; + int bclk_ratio = macaudio_get_runtime_bclk_ratio(substream); + int i; + + if (bclk_ratio) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_dai_set_sysclk(dai, 0, 0, SND_SOC_CLOCK_IN); + + snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT); + } +} + +static int macaudio_be_hw_free(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_speakers && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + /* + * Clear the DAI rates, so the next open can change the sample rate. + * This won't happen automatically if the sense PCM is open. + */ + for_each_rtd_dais(rtd, i, dai) { + dai->symmetric_rate = 0; + } + + /* Notify userspace that the speakers are closed */ + ma->speaker_sample_rate = 0; + snd_ctl_notify(ma->card.snd_card, SNDRV_CTL_EVENT_MASK_VALUE, + &ma->speaker_sample_rate_kctl->id); + } + + return 0; +} + +static int macaudio_be_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(rtd->card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (props->is_speakers && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + ma->bes_active |= BIT(rtd->dai_link->id); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_STOP: + ma->bes_active &= ~BIT(rtd->dai_link->id); + break; + default: + return -EINVAL; + } + + schedule_work(&ma->lock_update_work); + } + + return 0; +} + +static const struct snd_soc_ops macaudio_fe_ops = { + .startup = macaudio_fe_startup, + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_fe_hw_params, +}; + +static const struct snd_soc_ops macaudio_be_ops = { + .hw_free = macaudio_be_hw_free, + .shutdown = macaudio_dpcm_shutdown, + .hw_params = macaudio_dpcm_hw_params, + .trigger = macaudio_be_trigger, +}; + +static int macaudio_be_assign_tdm(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + unsigned int mask; + int nslots, ret, i; + + if (!props->tdm_mask) + return 0; + + mask = props->tdm_mask; + nslots = __fls(mask) + 1; + + if (rtd->dai_link->num_codecs == 1) { + ret = snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_codec(rtd, 0), mask, + 0, nslots, MACAUDIO_SLOTWIDTH); + + /* + * Headphones get a pass on -ENOTSUPP (see the comment + * around bclk_ratio value for primary FE). + */ + if (ret == -ENOTSUPP && props->is_headphones) + return 0; + + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + int slot = __ffs(mask); + + mask &= ~(1 << slot); + ret = snd_soc_dai_set_tdm_slot(dai, 1 << slot, 0, nslots, + MACAUDIO_SLOTWIDTH); + if (ret) + return ret; + } + + return 0; +} + +static int macaudio_be_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i, ret; + + ret = macaudio_be_assign_tdm(rtd); + if (ret < 0) + return ret; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, &ma->jack, NULL); + } + + return 0; +} + +static void macaudio_be_exit(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + struct snd_soc_dai *dai; + int i; + + if (props->is_headphones) { + for_each_rtd_codec_dais(rtd, i, dai) + snd_soc_component_set_jack(dai->component, NULL, NULL); + } +} + +static int macaudio_fe_init(struct snd_soc_pcm_runtime *rtd) +{ + struct snd_soc_card *card = rtd->card; + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + int nslots = props->bclk_ratio / MACAUDIO_SLOTWIDTH; + + if (props->is_sense) + return snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_cpu(rtd, 0), 0, 0xffff, 16, 16); + + return snd_soc_dai_set_tdm_slot(snd_soc_rtd_to_cpu(rtd, 0), (1 << nslots) - 1, + (1 << nslots) - 1, nslots, MACAUDIO_SLOTWIDTH); +} + +static struct snd_soc_jack_pin macaudio_jack_pins[] = { + { + .pin = "Headphone", + .mask = SND_JACK_HEADPHONE, + }, + { + .pin = "Headset Mic", + .mask = SND_JACK_MICROPHONE, + }, +}; + +static int macaudio_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + int ret; + + dev_dbg(card->dev, "%s!\n", __func__); + + ret = snd_soc_card_jack_new_pins(card, "Headphone Jack", + SND_JACK_HEADSET | SND_JACK_HEADPHONE, + &ma->jack, macaudio_jack_pins, + ARRAY_SIZE(macaudio_jack_pins)); + if (ret < 0) { + dev_err(card->dev, "jack creation failed: %d\n", ret); + return ret; + } + + return ret; +} + +static int macaudio_add_backend_dai_route(struct snd_soc_card *card, struct snd_soc_dai *dai, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card); + struct snd_soc_dapm_route *r; + int nroutes = 0; + int ret; + + memset(routes, 0, sizeof(routes)); + + dev_dbg(card->dev, "adding routes for '%s'\n", dai->name); + + r = &routes[nroutes++]; + if (is_speakers) + r->source = "Speaker Playback"; + else + r->source = "Headphone Playback"; + r->sink = dai->stream[SNDRV_PCM_STREAM_PLAYBACK].widget->name; + + /* If headphone jack, add capture path */ + if (!is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Headset Capture"; + } + + /* If speakers, add sense capture path */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = dai->stream[SNDRV_PCM_STREAM_CAPTURE].widget->name; + r->sink = "Speaker Sense Capture"; + } + + ret = snd_soc_dapm_add_routes(dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + dai->name); + return ret; +} + +static int macaudio_add_pin_routes(struct snd_soc_card *card, struct snd_soc_component *component, + bool is_speakers) +{ + struct snd_soc_dapm_route routes[2]; + struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card); + struct snd_soc_dapm_route *r; + int nroutes = 0; + char buf[32]; + int ret; + + memset(routes, 0, sizeof(routes)); + + /* Connect the far ends of CODECs to pins */ + if (is_speakers) { + r = &routes[nroutes++]; + r->source = "OUT"; + if (component->name_prefix) { + snprintf(buf, sizeof(buf) - 1, "%s OUT", component->name_prefix); + r->source = buf; + } + r->sink = "Speaker"; + } else { + r = &routes[nroutes++]; + r->source = "Jack HP"; + r->sink = "Headphone"; + r = &routes[nroutes++]; + r->source = "Headset Mic"; + r->sink = "Jack HS"; + } + + ret = snd_soc_dapm_add_routes(dapm, routes, nroutes); + if (ret) + dev_err(card->dev, "failed adding dynamic DAPM routes for %s\n", + component->name); + return ret; +} + +static int macaudio_late_probe(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + struct snd_soc_pcm_runtime *rtd; + struct snd_soc_dai *dai; + int ret, i; + + /* Add the dynamic DAPM routes */ + for_each_card_rtds(card, rtd) { + struct macaudio_link_props *props = &ma->link_props[rtd->dai_link->id]; + + if (!rtd->dai_link->no_pcm) + continue; + + for_each_rtd_cpu_dais(rtd, i, dai) { + ret = macaudio_add_backend_dai_route(card, dai, props->is_speakers); + + if (ret) + return ret; + } + + for_each_rtd_codec_dais(rtd, i, dai) { + ret = macaudio_add_pin_routes(card, dai->component, + props->is_speakers); + + if (ret) + return ret; + } + } + + if (ma->has_speakers) + ma->speaker_sample_rate_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Sample Rate"); + if (ma->has_safety) { + ma->speaker_lock_kctl = snd_soc_card_get_kcontrol(card, + "Speaker Volume Unlock"); + + mutex_lock(&ma->volume_lock_mutex); + macaudio_vlimit_unlock(ma, false); + mutex_unlock(&ma->volume_lock_mutex); + } + + return 0; +} + +#define CHECK(call, pattern, value, min) \ + { \ + int ret = call(card, pattern, value); \ + int err = (ret >= 0 && ret < min) ? -ERANGE : ret; \ + if (err < 0) { \ + dev_err(card->dev, "%s on '%s': %d\n", #call, pattern, \ + ret); \ + if (please_blow_up_my_speakers < 2) \ + return err; \ + } else { \ + dev_dbg(card->dev, "%s on '%s': %d hits\n", #call, \ + pattern, ret); \ + } \ + } + +#define CHECK_CONCAT(call, suffix, value) \ + { \ + snprintf(buf, sizeof(buf), "%s%s", prefix, suffix); \ + CHECK(call, buf, value, 1); \ + } + +static int macaudio_set_speaker(struct snd_soc_card *card, const char *prefix, bool tweeter) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + char buf[256]; + + if (!ma->has_speakers) + return 0; + + switch (ma->cfg->amp) { + case AMP_TAS5770: + if (ma->cfg->stereo) { + CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); + } + + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); + break; + case AMP_SN012776: + if (ma->cfg->stereo) { + CHECK_CONCAT(snd_soc_set_enum_kctl, "ASI1 Sel", "Left"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "ASI1 Sel", 0); + } + + CHECK_CONCAT(snd_soc_limit_volume, "Amp Gain Volume", ma->cfg->amp_gain); + CHECK_CONCAT(snd_soc_set_enum_kctl, "HPF Corner Frequency", + tweeter ? "800 Hz" : "2 Hz"); + + if (please_blow_up_my_speakers < 2) + CHECK_CONCAT(snd_soc_deactivate_kctl, "HPF Corner Frequency", 0); + + CHECK_CONCAT(snd_soc_set_enum_kctl, "OCE Handling", "Retry"); + CHECK_CONCAT(snd_soc_deactivate_kctl, "OCE Handling", 0); + break; + case AMP_SSM3515: + /* TODO: check */ + CHECK_CONCAT(snd_soc_set_enum_kctl, "DAC Analog Gain Select", "8.4 V Span"); + + if (please_blow_up_my_speakers < 2) + CHECK_CONCAT(snd_soc_deactivate_kctl, "DAC Analog Gain Select", 0); + + /* TODO: HPF, needs new call to set */ + break; + default: + return -EINVAL; + } + + return 0; +} + +static int macaudio_fixup_controls(struct snd_soc_card *card) +{ + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + const char *p; + + /* Set the card ID early to avoid races with udev */ + p = strrchr(card->name, ' '); + if (p) { + snprintf(card->snd_card->id, sizeof(card->snd_card->id), + "Apple%s", p + 1); + } + + if (!ma->has_speakers) + return 0; + + /* + * This needs some care to avoid matches against cs42l84's + * "Jack HPF Corner Frequency". + */ + switch(ma->cfg->speakers) { + case SPKR_NONE: + WARN_ON(please_blow_up_my_speakers < 2); + return please_blow_up_my_speakers >= 2 ? 0 : -EINVAL; + case SPKR_1W: + /* only 1W stereo system (J313) is uses cs42l83 */ + if (ma->cfg->stereo) { + CHECK(macaudio_set_speaker, "* ", false, 0); + } else { + CHECK(macaudio_set_speaker, "", false, 0); + } + break; + case SPKR_2W: + CHECK(macaudio_set_speaker, "* Front ", false, 0); + CHECK(macaudio_set_speaker, "* Rear ", false, 0); + break; + case SPKR_1W1T: + CHECK(macaudio_set_speaker, "* Tweeter ", true, 0); + CHECK(macaudio_set_speaker, "* Woofer ", false, 0); + break; + case SPKR_2W1T: + CHECK(macaudio_set_speaker, "* Tweeter ", true, 0); + CHECK(macaudio_set_speaker, "* Woofer 1 ", false, 0); + CHECK(macaudio_set_speaker, "* Woofer 2 ", false, 0); + break; + } + + return 0; +} + +static const char * const macaudio_spk_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_spk_mux_enum, macaudio_spk_mux_texts); + +static const struct snd_kcontrol_new macaudio_spk_mux = + SOC_DAPM_ENUM("Speaker Playback Mux", macaudio_spk_mux_enum); + +static const char * const macaudio_hp_mux_texts[] = { + "Primary", + "Secondary" +}; + +SOC_ENUM_SINGLE_VIRT_DECL(macaudio_hp_mux_enum, macaudio_hp_mux_texts); + +static const struct snd_kcontrol_new macaudio_hp_mux = + SOC_DAPM_ENUM("Headphones Playback Mux", macaudio_hp_mux_enum); + +static const struct snd_soc_dapm_widget macaudio_snd_widgets[] = { + SND_SOC_DAPM_SPK("Speaker", NULL), + SND_SOC_DAPM_SPK("Speaker (Static)", NULL), + SND_SOC_DAPM_HP("Headphone", NULL), + SND_SOC_DAPM_MIC("Headset Mic", NULL), + + SND_SOC_DAPM_MUX("Speaker Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_spk_mux), + SND_SOC_DAPM_MUX("Headphone Playback Mux", SND_SOC_NOPM, 0, 0, &macaudio_hp_mux), + + SND_SOC_DAPM_AIF_OUT("Speaker Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_OUT("Headphone Playback", NULL, 0, SND_SOC_NOPM, 0, 0), + + SND_SOC_DAPM_AIF_IN("Headset Capture", NULL, 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("Speaker Sense Capture", NULL, 0, SND_SOC_NOPM, 0, 0), +}; + +static int macaudio_sss_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = 0; + uinfo->value.integer.max = 192000; + + return 0; +} + +static int macaudio_sss_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + /* + * TODO: Check if any locking is in order here. I would + * assume there is some ALSA-level lock, but DAPM implementations + * of kcontrol ops do explicit locking, so look into it. + */ + uvalue->value.integer.value[0] = ma->speaker_sample_rate; + + return 0; +} + +static int macaudio_slk_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER; + uinfo->count = 1; + uinfo->value.integer.min = INT_MIN; + uinfo->value.integer.max = INT_MAX; + + return 0; +} + +static int macaudio_slk_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + if (!ma->speaker_lock_owner) + return -EPERM; + + if (uvalue->value.integer.value[0] != SPEAKER_MAGIC_VALUE) + return -EINVAL; + + /* Serves as a notification that the lock was lost at some point */ + if (ma->speaker_volume_was_locked) { + ma->speaker_volume_was_locked = false; + return -ETIMEDOUT; + } + + mutex_lock(&ma->volume_lock_mutex); + + cancel_delayed_work(&ma->lock_timeout_work); + + ma->speaker_lock_remain = ms_to_ktime(SPEAKER_LOCK_TIMEOUT); + ma->speaker_lock_timeout = ktime_add(ktime_get(), ma->speaker_lock_remain); + macaudio_vlimit_update(ma); + + if (ma->speaker_lock_timeout_enabled) { + dev_dbg(ma->card.dev, "Volume limit timeout ping: %ld us left\n", + (long)ktime_to_us(ma->speaker_lock_remain)); + schedule_delayed_work(&ma->lock_timeout_work, usecs_to_jiffies(ktime_to_us(ma->speaker_lock_remain))); + } + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static int macaudio_slk_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *uvalue) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + uvalue->value.integer.value[0] = ma->speaker_volume_unlocked ? 1 : 0; + + return 0; +} + +static int macaudio_slk_lock(struct snd_kcontrol *kcontrol, struct snd_ctl_file *owner) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + mutex_lock(&ma->volume_lock_mutex); + ma->speaker_lock_owner = owner; + macaudio_vlimit_update(ma); + + /* + * Reset the unintended lock flag when the control is first locked. + * At this point the state is locked and cannot be unlocked until + * userspace writes to this control, so this cannot spuriously become + * true again until that point. + */ + ma->speaker_volume_was_locked = false; + + mutex_unlock(&ma->volume_lock_mutex); + + return 0; +} + +static void macaudio_slk_unlock(struct snd_kcontrol *kcontrol) +{ + struct snd_soc_card *card = snd_kcontrol_chip(kcontrol); + struct macaudio_snd_data *ma = snd_soc_card_get_drvdata(card); + + ma->speaker_lock_owner = NULL; + ma->speaker_lock_timeout = 0; + macaudio_vlimit_update(ma); +} + +/* + * Speaker limit controls go last. We only drop the unlock control, + * leaving sample rate, since that can be useful for safety + * bring-up before the kernel-side caps are ready. + */ +#define MACAUDIO_NUM_SPEAKER_LIMIT_CONTROLS 1 +/* + * If there are no speakers configured at all, we can drop both + * controls. + */ +#define MACAUDIO_NUM_SPEAKER_CONTROLS 2 + +static const struct snd_kcontrol_new macaudio_controls[] = { + SOC_DAPM_PIN_SWITCH("Speaker"), + SOC_DAPM_PIN_SWITCH("Headphone"), + SOC_DAPM_PIN_SWITCH("Headset Mic"), + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Sample Rate", + .info = macaudio_sss_info, .get = macaudio_sss_get, + }, + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | + SNDRV_CTL_ELEM_ACCESS_WRITE | + SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "Speaker Volume Unlock", + .info = macaudio_slk_info, + .put = macaudio_slk_put, .get = macaudio_slk_get, + .lock = macaudio_slk_lock, .unlock = macaudio_slk_unlock, + }, +}; + +static const struct snd_soc_dapm_route macaudio_dapm_routes[] = { + /* Playback paths */ + { "Speaker Playback Mux", "Primary", "PCM0 TX" }, + { "Speaker Playback Mux", "Secondary", "PCM1 TX" }, + { "Speaker Playback", NULL, "Speaker Playback Mux"}, + + { "Headphone Playback Mux", "Primary", "PCM0 TX" }, + { "Headphone Playback Mux", "Secondary", "PCM1 TX" }, + { "Headphone Playback", NULL, "Headphone Playback Mux"}, + /* + * Additional paths (to specific I2S ports) are added dynamically. + */ + + /* Capture paths */ + { "PCM0 RX", NULL, "Headset Capture" }, + + /* Sense paths */ + { "PCM2 RX", NULL, "Speaker Sense Capture" }, +}; + +/* enable amp speakers stereo gain safe_vol */ +struct macaudio_platform_cfg macaudio_j180_cfg = { + false, AMP_SN012776, SPKR_1W1T, false, 10, -20, +}; +struct macaudio_platform_cfg macaudio_j274_cfg = { + true, AMP_TAS5770, SPKR_1W, false, 20, -20, +}; + +struct macaudio_platform_cfg macaudio_j293_cfg = { + true, AMP_TAS5770, SPKR_2W, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j313_cfg = { + true, AMP_TAS5770, SPKR_1W, true, 10, -20, +}; + +struct macaudio_platform_cfg macaudio_j314_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j316_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j37x_j47x_cfg = { + true, AMP_SN012776, SPKR_1W, false, 20, -20, +}; + +struct macaudio_platform_cfg macaudio_j413_cfg = { + true, AMP_SN012776, SPKR_1W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j415_cfg = { + true, AMP_SN012776, SPKR_2W1T, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_j45x_cfg = { + false, AMP_SSM3515, SPKR_1W1T, true, 9, -20, /* TODO: gain?? */ +}; + +struct macaudio_platform_cfg macaudio_j493_cfg = { + true, AMP_SN012776, SPKR_2W, true, 15, -20, +}; + +struct macaudio_platform_cfg macaudio_fallback_cfg = { + false, AMP_NONE, SPKR_NONE, false, 0, 0, +}; + +/* + * DT compatible/ID table rules: + * + * 1. Machines with **identical** speaker configurations (amps, models, chassis) + * are allowed to declare compatibility with the first model (chronologically), + * and are not enumerated in this array. + * + * 2. Machines with identical amps and speakers (=identical speaker protection + * rules) but a different chassis must use different compatibles, but may share + * the private data structure here. They are explicitly enumerated. + * + * 3. Machines with different amps or speaker layouts must use separate + * data structures. + * + * 4. Machines with identical speaker layouts and amps (but possibly different + * speaker models/chassis) may share the data structure, since only userspace + * cares about that (assuming our general -20dB safe level standard holds). + */ +static const struct of_device_id macaudio_snd_device_id[] = { + /* Model ID Amp Gain Speakers */ + /* j180 AID19 sn012776 10 1× 1W+1T */ + { .compatible = "apple,j180-macaudio", .data = &macaudio_j180_cfg }, + /* j274 AID6 tas5770 20 1× 1W */ + { .compatible = "apple,j274-macaudio", .data = &macaudio_j274_cfg }, + /* j293 AID3 tas5770 15 2× 2W */ + { .compatible = "apple,j293-macaudio", .data = &macaudio_j293_cfg }, + /* j313 AID4 tas5770 10 2× 1W */ + { .compatible = "apple,j313-macaudio", .data = &macaudio_j313_cfg }, + /* j314 AID8 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j314-macaudio", .data = &macaudio_j314_cfg }, + /* j316 AID9 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j316-macaudio", .data = &macaudio_j316_cfg }, + /* j375 AID10 sn012776 15 1× 1W */ + { .compatible = "apple,j375-macaudio", .data = &macaudio_j37x_j47x_cfg }, + /* j413 AID13 sn012776 15 2× 1W+1T */ + { .compatible = "apple,j413-macaudio", .data = &macaudio_j413_cfg }, + /* j414 AID14 sn012776 15 2× 2W+1T Compat: apple,j314-macaudio */ + /* j415 AID27 sn012776 15 2× 2W+1T */ + { .compatible = "apple,j415-macaudio", .data = &macaudio_j415_cfg }, + /* j416 AID15 sn012776 15 2× 2W+1T Compat: apple,j316-macaudio */ + /* j456 AID5 ssm3515 15 2× 1W+1T */ + { .compatible = "apple,j456-macaudio", .data = &macaudio_j45x_cfg }, + /* j457 AID7 ssm3515 15 2× 1W+1T Compat: apple,j456-macaudio */ + /* j473 AID12 sn012776 20 1× 1W */ + { .compatible = "apple,j473-macaudio", .data = &macaudio_j37x_j47x_cfg }, + /* j474 AID26 sn012776 20 1× 1W Compat: apple,j473-macaudio */ + /* j475 AID25 sn012776 20 1× 1W Compat: apple,j375-macaudio */ + /* j493 AID18 sn012776 15 2× 2W */ + { .compatible = "apple,j493-macaudio", .data = &macaudio_j493_cfg }, + /* Fallback, jack only */ + { .compatible = "apple,macaudio"}, + { } +}; +MODULE_DEVICE_TABLE(of, macaudio_snd_device_id); + +static int macaudio_snd_platform_probe(struct platform_device *pdev) +{ + struct snd_soc_card *card; + struct macaudio_snd_data *data; + struct device *dev = &pdev->dev; + struct snd_soc_dai_link *link; + const struct of_device_id *of_id; + int ret; + int i; + + of_id = of_match_device(macaudio_snd_device_id, dev); + if (!of_id) + return -EINVAL; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + card = &data->card; + snd_soc_card_set_drvdata(card, data); + dev_set_drvdata(&pdev->dev, data); + mutex_init(&data->volume_lock_mutex); + + card->owner = THIS_MODULE; + card->driver_name = "macaudio"; + card->dev = dev; + card->dapm_widgets = macaudio_snd_widgets; + card->num_dapm_widgets = ARRAY_SIZE(macaudio_snd_widgets); + card->dapm_routes = macaudio_dapm_routes; + card->num_dapm_routes = ARRAY_SIZE(macaudio_dapm_routes); + card->controls = macaudio_controls; + card->num_controls = ARRAY_SIZE(macaudio_controls); + card->probe = macaudio_probe; + card->late_probe = macaudio_late_probe; + card->component_chaining = true; + card->fully_routed = true; + + if (of_id->data) + data->cfg = of_id->data; + else + data->cfg = &macaudio_fallback_cfg; + + card->fixup_controls = macaudio_fixup_controls; + + ret = macaudio_parse_of(data); + if (ret) + return ret; + + /* Remove useless controls */ + if (!data->has_speakers) /* No speakers, remove both */ + card->num_controls -= MACAUDIO_NUM_SPEAKER_CONTROLS; + else if (!data->cfg->safe_vol) /* No safety, remove unlock */ + card->num_controls -= MACAUDIO_NUM_SPEAKER_LIMIT_CONTROLS; + else /* Speakers with safety, mark us as such */ + data->has_safety = true; + + for_each_card_prelinks(card, i, link) { + if (link->no_pcm) { + link->ops = &macaudio_be_ops; + link->init = macaudio_be_init; + link->exit = macaudio_be_exit; + } else { + link->ops = &macaudio_fe_ops; + link->init = macaudio_fe_init; + } + } + + INIT_WORK(&data->lock_update_work, macaudio_vlimit_update_work); + INIT_DELAYED_WORK(&data->lock_timeout_work, macaudio_vlimit_timeout_work); + + return devm_snd_soc_register_card(dev, card); +} + +static void macaudio_snd_platform_remove(struct platform_device *pdev) +{ + struct macaudio_snd_data *ma = dev_get_drvdata(&pdev->dev); + + cancel_delayed_work_sync(&ma->lock_timeout_work); +} + +static struct platform_driver macaudio_snd_driver = { + .probe = macaudio_snd_platform_probe, + .remove = macaudio_snd_platform_remove, + .driver = { + .name = DRIVER_NAME, + .of_match_table = macaudio_snd_device_id, + .pm = &snd_soc_pm_ops, + }, +}; +module_platform_driver(macaudio_snd_driver); + +MODULE_AUTHOR("Martin Povišer "); +MODULE_DESCRIPTION("Apple Silicon Macs machine-level sound driver"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index c4dcb2b545912c..01dacd10bd39ce 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -133,12 +133,17 @@ struct mca_cluster { struct clk *clk_parent; struct dma_chan *dma_chans[SNDRV_PCM_STREAM_LAST + 1]; - bool port_started[SNDRV_PCM_STREAM_LAST + 1]; - int port_driver; /* The cluster driving this cluster's port */ + bool clk_provider; + + bool port_clk_started[SNDRV_PCM_STREAM_LAST + 1]; + int port_clk_driver; /* The cluster driving this cluster's port */ bool clocks_in_use[SNDRV_PCM_STREAM_LAST + 1]; struct device_link *pd_link; + /* In case of clock consumer FE */ + int syncgen_in_use; + unsigned int bclk_ratio; /* Masks etc. picked up via the set_tdm_slot method */ @@ -157,7 +162,7 @@ struct mca_data { struct reset_control *rstc; struct device_link *pd_link; - /* Mutex for accessing port_driver of foreign clusters */ + /* Mutex for accessing port_clk_driver of foreign clusters */ struct mutex port_mutex; int nclusters; @@ -211,15 +216,21 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, SERDES_STATUS_RST); /* * Experiments suggest that it takes at most ~1 us - * for the bit to clear, so wait 2 us for good measure. + * for the bit to clear, so wait 5 us for good measure. */ - udelay(2); + udelay(50); WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) & SERDES_STATUS_RST); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, FIELD_PREP(SERDES_CONF_SYNC_SEL, 0)); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, FIELD_PREP(SERDES_CONF_SYNC_SEL, cl->no + 1)); + /* + * ADMAC gets started right after this. This delay seems + * to be needed for that to be reliable, e.g. ensure the + * clock is stable? + */ + udelay(100); break; default: break; @@ -256,11 +267,28 @@ static int mca_fe_trigger(struct snd_pcm_substream *substream, int cmd, return 0; } +static int mca_fe_get_portmask(struct snd_pcm_substream *substream) +{ + struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream); + struct snd_soc_dpcm *dpcm; + int mask = 0; + + for_each_dpcm_be(fe, substream->stream, dpcm) { + int no = mca_dai_to_cluster(snd_soc_rtd_to_cpu(dpcm->be, 0))->no; + mask |= 1 << no; + } + + return mask; +} + static int mca_fe_enable_clocks(struct mca_cluster *cl) { struct mca_data *mca = cl->host; int ret; + if (!cl->clk_provider) + return -EINVAL; + ret = clk_prepare_enable(cl->clk_parent); if (ret) { dev_err(mca->dev, @@ -274,6 +302,7 @@ static int mca_fe_enable_clocks(struct mca_cluster *cl) * the power state driver would error out on seeing the device * as clock-gated. */ + WARN_ON(cl->pd_link); cl->pd_link = device_link_add(mca->dev, cl->pd_dev, DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE); @@ -297,7 +326,11 @@ static void mca_fe_disable_clocks(struct mca_cluster *cl) mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, 0); - device_link_del(cl->pd_link); + if (cl->pd_link) { + device_link_del(cl->pd_link); + cl->pd_link = NULL; + } + clk_disable_unprepare(cl->clk_parent); } @@ -311,7 +344,7 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl) for (i = 0; i < mca->nclusters; i++) { be_cl = &mca->clusters[i]; - if (be_cl->port_driver != cl->no) + if (be_cl->port_clk_driver != cl->no) continue; for_each_pcm_streams(stream) { @@ -325,59 +358,55 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl) return false; } -static int mca_be_prepare(struct snd_pcm_substream *substream, +static int mca_fe_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct mca_cluster *cl = mca_dai_to_cluster(dai); struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - int ret; - if (cl->port_driver < 0) - return -EINVAL; + if (cl->clk_provider) + return 0; - fe_cl = &mca->clusters[cl->port_driver]; + if (!cl->syncgen_in_use) { + int port = ffs(mca_fe_get_portmask(substream)) - 1; - /* - * Typically the CODECs we are paired with will require clocks - * to be present at time of unmute with the 'mute_stream' op - * or at time of DAPM widget power-up. We need to enable clocks - * here at the latest (frontend prepare would be too late). - */ - if (!mca_fe_clocks_in_use(fe_cl)) { - ret = mca_fe_enable_clocks(fe_cl); - if (ret < 0) - return ret; - } + WARN_ON(cl->pd_link); + cl->pd_link = device_link_add(mca->dev, cl->pd_dev, + DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!cl->pd_link) { + dev_err(mca->dev, + "cluster %d: unable to prop-up power domain\n", cl->no); + return -EINVAL; + } - cl->clocks_in_use[substream->stream] = true; + writel_relaxed(port + 6 + 1, + cl->base + REG_SYNCGEN_MCLK_SEL); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, + SYNCGEN_STATUS_EN); + } + cl->syncgen_in_use |= 1 << substream->stream; return 0; } -static int mca_be_hw_free(struct snd_pcm_substream *substream, +static int mca_fe_hw_free(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_data *mca = cl->host; - struct mca_cluster *fe_cl; - - if (cl->port_driver < 0) - return -EINVAL; - /* - * We are operating on a foreign cluster here, but since we - * belong to the same PCM, accesses should have been - * synchronized at ASoC level. - */ - fe_cl = &mca->clusters[cl->port_driver]; - if (!mca_fe_clocks_in_use(fe_cl)) - return 0; /* Nothing to do */ + if (cl->clk_provider) + return 0; - cl->clocks_in_use[substream->stream] = false; + cl->syncgen_in_use &= ~(1 << substream->stream); + if (cl->syncgen_in_use) + return 0; - if (!mca_fe_clocks_in_use(fe_cl)) - mca_fe_disable_clocks(fe_cl); + mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0); + if (cl->pd_link) { + device_link_del(cl->pd_link); + cl->pd_link = NULL; + } return 0; } @@ -392,7 +421,7 @@ static unsigned int mca_crop_mask(unsigned int mask, int nchans) static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, unsigned int mask, int slots, int nchans, - int slot_width, bool is_tx, int port) + int slot_width, bool is_tx, int portmask) { __iomem void *serdes_base = cl->base + serdes_unit; u32 serdes_conf, serdes_conf_mask; @@ -451,7 +480,7 @@ static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit, serdes_base + REG_RX_SERDES_SLOTMASK); writel_relaxed(~((u32)mca_crop_mask(mask, nchans)), serdes_base + REG_RX_SERDES_SLOTMASK + 0x4); - writel_relaxed(1 << port, + writel_relaxed(portmask, serdes_base + REG_RX_SERDES_PORT); } @@ -507,9 +536,18 @@ static int mca_fe_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) u32 serdes_conf = 0; u32 bitstart; - if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) != - SND_SOC_DAIFMT_BP_FP) + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: + cl->clk_provider = true; + break; + + case SND_SOC_DAIFMT_BC_FC: + cl->clk_provider = false; + break; + + default: goto err; + } switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: @@ -566,24 +604,6 @@ static int mca_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) return 0; } -static int mca_fe_get_port(struct snd_pcm_substream *substream) -{ - struct snd_soc_pcm_runtime *fe = snd_soc_substream_to_rtd(substream); - struct snd_soc_pcm_runtime *be; - struct snd_soc_dpcm *dpcm; - - be = NULL; - for_each_dpcm_be(fe, substream->stream, dpcm) { - be = dpcm->be; - break; - } - - if (!be) - return -EINVAL; - - return mca_dai_to_cluster(snd_soc_rtd_to_cpu(be, 0))->no; -} - static int mca_fe_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params, struct snd_soc_dai *dai) @@ -597,7 +617,7 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, unsigned long bclk_ratio; unsigned int tdm_slots, tdm_slot_width, tdm_mask; u32 regval, pad; - int ret, port, nchans_ceiled; + int ret, portmask, nchans_ceiled; if (!cl->tdm_slot_width) { /* @@ -646,13 +666,13 @@ static int mca_fe_hw_params(struct snd_pcm_substream *substream, tdm_mask = (1 << tdm_slots) - 1; } - port = mca_fe_get_port(substream); - if (port < 0) - return port; + portmask = mca_fe_get_portmask(substream); + if (!portmask) + return -EINVAL; ret = mca_configure_serdes(cl, is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF, tdm_mask, tdm_slots, params_channels(params), - tdm_slot_width, is_tx, port); + tdm_slot_width, is_tx, portmask); if (ret) return ret; @@ -708,67 +728,123 @@ static const struct snd_soc_dai_ops mca_fe_ops = { .set_tdm_slot = mca_fe_set_tdm_slot, .hw_params = mca_fe_hw_params, .trigger = mca_fe_trigger, + .prepare = mca_fe_prepare, + .hw_free = mca_fe_hw_free, }; -static bool mca_be_started(struct mca_cluster *cl) +/* + * Is there a FE attached which will be feeding this port's clocks? + */ +static bool mca_be_clk_started(struct mca_cluster *cl) { int stream; for_each_pcm_streams(stream) - if (cl->port_started[stream]) + if (cl->port_clk_started[stream]) return true; return false; } -static int mca_be_startup(struct snd_pcm_substream *substream, +static struct snd_soc_pcm_runtime *mca_be_get_fe(struct snd_soc_pcm_runtime *be, + int stream) +{ + struct snd_soc_pcm_runtime *fe = NULL; + struct snd_soc_dpcm *dpcm; + + for_each_dpcm_fe(be, stream, dpcm) { + if (fe && dpcm->fe != fe) { + dev_err(be->dev, "many FE per one BE unsupported\n"); + return NULL; + } + + fe = dpcm->fe; + } + + return fe; +} + +static int mca_be_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); - struct snd_soc_pcm_runtime *fe; + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); struct mca_cluster *cl = mca_dai_to_cluster(dai); - struct mca_cluster *fe_cl; struct mca_data *mca = cl->host; - struct snd_soc_dpcm *dpcm; + struct mca_cluster *fe_cl, *fe_clk_cl; + int ret; - fe = NULL; + fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); - for_each_dpcm_fe(be, substream->stream, dpcm) { - if (fe && dpcm->fe != fe) { - dev_err(mca->dev, "many FE per one BE unsupported\n"); - return -EINVAL; - } + if (!fe_cl->clk_provider) + return 0; - fe = dpcm->fe; + if (cl->port_clk_driver < 0) + return 0; + + fe_clk_cl = &mca->clusters[cl->port_clk_driver]; + + /* + * Typically the CODECs we are paired with will require clocks + * to be present at time of unmute with the 'mute_stream' op + * or at time of DAPM widget power-up. We need to enable clocks + * here at the latest (frontend prepare would be too late). + */ + if (!mca_fe_clocks_in_use(fe_clk_cl)) { + ret = mca_fe_enable_clocks(fe_clk_cl); + if (ret < 0) + return ret; } + cl->clocks_in_use[substream->stream] = true; + + return 0; +} + +static int mca_be_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); + struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; + struct mca_data *mca = cl->host; + if (!fe) return -EINVAL; - fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); - if (mca_be_started(cl)) { + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), + cl->base + REG_PORT_DATA_SEL); + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, + PORT_ENABLES_TX_DATA); + } + + if (!fe_cl->clk_provider) + return 0; + + if (mca_be_clk_started(cl)) { /* * Port is already started in the other direction. * Make sure there isn't a conflict with another cluster - * driving the port. + * driving the port clocks. */ - if (cl->port_driver != fe_cl->no) + if (cl->port_clk_driver != fe_cl->no) return -EINVAL; - cl->port_started[substream->stream] = true; + cl->port_clk_started[substream->stream] = true; return 0; } - writel_relaxed(PORT_ENABLES_CLOCKS | PORT_ENABLES_TX_DATA, - cl->base + REG_PORT_ENABLES); writel_relaxed(FIELD_PREP(PORT_CLOCK_SEL, fe_cl->no + 1), cl->base + REG_PORT_CLOCK_SEL); - writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no), - cl->base + REG_PORT_DATA_SEL); + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, + PORT_ENABLES_CLOCKS); + mutex_lock(&mca->port_mutex); - cl->port_driver = fe_cl->no; + cl->port_clk_driver = fe_cl->no; mutex_unlock(&mca->port_mutex); - cl->port_started[substream->stream] = true; + cl->port_clk_started[substream->stream] = true; return 0; } @@ -776,27 +852,60 @@ static int mca_be_startup(struct snd_pcm_substream *substream, static void mca_be_shutdown(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { + struct snd_soc_pcm_runtime *be = snd_soc_substream_to_rtd(substream); + struct snd_soc_pcm_runtime *fe = mca_be_get_fe(be, substream->stream); struct mca_cluster *cl = mca_dai_to_cluster(dai); + struct mca_cluster *fe_cl; struct mca_data *mca = cl->host; - cl->port_started[substream->stream] = false; + if (cl->clocks_in_use[substream->stream] && + !WARN_ON(cl->port_clk_driver < 0)) { + struct mca_cluster *fe_cl = &mca->clusters[cl->port_clk_driver]; - if (!mca_be_started(cl)) { /* - * Were we the last direction to shutdown? - * Turn off the lights. + * Typically the CODECs we are paired with will require clocks + * to be present at time of mute with the 'mute_stream' op. + * We need to disable the clocks here at the earliest (hw_free + * would be too early). + * + * We are operating on a foreign cluster here, but since we + * belong to the same PCM, accesses should have been + * synchronized at ASoC level. */ - writel_relaxed(0, cl->base + REG_PORT_ENABLES); + cl->clocks_in_use[substream->stream] = false; + + if (!mca_fe_clocks_in_use(fe_cl)) + mca_fe_disable_clocks(fe_cl); + } + + if (!fe) + return; + fe_cl = mca_dai_to_cluster(snd_soc_rtd_to_cpu(fe, 0)); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_TX_DATA, 0); writel_relaxed(0, cl->base + REG_PORT_DATA_SEL); + } + + if (!fe_cl->clk_provider) + return; + + cl->port_clk_started[substream->stream] = false; + if (!mca_be_clk_started(cl)) { + /* + * Were we the last direction to shutdown? + * Turn off the lights (clocks). + */ + mca_modify(cl, REG_PORT_ENABLES, PORT_ENABLES_CLOCKS, 0); + writel_relaxed(0, cl->base + REG_PORT_CLOCK_SEL); mutex_lock(&mca->port_mutex); - cl->port_driver = -1; + cl->port_clk_driver = -1; mutex_unlock(&mca->port_mutex); } } static const struct snd_soc_dai_ops mca_be_ops = { .prepare = mca_be_prepare, - .hw_free = mca_be_hw_free, .startup = mca_be_startup, .shutdown = mca_be_shutdown, }; @@ -1020,8 +1129,10 @@ static void apple_mca_release(struct mca_data *mca) dev_pm_domain_detach(cl->pd_dev, true); } - if (mca->pd_link) + if (mca->pd_link) { device_link_del(mca->pd_link); + mca->pd_link = NULL; + } if (!IS_ERR_OR_NULL(mca->pd_dev)) dev_pm_domain_detach(mca->pd_dev, true); @@ -1096,7 +1207,7 @@ static int apple_mca_probe(struct platform_device *pdev) cl->host = mca; cl->no = i; cl->base = base + CLUSTER_STRIDE * i; - cl->port_driver = -1; + cl->port_clk_driver = -1; cl->clk_parent = of_clk_get(pdev->dev.of_node, i); if (IS_ERR(cl->clk_parent)) { dev_err(&pdev->dev, "unable to obtain clock %d: %ld\n", diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c index fdda1b747bf7e5..8ab2e60f80b4ff 100644 --- a/sound/soc/codecs/ab8500-codec.c +++ b/sound/soc/codecs/ab8500-codec.c @@ -2496,13 +2496,13 @@ static int ab8500_codec_probe(struct snd_soc_component *component) return status; } fc = (struct filter_control *) - &ab8500_filter_controls[AB8500_FILTER_ANC_FIR].private_value; + ab8500_filter_controls[AB8500_FILTER_ANC_FIR].private_value; drvdata->anc_fir_values = (long *)fc->value; fc = (struct filter_control *) - &ab8500_filter_controls[AB8500_FILTER_ANC_IIR].private_value; + ab8500_filter_controls[AB8500_FILTER_ANC_IIR].private_value; drvdata->anc_iir_values = (long *)fc->value; fc = (struct filter_control *) - &ab8500_filter_controls[AB8500_FILTER_SID_FIR].private_value; + ab8500_filter_controls[AB8500_FILTER_SID_FIR].private_value; drvdata->sid_fir_values = (long *)fc->value; snd_soc_dapm_disable_pin(dapm, "ANC Configure Input"); diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c index af87ebae98cb0b..0ddf9a8d39a072 100644 --- a/sound/soc/codecs/cs35l56-shared.c +++ b/sound/soc/codecs/cs35l56-shared.c @@ -108,8 +108,6 @@ int cs35l56_set_patch(struct cs35l56_base *cs35l56_base) EXPORT_SYMBOL_NS_GPL(cs35l56_set_patch, "SND_SOC_CS35L56_SHARED"); static const struct reg_default cs35l56_reg_defaults[] = { - /* no defaults for OTP_MEM - first read populates cache */ - { CS35L56_ASP1_ENABLES1, 0x00000000 }, { CS35L56_ASP1_CONTROL1, 0x00000028 }, { CS35L56_ASP1_CONTROL2, 0x18180200 }, @@ -138,8 +136,6 @@ static const struct reg_default cs35l56_reg_defaults[] = { }; static const struct reg_default cs35l63_reg_defaults[] = { - /* no defaults for OTP_MEM - first read populates cache */ - { CS35L56_ASP1_ENABLES1, 0x00000000 }, { CS35L56_ASP1_CONTROL1, 0x00000028 }, { CS35L56_ASP1_CONTROL2, 0x18180200 }, @@ -282,6 +278,9 @@ static bool cs35l56_common_volatile_reg(unsigned int reg) case CS35L56_GLOBAL_ENABLES: /* owned by firmware */ case CS35L56_BLOCK_ENABLES: /* owned by firmware */ case CS35L56_BLOCK_ENABLES2: /* owned by firmware */ + case CS35L56_OTP_MEM_53: + case CS35L56_OTP_MEM_54: + case CS35L56_OTP_MEM_55: case CS35L56_SYNC_GPIO1_CFG ... CS35L56_ASP2_DIO_GPIO13_CFG: case CS35L56_UPDATE_REGS: case CS35L56_REFCLK_INPUT: /* owned by firmware */ diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 2652a639a79ad2..1f2efce071912d 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -1148,7 +1148,6 @@ struct snd_soc_dai_driver cs42l42_dai = { .formats = CS42L42_FORMATS, }, .symmetric_rate = 1, - .symmetric_sample_bits = 1, .ops = &cs42l42_ops, }; EXPORT_SYMBOL_NS_GPL(cs42l42_dai, "SND_SOC_CS42L42_CORE"); @@ -1676,7 +1675,7 @@ irqreturn_t cs42l42_irq_thread(int irq, void *data) return IRQ_NONE; } - /* Read sticky registers to clear interurpt */ + /* Read sticky registers to clear interrupt */ for (i = 0; i < ARRAY_SIZE(stickies); i++) { regmap_read(cs42l42->regmap, irq_params_table[i].status_addr, &(stickies[i])); @@ -2419,6 +2418,16 @@ int cs42l42_init(struct cs42l42_private *cs42l42) (1 << CS42L42_ADC_PDN_SHIFT) | (0 << CS42L42_PDN_ALL_SHIFT)); + /* + * Configure a faster digital ramp time, to avoid an audible + * fade-in when streams start. + */ + regmap_update_bits(cs42l42->regmap, CS42L42_SFTRAMP_RATE, + CS42L42_SFTRAMP_ASR_RATE_MASK | + CS42L42_SFTRAMP_DSR_RATE_MASK, + (10 << CS42L42_SFTRAMP_ASR_RATE_SHIFT) | + (1 << CS42L42_SFTRAMP_DSR_RATE_SHIFT)); + ret = cs42l42_handle_device_data(cs42l42->dev, cs42l42); if (ret != 0) goto err_shutdown; diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 1e1307a16f8152..eba2c900b8e4c8 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -357,8 +357,11 @@ struct cs42l84_pll_params { * Common PLL Settings for given BCLK */ static const struct cs42l84_pll_params pll_ratio_table[] = { + { 2822400, 1, 0, 0x40, 0x000000, 0x03, 0x10, 11289600}, { 3072000, 1, 0, 0x40, 0x000000, 0x03, 0x10, 12288000}, + { 5644800, 1, 0, 0x40, 0x000000, 0x03, 0x10, 11289600}, { 6144000, 1, 1, 0x40, 0x000000, 0x03, 0x10, 12288000}, + { 11289600, 0, 0, 0, 0, 0, 0, 11289600}, { 12288000, 0, 0, 0, 0, 0, 0, 12288000}, { 24576000, 1, 3, 0x40, 0x000000, 0x03, 0x10, 12288000}, }; @@ -670,14 +673,18 @@ static struct snd_soc_dai_driver cs42l84_dai = { .stream_name = "Playback", .channels_min = 1, .channels_max = 2, - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000, + .rates = SNDRV_PCM_RATE_44100 | SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000 | + SNDRV_PCM_RATE_176400 | SNDRV_PCM_RATE_192000, .formats = CS42L84_FORMATS, }, .capture = { .stream_name = "Capture", .channels_min = 1, .channels_max = 1, - .rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000, + .rates = SNDRV_PCM_RATE_44100 | SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_88200 | SNDRV_PCM_RATE_96000 | + SNDRV_PCM_RATE_176400 | SNDRV_PCM_RATE_192000, .formats = CS42L84_FORMATS, }, .symmetric_rate = 1, diff --git a/sound/soc/codecs/es8389.c b/sound/soc/codecs/es8389.c index 8d418cae371a82..449d9574b03a8d 100644 --- a/sound/soc/codecs/es8389.c +++ b/sound/soc/codecs/es8389.c @@ -892,7 +892,7 @@ static int es8389_probe(struct snd_soc_component *component) return ret; } - es8389->mclk = devm_clk_get(component->dev, "mclk"); + es8389->mclk = devm_clk_get_optional(component->dev, "mclk"); if (IS_ERR(es8389->mclk)) return dev_err_probe(component->dev, PTR_ERR(es8389->mclk), "ES8389 is unable to get mclk\n"); diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 36e25e48b35463..f15df0a72358ac 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -34,6 +34,7 @@ struct tas2764_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; + struct regulator *sdz_reg; struct regmap *regmap; struct device *dev; int irq; @@ -41,6 +42,7 @@ struct tas2764_priv { int v_sense_slot; int i_sense_slot; + u32 sdout_zero_mask; bool dac_powered; bool unmuted; @@ -152,6 +154,8 @@ static int tas2764_codec_suspend(struct snd_soc_component *component) if (tas2764->sdz_gpio) gpiod_set_value_cansleep(tas2764->sdz_gpio, 0); + regulator_disable(tas2764->sdz_reg); + regcache_cache_only(tas2764->regmap, true); regcache_mark_dirty(tas2764->regmap); @@ -165,19 +169,26 @@ static int tas2764_codec_resume(struct snd_soc_component *component) struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); int ret; + ret = regulator_enable(tas2764->sdz_reg); + + if (ret) { + dev_err(tas2764->dev, "Failed to enable regulator\n"); + return ret; + } + if (tas2764->sdz_gpio) { gpiod_set_value_cansleep(tas2764->sdz_gpio, 1); - usleep_range(1000, 2000); } - ret = tas2764_update_pwr_ctrl(tas2764); + usleep_range(1000, 2000); + + regcache_cache_only(tas2764->regmap, false); + ret = regcache_sync(tas2764->regmap); if (ret < 0) return ret; - regcache_cache_only(tas2764->regmap, false); - - return regcache_sync(tas2764->regmap); + return tas2764_update_pwr_ctrl(tas2764); } #else #define tas2764_codec_suspend NULL @@ -210,7 +221,7 @@ static const struct snd_soc_dapm_widget tas2764_dapm_widgets[] = { SND_SOC_DAPM_DAC("DAC", NULL, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_OUTPUT("OUT"), SND_SOC_DAPM_SIGGEN("VMON"), - SND_SOC_DAPM_SIGGEN("IMON") + SND_SOC_DAPM_SIGGEN("IMON"), }; static const struct snd_soc_dapm_route tas2764_audio_map[] = { @@ -261,7 +272,6 @@ static int tas2764_mute(struct snd_soc_dai *dai, int mute, int direction) static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) { struct snd_soc_component *component = tas2764->component; - int sense_en; int val; int ret; @@ -296,28 +306,6 @@ static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth) if (val < 0) return val; - if (val & (1 << TAS2764_VSENSE_POWER_EN)) - sense_en = 0; - else - sense_en = TAS2764_TDM_CFG5_VSNS_ENABLE; - - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, - TAS2764_TDM_CFG5_VSNS_ENABLE, - sense_en); - if (ret < 0) - return ret; - - if (val & (1 << TAS2764_ISENSE_POWER_EN)) - sense_en = 0; - else - sense_en = TAS2764_TDM_CFG6_ISNS_ENABLE; - - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, - TAS2764_TDM_CFG6_ISNS_ENABLE, - sense_en); - if (ret < 0) - return ret; - return 0; } @@ -373,6 +361,44 @@ static int tas2764_hw_params(struct snd_pcm_substream *substream, return tas2764_set_samplerate(tas2764, params_rate(params)); } +static int tas2764_write_sdout_zero_mask(struct tas2764_priv *tas2764, int bclk_ratio) +{ + struct snd_soc_component *component = tas2764->component; + int nsense_slots = bclk_ratio / 8; + u32 cropped_mask; + int i, ret; + + if (!tas2764->sdout_zero_mask) + return 0; + + cropped_mask = tas2764->sdout_zero_mask & GENMASK(nsense_slots - 1, 0); + + for (i = 0; i < 4; i++) { + ret = snd_soc_component_write(component, TAS2764_SDOUT_HIZ_1 + i, + (cropped_mask >> (i * 8)) & 0xff); + + if (ret < 0) + return ret; + } + + ret = snd_soc_component_update_bits(component, TAS2764_SDOUT_HIZ_9, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN); + + if (ret < 0) + return ret; + + return 0; +} + +static int tas2764_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio) +{ + struct snd_soc_component *component = dai->component; + struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + + return tas2764_write_sdout_zero_mask(tas2764, ratio); +} + static int tas2764_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) { struct snd_soc_component *component = dai->component; @@ -447,7 +473,6 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); int left_slot, right_slot; int slots_cfg; int slot_size; @@ -494,15 +519,26 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_TDM_CFG5, + return 0; +} + +static int tas2764_set_ivsense_transmit(struct tas2764_priv *tas2764, int i_slot, int v_slot) +{ + int ret; + + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, + TAS2764_TDM_CFG5_VSNS_ENABLE | TAS2764_TDM_CFG5_50_MASK, - tas2764->v_sense_slot); + TAS2764_TDM_CFG5_VSNS_ENABLE | + v_slot); if (ret < 0) return ret; - ret = snd_soc_component_update_bits(component, TAS2764_TDM_CFG6, + ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, + TAS2764_TDM_CFG6_ISNS_ENABLE | TAS2764_TDM_CFG6_50_MASK, - tas2764->i_sense_slot); + TAS2764_TDM_CFG6_ISNS_ENABLE | + i_slot); if (ret < 0) return ret; @@ -512,6 +548,7 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, static const struct snd_soc_dai_ops tas2764_dai_ops = { .mute_stream = tas2764_mute, .hw_params = tas2764_hw_params, + .set_bclk_ratio = tas2764_set_bclk_ratio, .set_fmt = tas2764_set_fmt, .set_tdm_slot = tas2764_set_dai_tdm_slot, .no_capture_mute = 1, @@ -589,18 +626,33 @@ static int tas2764_read_die_temp(struct tas2764_priv *tas2764, long *result) * As per datasheet, subtract 93 from raw value to get degrees * Celsius. hwmon wants millidegrees. * - * NOTE: The chip will initialise the TAS2764_TEMP register to - * 2.6 *C to avoid triggering temperature protection. Since the - * ADC is powered down during software shutdown, this value will - * persist until the chip is fully powered up (e.g. the PCM it's - * attached to is opened). The ADC will power down again when - * the chip is put back into software shutdown, with the last - * value sampled persisting in the ADC's register. + * NOTE: The TAS2764 datasheet mentions initialising TAS2764_TEMP + * such that the temperature is 2.6 *C, however the register + * is actually initialised to 0. The ADC is also powered down during + * software shutdown. The last sampled temperature will persist + * in the register while the amp is in this power state. */ + if (reg == 0) + return -ENODATA; + *result = (reg - 93) * 1000; return 0; } +static int tas2764_hwmon_is_fault(struct tas2764_priv *tas2764, long *result) +{ + int ret; + long temp; + + ret = tas2764_read_die_temp(tas2764, &temp); + if (ret == -ENODATA) { + *result = true; + return 0; + } + + return ret; +} + static umode_t tas2764_hwmon_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel) @@ -610,6 +662,7 @@ static umode_t tas2764_hwmon_is_visible(const void *data, switch (attr) { case hwmon_temp_input: + case hwmon_temp_fault: return 0444; default: break; @@ -629,6 +682,9 @@ static int tas2764_hwmon_read(struct device *dev, case hwmon_temp_input: ret = tas2764_read_die_temp(tas2764, val); break; + case hwmon_temp_fault: + ret = tas2764_hwmon_is_fault(tas2764, val); + break; default: ret = -EOPNOTSUPP; break; @@ -638,7 +694,7 @@ static int tas2764_hwmon_read(struct device *dev, } static const struct hwmon_channel_info *const tas2764_hwmon_info[] = { - HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_FAULT), NULL }; @@ -659,11 +715,18 @@ static int tas2764_codec_probe(struct snd_soc_component *component) tas2764->component = component; + ret = regulator_enable(tas2764->sdz_reg); + if (ret != 0) { + dev_err(tas2764->dev, "Failed to enable regulator: %d\n", ret); + return ret; + } + if (tas2764->sdz_gpio) { gpiod_set_value_cansleep(tas2764->sdz_gpio, 1); - usleep_range(1000, 2000); } + usleep_range(1000, 2000); + tas2764_reset(tas2764); regmap_reinit_cache(tas2764->regmap, &tas2764_i2c_regmap); @@ -695,18 +758,33 @@ static int tas2764_codec_probe(struct snd_soc_component *component) dev_warn(tas2764->dev, "failed to request IRQ: %d\n", ret); } - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5, - TAS2764_TDM_CFG5_VSNS_ENABLE, 0); - if (ret < 0) - return ret; + if (tas2764->i_sense_slot != -1 && tas2764->v_sense_slot != -1) { + ret = tas2764_set_ivsense_transmit(tas2764, tas2764->i_sense_slot, + tas2764->v_sense_slot); - ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG6, - TAS2764_TDM_CFG6_ISNS_ENABLE, 0); - if (ret < 0) - return ret; + if (ret < 0) + return ret; + } switch (tas2764->devid) { case DEVID_SN012776: + if (tas2764->sdout_zero_mask) { + for (i = 0; i < 4; i++) { + ret = snd_soc_component_write(component, TAS2764_SDOUT_HIZ_1 + i, + (tas2764->sdout_zero_mask >> (i * 8)) & 0xff); + + if (ret < 0) + return ret; + } + + ret = snd_soc_component_update_bits(component, TAS2764_SDOUT_HIZ_9, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN, + TAS2764_SDOUT_HIZ_9_FORCE_0_EN); + + if (ret < 0) + return ret; + } + ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL, TAS2764_PWR_CTRL_BOP_SRC, TAS2764_PWR_CTRL_BOP_SRC); @@ -736,6 +814,13 @@ static int tas2764_codec_probe(struct snd_soc_component *component) return 0; } +static void tas2764_codec_remove(struct snd_soc_component *component) +{ + struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component); + + regulator_disable(tas2764->sdz_reg); +} + static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2764_playback_volume, -10050, 50, 1); @@ -767,6 +852,7 @@ static const struct snd_kcontrol_new tas2764_snd_controls[] = { static const struct snd_soc_component_driver soc_component_driver_tas2764 = { .probe = tas2764_codec_probe, + .remove = tas2764_codec_remove, .suspend = tas2764_codec_suspend, .resume = tas2764_codec_resume, .controls = tas2764_snd_controls, @@ -809,6 +895,7 @@ static bool tas2764_volatile_register(struct device *dev, unsigned int reg) { switch (reg) { case TAS2764_SW_RST: + case TAS2764_TEMP: case TAS2764_INT_LTCH0 ... TAS2764_INT_LTCH4: case TAS2764_INT_CLK_CFG: return true; @@ -836,6 +923,11 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) { int ret = 0; + tas2764->sdz_reg = devm_regulator_get(dev, "SDZ"); + if (IS_ERR(tas2764->sdz_reg)) + return dev_err_probe(dev, PTR_ERR(tas2764->sdz_reg), + "Failed to get SDZ supply\n"); + tas2764->reset_gpio = devm_gpiod_get_optional(tas2764->dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(tas2764->reset_gpio)) { @@ -856,12 +948,17 @@ static int tas2764_parse_dt(struct device *dev, struct tas2764_priv *tas2764) ret = fwnode_property_read_u32(dev->fwnode, "ti,imon-slot-no", &tas2764->i_sense_slot); if (ret) - tas2764->i_sense_slot = 0; + tas2764->i_sense_slot = -1; ret = fwnode_property_read_u32(dev->fwnode, "ti,vmon-slot-no", &tas2764->v_sense_slot); if (ret) - tas2764->v_sense_slot = 2; + tas2764->v_sense_slot = -1; + + ret = fwnode_property_read_u32(dev->fwnode, "ti,sdout-force-zero-mask", + &tas2764->sdout_zero_mask); + if (ret) + tas2764->sdout_zero_mask = 0; return 0; } diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h index 538290ed3d92ac..4a419c11d4b08e 100644 --- a/sound/soc/codecs/tas2764.h +++ b/sound/soc/codecs/tas2764.h @@ -126,4 +126,15 @@ #define TAS2764_BOP_CFG0 TAS2764_REG(0X0, 0x1d) +#define TAS2764_SDOUT_HIZ_1 TAS2764_REG(0x1, 0x3d) +#define TAS2764_SDOUT_HIZ_2 TAS2764_REG(0x1, 0x3e) +#define TAS2764_SDOUT_HIZ_3 TAS2764_REG(0x1, 0x3f) +#define TAS2764_SDOUT_HIZ_4 TAS2764_REG(0x1, 0x40) +#define TAS2764_SDOUT_HIZ_5 TAS2764_REG(0x1, 0x41) +#define TAS2764_SDOUT_HIZ_6 TAS2764_REG(0x1, 0x42) +#define TAS2764_SDOUT_HIZ_7 TAS2764_REG(0x1, 0x43) +#define TAS2764_SDOUT_HIZ_8 TAS2764_REG(0x1, 0x44) +#define TAS2764_SDOUT_HIZ_9 TAS2764_REG(0x1, 0x45) +#define TAS2764_SDOUT_HIZ_9_FORCE_0_EN BIT(7) + #endif /* __TAS2764__ */ diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index 6f878b01716f72..e4bdd3f11787fd 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -71,23 +71,21 @@ static int tas2770_codec_suspend(struct snd_soc_component *component) struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); int ret = 0; - regcache_cache_only(tas2770->regmap, true); - regcache_mark_dirty(tas2770->regmap); + ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, + TAS2770_PWR_CTRL_MASK, + TAS2770_PWR_CTRL_SHUTDOWN); + if (ret < 0) + return ret; - if (tas2770->sdz_gpio) { + if (tas2770->sdz_gpio) gpiod_set_value_cansleep(tas2770->sdz_gpio, 0); - } else { - ret = snd_soc_component_update_bits(component, TAS2770_PWR_CTRL, - TAS2770_PWR_CTRL_MASK, - TAS2770_PWR_CTRL_SHUTDOWN); - if (ret < 0) { - regcache_cache_only(tas2770->regmap, false); - regcache_sync(tas2770->regmap); - return ret; - } - ret = 0; - } + regulator_disable(tas2770->sdz_reg); + + regcache_cache_only(tas2770->regmap, true); + regcache_mark_dirty(tas2770->regmap); + + usleep_range(6000, 7000); return ret; } @@ -97,18 +95,26 @@ static int tas2770_codec_resume(struct snd_soc_component *component) struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); int ret; - if (tas2770->sdz_gpio) { - gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); - usleep_range(1000, 2000); - } else { - ret = tas2770_update_pwr_ctrl(tas2770); - if (ret < 0) - return ret; + ret = regulator_enable(tas2770->sdz_reg); + + if (ret) { + dev_err(tas2770->dev, "Failed to enable regulator\n"); + return ret; } + if (tas2770->sdz_gpio) + gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); + + + usleep_range(1000, 2000); + regcache_cache_only(tas2770->regmap, false); - return regcache_sync(tas2770->regmap); + ret = regcache_sync(tas2770->regmap); + if (ret < 0) + return ret; + + return tas2770_update_pwr_ctrl(tas2770); } #else #define tas2770_codec_suspend NULL @@ -549,7 +555,7 @@ static int tas2770_read_die_temp(struct tas2770_priv *tas2770, long *result) /* * As per datasheet: divide register by 16 and subtract 93 to get * degrees Celsius. hwmon requires millidegrees. Let's avoid rounding - * errors by subtracting 93 * 16 then multiplying by 1000 / 16. + * errors by subtracting 93 * 16 and scaling before dividing. * * NOTE: The ADC registers are initialised to 0 on reset. This means * that the temperature will read -93 *C until the chip is brought out @@ -558,10 +564,27 @@ static int tas2770_read_die_temp(struct tas2770_priv *tas2770, long *result) * value read back from its registers will be the last value sampled * before entering software shutdown. */ - *result = (reading - (93 * 16)) * (1000 / 16); + if (reading == 0) + return -ENODATA; + + *result = (reading - (93 * 16)) * 1000 / 16; return 0; } +static int tas2770_hwmon_is_fault(struct tas2770_priv *tas2770, long *result) +{ + int ret; + long temp; + + ret = tas2770_read_die_temp(tas2770, &temp); + if (ret == -ENODATA) { + *result = true; + return 0; + } + + return ret; +} + static umode_t tas2770_hwmon_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel) @@ -571,6 +594,7 @@ static umode_t tas2770_hwmon_is_visible(const void *data, switch (attr) { case hwmon_temp_input: + case hwmon_temp_fault: return 0444; default: break; @@ -590,6 +614,9 @@ static int tas2770_hwmon_read(struct device *dev, case hwmon_temp_input: ret = tas2770_read_die_temp(tas2770, val); break; + case hwmon_temp_fault: + ret = tas2770_hwmon_is_fault(tas2770, val); + break; default: ret = -EOPNOTSUPP; break; @@ -599,7 +626,7 @@ static int tas2770_hwmon_read(struct device *dev, } static const struct hwmon_channel_info *const tas2770_hwmon_info[] = { - HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_FAULT), NULL }; @@ -623,11 +650,18 @@ static int tas2770_codec_probe(struct snd_soc_component *component) tas2770->component = component; + ret = regulator_enable(tas2770->sdz_reg); + if (ret != 0) { + dev_err(tas2770->dev, "Failed to enable regulator: %d\n", ret); + return ret; + } + if (tas2770->sdz_gpio) { gpiod_set_value_cansleep(tas2770->sdz_gpio, 1); - usleep_range(1000, 2000); } + usleep_range(1000, 2000); + tas2770_reset(tas2770); regmap_reinit_cache(tas2770->regmap, &tas2770_i2c_regmap); @@ -641,14 +675,34 @@ static int tas2770_codec_probe(struct snd_soc_component *component) if (tas2770->pdm_slot != -1) { ret = tas2770_set_pdm_transmit(tas2770, tas2770->pdm_slot); - if (ret < 0) return ret; } + ret = snd_soc_component_update_bits(component, TAS2770_TDM_CFG_REG4, + TAS2770_TDM_CFG_REG4_TX_FILL, + tas2770->sdout_zfill ? 0 : + TAS2770_TDM_CFG_REG4_TX_FILL); + if (ret < 0) + return ret; + + ret = snd_soc_component_update_bits(component, TAS2770_DIN_PD, + TAS2770_DIN_PD_SDOUT, + tas2770->sdout_pd ? + TAS2770_DIN_PD_SDOUT : 0); + if (ret < 0) + return ret; + return 0; } +static void tas2770_codec_remove(struct snd_soc_component *component) +{ + struct tas2770_priv *tas2770 = snd_soc_component_get_drvdata(component); + + regulator_disable(tas2770->sdz_reg); +} + static DECLARE_TLV_DB_SCALE(tas2770_digital_tlv, 1100, 50, 0); static DECLARE_TLV_DB_SCALE(tas2770_playback_volume, -10050, 50, 0); @@ -661,6 +715,7 @@ static const struct snd_kcontrol_new tas2770_snd_controls[] = { static const struct snd_soc_component_driver soc_component_driver_tas2770 = { .probe = tas2770_codec_probe, + .remove = tas2770_codec_remove, .suspend = tas2770_codec_suspend, .resume = tas2770_codec_resume, .controls = tas2770_snd_controls, @@ -790,6 +845,14 @@ static int tas2770_parse_dt(struct device *dev, struct tas2770_priv *tas2770) if (rc) tas2770->pdm_slot = -1; + tas2770->sdout_pd = fwnode_property_read_bool(dev->fwnode, "ti,sdout-pull-down"); + tas2770->sdout_zfill = fwnode_property_read_bool(dev->fwnode, "ti,sdout-zero-fill"); + + tas2770->sdz_reg = devm_regulator_get(dev, "SDZ"); + if (IS_ERR(tas2770->sdz_reg)) + return dev_err_probe(dev, PTR_ERR(tas2770->sdz_reg), + "Failed to get SDZ supply\n"); + tas2770->sdz_gpio = devm_gpiod_get_optional(dev, "shutdown", GPIOD_OUT_HIGH); if (IS_ERR(tas2770->sdz_gpio)) { if (PTR_ERR(tas2770->sdz_gpio) == -EPROBE_DEFER) diff --git a/sound/soc/codecs/tas2770.h b/sound/soc/codecs/tas2770.h index 3fd2e7003c50b6..b309d19c58e1da 100644 --- a/sound/soc/codecs/tas2770.h +++ b/sound/soc/codecs/tas2770.h @@ -67,6 +67,14 @@ #define TAS2770_TDM_CFG_REG3_RXS_SHIFT 0x4 #define TAS2770_TDM_CFG_REG3_30_MASK GENMASK(3, 0) #define TAS2770_TDM_CFG_REG3_30_SHIFT 0 + /* TDM Configuration Reg4 */ +#define TAS2770_TDM_CFG_REG4 TAS2770_REG(0X0, 0x0E) +#define TAS2770_TDM_CFG_REG4_TX_LSB_CFG BIT(7) +#define TAS2770_TDM_CFG_REG4_TX_KEEPER_CFG BIT(6) +#define TAS2770_TDM_CFG_REG4_TX_KEEPER BIT(5) +#define TAS2770_TDM_CFG_REG4_TX_FILL BIT(4) +#define TAS2770_TDM_CFG_REG4_TX_OFFSET_MASK GENMASK(3, 1) +#define TAS2770_TDM_CFG_REG4_TX_EDGE_FALLING BIT(0) /* TDM Configuration Reg5 */ #define TAS2770_TDM_CFG_REG5 TAS2770_REG(0X0, 0x0F) #define TAS2770_TDM_CFG_REG5_VSNS_MASK BIT(6) @@ -115,6 +123,9 @@ #define TAS2770_TEMP_LSB TAS2770_REG(0X0, 0x2A) /* Interrupt Configuration */ #define TAS2770_INT_CFG TAS2770_REG(0X0, 0x30) + /* Data In Pull-Down */ +#define TAS2770_DIN_PD TAS2770_REG(0X0, 0x31) +#define TAS2770_DIN_PD_SDOUT BIT(7) /* Misc IRQ */ #define TAS2770_MISC_IRQ TAS2770_REG(0X0, 0x32) /* Clock Configuration */ @@ -139,11 +150,14 @@ struct tas2770_priv { struct snd_soc_component *component; struct gpio_desc *reset_gpio; struct gpio_desc *sdz_gpio; + struct regulator *sdz_reg; struct regmap *regmap; struct device *dev; int v_sense_slot; int i_sense_slot; int pdm_slot; + bool sdout_pd; + bool sdout_zfill; bool dac_powered; bool unmuted; }; diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c index 6c56134c60cc81..114a6c0b6b7301 100644 --- a/sound/soc/fsl/fsl_easrc.c +++ b/sound/soc/fsl/fsl_easrc.c @@ -54,6 +54,9 @@ static int fsl_easrc_iec958_put_bits(struct snd_kcontrol *kcontrol, unsigned int regval = ucontrol->value.integer.value[0]; int ret; + if (regval < EASRC_WIDTH_16_BIT || regval > EASRC_WIDTH_24_BIT) + return -EINVAL; + ret = (easrc_priv->bps_iec958[mc->regbase] != regval); easrc_priv->bps_iec958[mc->regbase] = regval; @@ -70,8 +73,16 @@ static int fsl_easrc_iec958_get_bits(struct snd_kcontrol *kcontrol, struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; - ucontrol->value.enumerated.item[0] = easrc_priv->bps_iec958[mc->regbase]; + ucontrol->value.integer.value[0] = easrc_priv->bps_iec958[mc->regbase]; + + return 0; +} +static int fsl_easrc_iec958_info(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_info *uinfo) +{ + uinfo->type = SNDRV_CTL_ELEM_TYPE_IEC958; + uinfo->count = 1; return 0; } @@ -81,11 +92,33 @@ static int fsl_easrc_get_reg(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_kcontrol_chip(kcontrol); struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; - unsigned int regval; + struct fsl_asrc *easrc = snd_soc_component_get_drvdata(component); + unsigned int *regval = (unsigned int *)ucontrol->value.iec958.status; + int ret; + + ret = regmap_read(easrc->regmap, REG_EASRC_CS0(mc->regbase), ®val[0]); + if (ret) + return ret; - regval = snd_soc_component_read(component, mc->regbase); + ret = regmap_read(easrc->regmap, REG_EASRC_CS1(mc->regbase), ®val[1]); + if (ret) + return ret; + + ret = regmap_read(easrc->regmap, REG_EASRC_CS2(mc->regbase), ®val[2]); + if (ret) + return ret; + + ret = regmap_read(easrc->regmap, REG_EASRC_CS3(mc->regbase), ®val[3]); + if (ret) + return ret; + + ret = regmap_read(easrc->regmap, REG_EASRC_CS4(mc->regbase), ®val[4]); + if (ret) + return ret; - ucontrol->value.integer.value[0] = regval; + ret = regmap_read(easrc->regmap, REG_EASRC_CS5(mc->regbase), ®val[5]); + if (ret) + return ret; return 0; } @@ -97,22 +130,62 @@ static int fsl_easrc_set_reg(struct snd_kcontrol *kcontrol, struct soc_mreg_control *mc = (struct soc_mreg_control *)kcontrol->private_value; struct fsl_asrc *easrc = snd_soc_component_get_drvdata(component); - unsigned int regval = ucontrol->value.integer.value[0]; - bool changed; + unsigned int *regval = (unsigned int *)ucontrol->value.iec958.status; + bool changed, changed_all = false; int ret; - ret = regmap_update_bits_check(easrc->regmap, mc->regbase, - GENMASK(31, 0), regval, &changed); - if (ret != 0) + ret = pm_runtime_resume_and_get(component->dev); + if (ret) return ret; - return changed; + ret = regmap_update_bits_check(easrc->regmap, REG_EASRC_CS0(mc->regbase), + GENMASK(31, 0), regval[0], &changed); + if (ret != 0) + goto err; + changed_all |= changed; + + ret = regmap_update_bits_check(easrc->regmap, REG_EASRC_CS1(mc->regbase), + GENMASK(31, 0), regval[1], &changed); + if (ret != 0) + goto err; + changed_all |= changed; + + ret = regmap_update_bits_check(easrc->regmap, REG_EASRC_CS2(mc->regbase), + GENMASK(31, 0), regval[2], &changed); + if (ret != 0) + goto err; + changed_all |= changed; + + ret = regmap_update_bits_check(easrc->regmap, REG_EASRC_CS3(mc->regbase), + GENMASK(31, 0), regval[3], &changed); + if (ret != 0) + goto err; + changed_all |= changed; + + ret = regmap_update_bits_check(easrc->regmap, REG_EASRC_CS4(mc->regbase), + GENMASK(31, 0), regval[4], &changed); + if (ret != 0) + goto err; + changed_all |= changed; + + ret = regmap_update_bits_check(easrc->regmap, REG_EASRC_CS5(mc->regbase), + GENMASK(31, 0), regval[5], &changed); + if (ret != 0) + goto err; + changed_all |= changed; +err: + pm_runtime_put_autosuspend(component->dev); + + if (ret != 0) + return ret; + else + return changed_all; } #define SOC_SINGLE_REG_RW(xname, xreg) \ { .iface = SNDRV_CTL_ELEM_IFACE_PCM, .name = (xname), \ .access = SNDRV_CTL_ELEM_ACCESS_READWRITE, \ - .info = snd_soc_info_xr_sx, .get = fsl_easrc_get_reg, \ + .info = fsl_easrc_iec958_info, .get = fsl_easrc_get_reg, \ .put = fsl_easrc_set_reg, \ .private_value = (unsigned long)&(struct soc_mreg_control) \ { .regbase = xreg, .regcount = 1, .nbits = 32, \ @@ -143,30 +216,10 @@ static const struct snd_kcontrol_new fsl_easrc_snd_controls[] = { SOC_SINGLE_VAL_RW("Context 2 IEC958 Bits Per Sample", 2), SOC_SINGLE_VAL_RW("Context 3 IEC958 Bits Per Sample", 3), - SOC_SINGLE_REG_RW("Context 0 IEC958 CS0", REG_EASRC_CS0(0)), - SOC_SINGLE_REG_RW("Context 1 IEC958 CS0", REG_EASRC_CS0(1)), - SOC_SINGLE_REG_RW("Context 2 IEC958 CS0", REG_EASRC_CS0(2)), - SOC_SINGLE_REG_RW("Context 3 IEC958 CS0", REG_EASRC_CS0(3)), - SOC_SINGLE_REG_RW("Context 0 IEC958 CS1", REG_EASRC_CS1(0)), - SOC_SINGLE_REG_RW("Context 1 IEC958 CS1", REG_EASRC_CS1(1)), - SOC_SINGLE_REG_RW("Context 2 IEC958 CS1", REG_EASRC_CS1(2)), - SOC_SINGLE_REG_RW("Context 3 IEC958 CS1", REG_EASRC_CS1(3)), - SOC_SINGLE_REG_RW("Context 0 IEC958 CS2", REG_EASRC_CS2(0)), - SOC_SINGLE_REG_RW("Context 1 IEC958 CS2", REG_EASRC_CS2(1)), - SOC_SINGLE_REG_RW("Context 2 IEC958 CS2", REG_EASRC_CS2(2)), - SOC_SINGLE_REG_RW("Context 3 IEC958 CS2", REG_EASRC_CS2(3)), - SOC_SINGLE_REG_RW("Context 0 IEC958 CS3", REG_EASRC_CS3(0)), - SOC_SINGLE_REG_RW("Context 1 IEC958 CS3", REG_EASRC_CS3(1)), - SOC_SINGLE_REG_RW("Context 2 IEC958 CS3", REG_EASRC_CS3(2)), - SOC_SINGLE_REG_RW("Context 3 IEC958 CS3", REG_EASRC_CS3(3)), - SOC_SINGLE_REG_RW("Context 0 IEC958 CS4", REG_EASRC_CS4(0)), - SOC_SINGLE_REG_RW("Context 1 IEC958 CS4", REG_EASRC_CS4(1)), - SOC_SINGLE_REG_RW("Context 2 IEC958 CS4", REG_EASRC_CS4(2)), - SOC_SINGLE_REG_RW("Context 3 IEC958 CS4", REG_EASRC_CS4(3)), - SOC_SINGLE_REG_RW("Context 0 IEC958 CS5", REG_EASRC_CS5(0)), - SOC_SINGLE_REG_RW("Context 1 IEC958 CS5", REG_EASRC_CS5(1)), - SOC_SINGLE_REG_RW("Context 2 IEC958 CS5", REG_EASRC_CS5(2)), - SOC_SINGLE_REG_RW("Context 3 IEC958 CS5", REG_EASRC_CS5(3)), + SOC_SINGLE_REG_RW("Context 0 IEC958 CS", 0), + SOC_SINGLE_REG_RW("Context 1 IEC958 CS", 1), + SOC_SINGLE_REG_RW("Context 2 IEC958 CS", 2), + SOC_SINGLE_REG_RW("Context 3 IEC958 CS", 3), }; /* @@ -1286,7 +1339,7 @@ static int fsl_easrc_request_context(int channels, struct fsl_asrc_pair *ctx) /* * Release the context * - * This funciton is mainly doing the revert thing in request context + * This function is mainly doing the revert thing in request context */ static void fsl_easrc_release_context(struct fsl_asrc_pair *ctx) { diff --git a/sound/soc/fsl/fsl_micfil.c b/sound/soc/fsl/fsl_micfil.c index d6cde2757c6d55..2e887f1f1f361e 100644 --- a/sound/soc/fsl/fsl_micfil.c +++ b/sound/soc/fsl/fsl_micfil.c @@ -210,15 +210,23 @@ static int micfil_range_set(struct snd_kcontrol *kcontrol, (struct soc_mixer_control *)kcontrol->private_value; unsigned int shift = mc->shift; int max_range, new_range; + int ret; new_range = ucontrol->value.integer.value[0]; max_range = micfil_get_max_range(micfil); if (new_range > max_range) dev_warn(&micfil->pdev->dev, "range makes channel %d data unreliable\n", shift / 4); - regmap_update_bits(micfil->regmap, REG_MICFIL_OUT_CTRL, 0xF << shift, new_range << shift); + ret = pm_runtime_resume_and_get(cmpnt->dev); + if (ret) + return ret; - return 0; + ret = snd_soc_component_update_bits(cmpnt, REG_MICFIL_OUT_CTRL, 0xF << shift, + new_range << shift); + + pm_runtime_put_autosuspend(cmpnt->dev); + + return ret; } static int micfil_set_quality(struct fsl_micfil *micfil) @@ -281,10 +289,34 @@ static int micfil_quality_set(struct snd_kcontrol *kcontrol, { struct snd_soc_component *cmpnt = snd_kcontrol_chip(kcontrol); struct fsl_micfil *micfil = snd_soc_component_get_drvdata(cmpnt); + int val = ucontrol->value.integer.value[0]; + bool change = false; + int old_val; + int ret; + + if (val < QUALITY_HIGH || val > QUALITY_VLOW2) + return -EINVAL; + + if (micfil->quality != val) { + ret = pm_runtime_resume_and_get(cmpnt->dev); + if (ret) + return ret; + + old_val = micfil->quality; + micfil->quality = val; + ret = micfil_set_quality(micfil); - micfil->quality = ucontrol->value.integer.value[0]; + pm_runtime_put_autosuspend(cmpnt->dev); + + if (ret) { + micfil->quality = old_val; + return ret; + } - return micfil_set_quality(micfil); + change = true; + } + + return change; } static const char * const micfil_hwvad_enable[] = { @@ -343,6 +375,10 @@ static int micfil_put_dc_remover_state(struct snd_kcontrol *kcontrol, if (val < 0 || val > 3) return -EINVAL; + ret = pm_runtime_resume_and_get(comp->dev); + if (ret) + return ret; + micfil->dc_remover = val; /* Calculate total value for all channels */ @@ -352,10 +388,10 @@ static int micfil_put_dc_remover_state(struct snd_kcontrol *kcontrol, /* Update DC Remover mode for all channels */ ret = snd_soc_component_update_bits(comp, REG_MICFIL_DC_CTRL, MICFIL_DC_CTRL_CONFIG, reg_val); - if (ret < 0) - return ret; - return 0; + pm_runtime_put_autosuspend(comp->dev); + + return ret; } static int micfil_get_dc_remover_state(struct snd_kcontrol *kcontrol, @@ -377,10 +413,15 @@ static int hwvad_put_enable(struct snd_kcontrol *kcontrol, unsigned int *item = ucontrol->value.enumerated.item; struct fsl_micfil *micfil = snd_soc_component_get_drvdata(comp); int val = snd_soc_enum_item_to_val(e, item[0]); + bool change = false; + + if (val < 0 || val > 1) + return -EINVAL; + change = (micfil->vad_enabled != val); micfil->vad_enabled = val; - return 0; + return change; } static int hwvad_get_enable(struct snd_kcontrol *kcontrol, @@ -402,13 +443,18 @@ static int hwvad_put_init_mode(struct snd_kcontrol *kcontrol, unsigned int *item = ucontrol->value.enumerated.item; struct fsl_micfil *micfil = snd_soc_component_get_drvdata(comp); int val = snd_soc_enum_item_to_val(e, item[0]); + bool change = false; + + if (val < MICFIL_HWVAD_ENVELOPE_MODE || val > MICFIL_HWVAD_ENERGY_MODE) + return -EINVAL; /* 0 - Envelope-based Mode * 1 - Energy-based Mode */ + change = (micfil->vad_init_mode != val); micfil->vad_init_mode = val; - return 0; + return change; } static int hwvad_get_init_mode(struct snd_kcontrol *kcontrol, @@ -503,7 +549,13 @@ static const struct snd_kcontrol_new fsl_micfil_snd_controls[] = { SOC_SINGLE("HWVAD ZCD Adjustment", REG_MICFIL_VAD0_ZCD, 8, 15, 0), SOC_SINGLE("HWVAD ZCD And Behavior Switch", REG_MICFIL_VAD0_ZCD, 4, 1, 0), - SOC_SINGLE_BOOL_EXT("VAD Detected", 0, hwvad_detected, NULL), + { + .iface = SNDRV_CTL_ELEM_IFACE_MIXER, + .access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, + .name = "VAD Detected", + .info = snd_soc_info_bool_ext, + .get = hwvad_detected, + }, }; static int fsl_micfil_use_verid(struct device *dev) diff --git a/sound/soc/fsl/fsl_xcvr.c b/sound/soc/fsl/fsl_xcvr.c index a268fb81a2f868..d7a823384c08ae 100644 --- a/sound/soc/fsl/fsl_xcvr.c +++ b/sound/soc/fsl/fsl_xcvr.c @@ -115,10 +115,17 @@ static int fsl_xcvr_arc_mode_put(struct snd_kcontrol *kcontrol, struct fsl_xcvr *xcvr = snd_soc_dai_get_drvdata(dai); struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; unsigned int *item = ucontrol->value.enumerated.item; + int val = snd_soc_enum_item_to_val(e, item[0]); + int ret; - xcvr->arc_mode = snd_soc_enum_item_to_val(e, item[0]); + if (val < 0 || val > 1) + return -EINVAL; - return 0; + ret = (xcvr->arc_mode != val); + + xcvr->arc_mode = val; + + return ret; } static int fsl_xcvr_arc_mode_get(struct snd_kcontrol *kcontrol, @@ -218,10 +225,17 @@ static int fsl_xcvr_mode_put(struct snd_kcontrol *kcontrol, struct fsl_xcvr *xcvr = snd_soc_dai_get_drvdata(dai); struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; unsigned int *item = ucontrol->value.enumerated.item; + int val = snd_soc_enum_item_to_val(e, item[0]); struct snd_soc_card *card = dai->component->card; struct snd_soc_pcm_runtime *rtd; + int ret; + + if (val < FSL_XCVR_MODE_SPDIF || val > FSL_XCVR_MODE_EARC) + return -EINVAL; - xcvr->mode = snd_soc_enum_item_to_val(e, item[0]); + ret = (xcvr->mode != val); + + xcvr->mode = val; fsl_xcvr_activate_ctl(dai, fsl_xcvr_arc_mode_kctl.name, (xcvr->mode == FSL_XCVR_MODE_ARC)); @@ -231,7 +245,7 @@ static int fsl_xcvr_mode_put(struct snd_kcontrol *kcontrol, rtd = snd_soc_get_pcm_runtime(card, card->dai_link); rtd->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream_count = (xcvr->mode == FSL_XCVR_MODE_SPDIF ? 1 : 0); - return 0; + return ret; } static int fsl_xcvr_mode_get(struct snd_kcontrol *kcontrol, diff --git a/sound/soc/generic/audio-graph-card.c b/sound/soc/generic/audio-graph-card.c index 8a5f4170473973..74e8f2ab7ffc9d 100644 --- a/sound/soc/generic/audio-graph-card.c +++ b/sound/soc/generic/audio-graph-card.c @@ -77,6 +77,7 @@ static bool soc_component_is_pcm(struct snd_soc_dai_link_component *dlc) struct snd_soc_dai *dai = snd_soc_find_dai_with_mutex(dlc); if (dai && (dai->component->driver->pcm_construct || + dai->component->driver->pcm_new || (dai->driver->ops && dai->driver->ops->pcm_new))) return true; diff --git a/sound/soc/intel/Kconfig b/sound/soc/intel/Kconfig index 412555e626b81d..63367364916ae0 100644 --- a/sound/soc/intel/Kconfig +++ b/sound/soc/intel/Kconfig @@ -95,7 +95,7 @@ config SND_SOC_INTEL_KEEMBAY config SND_SOC_INTEL_AVS tristate "Intel AVS driver" - depends on X86 || COMPILE_TEST + depends on X86 depends on PCI depends on COMMON_CLK select ACPI_NHLT if ACPI diff --git a/sound/soc/intel/avs/tgl.c b/sound/soc/intel/avs/tgl.c index afb06651610101..a7123639de4311 100644 --- a/sound/soc/intel/avs/tgl.c +++ b/sound/soc/intel/avs/tgl.c @@ -7,12 +7,11 @@ // #include +#include #include "avs.h" #include "debug.h" #include "messages.h" -#define CPUID_TSC_LEAF 0x15 - static int avs_tgl_dsp_core_power(struct avs_dev *adev, u32 core_mask, bool power) { core_mask &= AVS_MAIN_CORE_MASK; @@ -40,22 +39,37 @@ static int avs_tgl_dsp_core_stall(struct avs_dev *adev, u32 core_mask, bool stal return avs_dsp_core_stall(adev, core_mask, stall); } -static int avs_tgl_config_basefw(struct avs_dev *adev) +/* + * Succeed if CPUID(0x15) is not available, or if the nominal core crystal clock + * frequency cannot be enumerated from it. There is nothing to do in both cases. + */ +static int avs_tgl_set_xtal_freq(struct avs_dev *adev) { - struct pci_dev *pci = adev->base.pci; - struct avs_bus_hwid hwid; + unsigned int freq; int ret; -#ifdef CONFIG_X86 - unsigned int ecx; -#include - ecx = cpuid_ecx(CPUID_TSC_LEAF); - if (ecx) { - ret = avs_ipc_set_fw_config(adev, 1, AVS_FW_CFG_XTAL_FREQ_HZ, sizeof(ecx), &ecx); + if (boot_cpu_data.cpuid_level < CPUID_LEAF_TSC) + return 0; + + freq = cpuid_ecx(CPUID_LEAF_TSC); + if (freq) { + ret = avs_ipc_set_fw_config(adev, 1, AVS_FW_CFG_XTAL_FREQ_HZ, sizeof(freq), &freq); if (ret) return AVS_IPC_RET(ret); } -#endif + + return 0; +} + +static int avs_tgl_config_basefw(struct avs_dev *adev) +{ + struct pci_dev *pci = adev->base.pci; + struct avs_bus_hwid hwid; + int ret; + + ret = avs_tgl_set_xtal_freq(adev); + if (ret) + return ret; hwid.device = pci->device; hwid.subsystem = pci->subsystem_vendor | (pci->subsystem_device << 16); diff --git a/sound/soc/intel/boards/bytcr_wm5102.c b/sound/soc/intel/boards/bytcr_wm5102.c index 4879f79aef292b..4aa0cf49b03359 100644 --- a/sound/soc/intel/boards/bytcr_wm5102.c +++ b/sound/soc/intel/boards/bytcr_wm5102.c @@ -170,6 +170,7 @@ static int platform_clock_control(struct snd_soc_dapm_widget *w, ret = byt_wm5102_prepare_and_enable_pll1(codec_dai, 48000); if (ret) { dev_err(card->dev, "Error setting codec sysclk: %d\n", ret); + clk_disable_unprepare(priv->mclk); return ret; } } else { diff --git a/sound/soc/mediatek/mt8188/mt8188-mt6359.c b/sound/soc/mediatek/mt8188/mt8188-mt6359.c index 55ebac0c3cef2a..5a69378545b861 100644 --- a/sound/soc/mediatek/mt8188/mt8188-mt6359.c +++ b/sound/soc/mediatek/mt8188/mt8188-mt6359.c @@ -1280,7 +1280,7 @@ static struct snd_soc_dai_link mt8188_mt6359_dai_links[] = { }, }; -static void mt8188_fixup_controls(struct snd_soc_card *card) +static int mt8188_fixup_controls(struct snd_soc_card *card) { struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(card); struct mtk_platform_card_data *card_data = soc_card_data->card_data; @@ -1302,6 +1302,8 @@ static void mt8188_fixup_controls(struct snd_soc_card *card) else dev_warn(card->dev, "Cannot find ctl : Headphone Switch\n"); } + + return 0; } static struct snd_soc_card mt8188_mt6359_soc_card = { diff --git a/sound/soc/qcom/qdsp6/audioreach.c b/sound/soc/qcom/qdsp6/audioreach.c index 241c3b4479c6f6..ff8cd55b0d8984 100644 --- a/sound/soc/qcom/qdsp6/audioreach.c +++ b/sound/soc/qcom/qdsp6/audioreach.c @@ -1365,9 +1365,14 @@ int audioreach_set_media_format(struct q6apm_graph *graph, case MODULE_ID_SPEAKER_PROTECTION: rc = audioreach_speaker_protection(graph, module, PARAM_ID_SP_OP_MODE_NORMAL); + if (!rc) + rc = audioreach_module_enable(graph, module, true); + break; case MODULE_ID_SPEAKER_PROTECTION_VI: rc = audioreach_speaker_protection_vi(graph, module, cfg); + if (!rc) + rc = audioreach_module_enable(graph, module, true); break; default: diff --git a/sound/soc/qcom/qdsp6/q6apm-dai.c b/sound/soc/qcom/qdsp6/q6apm-dai.c index 168c166c960d7e..fd4f24ff1eac65 100644 --- a/sound/soc/qcom/qdsp6/q6apm-dai.c +++ b/sound/soc/qcom/qdsp6/q6apm-dai.c @@ -317,6 +317,7 @@ static int q6apm_dai_trigger(struct snd_soc_component *component, case SNDRV_PCM_TRIGGER_STOP: /* TODO support be handled via SoftPause Module */ prtd->state = Q6APM_STREAM_STOPPED; + prtd->queue_ptr = 0; break; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: diff --git a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c index 5be37eeea329fc..ba64117b8cfe13 100644 --- a/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c +++ b/sound/soc/qcom/qdsp6/q6apm-lpass-dais.c @@ -181,7 +181,7 @@ static int q6apm_lpass_dai_prepare(struct snd_pcm_substream *substream, struct s * It is recommend to load DSP with source graph first and then sink * graph, so sequence for playback and capture will be different */ - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && dai_data->graph[dai->id] == NULL) { graph = q6apm_graph_open(dai->dev, NULL, dai->dev, graph_id); if (IS_ERR(graph)) { dev_err(dai->dev, "Failed to open graph (%d)\n", graph_id); diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 970b08c89bb355..5751e80b3b9289 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -215,6 +215,8 @@ int q6apm_map_memory_regions(struct q6apm_graph *graph, unsigned int dir, phys_a mutex_lock(&graph->lock); + data->dsp_buf = 0; + if (data->buf) { mutex_unlock(&graph->lock); return 0; @@ -747,13 +749,23 @@ static int apm_probe(gpr_device_t *gdev) q6apm_get_apm_state(apm); - ret = devm_snd_soc_register_component(dev, &q6apm_audio_component, NULL, 0); + ret = snd_soc_register_component(dev, &q6apm_audio_component, NULL, 0); if (ret < 0) { dev_err(dev, "failed to register q6apm: %d\n", ret); return ret; } - return of_platform_populate(dev->of_node, NULL, NULL, dev); + ret = of_platform_populate(dev->of_node, NULL, NULL, dev); + if (ret) + snd_soc_unregister_component(dev); + + return ret; +} + +static void apm_remove(gpr_device_t *gdev) +{ + of_platform_depopulate(&gdev->dev); + snd_soc_unregister_component(&gdev->dev); } struct audioreach_module *q6apm_find_module_by_mid(struct q6apm_graph *graph, uint32_t mid) @@ -820,6 +832,7 @@ MODULE_DEVICE_TABLE(of, apm_device_id); static gpr_driver_t apm_driver = { .probe = apm_probe, + .remove = apm_remove, .gpr_callback = apm_callback, .driver = { .name = "qcom-apm", diff --git a/sound/soc/qcom/qdsp6/topology.c b/sound/soc/qcom/qdsp6/topology.c index e732fac9b8ca05..1f69fba6de26d1 100644 --- a/sound/soc/qcom/qdsp6/topology.c +++ b/sound/soc/qcom/qdsp6/topology.c @@ -952,9 +952,6 @@ static int audioreach_widget_unload(struct snd_soc_component *scomp, struct audioreach_container *cont; struct audioreach_module *mod; - mod = dobj->private; - cont = mod->container; - if (w->id == snd_soc_dapm_mixer) { /* virtual widget */ struct snd_ar_control *scontrol = dobj->private; @@ -963,6 +960,11 @@ static int audioreach_widget_unload(struct snd_soc_component *scomp, kfree(scontrol); return 0; } + mod = dobj->private; + if (!mod) + return 0; + + cont = mod->container; mutex_lock(&apm->lock); idr_remove(&apm->modules_idr, mod->instance_id); diff --git a/sound/soc/renesas/rcar/core.c b/sound/soc/renesas/rcar/core.c index 69fb19964a71d2..2dc078358612d7 100644 --- a/sound/soc/renesas/rcar/core.c +++ b/sound/soc/renesas/rcar/core.c @@ -1974,7 +1974,7 @@ static int rsnd_probe(struct platform_device *pdev) * asoc register */ ci = 0; - for (i = 0; priv->component_dais[i] > 0; i++) { + for (i = 0; i < RSND_MAX_COMPONENT && priv->component_dais[i] > 0; i++) { int nr = priv->component_dais[i]; ret = devm_snd_soc_register_component(dev, &rsnd_soc_component, diff --git a/sound/soc/rockchip/rockchip_sai.c b/sound/soc/rockchip/rockchip_sai.c index 1bf614dbdf4d0f..ed393e5034a49d 100644 --- a/sound/soc/rockchip/rockchip_sai.c +++ b/sound/soc/rockchip/rockchip_sai.c @@ -628,6 +628,10 @@ static int rockchip_sai_hw_params(struct snd_pcm_substream *substream, regmap_update_bits(sai->regmap, reg, SAI_XCR_VDW_MASK | SAI_XCR_CSR_MASK, val); + if (!sai->is_tdm) + regmap_update_bits(sai->regmap, reg, SAI_XCR_SBW_MASK, + SAI_XCR_SBW(params_physical_width(params))); + regmap_read(sai->regmap, reg, &val); slot_width = SAI_XCR_SBW_V(val); diff --git a/sound/soc/sdca/sdca_asoc.c b/sound/soc/sdca/sdca_asoc.c index a0191e5a5a7dd0..b6536eeecf58f6 100644 --- a/sound/soc/sdca/sdca_asoc.c +++ b/sound/soc/sdca/sdca_asoc.c @@ -51,6 +51,25 @@ static bool readonly_control(struct sdca_control *control) return control->has_fixed || control->mode == SDCA_ACCESS_MODE_RO; } +static int ge_count_routes(struct sdca_entity *entity) +{ + int count = 0; + int i, j; + + for (i = 0; i < entity->ge.num_modes; i++) { + struct sdca_ge_mode *mode = &entity->ge.modes[i]; + + for (j = 0; j < mode->num_controls; j++) { + struct sdca_ge_control *affected = &mode->controls[j]; + + if (affected->sel != SDCA_CTL_SU_SELECTOR || affected->val) + count++; + } + } + + return count; +} + /** * sdca_asoc_count_component - count the various component parts * @dev: Pointer to the device against which allocations will be done. @@ -74,6 +93,7 @@ int sdca_asoc_count_component(struct device *dev, struct sdca_function_data *fun int *num_widgets, int *num_routes, int *num_controls, int *num_dais) { + struct sdca_control *control; int i, j; *num_widgets = function->num_entities - 1; @@ -83,6 +103,7 @@ int sdca_asoc_count_component(struct device *dev, struct sdca_function_data *fun for (i = 0; i < function->num_entities - 1; i++) { struct sdca_entity *entity = &function->entities[i]; + bool skip_primary_routes = false; /* Add supply/DAI widget connections */ switch (entity->type) { @@ -96,6 +117,17 @@ int sdca_asoc_count_component(struct device *dev, struct sdca_function_data *fun case SDCA_ENTITY_TYPE_PDE: *num_routes += entity->pde.num_managed; break; + case SDCA_ENTITY_TYPE_GE: + *num_routes += ge_count_routes(entity); + skip_primary_routes = true; + break; + case SDCA_ENTITY_TYPE_SU: + control = sdca_selector_find_control(dev, entity, SDCA_CTL_SU_SELECTOR); + if (!control) + return -EINVAL; + + skip_primary_routes = (control->layers == SDCA_ACCESS_LAYER_DEVICE); + break; default: break; } @@ -104,7 +136,8 @@ int sdca_asoc_count_component(struct device *dev, struct sdca_function_data *fun (*num_routes)++; /* Add primary entity connections from DisCo */ - *num_routes += entity->num_sources; + if (!skip_primary_routes) + *num_routes += entity->num_sources; for (j = 0; j < entity->num_controls; j++) { if (exported_control(entity, &entity->controls[j])) @@ -442,7 +475,6 @@ static int entity_parse_su_device(struct device *dev, struct snd_soc_dapm_route **route) { struct sdca_control_range *range; - int num_routes = 0; int i, j; if (!entity->group) { @@ -478,11 +510,6 @@ static int entity_parse_su_device(struct device *dev, return -EINVAL; } - if (++num_routes > entity->num_sources) { - dev_err(dev, "%s: too many input routes\n", entity->label); - return -EINVAL; - } - term = sdca_range_search(range, SDCA_SELECTED_MODE_INDEX, mode->val, SDCA_SELECTED_MODE_TERM_TYPE); if (!term) { diff --git a/sound/soc/sdca/sdca_class.c b/sound/soc/sdca/sdca_class.c index 918b638acb5774..5def6ae2d99f04 100644 --- a/sound/soc/sdca/sdca_class.c +++ b/sound/soc/sdca/sdca_class.c @@ -137,6 +137,13 @@ static const struct regmap_config class_dev_regmap_config = { .unlock = class_regmap_unlock, }; +static void class_remove_functions(void *data) +{ + struct sdca_class_drv *drv = data; + + sdca_dev_unregister_functions(drv->sdw); +} + static void class_boot_work(struct work_struct *work) { struct sdca_class_drv *drv = container_of(work, @@ -157,6 +164,11 @@ static void class_boot_work(struct work_struct *work) if (ret) goto err; + /* Ensure function drivers are removed before the IRQ is destroyed */ + ret = devm_add_action_or_reset(drv->dev, class_remove_functions, drv); + if (ret) + goto err; + dev_dbg(drv->dev, "boot work complete\n"); pm_runtime_mark_last_busy(drv->dev); @@ -168,15 +180,6 @@ static void class_boot_work(struct work_struct *work) pm_runtime_put_sync(drv->dev); } -static void class_dev_remove(void *data) -{ - struct sdca_class_drv *drv = data; - - cancel_work_sync(&drv->boot_work); - - sdca_dev_unregister_functions(drv->sdw); -} - static int class_sdw_probe(struct sdw_slave *sdw, const struct sdw_device_id *id) { struct device *dev = &sdw->dev; @@ -230,15 +233,19 @@ static int class_sdw_probe(struct sdw_slave *sdw, const struct sdw_device_id *id if (ret) return ret; - ret = devm_add_action_or_reset(dev, class_dev_remove, drv); - if (ret) - return ret; - queue_work(system_long_wq, &drv->boot_work); return 0; } +static void class_sdw_remove(struct sdw_slave *sdw) +{ + struct device *dev = &sdw->dev; + struct sdca_class_drv *drv = dev_get_drvdata(dev); + + cancel_work_sync(&drv->boot_work); +} + static int class_suspend(struct device *dev) { struct sdca_class_drv *drv = dev_get_drvdata(dev); @@ -328,6 +335,7 @@ static struct sdw_driver class_sdw_driver = { }, .probe = class_sdw_probe, + .remove = class_sdw_remove, .id_table = class_sdw_id, .ops = &class_sdw_ops, }; diff --git a/sound/soc/sdca/sdca_fdl.c b/sound/soc/sdca/sdca_fdl.c index 07892bc3a44e6e..994821a6df617d 100644 --- a/sound/soc/sdca/sdca_fdl.c +++ b/sound/soc/sdca/sdca_fdl.c @@ -46,11 +46,6 @@ int sdca_reset_function(struct device *dev, struct sdca_function_data *function, if (ret) // Allowed for function reset to not be implemented return 0; - if (!function->reset_max_delay) { - dev_err(dev, "No reset delay specified in DisCo\n"); - return -EINVAL; - } - /* * Poll up to 16 times but no more than once per ms, these are just * arbitrarily selected values, so may be fine tuned in future. diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c index dca60ee8e62c3c..fd6a254c953050 100644 --- a/sound/soc/sdca/sdca_functions.c +++ b/sound/soc/sdca/sdca_functions.c @@ -2176,8 +2176,12 @@ int sdca_parse_function(struct device *dev, struct sdw_slave *sdw, ret = fwnode_property_read_u32(function_desc->node, "mipi-sdca-function-reset-max-delay", &tmp); - if (!ret) + if (ret || tmp == 0) { + dev_dbg(dev, "reset delay missing, defaulting to 100mS\n"); + function->reset_max_delay = 100000; + } else { function->reset_max_delay = tmp; + } dev_dbg(dev, "%pfwP: name %s busy delay %dus reset delay %dus\n", function->desc->node, function->desc->name, diff --git a/sound/soc/soc-card.c b/sound/soc/soc-card.c index 235427d6906173..bc02c7b864e295 100644 --- a/sound/soc/soc-card.c +++ b/sound/soc/soc-card.c @@ -184,10 +184,16 @@ int snd_soc_card_late_probe(struct snd_soc_card *card) return 0; } -void snd_soc_card_fixup_controls(struct snd_soc_card *card) +int snd_soc_card_fixup_controls(struct snd_soc_card *card) { - if (card->fixup_controls) - card->fixup_controls(card); + if (card->fixup_controls) { + int ret = card->fixup_controls(card); + + if (ret < 0) + return soc_card_ret(card, ret); + } + + return 0; } int snd_soc_card_remove(struct snd_soc_card *card) diff --git a/sound/soc/soc-component.c b/sound/soc/soc-component.c index 89f236ab303414..77ad3338397445 100644 --- a/sound/soc/soc-component.c +++ b/sound/soc/soc-component.c @@ -1042,6 +1042,11 @@ int snd_soc_pcm_component_new(struct snd_soc_pcm_runtime *rtd) if (ret < 0) return soc_component_ret(component, ret); } + if (component->driver->pcm_new) { + ret = component->driver->pcm_new(component, rtd); + if (ret < 0) + return soc_component_ret(component, ret); + } } return 0; @@ -1055,9 +1060,12 @@ void snd_soc_pcm_component_free(struct snd_soc_pcm_runtime *rtd) if (!rtd->pcm) return; - for_each_rtd_components(rtd, i, component) + for_each_rtd_components(rtd, i, component) { if (component->driver->pcm_destruct) component->driver->pcm_destruct(component, rtd->pcm); + if (component->driver->pcm_free) + component->driver->pcm_free(component, rtd->pcm); + } } int snd_soc_pcm_component_prepare(struct snd_pcm_substream *substream) diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c index 7b81dffc6a935a..b8402802ae7848 100644 --- a/sound/soc/soc-compress.c +++ b/sound/soc/soc-compress.c @@ -69,10 +69,10 @@ static int soc_compr_clean(struct snd_compr_stream *cstream, int rollback) snd_soc_dai_digital_mute(codec_dai, 1, stream); if (!snd_soc_dai_active(cpu_dai)) - cpu_dai->symmetric_rate = 0; + soc_pcm_set_dai_params(cpu_dai, NULL); if (!snd_soc_dai_active(codec_dai)) - codec_dai->symmetric_rate = 0; + soc_pcm_set_dai_params(codec_dai, NULL); snd_soc_link_compr_shutdown(cstream, rollback); diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index ff6eb6bfc63bc1..5e2172198e5d61 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -2293,7 +2293,10 @@ static int snd_soc_bind_card(struct snd_soc_card *card) goto probe_end; snd_soc_dapm_new_widgets(card); - snd_soc_card_fixup_controls(card); + + ret = snd_soc_card_fixup_controls(card); + if (ret < 0) + goto probe_end; ret = snd_card_register(card->snd_card); if (ret < 0) { diff --git a/sound/soc/soc-ops.c b/sound/soc/soc-ops.c index f966d4e13c7fc3..c8acd6a55ae764 100644 --- a/sound/soc/soc-ops.c +++ b/sound/soc/soc-ops.c @@ -445,6 +445,30 @@ int snd_soc_put_volsw_sx(struct snd_kcontrol *kcontrol, } EXPORT_SYMBOL_GPL(snd_soc_put_volsw_sx); +bool snd_soc_control_matches(struct snd_kcontrol *kctl, + const char *pattern) +{ + const char *name = kctl->id.name; + + if (pattern[0] == '*') { + int namelen; + int patternlen; + + pattern++; + if (pattern[0] == ' ') + pattern++; + + namelen = strlen(name); + patternlen = strlen(pattern); + + if (namelen > patternlen) + name += namelen - patternlen; + } + + return !strcmp(name, pattern); +} +EXPORT_SYMBOL_GPL(snd_soc_control_matches); + static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) { struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; @@ -476,39 +500,165 @@ static int snd_soc_clip_to_platform_max(struct snd_kcontrol *kctl) return ret; } +static int soc_limit_volume(struct snd_kcontrol *kctl, int max) +{ + struct soc_mixer_control *mc = (struct soc_mixer_control *)kctl->private_value; + + if (max <= 0 || max > mc->max - mc->min) + return -EINVAL; + mc->platform_max = max; + return snd_soc_clip_to_platform_max(kctl); +} + /** - * snd_soc_limit_volume - Set new limit to an existing volume control. + * snd_soc_limit_volume - Set new limit to existing volume controls * * @card: where to look for the control - * @name: Name of the control + * @name: name pattern * @max: new maximum limit + * + * Finds controls matching the given name (which can be either a name + * verbatim, or a pattern starting with the wildcard '*') and sets + * a platform volume limit on them. * - * Return 0 for success, else error. + * Return number of matching controls on success, else error. At least + * one control needs to match the pattern. */ int snd_soc_limit_volume(struct snd_soc_card *card, const char *name, int max) { struct snd_kcontrol *kctl; - int ret = -EINVAL; + int hits = 0; + int ret; - /* Sanity check for name and max */ - if (unlikely(!name || max <= 0)) + /* Sanity check for name */ + if (unlikely(!name)) return -EINVAL; - kctl = snd_soc_card_get_kcontrol(card, name); - if (kctl) { - struct soc_mixer_control *mc = - (struct soc_mixer_control *)kctl->private_value; + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; - if (max <= mc->max - mc->min) { - mc->platform_max = max; - ret = snd_soc_clip_to_platform_max(kctl); - } + ret = soc_limit_volume(kctl, max); + if (ret < 0) + return ret; + hits++; } - return ret; + if (!hits) + return -EINVAL; + + return hits; } EXPORT_SYMBOL_GPL(snd_soc_limit_volume); +/** + * snd_soc_deactivate_kctl - Activate/deactive controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @active: non-zero to activate, zero to deactivate + * + * Return number of matching controls on success, else error. + * No controls need to match. + */ +int snd_soc_deactivate_kctl(struct snd_soc_card *card, + const char *name, int active) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + ret = snd_ctl_activate_id(card->snd_card, &kctl->id, active); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_deactivate_kctl); + +static int soc_set_enum_kctl(struct snd_kcontrol *kctl, const char *strval) +{ + struct snd_ctl_elem_value value; + struct snd_ctl_elem_info info; + int sel, i, ret; + + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (info.type != SNDRV_CTL_ELEM_TYPE_ENUMERATED) + return -EINVAL; + + for (sel = 0; sel < info.value.enumerated.items; sel++) { + info.value.enumerated.item = sel; + ret = kctl->info(kctl, &info); + if (ret < 0) + return ret; + + if (!strcmp(strval, info.value.enumerated.name)) + break; + } + + if (sel == info.value.enumerated.items) + return -EINVAL; + + for (i = 0; i < info.count; i++) + value.value.enumerated.item[i] = sel; + + return kctl->put(kctl, &value); +} + +/** + * snd_soc_set_enum_kctl - Set enumerated controls matching a pattern + * + * @card: where to look for the controls + * @name: name pattern + * @value: string value to set the controls to + * + * Return number of matching and set controls on success, else error. + * No controls need to match. + */ +int snd_soc_set_enum_kctl(struct snd_soc_card *card, + const char *name, const char *value) +{ + struct snd_kcontrol *kctl; + int hits = 0; + int ret; + + /* Sanity check for name */ + if (unlikely(!name)) + return -EINVAL; + + list_for_each_entry(kctl, &card->snd_card->controls, list) { + if (!snd_soc_control_matches(kctl, name)) + continue; + + ret = soc_set_enum_kctl(kctl, value); + if (ret < 0) + return ret; + hits++; + } + + if (!hits) + return -EINVAL; + + return hits; +} +EXPORT_SYMBOL_GPL(snd_soc_set_enum_kctl); + int snd_soc_bytes_info(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index afa9fad4457f2c..9b12eedb77c331 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -423,8 +423,8 @@ void dpcm_dapm_stream_event(struct snd_soc_pcm_runtime *fe, int dir, int event) snd_soc_dapm_stream_event(fe, dir, event); } -static void soc_pcm_set_dai_params(struct snd_soc_dai *dai, - struct snd_pcm_hw_params *params) +void soc_pcm_set_dai_params(struct snd_soc_dai *dai, + struct snd_pcm_hw_params *params) { if (params) { dai->symmetric_rate = params_rate(params); diff --git a/sound/soc/sof/compress.c b/sound/soc/sof/compress.c index 96570121aae01b..a676ffc2379df8 100644 --- a/sound/soc/sof/compress.c +++ b/sound/soc/sof/compress.c @@ -255,6 +255,7 @@ static int sof_compr_set_params(struct snd_soc_component *component, sstream->sampling_rate = params->codec.sample_rate; sstream->channels = params->codec.ch_out; sstream->sample_container_bytes = pcm->params.sample_container_bytes; + sstream->codec_params = params->codec; spcm->prepared[cstream->direction] = true; @@ -267,9 +268,10 @@ static int sof_compr_set_params(struct snd_soc_component *component, static int sof_compr_get_params(struct snd_soc_component *component, struct snd_compr_stream *cstream, struct snd_codec *params) { - /* TODO: we don't query the supported codecs for now, if the - * application asks for an unsupported codec the set_params() will fail. - */ + struct sof_compr_stream *sstream = cstream->runtime->private_data; + + *params = sstream->codec_params; + return 0; } @@ -379,6 +381,9 @@ static int sof_compr_pointer(struct snd_soc_component *component, if (!spcm) return -EINVAL; + if (!sstream->channels || !sstream->sample_container_bytes) + return -EBUSY; + tstamp->sampling_rate = sstream->sampling_rate; tstamp->copied_total = sstream->copied_total; tstamp->pcm_io_frames = div_u64(spcm->stream[cstream->direction].posn.dai_posn, diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c index 1c04b5d9c0d8b6..5c1f3b427cdb86 100644 --- a/sound/soc/sof/intel/hda-stream.c +++ b/sound/soc/sof/intel/hda-stream.c @@ -480,16 +480,20 @@ int hda_dsp_iccmax_stream_hw_params(struct snd_sof_dev *sdev, struct hdac_ext_st struct snd_dma_buffer *dmab, struct snd_pcm_hw_params *params) { - struct hdac_stream *hstream = &hext_stream->hstream; - int sd_offset = SOF_STREAM_SD_OFFSET(hstream); + struct hdac_stream *hstream; + int sd_offset; int ret; - u32 mask = 0x1 << hstream->index; + u32 mask; if (!hext_stream) { dev_err(sdev->dev, "error: no stream available\n"); return -ENODEV; } + hstream = &hext_stream->hstream; + sd_offset = SOF_STREAM_SD_OFFSET(hstream); + mask = 0x1 << hstream->index; + if (!dmab) { dev_err(sdev->dev, "error: no dma buffer allocated!\n"); return -ENODEV; diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 8a240dcb7fcb39..5fa773bb267883 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -1399,7 +1399,8 @@ static struct snd_soc_acpi_mach *hda_sdw_machine_select(struct snd_sof_dev *sdev link_mask |= BIT(peripherals->array[i]->bus->link_id); link_num = hweight32(link_mask); - links = devm_kcalloc(sdev->dev, link_num, sizeof(*links), GFP_KERNEL); + /* An empty adr_link is needed to terminate the adr_link loop */ + links = devm_kcalloc(sdev->dev, link_num + 1, sizeof(*links), GFP_KERNEL); if (!links) return NULL; diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h index 693d063830faed..38753b088fc1dd 100644 --- a/sound/soc/sof/sof-priv.h +++ b/sound/soc/sof/sof-priv.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -111,6 +112,7 @@ struct sof_compr_stream { u32 sampling_rate; u16 channels; u16 sample_container_bytes; + struct snd_codec codec_params; size_t posn_offset; }; diff --git a/sound/soc/sti/uniperif_player.c b/sound/soc/sti/uniperif_player.c index 6d1ce030963c62..45d35b887e4ebf 100644 --- a/sound/soc/sti/uniperif_player.c +++ b/sound/soc/sti/uniperif_player.c @@ -1028,8 +1028,13 @@ static int uni_player_parse_dt_audio_glue(struct platform_device *pdev, return PTR_ERR(regmap); } - player->clk_sel = regmap_field_alloc(regmap, regfield[0]); - player->valid_sel = regmap_field_alloc(regmap, regfield[1]); + player->clk_sel = devm_regmap_field_alloc(&pdev->dev, regmap, regfield[0]); + if (IS_ERR(player->clk_sel)) + return PTR_ERR(player->clk_sel); + + player->valid_sel = devm_regmap_field_alloc(&pdev->dev, regmap, regfield[1]); + if (IS_ERR(player->valid_sel)) + return PTR_ERR(player->valid_sel); return 0; } diff --git a/sound/usb/6fire/chip.c b/sound/usb/6fire/chip.c index 5ff78814e68750..874f6cd503ca59 100644 --- a/sound/usb/6fire/chip.c +++ b/sound/usb/6fire/chip.c @@ -53,11 +53,6 @@ static void usb6fire_chip_abort(struct sfire_chip *chip) usb6fire_comm_abort(chip); if (chip->control) usb6fire_control_abort(chip); - if (chip->card) { - snd_card_disconnect(chip->card); - snd_card_free_when_closed(chip->card); - chip->card = NULL; - } } } @@ -168,6 +163,7 @@ static int usb6fire_chip_probe(struct usb_interface *intf, static void usb6fire_chip_disconnect(struct usb_interface *intf) { struct sfire_chip *chip; + struct snd_card *card; chip = usb_get_intfdata(intf); if (chip) { /* if !chip, fw upload has been performed */ @@ -178,8 +174,19 @@ static void usb6fire_chip_disconnect(struct usb_interface *intf) chips[chip->regidx] = NULL; } + /* + * Save card pointer before teardown. + * snd_card_free_when_closed() may free card (and + * the embedded chip) immediately, so it must be + * called last and chip must not be accessed after. + */ + card = chip->card; chip->shutdown = true; + if (card) + snd_card_disconnect(card); usb6fire_chip_abort(chip); + if (card) + snd_card_free_when_closed(card); } } } diff --git a/sound/usb/6fire/control.c b/sound/usb/6fire/control.c index dd25a6407b6385..c77a21a9acd796 100644 --- a/sound/usb/6fire/control.c +++ b/sound/usb/6fire/control.c @@ -290,15 +290,17 @@ static int usb6fire_control_input_vol_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct control_runtime *rt = snd_kcontrol_chip(kcontrol); + int vol0 = ucontrol->value.integer.value[0] - 15; + int vol1 = ucontrol->value.integer.value[1] - 15; int changed = 0; - if (rt->input_vol[0] != ucontrol->value.integer.value[0]) { - rt->input_vol[0] = ucontrol->value.integer.value[0] - 15; + if (rt->input_vol[0] != vol0) { + rt->input_vol[0] = vol0; rt->ivol_updated &= ~(1 << 0); changed = 1; } - if (rt->input_vol[1] != ucontrol->value.integer.value[1]) { - rt->input_vol[1] = ucontrol->value.integer.value[1] - 15; + if (rt->input_vol[1] != vol1) { + rt->input_vol[1] = vol1; rt->ivol_updated &= ~(1 << 1); changed = 1; } diff --git a/sound/usb/caiaq/control.c b/sound/usb/caiaq/control.c index af459c49baf4c2..4598fb7e8be025 100644 --- a/sound/usb/caiaq/control.c +++ b/sound/usb/caiaq/control.c @@ -87,6 +87,7 @@ static int control_put(struct snd_kcontrol *kcontrol, struct snd_usb_caiaqdev *cdev = caiaqdev(chip->card); int pos = kcontrol->private_value; int v = ucontrol->value.integer.value[0]; + int ret; unsigned char cmd; switch (cdev->chip.usb_id) { @@ -103,6 +104,10 @@ static int control_put(struct snd_kcontrol *kcontrol, if (pos & CNT_INTVAL) { int i = pos & ~CNT_INTVAL; + unsigned char old = cdev->control_state[i]; + + if (old == v) + return 0; cdev->control_state[i] = v; @@ -113,10 +118,11 @@ static int control_put(struct snd_kcontrol *kcontrol, cdev->ep8_out_buf[0] = i; cdev->ep8_out_buf[1] = v; - usb_bulk_msg(cdev->chip.dev, - usb_sndbulkpipe(cdev->chip.dev, 8), - cdev->ep8_out_buf, sizeof(cdev->ep8_out_buf), - &actual_len, 200); + ret = usb_bulk_msg(cdev->chip.dev, + usb_sndbulkpipe(cdev->chip.dev, 8), + cdev->ep8_out_buf, + sizeof(cdev->ep8_out_buf), + &actual_len, 200); } else if (cdev->chip.usb_id == USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_MASCHINECONTROLLER)) { @@ -128,21 +134,36 @@ static int control_put(struct snd_kcontrol *kcontrol, offset = MASCHINE_BANK_SIZE; } - snd_usb_caiaq_send_command_bank(cdev, cmd, bank, - cdev->control_state + offset, - MASCHINE_BANK_SIZE); + ret = snd_usb_caiaq_send_command_bank(cdev, cmd, bank, + cdev->control_state + offset, + MASCHINE_BANK_SIZE); } else { - snd_usb_caiaq_send_command(cdev, cmd, - cdev->control_state, sizeof(cdev->control_state)); + ret = snd_usb_caiaq_send_command(cdev, cmd, + cdev->control_state, + sizeof(cdev->control_state)); + } + + if (ret < 0) { + cdev->control_state[i] = old; + return ret; } } else { - if (v) - cdev->control_state[pos / 8] |= 1 << (pos % 8); - else - cdev->control_state[pos / 8] &= ~(1 << (pos % 8)); + int idx = pos / 8; + unsigned char mask = 1 << (pos % 8); + unsigned char old = cdev->control_state[idx]; + unsigned char val = v ? (old | mask) : (old & ~mask); - snd_usb_caiaq_send_command(cdev, cmd, - cdev->control_state, sizeof(cdev->control_state)); + if (old == val) + return 0; + + cdev->control_state[idx] = val; + ret = snd_usb_caiaq_send_command(cdev, cmd, + cdev->control_state, + sizeof(cdev->control_state)); + if (ret < 0) { + cdev->control_state[idx] = old; + return ret; + } } return 1; @@ -640,4 +661,3 @@ int snd_usb_caiaq_control_init(struct snd_usb_caiaqdev *cdev) return ret; } - diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index 3a71bab8a47749..b20aae0caf60a4 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -290,7 +290,7 @@ int snd_usb_caiaq_set_auto_msg(struct snd_usb_caiaqdev *cdev, tmp, sizeof(tmp)); } -static void setup_card(struct snd_usb_caiaqdev *cdev) +static int setup_card(struct snd_usb_caiaqdev *cdev) { int ret; char val[4]; @@ -325,8 +325,10 @@ static void setup_card(struct snd_usb_caiaqdev *cdev) snd_usb_caiaq_send_command(cdev, EP1_CMD_READ_IO, NULL, 0); if (!wait_event_timeout(cdev->ep1_wait_queue, - cdev->control_state[0] != 0xff, HZ)) - return; + cdev->control_state[0] != 0xff, HZ)) { + dev_err(dev, "Read timeout for control state\n"); + return -EINVAL; + } /* fix up some defaults */ if ((cdev->control_state[1] != 2) || @@ -347,33 +349,43 @@ static void setup_card(struct snd_usb_caiaqdev *cdev) cdev->spec.num_digital_audio_out + cdev->spec.num_digital_audio_in > 0) { ret = snd_usb_caiaq_audio_init(cdev); - if (ret < 0) + if (ret < 0) { dev_err(dev, "Unable to set up audio system (ret=%d)\n", ret); + return ret; + } } if (cdev->spec.num_midi_in + cdev->spec.num_midi_out > 0) { ret = snd_usb_caiaq_midi_init(cdev); - if (ret < 0) + if (ret < 0) { dev_err(dev, "Unable to set up MIDI system (ret=%d)\n", ret); + return ret; + } } #ifdef CONFIG_SND_USB_CAIAQ_INPUT ret = snd_usb_caiaq_input_init(cdev); - if (ret < 0) + if (ret < 0 && ret != -ENODEV) { dev_err(dev, "Unable to set up input system (ret=%d)\n", ret); + return ret; + } #endif /* finally, register the card and all its sub-instances */ ret = snd_card_register(cdev->chip.card); if (ret < 0) { dev_err(dev, "snd_card_register() returned %d\n", ret); - snd_card_free(cdev->chip.card); + return ret; } ret = snd_usb_caiaq_control_init(cdev); - if (ret < 0) + if (ret < 0) { dev_err(dev, "Unable to set up control system (ret=%d)\n", ret); + return ret; + } + + return 0; } static void card_free(struct snd_card *card) @@ -384,7 +396,7 @@ static void card_free(struct snd_card *card) snd_usb_caiaq_input_free(cdev); #endif snd_usb_caiaq_audio_free(cdev); - usb_reset_device(cdev->chip.dev); + usb_put_dev(cdev->chip.dev); } static int create_card(struct usb_device *usb_dev, @@ -410,7 +422,8 @@ static int create_card(struct usb_device *usb_dev, return err; cdev = caiaqdev(card); - cdev->chip.dev = usb_dev; + cdev->chip.dev = usb_get_dev(usb_dev); + card->private_free = card_free; cdev->chip.card = card; cdev->chip.usb_id = USB_ID(le16_to_cpu(usb_dev->descriptor.idVendor), le16_to_cpu(usb_dev->descriptor.idProduct)); @@ -499,8 +512,10 @@ static int init_card(struct snd_usb_caiaqdev *cdev) scnprintf(card->longname, sizeof(card->longname), "%s %s (%s)", cdev->vendor_name, cdev->product_name, usbpath); - setup_card(cdev); - card->private_free = card_free; + err = setup_card(cdev); + if (err < 0) + goto err_kill_urb; + return 0; err_kill_urb: diff --git a/sound/usb/caiaq/input.c b/sound/usb/caiaq/input.c index a9130891bb696d..5c70fdf61cc139 100644 --- a/sound/usb/caiaq/input.c +++ b/sound/usb/caiaq/input.c @@ -804,7 +804,7 @@ int snd_usb_caiaq_input_init(struct snd_usb_caiaqdev *cdev) default: /* no input methods supported on this device */ - ret = -EINVAL; + ret = -ENODEV; goto exit_free_idev; } diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index bf4401aba76ca8..6fbcb117555c6b 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -1379,9 +1379,6 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, return -EINVAL; } - ep->packsize[0] = min(ep->packsize[0], ep->maxframesize); - ep->packsize[1] = min(ep->packsize[1], ep->maxframesize); - /* calculate the frequency in 16.16 format */ ep->freqm = ep->freqn; ep->freqshift = INT_MIN; @@ -1408,6 +1405,9 @@ int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, ep->maxframesize = ep->maxpacksize / ep->cur_frame_bytes; ep->curframesize = ep->curpacksize / ep->cur_frame_bytes; + ep->packsize[0] = min(ep->packsize[0], ep->maxframesize); + ep->packsize[1] = min(ep->packsize[1], ep->maxframesize); + err = update_clock_ref_rate(chip, ep); if (err >= 0) { ep->need_setup = false; diff --git a/sound/usb/format.c b/sound/usb/format.c index 1207c507882add..0fa2f3f3dd2b50 100644 --- a/sound/usb/format.c +++ b/sound/usb/format.c @@ -466,7 +466,7 @@ static int parse_uac2_sample_rate_range(struct snd_usb_audio *chip, nr_rates++; if (nr_rates >= MAX_NR_RATES) { usb_audio_err(chip, "invalid uac2 rates\n"); - break; + return nr_rates; } skip_rate: diff --git a/sound/usb/midi.c b/sound/usb/midi.c index a8bddc90c0ed6b..13d7380bb3927f 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -1947,15 +1947,17 @@ static struct usb_ms_endpoint_descriptor *find_usb_ms_endpoint_descriptor( while (extralen > 3) { struct usb_ms_endpoint_descriptor *ms_ep = (struct usb_ms_endpoint_descriptor *)extra; + int length = ms_ep->bLength; - if (ms_ep->bLength > 3 && + if (!length || length > extralen) + break; + + if (length > 3 && ms_ep->bDescriptorType == USB_DT_CS_ENDPOINT && ms_ep->bDescriptorSubtype == UAC_MS_GENERAL) return ms_ep; - if (!extra[0]) - break; - extralen -= extra[0]; - extra += extra[0]; + extralen -= length; + extra += length; } return NULL; } diff --git a/sound/usb/midi2.c b/sound/usb/midi2.c index ef602e81576d9e..44d1deacacdf8f 100644 --- a/sound/usb/midi2.c +++ b/sound/usb/midi2.c @@ -227,7 +227,7 @@ static void kill_midi_urbs(struct snd_usb_midi2_endpoint *ep, bool suspending) if (!ep) return; if (suspending) - ep->suspended = ep->running; + atomic_set(&ep->suspended, atomic_read(&ep->running)); atomic_set(&ep->running, 0); for (i = 0; i < ep->num_urbs; i++) { if (!ep->urbs[i].urb) @@ -496,15 +496,17 @@ static void *find_usb_ms_endpoint_descriptor(struct usb_host_endpoint *hostep, while (extralen > 3) { struct usb_ms_endpoint_descriptor *ms_ep = (struct usb_ms_endpoint_descriptor *)extra; + int length = ms_ep->bLength; - if (ms_ep->bLength > 3 && + if (!length || length > extralen) + break; + + if (length > 3 && ms_ep->bDescriptorType == USB_DT_CS_ENDPOINT && ms_ep->bDescriptorSubtype == subtype) return ms_ep; - if (!extra[0]) - break; - extralen -= extra[0]; - extra += extra[0]; + extralen -= length; + extra += length; } return NULL; } @@ -1190,10 +1192,11 @@ void snd_usb_midi_v2_suspend_all(struct snd_usb_audio *chip) static void resume_midi2_endpoint(struct snd_usb_midi2_endpoint *ep) { - ep->running = ep->suspended; - if (ep->direction == STR_IN) + atomic_set(&ep->running, atomic_read(&ep->suspended)); + atomic_set(&ep->suspended, 0); + + if (ep->direction == STR_IN || atomic_read(&ep->running)) submit_io_urbs(ep); - /* FIXME: does it all? */ } void snd_usb_midi_v2_resume_all(struct snd_usb_audio *chip) diff --git a/sound/usb/misc/ua101.c b/sound/usb/misc/ua101.c index 49b3dd8d827dae..d129b42eb979d0 100644 --- a/sound/usb/misc/ua101.c +++ b/sound/usb/misc/ua101.c @@ -974,6 +974,13 @@ static int detect_usb_format(struct ua101 *ua) ua->capture.channels = fmt_capture->bNrChannels; ua->playback.channels = fmt_playback->bNrChannels; + if (!ua->capture.channels || !ua->playback.channels) { + dev_err(&ua->dev->dev, + "invalid channel count: capture %u, playback %u\n", + ua->capture.channels, ua->playback.channels); + return -EINVAL; + } + ua->capture.frame_bytes = fmt_capture->bSubframeSize * ua->capture.channels; ua->playback.frame_bytes = diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index ac8c71ba948349..21191da4c04962 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1204,6 +1204,13 @@ static void volume_control_quirks(struct usb_mixer_elem_info *cval, cval->min = -11264; /* Mute under it */ } break; + case USB_ID(0x31b2, 0x0111): /* MOONDROP JU Jiu */ + if (!strcmp(kctl->id.name, "PCM Playback Volume")) { + usb_audio_info(chip, + "set volume quirk for MOONDROP JU Jiu\n"); + cval->min = -10880; /* Mute under it */ + } + break; } } @@ -1813,10 +1820,11 @@ static void __build_feature_ctl(struct usb_mixer_interface *mixer, range = (cval->max - cval->min) / cval->res; /* - * There are definitely devices with a range of ~20,000, so let's be - * conservative and allow for a bit more. + * Are there devices with volume range more than 255? I use a bit more + * to be sure. 384 is a resolution magic number found on Logitech + * devices. It will definitively catch all buggy Logitech devices. */ - if (range > 65535) { + if (range > 384) { usb_audio_warn(mixer->chip, "Warning! Unlikely big volume range (=%u), cval->res is probably wrong.", range); diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index 11e205da7964de..50c42a477030f2 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -1538,15 +1538,17 @@ void snd_emuusb_set_samplerate(struct snd_usb_audio *chip, { struct usb_mixer_interface *mixer; struct usb_mixer_elem_info *cval; + int err; int unitid = 12; /* SampleRate ExtensionUnit ID */ list_for_each_entry(mixer, &chip->mixer_list, list) { if (mixer->id_elems[unitid]) { cval = mixer_elem_list_to_info(mixer->id_elems[unitid]); - snd_usb_mixer_set_ctl_value(cval, UAC_SET_CUR, - cval->control << 8, - samplerate_id); - snd_usb_mixer_notify_id(mixer, unitid); + err = snd_usb_mixer_set_ctl_value(cval, UAC_SET_CUR, + cval->control << 8, + samplerate_id); + if (!err) + snd_usb_mixer_notify_id(mixer, unitid); break; } } @@ -2025,7 +2027,7 @@ static int snd_microii_spdif_switch_put(struct snd_kcontrol *kcontrol, int err; reg = ucontrol->value.integer.value[0] ? 0x28 : 0x2a; - if (reg != list->kctl->private_value) + if (reg == list->kctl->private_value) return 0; kcontrol->private_value = reg; diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index fd1fb668929a25..8eaa962227596f 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -2262,7 +2262,7 @@ static const struct scarlett2_device_entry scarlett2_devices[] = { { USB_ID(0x1235, 0x820c), &clarett_8pre_info, "Clarett+" }, /* End of list */ - { 0, NULL }, + { 0, NULL, NULL }, }; /* get the starting port index number for a given port type/direction */ diff --git a/sound/usb/qcom/qc_audio_offload.c b/sound/usb/qcom/qc_audio_offload.c index 2ac813d57f4f53..a0009503b2c592 100644 --- a/sound/usb/qcom/qc_audio_offload.c +++ b/sound/usb/qcom/qc_audio_offload.c @@ -565,6 +565,7 @@ static unsigned long uaudio_iommu_map_pa(enum mem_type mtype, bool dma_coherent, unsigned long iova = 0; bool map = true; int prot = uaudio_iommu_map_prot(dma_coherent); + int ret; switch (mtype) { case MEM_EVENT_RING: @@ -582,10 +583,24 @@ static unsigned long uaudio_iommu_map_pa(enum mem_type mtype, bool dma_coherent, dev_err(uaudio_qdev->data->dev, "unknown mem type %d\n", mtype); } - if (!iova || !map) + if (!iova) return 0; - iommu_map(uaudio_qdev->data->domain, iova, pa, size, prot, GFP_KERNEL); + if (!map) + return iova; + + ret = iommu_map(uaudio_qdev->data->domain, iova, pa, size, prot, + GFP_KERNEL); + if (ret) { + dev_err(uaudio_qdev->data->dev, + "failed to map %zu bytes at iova 0x%08lx: %d\n", + size, iova, ret); + if (mtype == MEM_XFER_RING) + uaudio_put_iova(iova, size, + &uaudio_qdev->xfer_ring_list, + &uaudio_qdev->xfer_ring_iova_size); + return 0; + } return iova; } @@ -948,7 +963,7 @@ static int enable_audio_stream(struct snd_usb_substream *subs, _snd_pcm_hw_params_any(¶ms); m = hw_param_mask(¶ms, SNDRV_PCM_HW_PARAM_FORMAT); - snd_mask_leave(m, pcm_format); + snd_mask_leave(m, (__force unsigned int)pcm_format); i = hw_param_interval(¶ms, SNDRV_PCM_HW_PARAM_CHANNELS); snd_interval_setinteger(i); @@ -1054,15 +1069,17 @@ static int uaudio_transfer_buffer_setup(struct snd_usb_substream *subs, if (!xfer_buf) return -ENOMEM; - dma_get_sgtable(subs->dev->bus->sysdev, &xfer_buf_sgt, xfer_buf, - xfer_buf_dma, len); + ret = dma_get_sgtable(subs->dev->bus->sysdev, &xfer_buf_sgt, xfer_buf, + xfer_buf_dma, len); + if (ret) + goto free_xfer_buf; /* map the physical buffer into sysdev as well */ xfer_buf_dma_sysdev = uaudio_iommu_map_xfer_buf(dma_coherent, len, &xfer_buf_sgt); if (!xfer_buf_dma_sysdev) { ret = -ENOMEM; - goto unmap_sync; + goto free_sgt; } mem_info->dma = xfer_buf_dma; @@ -1073,7 +1090,9 @@ static int uaudio_transfer_buffer_setup(struct snd_usb_substream *subs, return 0; -unmap_sync: +free_sgt: + sg_free_table(&xfer_buf_sgt); +free_xfer_buf: usb_free_coherent(subs->dev, len, xfer_buf, xfer_buf_dma); return ret; diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 4cfa24c06fcdf2..e8ae3464887b23 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -122,7 +122,7 @@ static int add_audio_stream_from_fixed_fmt(struct snd_usb_audio *chip, snd_usb_audioformat_set_sync_ep(chip, fp); - err = snd_usb_add_audio_stream(chip, stream, fp); + err = snd_usb_add_audio_stream(chip, stream, fp, NULL); if (err < 0) return err; @@ -2435,6 +2435,7 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_VALIDATE_RATES), DEVICE_FLG(0x1235, 0x8006, 0), /* Focusrite Scarlett 2i2 1st Gen */ DEVICE_FLG(0x1235, 0x800a, 0), /* Focusrite Scarlett 2i4 1st Gen */ + DEVICE_FLG(0x1235, 0x800c, 0), /* Focusrite Scarlett 18i20 1st Gen */ DEVICE_FLG(0x1235, 0x8016, 0), /* Focusrite Scarlett 2i2 1st Gen */ DEVICE_FLG(0x1235, 0x801c, 0), /* Focusrite Scarlett Solo 1st Gen */ VENDOR_FLG(0x1235, /* Focusrite Novation */ diff --git a/sound/usb/stream.c b/sound/usb/stream.c index d38c39e28f38fa..03a939cdd07ac6 100644 --- a/sound/usb/stream.c +++ b/sound/usb/stream.c @@ -79,7 +79,7 @@ static void snd_usb_audio_pcm_free(struct snd_pcm *pcm) static void snd_usb_init_substream(struct snd_usb_stream *as, int stream, struct audioformat *fp, - struct snd_usb_power_domain *pd) + struct snd_usb_power_domain **pdptr) { struct snd_usb_substream *subs = &as->substream[stream]; @@ -105,10 +105,11 @@ static void snd_usb_init_substream(struct snd_usb_stream *as, if (fp->channels > subs->channels_max) subs->channels_max = fp->channels; - if (pd) { - subs->str_pd = pd; + if (pdptr && *pdptr) { + subs->str_pd = *pdptr; + *pdptr = NULL; /* assigned */ /* Initialize Power Domain to idle status D1 */ - snd_usb_power_domain_set(subs->stream->chip, pd, + snd_usb_power_domain_set(subs->stream->chip, subs->str_pd, UAC3_PD_STATE_D1); } @@ -352,6 +353,8 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor if (len < sizeof(*cs_desc)) break; cs_len = le16_to_cpu(cs_desc->wLength); + if (cs_len < sizeof(*cs_desc)) + break; if (len < cs_len) break; cs_type = cs_desc->bSegmentType; @@ -484,11 +487,14 @@ snd_pcm_chmap_elem *convert_chmap_v3(struct uac3_cluster_header_descriptor * if not, create a new pcm stream. note, fp is added to the substream * fmt_list and will be freed on the chip instance release. do not free * fp or do remove it from the substream fmt_list to avoid double-free. + * + * pdptr is optional and can be NULL. When it's non-NULL and the PD gets + * assigned to the stream, *pdptr is cleared to NULL upon return. */ -static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, - int stream, - struct audioformat *fp, - struct snd_usb_power_domain *pd) +int snd_usb_add_audio_stream(struct snd_usb_audio *chip, + int stream, + struct audioformat *fp, + struct snd_usb_power_domain **pdptr) { struct snd_usb_stream *as; @@ -521,7 +527,7 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, err = snd_pcm_new_stream(as->pcm, stream, 1); if (err < 0) return err; - snd_usb_init_substream(as, stream, fp, pd); + snd_usb_init_substream(as, stream, fp, pdptr); return add_chmap(as->pcm, stream, subs); } @@ -550,7 +556,7 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, else strscpy(pcm->name, "USB Audio"); - snd_usb_init_substream(as, stream, fp, pd); + snd_usb_init_substream(as, stream, fp, pdptr); /* * Keep using head insertion for M-Audio Audiophile USB (tm) which has a @@ -568,21 +574,6 @@ static int __snd_usb_add_audio_stream(struct snd_usb_audio *chip, return add_chmap(pcm, stream, &as->substream[stream]); } -int snd_usb_add_audio_stream(struct snd_usb_audio *chip, - int stream, - struct audioformat *fp) -{ - return __snd_usb_add_audio_stream(chip, stream, fp, NULL); -} - -static int snd_usb_add_audio_stream_v3(struct snd_usb_audio *chip, - int stream, - struct audioformat *fp, - struct snd_usb_power_domain *pd) -{ - return __snd_usb_add_audio_stream(chip, stream, fp, pd); -} - static int parse_uac_endpoint_attributes(struct snd_usb_audio *chip, struct usb_host_interface *alts, int protocol, int iface_no) @@ -995,7 +986,7 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip, * and request Cluster Descriptor */ wLength = le16_to_cpu(hc_header.wLength); - if (wLength < sizeof(cluster)) + if (wLength < sizeof(*cluster)) return NULL; cluster = kzalloc(wLength, GFP_KERNEL); if (!cluster) @@ -1105,8 +1096,7 @@ snd_usb_get_audioformat_uac3(struct snd_usb_audio *chip, } } - if (pd) - *pd_out = pd; + *pd_out = pd; return fp; } @@ -1121,7 +1111,6 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, struct usb_interface_descriptor *altsd; int i, altno, err, stream; struct audioformat *fp = NULL; - struct snd_usb_power_domain *pd = NULL; bool set_iface_first; int num, protocol; @@ -1163,6 +1152,12 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, if (snd_usb_apply_interface_quirk(chip, iface_no, altno)) continue; + /* pd may be allocated at snd_usb_get_audioformat_uac3() and + * assigned at snd_usb_add_audio_stream(); otherwise it'll be + * freed automatically by cleanup at each loop. + */ + struct snd_usb_power_domain *pd __free(kfree) = NULL; + /* * Roland audio streaming interfaces are marked with protocols * 0/1/2, but are UAC 1 compatible. @@ -1218,23 +1213,16 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip, *has_non_pcm = true; if ((fp->fmt_type == UAC_FORMAT_TYPE_I) == non_pcm) { audioformat_free(fp); - kfree(pd); fp = NULL; - pd = NULL; continue; } snd_usb_audioformat_set_sync_ep(chip, fp); dev_dbg(&dev->dev, "%u:%d: add audio endpoint %#x\n", iface_no, altno, fp->endpoint); - if (protocol == UAC_VERSION_3) - err = snd_usb_add_audio_stream_v3(chip, stream, fp, pd); - else - err = snd_usb_add_audio_stream(chip, stream, fp); - + err = snd_usb_add_audio_stream(chip, stream, fp, &pd); if (err < 0) { audioformat_free(fp); - kfree(pd); return err; } diff --git a/sound/usb/stream.h b/sound/usb/stream.h index d92e18d5818fee..61b9a133da018a 100644 --- a/sound/usb/stream.h +++ b/sound/usb/stream.h @@ -7,7 +7,8 @@ int snd_usb_parse_audio_interface(struct snd_usb_audio *chip, int snd_usb_add_audio_stream(struct snd_usb_audio *chip, int stream, - struct audioformat *fp); + struct audioformat *fp, + struct snd_usb_power_domain **pdptr); #endif /* __USBAUDIO_STREAM_H */ diff --git a/sound/usb/usx2y/us144mkii.c b/sound/usb/usx2y/us144mkii.c index 0cf4fa74e210ac..94553b61013c52 100644 --- a/sound/usb/usx2y/us144mkii.c +++ b/sound/usb/usx2y/us144mkii.c @@ -420,7 +420,11 @@ static int tascam_probe(struct usb_interface *intf, /* The device has two interfaces; we drive both from this driver. */ if (intf->cur_altsetting->desc.bInterfaceNumber == 1) { - tascam = usb_get_intfdata(usb_ifnum_to_if(dev, 0)); + struct usb_interface *intf_zero = usb_ifnum_to_if(dev, 0); + + if (!intf_zero) + return -ENODEV; + tascam = usb_get_intfdata(intf_zero); if (tascam) { usb_set_intfdata(intf, tascam); tascam->iface1 = intf; diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 50792df2770775..368a622ca0273f 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -60,7 +60,7 @@ int print_task_context_switch_counts; } /* Maximum size of response requested or message sent */ -#define MAX_MSG_SIZE 1024 +#define MAX_MSG_SIZE 2048 /* Maximum number of cpus expected to be specified in a cpumask */ #define MAX_CPUS 32 @@ -115,6 +115,32 @@ static int create_nl_socket(int protocol) return -1; } +static int recv_taskstats_msg(int sd, struct msgtemplate *msg) +{ + struct sockaddr_nl nladdr; + struct iovec iov = { + .iov_base = msg, + .iov_len = sizeof(*msg), + }; + struct msghdr hdr = { + .msg_name = &nladdr, + .msg_namelen = sizeof(nladdr), + .msg_iov = &iov, + .msg_iovlen = 1, + }; + int ret; + + ret = recvmsg(sd, &hdr, 0); + if (ret < 0) + return -1; + if (hdr.msg_flags & MSG_TRUNC) { + errno = EMSGSIZE; + return -1; + } + + return ret; +} + static int send_cmd(int sd, __u16 nlmsg_type, __u32 nlmsg_pid, __u8 genl_cmd, __u16 nla_type, @@ -633,12 +659,16 @@ int main(int argc, char *argv[]) } do { - rep_len = recv(nl_sd, &msg, sizeof(msg), 0); + rep_len = recv_taskstats_msg(nl_sd, &msg); PRINTF("received %d bytes\n", rep_len); if (rep_len < 0) { - fprintf(stderr, "nonfatal reply error: errno %d\n", - errno); + if (errno == EMSGSIZE) + fprintf(stderr, + "dropped truncated taskstats netlink message, please increase MAX_MSG_SIZE\n"); + else + fprintf(stderr, "nonfatal reply error: errno %d\n", + errno); continue; } if (msg.n.nlmsg_type == NLMSG_ERROR || @@ -680,6 +710,9 @@ int main(int argc, char *argv[]) printf("TGID\t%d\n", rtid); break; case TASKSTATS_TYPE_STATS: + PRINTF("version %u\n", + ((struct taskstats *) + NLA_DATA(na))->version); if (print_delays) print_delayacct((struct taskstats *) NLA_DATA(na)); if (print_io_accounting) diff --git a/tools/accounting/procacct.c b/tools/accounting/procacct.c index e8dee05a626464..46e5986ad9277a 100644 --- a/tools/accounting/procacct.c +++ b/tools/accounting/procacct.c @@ -71,7 +71,7 @@ int print_task_context_switch_counts; } /* Maximum size of response requested or message sent */ -#define MAX_MSG_SIZE 1024 +#define MAX_MSG_SIZE 2048 /* Maximum number of cpus expected to be specified in a cpumask */ #define MAX_CPUS 32 @@ -121,6 +121,32 @@ static int create_nl_socket(int protocol) return -1; } +static int recv_taskstats_msg(int sd, struct msgtemplate *msg) +{ + struct sockaddr_nl nladdr; + struct iovec iov = { + .iov_base = msg, + .iov_len = sizeof(*msg), + }; + struct msghdr hdr = { + .msg_name = &nladdr, + .msg_namelen = sizeof(nladdr), + .msg_iov = &iov, + .msg_iovlen = 1, + }; + int ret; + + ret = recvmsg(sd, &hdr, 0); + if (ret < 0) + return -1; + if (hdr.msg_flags & MSG_TRUNC) { + errno = EMSGSIZE; + return -1; + } + + return ret; +} + static int send_cmd(int sd, __u16 nlmsg_type, __u32 nlmsg_pid, __u8 genl_cmd, __u16 nla_type, @@ -239,6 +265,8 @@ void handle_aggr(int mother, struct nlattr *na, int fd) PRINTF("TGID\t%d\n", rtid); break; case TASKSTATS_TYPE_STATS: + PRINTF("version %u\n", + ((struct taskstats *)NLA_DATA(na))->version); if (mother == TASKSTATS_TYPE_AGGR_PID) print_procacct((struct taskstats *) NLA_DATA(na)); if (fd) { @@ -347,12 +375,16 @@ int main(int argc, char *argv[]) } do { - rep_len = recv(nl_sd, &msg, sizeof(msg), 0); + rep_len = recv_taskstats_msg(nl_sd, &msg); PRINTF("received %d bytes\n", rep_len); if (rep_len < 0) { - fprintf(stderr, "nonfatal reply error: errno %d\n", - errno); + if (errno == EMSGSIZE) + fprintf(stderr, + "dropped truncated taskstats netlink message, please increase MAX_MSG_SIZE\n"); + else + fprintf(stderr, "nonfatal reply error: errno %d\n", + errno); continue; } if (msg.n.nlmsg_type == NLMSG_ERROR || diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index 6673601246b382..eff29645719bc7 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -793,9 +793,10 @@ #define MSR_AMD64_LBR_SELECT 0xc000010e /* Zen4 */ -#define MSR_ZEN4_BP_CFG 0xc001102e +#define MSR_ZEN4_BP_CFG 0xc001102e #define MSR_ZEN4_BP_CFG_BP_SPEC_REDUCE_BIT 4 #define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5 +#define MSR_ZEN2_BP_CFG_BUG_FIX_BIT 33 /* Fam 19h MSRs */ #define MSR_F19H_UMC_PERF_CTL 0xc0010800 diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 1fbcb3ce74d217..f163a245837a63 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -103,12 +103,18 @@ else endif endif +ifeq ($(findstring -static,${LDFLAGS}),-static) + PKG_CONFIG += --static +endif + all: $(FILES) __BUILD = $(CC) $(CFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.c,$(@F)) $(LDFLAGS) BUILD = $(__BUILD) > $(@:.bin=.make.output) 2>&1 BUILD_BFD = $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl - BUILD_ALL = $(BUILD) -fstack-protector-all -O2 -D_FORTIFY_SOURCE=2 -ldw -lelf -lnuma -lelf -lslang $(FLAGS_PERL_EMBED) $(FLAGS_PYTHON_EMBED) -ldl -lz -llzma -lzstd -lssl + BUILD_ALL = $(BUILD) -fstack-protector-all -O2 -D_FORTIFY_SOURCE=2 -ldw -lelf -lnuma -lelf -lslang \ + $(FLAGS_PERL_EMBED) $(FLAGS_PYTHON_EMBED) -ldl -lz -llzma -lzstd \ + $(shell $(PKG_CONFIG) --libs --cflags openssl 2>/dev/null) __BUILDXX = $(CXX) $(CXXFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.cpp,$(@F)) $(LDFLAGS) BUILDXX = $(__BUILDXX) > $(@:.bin=.make.output) 2>&1 @@ -384,7 +390,7 @@ $(OUTPUT)test-libpfm4.bin: $(BUILD) -lpfm $(OUTPUT)test-libopenssl.bin: - $(BUILD) -lssl + $(BUILD) $(shell $(PKG_CONFIG) --libs --cflags openssl 2>/dev/null) $(OUTPUT)test-bpftool-skeletons.bin: $(SYSTEM_BPFTOOL) version | grep '^features:.*skeletons' \ diff --git a/tools/hv/Makefile b/tools/hv/Makefile index 34ffcec264ab0f..016753f3dd7f61 100644 --- a/tools/hv/Makefile +++ b/tools/hv/Makefile @@ -2,7 +2,7 @@ # Makefile for Hyper-V tools include ../scripts/Makefile.include -ARCH := $(shell uname -m 2>/dev/null) +ARCH ?= $(shell uname -m 2>/dev/null) sbindir ?= /usr/sbin libexecdir ?= /usr/libexec sharedstatedir ?= /var/lib @@ -20,7 +20,7 @@ override CFLAGS += -O2 -Wall -g -D_GNU_SOURCE -I$(OUTPUT)include override CFLAGS += -Wno-address-of-packed-member ALL_TARGETS := hv_kvp_daemon hv_vss_daemon -ifneq ($(ARCH), aarch64) +ifneq ($(filter x86_64 x86,$(ARCH)),) ALL_TARGETS += hv_fcopy_uio_daemon endif ALL_PROGRAMS := $(patsubst %,$(OUTPUT)%,$(ALL_TARGETS)) diff --git a/tools/include/nolibc/arch-mips.h b/tools/include/nolibc/arch-mips.h index a72506ceec6bd3..0c5818149f17e5 100644 --- a/tools/include/nolibc/arch-mips.h +++ b/tools/include/nolibc/arch-mips.h @@ -39,11 +39,19 @@ * - stack is 16-byte aligned */ +#if !defined(__mips_isa_rev) || __mips_isa_rev < 6 +#define _NOLIBC_SYSCALL_CLOBBER_HI_LO "hi", "lo" +#else +#define _NOLIBC_SYSCALL_CLOBBER_HI_LO "$0" +#endif + #if defined(_ABIO32) #define _NOLIBC_SYSCALL_CLOBBERLIST \ - "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" + "memory", "cc", "at", "v1", \ + "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9", \ + _NOLIBC_SYSCALL_CLOBBER_HI_LO + #define _NOLIBC_SYSCALL_STACK_RESERVE "addiu $sp, $sp, -32\n" #define _NOLIBC_SYSCALL_STACK_UNRESERVE "addiu $sp, $sp, 32\n" @@ -52,7 +60,8 @@ /* binutils, GCC and clang disagree about register aliases, use numbers instead. */ #define _NOLIBC_SYSCALL_CLOBBERLIST \ "memory", "cc", "at", "v1", \ - "10", "11", "12", "13", "14", "15", "24", "25" + "10", "11", "12", "13", "14", "15", "24", "25", \ + _NOLIBC_SYSCALL_CLOBBER_HI_LO #define _NOLIBC_SYSCALL_STACK_RESERVE #define _NOLIBC_SYSCALL_STACK_UNRESERVE diff --git a/tools/include/nolibc/compiler.h b/tools/include/nolibc/compiler.h index a8c7619dcdde2f..a8d5ca58dd6324 100644 --- a/tools/include/nolibc/compiler.h +++ b/tools/include/nolibc/compiler.h @@ -47,6 +47,12 @@ # define __nolibc_fallthrough do { } while (0) #endif /* __nolibc_has_attribute(fallthrough) */ +#if defined(__STDC_VERSION__) +# define __nolibc_stdc_version __STDC_VERSION__ +#else +# define __nolibc_stdc_version 0 +#endif + #define __nolibc_version(_major, _minor, _patch) ((_major) * 10000 + (_minor) * 100 + (_patch)) #ifdef __GNUC__ @@ -63,7 +69,7 @@ # define __nolibc_clang_version 0 #endif /* __clang__ */ -#if __STDC_VERSION__ >= 201112L || \ +#if __nolibc_stdc_version >= 201112L || \ __nolibc_gnuc_version >= __nolibc_version(4, 6, 0) || \ __nolibc_clang_version >= __nolibc_version(3, 0, 0) # define __nolibc_static_assert(_t) _Static_assert(_t, "") diff --git a/tools/include/nolibc/stdio.h b/tools/include/nolibc/stdio.h index 233318b0d0f016..a4df72d9a2d37e 100644 --- a/tools/include/nolibc/stdio.h +++ b/tools/include/nolibc/stdio.h @@ -295,22 +295,31 @@ int fseek(FILE *stream, long offset, int whence) * - %[l*]{d,u,c,x,p} * - %s * - unknown modifiers are ignored. + * + * Called by vfprintf() and snprintf() to do the actual formatting. + * The callers provide a callback function to save the formatted data. + * The callback function is called multiple times: + * - for each group of literal characters in the format string. + * - for field padding. + * - for each conversion specifier. + * - with (NULL, 0) at the end of the __nolibc_printf. + * If the callback returns non-zero __nolibc_printf() immediately returns -1. */ -typedef int (*__nolibc_printf_cb)(intptr_t state, const char *buf, size_t size); +typedef int (*__nolibc_printf_cb)(void *state, const char *buf, size_t size); -static __attribute__((unused, format(printf, 4, 0))) -int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char *fmt, va_list args) +static __attribute__((unused, format(printf, 3, 0))) +int __nolibc_printf(__nolibc_printf_cb cb, void *state, const char *fmt, va_list args) { - char escape, lpref, c; + char escape, lpref, ch; unsigned long long v; unsigned int written, width; - size_t len, ofs, w; - char tmpbuf[21]; + size_t len, ofs; + char outbuf[21]; const char *outstr; written = ofs = escape = lpref = 0; while (1) { - c = fmt[ofs++]; + ch = fmt[ofs++]; width = 0; if (escape) { @@ -318,17 +327,17 @@ int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char escape = 0; /* width */ - while (c >= '0' && c <= '9') { + while (ch >= '0' && ch <= '9') { width *= 10; - width += c - '0'; + width += ch - '0'; - c = fmt[ofs++]; + ch = fmt[ofs++]; } - if (c == 'c' || c == 'd' || c == 'u' || c == 'x' || c == 'p') { - char *out = tmpbuf; + if (ch == 'c' || ch == 'd' || ch == 'u' || ch == 'x' || ch == 'p') { + char *out = outbuf; - if (c == 'p') + if (ch == 'p') v = va_arg(args, unsigned long); else if (lpref) { if (lpref > 1) @@ -338,7 +347,7 @@ int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char } else v = va_arg(args, unsigned int); - if (c == 'd') { + if (ch == 'd') { /* sign-extend the value */ if (lpref == 0) v = (long long)(int)v; @@ -346,7 +355,7 @@ int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char v = (long long)(long)v; } - switch (c) { + switch (ch) { case 'c': out[0] = v; out[1] = 0; @@ -365,30 +374,30 @@ int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char u64toh_r(v, out); break; } - outstr = tmpbuf; + outstr = outbuf; } - else if (c == 's') { + else if (ch == 's') { outstr = va_arg(args, char *); if (!outstr) outstr="(null)"; } - else if (c == 'm') { + else if (ch == 'm') { #ifdef NOLIBC_IGNORE_ERRNO outstr = "unknown error"; #else outstr = strerror(errno); #endif /* NOLIBC_IGNORE_ERRNO */ } - else if (c == '%') { + else if (ch == '%') { /* queue it verbatim */ continue; } else { /* modifiers or final 0 */ - if (c == 'l') { + if (ch == 'l') { /* long format prefix, maintain the escape */ lpref++; - } else if (c == 'j') { + } else if (ch == 'j') { lpref = 2; } escape = 1; @@ -399,28 +408,24 @@ int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char } /* not an escape sequence */ - if (c == 0 || c == '%') { + if (ch == 0 || ch == '%') { /* flush pending data on escape or end */ escape = 1; lpref = 0; outstr = fmt; len = ofs - 1; flush_str: - if (n) { - w = len < n ? len : n; - n -= w; - while (width-- > w) { - if (cb(state, " ", 1) != 0) - return -1; - written += 1; - } - if (cb(state, outstr, w) != 0) + while (width-- > len) { + if (cb(state, " ", 1) != 0) return -1; + written += 1; } + if (cb(state, outstr, len) != 0) + return -1; written += len; do_escape: - if (c == 0) + if (ch == 0) break; fmt += ofs; ofs = 0; @@ -429,18 +434,25 @@ int __nolibc_printf(__nolibc_printf_cb cb, intptr_t state, size_t n, const char /* literal char, just queue it */ } + + /* Request a final '\0' be added to the snprintf() output. + * This may be the only call of the cb() function. + */ + if (cb(state, NULL, 0) != 0) + return -1; + return written; } -static int __nolibc_fprintf_cb(intptr_t state, const char *buf, size_t size) +static int __nolibc_fprintf_cb(void *stream, const char *buf, size_t size) { - return _fwrite(buf, size, (FILE *)state); + return _fwrite(buf, size, stream); } static __attribute__((unused, format(printf, 2, 0))) int vfprintf(FILE *stream, const char *fmt, va_list args) { - return __nolibc_printf(__nolibc_fprintf_cb, (intptr_t)stream, SIZE_MAX, fmt, args); + return __nolibc_printf(__nolibc_fprintf_cb, stream, fmt, args); } static __attribute__((unused, format(printf, 1, 0))) @@ -498,26 +510,54 @@ int dprintf(int fd, const char *fmt, ...) return ret; } -static int __nolibc_sprintf_cb(intptr_t _state, const char *buf, size_t size) +struct __nolibc_sprintf_cb_state { + char *buf; + size_t space; +}; + +static int __nolibc_sprintf_cb(void *v_state, const char *buf, size_t size) { - char **state = (char **)_state; + struct __nolibc_sprintf_cb_state *state = v_state; + size_t space = state->space; + char *tgt; + + /* Truncate the request to fit in the output buffer space. + * The last byte is reserved for the terminating '\0'. + * state->space can only be zero for snprintf(NULL, 0, fmt, args) + * so this normally lets through calls with 'size == 0'. + */ + if (size >= space) { + if (space <= 1) + return 0; + size = space - 1; + } + tgt = state->buf; + + /* __nolibc_printf() ends with cb(state, NULL, 0) to request the output + * buffer be '\0' terminated. + * That will be the only cb() call for, eg, snprintf(buf, sz, ""). + * Zero lengths can occur at other times (eg "%s" for an empty string). + * Unconditionally write the '\0' byte to reduce code size, it is + * normally overwritten by the data being output. + * There is no point adding a '\0' after copied data - there is always + * another call. + */ + *tgt = '\0'; + if (size) { + state->space = space - size; + state->buf = tgt + size; + memcpy(tgt, buf, size); + } - memcpy(*state, buf, size); - *state += size; return 0; } static __attribute__((unused, format(printf, 3, 0))) int vsnprintf(char *buf, size_t size, const char *fmt, va_list args) { - char *state = buf; - int ret; + struct __nolibc_sprintf_cb_state state = { .buf = buf, .space = size }; - ret = __nolibc_printf(__nolibc_sprintf_cb, (intptr_t)&state, size, fmt, args); - if (ret < 0) - return ret; - buf[(size_t)ret < size ? (size_t)ret : size - 1] = '\0'; - return ret; + return __nolibc_printf(__nolibc_sprintf_cb, &state, fmt, args); } static __attribute__((unused, format(printf, 3, 4))) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 0be7017800feeb..ef7e7f3e31b758 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -5798,11 +5798,12 @@ static int load_module_btfs(struct bpf_object *obj) info.name = ptr_to_u64(name); info.name_len = sizeof(name); + btf = NULL; err = bpf_btf_get_info_by_fd(fd, &info, &len); if (err) { err = -errno; pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err)); - goto err_out; + break; } /* ignore non-module BTFs */ @@ -5816,15 +5817,15 @@ static int load_module_btfs(struct bpf_object *obj) if (err) { pr_warn("failed to load module [%s]'s BTF object #%d: %s\n", name, id, errstr(err)); - goto err_out; + break; } err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); if (err) - goto err_out; + break; - mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; + mod_btf = &obj->btf_modules[obj->btf_module_cnt]; mod_btf->btf = btf; mod_btf->id = id; @@ -5832,16 +5833,16 @@ static int load_module_btfs(struct bpf_object *obj) mod_btf->name = strdup(name); if (!mod_btf->name) { err = -ENOMEM; - goto err_out; + break; } - continue; + obj->btf_module_cnt++; + } -err_out: + if (err) { + btf__free(btf); close(fd); - return err; } - - return 0; + return err; } static struct bpf_core_cand_list * diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 6eea5edba58a58..0ccc8f548cbaab 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -292,6 +292,8 @@ int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, ++spec_str; if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) return -EINVAL; + if (access_idx < 0) + return -EINVAL; if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) return -E2BIG; spec_str += parsed_len; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index b6765e8765074d..731fd595ac45b9 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -1301,7 +1301,7 @@ static const char *uaccess_safe_builtin[] = { "copy_mc_enhanced_fast_string", "rep_stos_alternative", "rep_movs_alternative", - "__copy_user_nocache", + "copy_to_nontemporal", NULL }; diff --git a/tools/perf/arch/loongarch/util/Build b/tools/perf/arch/loongarch/util/Build index 3ad73d0289f3ea..8d91e78d31c94e 100644 --- a/tools/perf/arch/loongarch/util/Build +++ b/tools/perf/arch/loongarch/util/Build @@ -1,4 +1,3 @@ perf-util-y += header.o perf-util-$(CONFIG_LOCAL_LIBUNWIND) += unwind-libunwind.o -perf-util-$(CONFIG_LIBDW_DWARF_UNWIND) += unwind-libdw.o diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c index e8962c985d34a1..5585aeb97684dd 100644 --- a/tools/perf/builtin-lock.c +++ b/tools/perf/builtin-lock.c @@ -2250,7 +2250,7 @@ static int parse_map_entry(const struct option *opt, const char *str, static int parse_max_stack(const struct option *opt, const char *str, int unset __maybe_unused) { - unsigned long *len = (unsigned long *)opt->value; + int *len = opt->value; long val; char *endptr; diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 73c2ba7e30760e..6a12c1068d8a01 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -164,7 +164,7 @@ struct opt_aggr_mode { }; /* Turn command line option into most generic aggregation mode setting. */ -static enum aggr_mode opt_aggr_mode_to_aggr_mode(struct opt_aggr_mode *opt_mode) +static enum aggr_mode opt_aggr_mode_to_aggr_mode(const struct opt_aggr_mode *opt_mode) { enum aggr_mode mode = AGGR_GLOBAL; @@ -1219,8 +1219,8 @@ static int parse_cache_level(const struct option *opt, int unset __maybe_unused) { int level; - struct opt_aggr_mode *opt_aggr_mode = (struct opt_aggr_mode *)opt->value; - u32 *aggr_level = (u32 *)opt->data; + bool *per_cache = opt->value; + u32 *aggr_level = opt->data; /* * If no string is specified, aggregate based on the topology of @@ -1258,7 +1258,7 @@ static int parse_cache_level(const struct option *opt, return -EINVAL; } out: - opt_aggr_mode->cache = true; + *per_cache = true; *aggr_level = level; return 0; } @@ -1917,25 +1917,33 @@ static int default_evlist_evsel_cmp(void *priv __maybe_unused, const struct evsel *lhs = container_of(lhs_core, struct evsel, core); const struct perf_evsel *rhs_core = container_of(r, struct perf_evsel, node); const struct evsel *rhs = container_of(rhs_core, struct evsel, core); + const struct evsel *lhs_leader = evsel__leader(lhs); + const struct evsel *rhs_leader = evsel__leader(rhs); - if (evsel__leader(lhs) == evsel__leader(rhs)) { + if (lhs_leader == rhs_leader) { /* Within the same group, respect the original order. */ return lhs_core->idx - rhs_core->idx; } + /* + * Compare using leader's attributes so that all members of a group + * stay together. This ensures leaders are opened before their members. + */ + /* Sort default metrics evsels first, and default show events before those. */ - if (lhs->default_metricgroup != rhs->default_metricgroup) - return lhs->default_metricgroup ? -1 : 1; + if (lhs_leader->default_metricgroup != rhs_leader->default_metricgroup) + return lhs_leader->default_metricgroup ? -1 : 1; - if (lhs->default_show_events != rhs->default_show_events) - return lhs->default_show_events ? -1 : 1; + if (lhs_leader->default_show_events != rhs_leader->default_show_events) + return lhs_leader->default_show_events ? -1 : 1; /* Sort by PMU type (prefers legacy types first). */ - if (lhs->pmu != rhs->pmu) - return lhs->pmu->type - rhs->pmu->type; + if (lhs_leader->pmu != rhs_leader->pmu) + return lhs_leader->pmu->type - rhs_leader->pmu->type; - /* Sort by name. */ - return strcmp(evsel__name((struct evsel *)lhs), evsel__name((struct evsel *)rhs)); + /* Sort by leader's name. */ + return strcmp(evsel__name((struct evsel *)lhs_leader), + evsel__name((struct evsel *)rhs_leader)); } /* @@ -2305,24 +2313,23 @@ static struct perf_stat perf_stat = { static int __cmd_report(int argc, const char **argv) { struct perf_session *session; + struct opt_aggr_mode opt_mode = {}; const struct option options[] = { OPT_STRING('i', "input", &input_name, "file", "input file name"), - OPT_SET_UINT(0, "per-socket", &perf_stat.aggr_mode, - "aggregate counts per processor socket", AGGR_SOCKET), - OPT_SET_UINT(0, "per-die", &perf_stat.aggr_mode, - "aggregate counts per processor die", AGGR_DIE), - OPT_SET_UINT(0, "per-cluster", &perf_stat.aggr_mode, - "aggregate counts perf processor cluster", AGGR_CLUSTER), - OPT_CALLBACK_OPTARG(0, "per-cache", &perf_stat.aggr_mode, &perf_stat.aggr_level, - "cache level", - "aggregate count at this cache level (Default: LLC)", + OPT_BOOLEAN(0, "per-thread", &opt_mode.thread, "aggregate counts per thread"), + OPT_BOOLEAN(0, "per-socket", &opt_mode.socket, + "aggregate counts per processor socket"), + OPT_BOOLEAN(0, "per-die", &opt_mode.die, "aggregate counts per processor die"), + OPT_BOOLEAN(0, "per-cluster", &opt_mode.cluster, + "aggregate counts per processor cluster"), + OPT_CALLBACK_OPTARG(0, "per-cache", &opt_mode.cache, &perf_stat.aggr_level, + "cache level", "aggregate count at this cache level (Default: LLC)", parse_cache_level), - OPT_SET_UINT(0, "per-core", &perf_stat.aggr_mode, - "aggregate counts per physical processor core", AGGR_CORE), - OPT_SET_UINT(0, "per-node", &perf_stat.aggr_mode, - "aggregate counts per numa node", AGGR_NODE), - OPT_SET_UINT('A', "no-aggr", &perf_stat.aggr_mode, - "disable CPU count aggregation", AGGR_NONE), + OPT_BOOLEAN(0, "per-core", &opt_mode.core, + "aggregate counts per physical processor core"), + OPT_BOOLEAN(0, "per-node", &opt_mode.node, "aggregate counts per numa node"), + OPT_BOOLEAN('A', "no-aggr", &opt_mode.no_aggr, + "disable aggregation across CPUs or PMUs"), OPT_END() }; struct stat st; @@ -2330,6 +2337,10 @@ static int __cmd_report(int argc, const char **argv) argc = parse_options(argc, argv, options, stat_report_usage, 0); + perf_stat.aggr_mode = opt_aggr_mode_to_aggr_mode(&opt_mode); + if (perf_stat.aggr_mode == AGGR_GLOBAL) + perf_stat.aggr_mode = AGGR_UNSET; /* No option found so leave unset. */ + if (!input_name || !strlen(input_name)) { if (!fstat(STDIN_FILENO, &st) && S_ISFIFO(st.st_mode)) input_name = "-"; @@ -2506,7 +2517,7 @@ int cmd_stat(int argc, const char **argv) OPT_BOOLEAN(0, "per-die", &opt_mode.die, "aggregate counts per processor die"), OPT_BOOLEAN(0, "per-cluster", &opt_mode.cluster, "aggregate counts per processor cluster"), - OPT_CALLBACK_OPTARG(0, "per-cache", &opt_mode, &stat_config.aggr_level, + OPT_CALLBACK_OPTARG(0, "per-cache", &opt_mode.cache, &stat_config.aggr_level, "cache level", "aggregate count at this cache level (Default: LLC)", parse_cache_level), OPT_BOOLEAN(0, "per-core", &opt_mode.core, diff --git a/tools/perf/builtin-trace.c b/tools/perf/builtin-trace.c index 311d9da9896a41..7ff85fa90d988a 100644 --- a/tools/perf/builtin-trace.c +++ b/tools/perf/builtin-trace.c @@ -1565,7 +1565,9 @@ static bool syscall_id_equal(long key1, long key2, void *ctx __maybe_unused) static struct hashmap *alloc_syscall_stats(void) { - return hashmap__new(syscall_id_hash, syscall_id_equal, NULL); + struct hashmap *result = hashmap__new(syscall_id_hash, syscall_id_equal, NULL); + + return IS_ERR(result) ? NULL : result; } static void delete_syscall_stats(struct hashmap *syscall_stats) @@ -1573,7 +1575,7 @@ static void delete_syscall_stats(struct hashmap *syscall_stats) struct hashmap_entry *pos; size_t bkt; - if (syscall_stats == NULL) + if (!syscall_stats) return; hashmap__for_each_entry(syscall_stats, pos, bkt) @@ -1589,7 +1591,7 @@ static struct thread_trace *thread_trace__new(struct trace *trace) ttrace->files.max = -1; if (trace->summary) { ttrace->syscall_stats = alloc_syscall_stats(); - if (IS_ERR(ttrace->syscall_stats)) + if (!ttrace->syscall_stats) zfree(&ttrace); } } @@ -4464,7 +4466,7 @@ static int trace__run(struct trace *trace, int argc, const char **argv) if (trace->summary_mode == SUMMARY__BY_TOTAL && !trace->summary_bpf) { trace->syscall_stats = alloc_syscall_stats(); - if (IS_ERR(trace->syscall_stats)) + if (!trace->syscall_stats) goto out_delete_evlist; } @@ -4771,7 +4773,7 @@ static int trace__replay(struct trace *trace) if (trace->summary_mode == SUMMARY__BY_TOTAL) { trace->syscall_stats = alloc_syscall_stats(); - if (IS_ERR(trace->syscall_stats)) + if (!trace->syscall_stats) goto out; } diff --git a/tools/perf/pmu-events/Build b/tools/perf/pmu-events/Build index dc5f94862a3bc4..dc1df2d57ddc17 100644 --- a/tools/perf/pmu-events/Build +++ b/tools/perf/pmu-events/Build @@ -211,10 +211,10 @@ ifneq ($(strip $(ORPHAN_FILES)),) # Message for $(call echo-cmd,rm). Generally cleaning files isn't part # of a build step. -quiet_cmd_rm = RM $^ +quiet_cmd_rm = RM ...$(words $^) orphan file(s)... +# The list of files can be long. Use xargs to prevent issues. prune_orphans: $(ORPHAN_FILES) - # The list of files can be long. Use xargs to prevent issues. $(Q)$(call echo-cmd,rm)echo "$^" | xargs rm -f JEVENTS_DEPS += prune_orphans diff --git a/tools/perf/pmu-events/arch/common/common/metrics.json b/tools/perf/pmu-events/arch/common/common/metrics.json index 0d010b3ebc6d66..cefc8bfe783027 100644 --- a/tools/perf/pmu-events/arch/common/common/metrics.json +++ b/tools/perf/pmu-events/arch/common/common/metrics.json @@ -46,14 +46,14 @@ }, { "BriefDescription": "Max front or backend stalls per instruction", - "MetricExpr": "max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions", + "MetricExpr": "(max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions) if (has_event(stalled\\-cycles\\-frontend) & has_event(stalled\\-cycles\\-backend)) else ((stalled\\-cycles\\-frontend / instructions) if has_event(stalled\\-cycles\\-frontend) else ((stalled\\-cycles\\-backend / instructions) if has_event(stalled\\-cycles\\-backend) else 0))", "MetricGroup": "Default", "MetricName": "stalled_cycles_per_instruction", "DefaultShowEvents": "1" }, { "BriefDescription": "Frontend stalls per cycle", - "MetricExpr": "stalled\\-cycles\\-frontend / cpu\\-cycles", + "MetricExpr": "(stalled\\-cycles\\-frontend / cpu\\-cycles) if has_event(stalled\\-cycles\\-frontend) else 0", "MetricGroup": "Default", "MetricName": "frontend_cycles_idle", "MetricThreshold": "frontend_cycles_idle > 0.1", @@ -61,7 +61,7 @@ }, { "BriefDescription": "Backend stalls per cycle", - "MetricExpr": "stalled\\-cycles\\-backend / cpu\\-cycles", + "MetricExpr": "(stalled\\-cycles\\-backend / cpu\\-cycles) if has_event(stalled\\-cycles\\-backend) else 0", "MetricGroup": "Default", "MetricName": "backend_cycles_idle", "MetricThreshold": "backend_cycles_idle > 0.2", diff --git a/tools/perf/pmu-events/empty-pmu-events.c b/tools/perf/pmu-events/empty-pmu-events.c index 76c395cf513cb4..a92dd0424f7905 100644 --- a/tools/perf/pmu-events/empty-pmu-events.c +++ b/tools/perf/pmu-events/empty-pmu-events.c @@ -1310,33 +1310,33 @@ static const char *const big_c_string = /* offset=128375 */ "migrations_per_second\000Default\000software@cpu\\-migrations\\,name\\=cpu\\-migrations@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Process migrations to a new CPU per CPU second\000\0001migrations/sec\000\000\000\000011" /* offset=128635 */ "page_faults_per_second\000Default\000software@page\\-faults\\,name\\=page\\-faults@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Page faults per CPU second\000\0001faults/sec\000\000\000\000011" /* offset=128866 */ "insn_per_cycle\000Default\000instructions / cpu\\-cycles\000insn_per_cycle < 1\000Instructions Per Cycle\000\0001instructions\000\000\000\000001" -/* offset=128979 */ "stalled_cycles_per_instruction\000Default\000max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001" -/* offset=129143 */ "frontend_cycles_idle\000Default\000stalled\\-cycles\\-frontend / cpu\\-cycles\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001" -/* offset=129273 */ "backend_cycles_idle\000Default\000stalled\\-cycles\\-backend / cpu\\-cycles\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001" -/* offset=129399 */ "cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011" -/* offset=129575 */ "branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011" -/* offset=129755 */ "branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001" -/* offset=129859 */ "l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001" -/* offset=129975 */ "llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001" -/* offset=130076 */ "l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001" -/* offset=130191 */ "dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001" -/* offset=130297 */ "itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001" -/* offset=130403 */ "l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001" -/* offset=130551 */ "CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000" -/* offset=130574 */ "IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000" -/* offset=130638 */ "Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000" -/* offset=130805 */ "dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" -/* offset=130870 */ "icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" -/* offset=130938 */ "cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000" -/* offset=131010 */ "DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000" -/* offset=131105 */ "DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000" -/* offset=131240 */ "DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000" -/* offset=131305 */ "DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000" -/* offset=131374 */ "DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000" -/* offset=131445 */ "M1\000\000ipc + M2\000\000\000\000\000\000\000\000000" -/* offset=131468 */ "M2\000\000ipc + M1\000\000\000\000\000\000\000\000000" -/* offset=131491 */ "M3\000\0001 / M3\000\000\000\000\000\000\000\000000" -/* offset=131512 */ "L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000" +/* offset=128979 */ "stalled_cycles_per_instruction\000Default\000(max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions if has_event(stalled\\-cycles\\-frontend) & has_event(stalled\\-cycles\\-backend) else (stalled\\-cycles\\-frontend / instructions if has_event(stalled\\-cycles\\-frontend) else (stalled\\-cycles\\-backend / instructions if has_event(stalled\\-cycles\\-backend) else 0)))\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001" +/* offset=129404 */ "frontend_cycles_idle\000Default\000(stalled\\-cycles\\-frontend / cpu\\-cycles if has_event(stalled\\-cycles\\-frontend) else 0)\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001" +/* offset=129583 */ "backend_cycles_idle\000Default\000(stalled\\-cycles\\-backend / cpu\\-cycles if has_event(stalled\\-cycles\\-backend) else 0)\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001" +/* offset=129757 */ "cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011" +/* offset=129933 */ "branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011" +/* offset=130113 */ "branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001" +/* offset=130217 */ "l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001" +/* offset=130333 */ "llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001" +/* offset=130434 */ "l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001" +/* offset=130549 */ "dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001" +/* offset=130655 */ "itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001" +/* offset=130761 */ "l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001" +/* offset=130909 */ "CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000" +/* offset=130932 */ "IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000" +/* offset=130996 */ "Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000" +/* offset=131163 */ "dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" +/* offset=131228 */ "icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000" +/* offset=131296 */ "cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000" +/* offset=131368 */ "DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000" +/* offset=131463 */ "DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000" +/* offset=131598 */ "DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000" +/* offset=131663 */ "DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000" +/* offset=131732 */ "DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000" +/* offset=131803 */ "M1\000\000ipc + M2\000\000\000\000\000\000\000\000000" +/* offset=131826 */ "M2\000\000ipc + M1\000\000\000\000\000\000\000\000000" +/* offset=131849 */ "M3\000\0001 / M3\000\000\000\000\000\000\000\000000" +/* offset=131870 */ "L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000" ; static const struct compact_pmu_event pmu_events__common_default_core[] = { @@ -2626,22 +2626,22 @@ static const struct pmu_table_entry pmu_events__common[] = { static const struct compact_pmu_event pmu_metrics__common_default_core[] = { { 127956 }, /* CPUs_utilized\000Default\000(software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@) / (duration_time * 1e9)\000\000Average CPU utilization\000\0001CPUs\000\000\000\000011 */ -{ 129273 }, /* backend_cycles_idle\000Default\000stalled\\-cycles\\-backend / cpu\\-cycles\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001 */ -{ 129575 }, /* branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011 */ -{ 129755 }, /* branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001 */ +{ 129583 }, /* backend_cycles_idle\000Default\000(stalled\\-cycles\\-backend / cpu\\-cycles if has_event(stalled\\-cycles\\-backend) else 0)\000backend_cycles_idle > 0.2\000Backend stalls per cycle\000\000\000\000\000\000001 */ +{ 129933 }, /* branch_frequency\000Default\000branches / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Branches per CPU second\000\0001000M/sec\000\000\000\000011 */ +{ 130113 }, /* branch_miss_rate\000Default\000branch\\-misses / branches\000branch_miss_rate > 0.05\000Branch miss rate\000\000100%\000\000\000\000001 */ { 128142 }, /* cs_per_second\000Default\000software@context\\-switches\\,name\\=context\\-switches@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Context switches per CPU second\000\0001cs/sec\000\000\000\000011 */ -{ 129399 }, /* cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011 */ -{ 130191 }, /* dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001 */ -{ 129143 }, /* frontend_cycles_idle\000Default\000stalled\\-cycles\\-frontend / cpu\\-cycles\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001 */ +{ 129757 }, /* cycles_frequency\000Default\000cpu\\-cycles / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Cycles per CPU second\000\0001GHz\000\000\000\000011 */ +{ 130549 }, /* dtlb_miss_rate\000Default3\000dTLB\\-load\\-misses / dTLB\\-loads\000dtlb_miss_rate > 0.05\000dTLB miss rate\000\000100%\000\000\000\000001 */ +{ 129404 }, /* frontend_cycles_idle\000Default\000(stalled\\-cycles\\-frontend / cpu\\-cycles if has_event(stalled\\-cycles\\-frontend) else 0)\000frontend_cycles_idle > 0.1\000Frontend stalls per cycle\000\000\000\000\000\000001 */ { 128866 }, /* insn_per_cycle\000Default\000instructions / cpu\\-cycles\000insn_per_cycle < 1\000Instructions Per Cycle\000\0001instructions\000\000\000\000001 */ -{ 130297 }, /* itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001 */ -{ 130403 }, /* l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001 */ -{ 129859 }, /* l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001 */ -{ 130076 }, /* l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001 */ -{ 129975 }, /* llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001 */ +{ 130655 }, /* itlb_miss_rate\000Default3\000iTLB\\-load\\-misses / iTLB\\-loads\000itlb_miss_rate > 0.05\000iTLB miss rate\000\000100%\000\000\000\000001 */ +{ 130761 }, /* l1_prefetch_miss_rate\000Default4\000L1\\-dcache\\-prefetch\\-misses / L1\\-dcache\\-prefetches\000l1_prefetch_miss_rate > 0.05\000L1 prefetch miss rate\000\000100%\000\000\000\000001 */ +{ 130217 }, /* l1d_miss_rate\000Default2\000L1\\-dcache\\-load\\-misses / L1\\-dcache\\-loads\000l1d_miss_rate > 0.05\000L1D miss rate\000\000100%\000\000\000\000001 */ +{ 130434 }, /* l1i_miss_rate\000Default3\000L1\\-icache\\-load\\-misses / L1\\-icache\\-loads\000l1i_miss_rate > 0.05\000L1I miss rate\000\000100%\000\000\000\000001 */ +{ 130333 }, /* llc_miss_rate\000Default2\000LLC\\-load\\-misses / LLC\\-loads\000llc_miss_rate > 0.05\000LLC miss rate\000\000100%\000\000\000\000001 */ { 128375 }, /* migrations_per_second\000Default\000software@cpu\\-migrations\\,name\\=cpu\\-migrations@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Process migrations to a new CPU per CPU second\000\0001migrations/sec\000\000\000\000011 */ { 128635 }, /* page_faults_per_second\000Default\000software@page\\-faults\\,name\\=page\\-faults@ * 1e9 / (software@cpu\\-clock\\,name\\=cpu\\-clock@ if #target_cpu else software@task\\-clock\\,name\\=task\\-clock@)\000\000Page faults per CPU second\000\0001faults/sec\000\000\000\000011 */ -{ 128979 }, /* stalled_cycles_per_instruction\000Default\000max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001 */ +{ 128979 }, /* stalled_cycles_per_instruction\000Default\000(max(stalled\\-cycles\\-frontend, stalled\\-cycles\\-backend) / instructions if has_event(stalled\\-cycles\\-frontend) & has_event(stalled\\-cycles\\-backend) else (stalled\\-cycles\\-frontend / instructions if has_event(stalled\\-cycles\\-frontend) else (stalled\\-cycles\\-backend / instructions if has_event(stalled\\-cycles\\-backend) else 0)))\000\000Max front or backend stalls per instruction\000\000\000\000\000\000001 */ }; @@ -2714,21 +2714,21 @@ static const struct pmu_table_entry pmu_events__test_soc_cpu[] = { }; static const struct compact_pmu_event pmu_metrics__test_soc_cpu_default_core[] = { -{ 130551 }, /* CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000 */ -{ 131240 }, /* DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000 */ -{ 131010 }, /* DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000 */ -{ 131105 }, /* DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000 */ -{ 131305 }, /* DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000 */ -{ 131374 }, /* DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000 */ -{ 130638 }, /* Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000 */ -{ 130574 }, /* IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000 */ -{ 131512 }, /* L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000 */ -{ 131445 }, /* M1\000\000ipc + M2\000\000\000\000\000\000\000\000000 */ -{ 131468 }, /* M2\000\000ipc + M1\000\000\000\000\000\000\000\000000 */ -{ 131491 }, /* M3\000\0001 / M3\000\000\000\000\000\000\000\000000 */ -{ 130938 }, /* cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000 */ -{ 130805 }, /* dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000 */ -{ 130870 }, /* icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000 */ +{ 130909 }, /* CPI\000\0001 / IPC\000\000\000\000\000\000\000\000000 */ +{ 131598 }, /* DCache_L2_All\000\000DCache_L2_All_Hits + DCache_L2_All_Miss\000\000\000\000\000\000\000\000000 */ +{ 131368 }, /* DCache_L2_All_Hits\000\000l2_rqsts.demand_data_rd_hit + l2_rqsts.pf_hit + l2_rqsts.rfo_hit\000\000\000\000\000\000\000\000000 */ +{ 131463 }, /* DCache_L2_All_Miss\000\000max(l2_rqsts.all_demand_data_rd - l2_rqsts.demand_data_rd_hit, 0) + l2_rqsts.pf_miss + l2_rqsts.rfo_miss\000\000\000\000\000\000\000\000000 */ +{ 131663 }, /* DCache_L2_Hits\000\000d_ratio(DCache_L2_All_Hits, DCache_L2_All)\000\000\000\000\000\000\000\000000 */ +{ 131732 }, /* DCache_L2_Misses\000\000d_ratio(DCache_L2_All_Miss, DCache_L2_All)\000\000\000\000\000\000\000\000000 */ +{ 130996 }, /* Frontend_Bound_SMT\000\000idq_uops_not_delivered.core / (4 * (cpu_clk_unhalted.thread / 2 * (1 + cpu_clk_unhalted.one_thread_active / cpu_clk_unhalted.ref_xclk)))\000\000\000\000\000\000\000\000000 */ +{ 130932 }, /* IPC\000group1\000inst_retired.any / cpu_clk_unhalted.thread\000\000\000\000\000\000\000\000000 */ +{ 131870 }, /* L1D_Cache_Fill_BW\000\00064 * l1d.replacement / 1e9 / duration_time\000\000\000\000\000\000\000\000000 */ +{ 131803 }, /* M1\000\000ipc + M2\000\000\000\000\000\000\000\000000 */ +{ 131826 }, /* M2\000\000ipc + M1\000\000\000\000\000\000\000\000000 */ +{ 131849 }, /* M3\000\0001 / M3\000\000\000\000\000\000\000\000000 */ +{ 131296 }, /* cache_miss_cycles\000group1\000dcache_miss_cpi + icache_miss_cycles\000\000\000\000\000\000\000\000000 */ +{ 131163 }, /* dcache_miss_cpi\000\000l1d\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000 */ +{ 131228 }, /* icache_miss_cycles\000\000l1i\\-loads\\-misses / inst_retired.any\000\000\000\000\000\000\000\000000 */ }; diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c index 1d3cc224fbc273..05c3e899b42517 100644 --- a/tools/perf/tests/parse-events.c +++ b/tools/perf/tests/parse-events.c @@ -1796,31 +1796,38 @@ static bool test__acr_valid(void) static int test__ratio_to_prev(struct evlist *evlist) { - struct evsel *evsel; + struct evsel *evsel, *leader; TEST_ASSERT_VAL("wrong number of entries", 2 * perf_pmus__num_core_pmus() == evlist->core.nr_entries); - evlist__for_each_entry(evlist, evsel) { - if (!perf_pmu__has_format(evsel->pmu, "acr_mask")) - return TEST_OK; - - if (evsel == evlist__first(evlist)) { - TEST_ASSERT_VAL("wrong config2", 0 == evsel->core.attr.config2); - TEST_ASSERT_VAL("wrong leader", evsel__is_group_leader(evsel)); - TEST_ASSERT_VAL("wrong core.nr_members", evsel->core.nr_members == 2); - TEST_ASSERT_VAL("wrong group_idx", evsel__group_idx(evsel) == 0); - TEST_ASSERT_EVSEL("unexpected event", - evsel__match(evsel, HARDWARE, HW_CPU_CYCLES), - evsel); - } else { - TEST_ASSERT_VAL("wrong config2", 0 == evsel->core.attr.config2); - TEST_ASSERT_VAL("wrong leader", !evsel__is_group_leader(evsel)); - TEST_ASSERT_VAL("wrong core.nr_members", evsel->core.nr_members == 0); - TEST_ASSERT_VAL("wrong group_idx", evsel__group_idx(evsel) == 1); - TEST_ASSERT_EVSEL("unexpected event", - evsel__match(evsel, HARDWARE, HW_INSTRUCTIONS), - evsel); + evlist__for_each_entry(evlist, evsel) { + if (evsel != evsel__leader(evsel) || + !perf_pmu__has_format(evsel->pmu, "acr_mask")) { + continue; } + leader = evsel; + /* cycles */ + TEST_ASSERT_VAL("wrong config2", 0 == leader->core.attr.config2); + TEST_ASSERT_VAL("wrong core.nr_members", leader->core.nr_members == 2); + TEST_ASSERT_VAL("wrong group_idx", evsel__group_idx(leader) == 0); + TEST_ASSERT_EVSEL("unexpected event", + evsel__match(leader, HARDWARE, HW_CPU_CYCLES), + leader); + /* + * The period value gets configured within evlist__config, + * while this test executes only parse events method. + */ + TEST_ASSERT_VAL("wrong period", 0 == leader->core.attr.sample_period); + + /* instructions/period=200000,ratio-to-prev=2.0/ */ + evsel = evsel__next(evsel); + TEST_ASSERT_VAL("wrong config2", 0 == evsel->core.attr.config2); + TEST_ASSERT_VAL("wrong leader", evsel__has_leader(evsel, leader)); + TEST_ASSERT_VAL("wrong core.nr_members", evsel->core.nr_members == 0); + TEST_ASSERT_VAL("wrong group_idx", evsel__group_idx(evsel) == 1); + TEST_ASSERT_EVSEL("unexpected event", + evsel__match(evsel, HARDWARE, HW_INSTRUCTIONS), + evsel); /* * The period value gets configured within evlist__config, * while this test executes only parse events method. diff --git a/tools/perf/tests/shell/data_type_profiling.sh b/tools/perf/tests/shell/data_type_profiling.sh index 2a7f8f7c42d094..eca694600a0478 100755 --- a/tools/perf/tests/shell/data_type_profiling.sh +++ b/tools/perf/tests/shell/data_type_profiling.sh @@ -8,13 +8,17 @@ set -e # data type profiling manifestation # Values in testtypes and testprogs should match -testtypes=("# data-type: struct Buf" "# data-type: struct _buf") +testtypes=("# data-type: struct Buf" "# data-type: struct buf") testprogs=("perf test -w code_with_type" "perf test -w datasym") err=0 perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) perfout=$(mktemp /tmp/__perf_test.perf.out.XXXXX) +# Check for support of perf mem before trap handler +perf mem record -o /dev/null -- true 2>&1 | \ + grep -q "failed: no PMU supports the memory events" && exit 2 + cleanup() { rm -rf "${perfdata}" "${perfout}" rm -rf "${perfdata}".old diff --git a/tools/perf/tests/workloads/datasym.c b/tools/perf/tests/workloads/datasym.c index 1d0b7d64e1ba1a..19242c7255c0c0 100644 --- a/tools/perf/tests/workloads/datasym.c +++ b/tools/perf/tests/workloads/datasym.c @@ -4,14 +4,14 @@ #include #include "../tests.h" -typedef struct _buf { +struct buf { char data1; char reserved[55]; char data2; -} buf __attribute__((aligned(64))); +} __attribute__((aligned(64))); /* volatile to try to avoid the compiler seeing reserved as unused. */ -static volatile buf workload_datasym_buf1 = { +static volatile struct buf workload_datasym_buf1 = { /* to have this in the data section */ .reserved[0] = 1, }; diff --git a/tools/perf/util/annotate-arch/annotate-loongarch.c b/tools/perf/util/annotate-arch/annotate-loongarch.c index 950f34e59e5cd9..c2addca77320be 100644 --- a/tools/perf/util/annotate-arch/annotate-loongarch.c +++ b/tools/perf/util/annotate-arch/annotate-loongarch.c @@ -110,6 +110,7 @@ static int loongarch_jump__parse(const struct arch *arch, struct ins_operands *o } static const struct ins_ops loongarch_jump_ops = { + .free = jump__delete, .parse = loongarch_jump__parse, .scnprintf = jump__scnprintf, .is_jump = true, diff --git a/tools/perf/util/branch.h b/tools/perf/util/branch.h index 7429530fa77490..a1d4736497c407 100644 --- a/tools/perf/util/branch.h +++ b/tools/perf/util/branch.h @@ -66,6 +66,9 @@ static inline struct branch_entry *perf_sample__branch_entries(struct perf_sampl { u64 *entry = (u64 *)sample->branch_stack; + if (entry == NULL) + return NULL; + entry++; if (sample->no_hw_idx) return (struct branch_entry *)entry; diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index 040eb75f080486..1b5664d1481f53 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -417,7 +417,6 @@ static bool has_pattern_string(const char *str) int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgroup) { struct evlist *orig_list, *tmp_list; - struct evsel *pos, *evsel, *leader; struct rblist orig_metric_events; struct cgroup *cgrp = NULL; struct cgroup_name *cn; @@ -452,6 +451,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro goto out_err; list_for_each_entry(cn, &cgroup_list, list) { + struct evsel *pos; char *name; if (!cn->used) @@ -467,21 +467,37 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgro if (cgrp == NULL) continue; - leader = NULL; + /* copy the list and set to the new cgroup. */ evlist__for_each_entry(orig_list, pos) { - evsel = evsel__clone(/*dest=*/NULL, pos); + struct evsel *evsel = evsel__clone(/*dest=*/NULL, pos); + if (evsel == NULL) goto out_err; + /* stash the copy during the copying. */ + pos->priv = evsel; cgroup__put(evsel->cgrp); evsel->cgrp = cgroup__get(cgrp); - if (evsel__is_group_leader(pos)) - leader = evsel; - evsel__set_leader(evsel, leader); - evlist__add(tmp_list, evsel); } + /* update leader information using stashed pointer to copy. */ + evlist__for_each_entry(orig_list, pos) { + struct evsel *evsel = pos->priv; + + if (evsel__leader(pos)) + evsel__set_leader(evsel, evsel__leader(pos)->priv); + + if (pos->metric_leader) + evsel->metric_leader = pos->metric_leader->priv; + + if (pos->first_wildcard_match) + evsel->first_wildcard_match = pos->first_wildcard_match->priv; + } + /* the stashed copy is no longer used. */ + evlist__for_each_entry(orig_list, pos) + pos->priv = NULL; + /* cgroup__new() has a refcount, release it here */ cgroup__put(cgrp); nr_cgroups++; diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index 212f17a3dc72df..310af407511036 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -237,46 +237,24 @@ cs_etm_decoder__init_def_logger_printing(struct cs_etm_decoder_params *d_params, (void *)decoder, cs_etm_decoder__print_str_cb); if (ret != 0) - ret = -1; - - return 0; -} + return -1; #ifdef CS_LOG_RAW_FRAMES -static void -cs_etm_decoder__init_raw_frame_logging(struct cs_etm_decoder_params *d_params, - struct cs_etm_decoder *decoder) -{ - /* Only log these during a --dump operation */ - if (d_params->operation == CS_ETM_OPERATION_PRINT) { - /* set up a library default logger to process the - * raw frame printer we add later - */ - ocsd_def_errlog_init(OCSD_ERR_SEV_ERROR, 1); - - /* no stdout / err / file output */ - ocsd_def_errlog_config_output(C_API_MSGLOGOUT_FLG_NONE, NULL); - - /* set the string CB for the default logger, - * passes strings to perf print logger. - */ - ocsd_def_errlog_set_strprint_cb(decoder->dcd_tree, - (void *)decoder, - cs_etm_decoder__print_str_cb); - + /* + * Only log raw frames if --dump operation and hardware is actually + * generating formatted CoreSight trace frames + */ + if ((d_params->operation == CS_ETM_OPERATION_PRINT) && + (d_params->formatted == true)) { /* use the built in library printer for the raw frames */ - ocsd_dt_set_raw_frame_printer(decoder->dcd_tree, - CS_RAW_DEBUG_FLAGS); + ret = ocsd_dt_set_raw_frame_printer(decoder->dcd_tree, + CS_RAW_DEBUG_FLAGS); + if (ret != 0) + return -1; } -} -#else -static void -cs_etm_decoder__init_raw_frame_logging( - struct cs_etm_decoder_params *d_params __maybe_unused, - struct cs_etm_decoder *decoder __maybe_unused) -{ -} #endif + return 0; +} static ocsd_datapath_resp_t cs_etm_decoder__do_soft_timestamp(struct cs_etm_queue *etmq, @@ -738,9 +716,6 @@ cs_etm_decoder__new(int decoders, struct cs_etm_decoder_params *d_params, if (ret != 0) goto err_free_decoder; - /* init raw frame logging if required */ - cs_etm_decoder__init_raw_frame_logging(d_params, decoder); - for (i = 0; i < decoders; i++) { ret = cs_etm_decoder__create_etm_decoder(d_params, &t_params[i], diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c index 9e0420e14be190..62bd8c3e53051d 100644 --- a/tools/perf/util/disasm.c +++ b/tools/perf/util/disasm.c @@ -451,7 +451,7 @@ int jump__scnprintf(const struct ins *ins, char *bf, size_t size, ops->target.offset); } -static void jump__delete(struct ins_operands *ops __maybe_unused) +void jump__delete(struct ins_operands *ops __maybe_unused) { /* * The ops->jump.raw_comment and ops->jump.raw_func_start belong to the diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h index a6e478caf61a95..25756e3f47e472 100644 --- a/tools/perf/util/disasm.h +++ b/tools/perf/util/disasm.h @@ -161,6 +161,8 @@ int jump__scnprintf(const struct ins *ins, char *bf, size_t size, int mov__scnprintf(const struct ins *ins, char *bf, size_t size, struct ins_operands *ops, int max_ins_name); +void jump__delete(struct ins_operands *ops); + int symbol__disassemble(struct symbol *sym, struct annotate_args *args); char *expand_tabs(char *line, char **storage, size_t *storage_len); diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c index 465fe2e9bbbeff..b7664cb68554b2 100644 --- a/tools/perf/util/expr.c +++ b/tools/perf/util/expr.c @@ -376,7 +376,8 @@ int expr__find_ids(const char *expr, const char *one, if (one) expr__del_id(ctx, one); - return ret; + /* A positive value means syntax error, convert to -EINVAL */ + return ret > 0 ? -EINVAL : ret; } double expr_id_data__value(const struct expr_id_data *data) diff --git a/tools/perf/util/maps.c b/tools/perf/util/maps.c index 4092211cff62b4..75b399a20b262c 100644 --- a/tools/perf/util/maps.c +++ b/tools/perf/util/maps.c @@ -956,6 +956,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) if (maps_by_name) { map__put(maps_by_name[ni]); maps_by_name[ni] = map__get(new); + maps__set_maps_by_name_sorted(maps, false); } err = __maps__insert_sorted(maps, i + 1, after, NULL); @@ -982,6 +983,7 @@ static int __maps__fixup_overlap_and_insert(struct maps *maps, struct map *new) if (maps_by_name) { map__put(maps_by_name[ni]); maps_by_name[ni] = map__get(new); + maps__set_maps_by_name_sorted(maps, false); } check_invariants(maps); @@ -1080,16 +1082,9 @@ int maps__copy_from(struct maps *dest, struct maps *parent) map__put(new); } maps__set_maps_by_address_sorted(dest, maps__maps_by_address_sorted(parent)); - if (!err) { - RC_CHK_ACCESS(dest)->last_search_by_name_idx = - RC_CHK_ACCESS(parent)->last_search_by_name_idx; - maps__set_maps_by_name_sorted(dest, - dest_maps_by_name && - maps__maps_by_name_sorted(parent)); - } else { - RC_CHK_ACCESS(dest)->last_search_by_name_idx = 0; - maps__set_maps_by_name_sorted(dest, false); - } + RC_CHK_ACCESS(dest)->last_search_by_name_idx = 0; + /* Values were copied into the name array in address order. */ + maps__set_maps_by_name_sorted(dest, false); } else { /* Unexpected copying to a maps containing entries. */ for (unsigned int i = 0; !err && i < n; i++) { diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 76912c62b6a070..968e269d9be1f8 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1356,8 +1356,12 @@ static int dso__process_kernel_symbol(struct dso *dso, struct map *map, char dso_name[PATH_MAX]; /* Adjust symbol to map to file offset */ - if (adjust_kernel_syms) - sym->st_value -= shdr->sh_addr - shdr->sh_offset; + if (adjust_kernel_syms) { + if (dso__rel(dso)) + sym->st_value += shdr->sh_offset; + else + sym->st_value -= shdr->sh_addr - shdr->sh_offset; + } if (strcmp(section_name, (dso__short_name(curr_dso) + dso__short_name_len(dso))) == 0) return 0; diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h index 394dbfa944ac70..e935438451b81f 100644 --- a/tools/perf/util/util.h +++ b/tools/perf/util/util.h @@ -30,7 +30,6 @@ extern bool perf_guest; /* General helper functions */ void usage(const char *err) __noreturn; -void die(const char *err, ...) __noreturn __printf(1, 2); struct dirent; struct strlist; diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index e9e8ef72395a94..e609272ed80b59 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -2427,11 +2427,17 @@ char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec"; int cpu_is_not_present(int cpu) { + if (cpu < 0) + return 1; + return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set); } int cpu_is_not_allowed(int cpu) { + if (cpu < 0) + return 1; + return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set); } @@ -2443,6 +2449,22 @@ int cpu_is_not_allowed(int cpu) #define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p +int has_allowed_lower_ht_sibling(int cpu) +{ + int i; + + for (i = 0; i <= cpus[cpu].ht_id; ++i) { + int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i]; + + if (sibling_cpu_id == cpu) + return 0; + + if (!cpu_is_not_allowed(sibling_cpu_id)) + return 1; + } + return 0; +} + int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *), struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base) { @@ -2460,7 +2482,7 @@ int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pk if (cpu_is_not_allowed(cpu)) continue; - if (cpus[cpu].ht_id > 0) /* skip HT sibling */ + if (has_allowed_lower_ht_sibling(cpu)) /* skip HT sibling */ continue; t = &thread_base[cpu]; @@ -2469,13 +2491,22 @@ int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pk retval |= func(t, c, p); - /* Handle HT sibling now */ + /* Handle other HT siblings now */ int i; - for (i = MAX_HT_ID; i > 0; --i) { /* ht_id 0 is self */ - if (cpus[cpu].ht_sibling_cpu_id[i] <= 0) + for (i = 0; i <= MAX_HT_ID; ++i) { + int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i]; + + if (sibling_cpu_id < 0) + break; + + if (sibling_cpu_id == cpu) continue; - t = &thread_base[cpus[cpu].ht_sibling_cpu_id[i]]; + + if (cpu_is_not_allowed(sibling_cpu_id)) + continue; + + t = &thread_base[sibling_cpu_id]; retval |= func(t, c, p); } @@ -5155,7 +5186,7 @@ static inline int get_rapl_num_domains(void) if (!platform->has_per_core_rapl) return topo.num_packages; - return topo.num_cores; + return GLOBAL_CORE_ID(topo.max_core_id, topo.num_packages) + 1; } static inline int get_rapl_domain_id(int cpu) @@ -6169,11 +6200,11 @@ int set_thread_siblings(struct cpu_topology *thiscpu) int cpu = thiscpu->cpu_id; int offset = topo.max_cpu_num + 1; size_t size; - int thread_id = 0; + int ht_id = 0; thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1)); if (thiscpu->ht_id < 0) - thiscpu->ht_id = thread_id++; + thiscpu->ht_id = 0; /* first CPU in core */ if (!thiscpu->put_ids) return -1; @@ -6197,13 +6228,9 @@ int set_thread_siblings(struct cpu_topology *thiscpu) sib_core = get_core_id(so); if (sib_core == thiscpu->core_id) { CPU_SET_S(so, size, thiscpu->put_ids); - if ((so != cpu) && (cpus[so].ht_id < 0)) { - cpus[so].ht_id = thread_id; - cpus[cpu].ht_sibling_cpu_id[thread_id] = so; - if (debug) - fprintf(stderr, "%s: cpu%d.ht_sibling_cpu_id[%d] = %d\n", __func__, cpu, thread_id, so); - thread_id += 1; - } + cpus[so].ht_id = ht_id; + cpus[cpu].ht_sibling_cpu_id[ht_id] = so; + ht_id += 1; } } } @@ -6236,7 +6263,7 @@ int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *, if (cpu_is_not_allowed(cpu)) continue; - if (cpus[cpu].ht_id > 0) /* skip HT sibling */ + if (has_allowed_lower_ht_sibling(cpu)) /* skip HT sibling */ continue; t = &thread_base[cpu]; @@ -6251,11 +6278,20 @@ int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *, /* Handle HT sibling now */ int i; - for (i = MAX_HT_ID; i > 0; --i) { /* ht_id 0 is self */ - if (cpus[cpu].ht_sibling_cpu_id[i] <= 0) + for (i = 0; i <= MAX_HT_ID; ++i) { + int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i]; + + if (sibling_cpu_id < 0) + break; + + if (sibling_cpu_id == cpu) continue; - t = &thread_base[cpus[cpu].ht_sibling_cpu_id[i]]; - t2 = &thread_base2[cpus[cpu].ht_sibling_cpu_id[i]]; + + if (cpu_is_not_allowed(sibling_cpu_id)) + continue; + + t = &thread_base[sibling_cpu_id]; + t2 = &thread_base2[sibling_cpu_id]; retval |= func(t, c, p, t2, c2, p2); } @@ -9505,6 +9541,8 @@ void topology_probe(bool startup) cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_present_setsize, cpu_present_set); for_all_proc_cpus(mark_cpu_present); + if (debug) + print_cpu_set("present set", cpu_present_set); /* * Allocate and initialize cpu_possible_set @@ -9515,6 +9553,8 @@ void topology_probe(bool startup) cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set); initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible"); + if (debug) + print_cpu_set("possible set", cpu_possible_set); /* * Allocate and initialize cpu_effective_set @@ -9525,6 +9565,8 @@ void topology_probe(bool startup) cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1)); CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set); update_effective_set(startup); + if (debug) + print_cpu_set("effective set", cpu_effective_set); /* * Allocate and initialize cpu_allowed_set @@ -9568,6 +9610,8 @@ void topology_probe(bool startup) CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set); } + if (debug) + print_cpu_set("allowed set", cpu_allowed_set); if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set)) err(-ENODEV, "No valid cpus found"); @@ -9671,12 +9715,18 @@ void topology_probe(bool startup) return; for (i = 0; i <= topo.max_cpu_num; ++i) { + int ht_id; + if (cpu_is_not_present(i)) continue; fprintf(outf, - "cpu %d pkg %d die %d l3 %d node %d lnode %d core %d thread %d\n", + "cpu %d pkg %d die %d l3 %d node %d lnode %d core %d ht_id %d", i, cpus[i].package_id, cpus[i].die_id, cpus[i].l3_id, cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].core_id, cpus[i].ht_id); + fprintf(outf, " siblings"); + for (ht_id = 0; ht_id <= MAX_HT_ID; ++ht_id) + fprintf(outf, " %d", cpus[i].ht_sibling_cpu_id[ht_id]); + fprintf(outf, "\n"); } } @@ -9817,6 +9867,8 @@ void topology_update(void) topo.allowed_cores = 0; topo.allowed_packages = 0; for_all_cpus(update_topo, ODD_COUNTERS); + if (debug) + fprintf(stderr, "allowed_cpus %d allowed_cores %d allowed_packages %d\n", topo.allowed_cpus, topo.allowed_cores, topo.allowed_packages); } void setup_all_buffers(bool startup) @@ -11449,7 +11501,7 @@ void cmdline(int argc, char **argv) } optind = 0; - while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMST:v", long_options, &option_index)) != -1) { + while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMPST:v", long_options, &option_index)) != -1) { switch (opt) { case 'a': parse_add_command(optarg); diff --git a/tools/sched_ext/scx_pair.c b/tools/sched_ext/scx_pair.c index 2e509391f3dab9..eb1bea2dd0ccc9 100644 --- a/tools/sched_ext/scx_pair.c +++ b/tools/sched_ext/scx_pair.c @@ -48,6 +48,7 @@ int main(int argc, char **argv) struct bpf_link *link; __u64 seq = 0, ecode; __s32 stride, i, opt, outer_fd; + __u32 pair_id = 0; libbpf_set_print(libbpf_print_fn); signal(SIGINT, sigint_handler); @@ -82,6 +83,14 @@ int main(int argc, char **argv) scx_pair__destroy(skel); return -1; } + + if (skel->rodata->nr_cpu_ids & 1) { + fprintf(stderr, "scx_pair requires an even CPU count, got %u\n", + skel->rodata->nr_cpu_ids); + scx_pair__destroy(skel); + return -1; + } + bpf_map__set_max_entries(skel->maps.pair_ctx, skel->rodata->nr_cpu_ids / 2); /* Resize arrays so their element count is equal to cpu count. */ @@ -109,10 +118,11 @@ int main(int argc, char **argv) skel->rodata_pair_cpu->pair_cpu[i] = j; skel->rodata_pair_cpu->pair_cpu[j] = i; - skel->rodata_pair_id->pair_id[i] = i; - skel->rodata_pair_id->pair_id[j] = i; + skel->rodata_pair_id->pair_id[i] = pair_id; + skel->rodata_pair_id->pair_id[j] = pair_id; skel->rodata_in_pair_idx->in_pair_idx[i] = 0; skel->rodata_in_pair_idx->in_pair_idx[j] = 1; + pair_id++; printf("[%d, %d] ", i, j); } diff --git a/tools/sched_ext/scx_sdt.bpf.c b/tools/sched_ext/scx_sdt.bpf.c index 31b09958e8d5ff..2e2179d0f509e0 100644 --- a/tools/sched_ext/scx_sdt.bpf.c +++ b/tools/sched_ext/scx_sdt.bpf.c @@ -317,7 +317,8 @@ int scx_alloc_free_idx(struct scx_allocator *alloc, __u64 idx) }; /* Zero out one word at a time. */ - for (i = zero; i < alloc->pool.elem_size / 8 && can_loop; i++) { + for (i = zero; i < (alloc->pool.elem_size - sizeof(struct sdt_data)) / 8 + && can_loop; i++) { data->payload[i] = 0; } } diff --git a/tools/testing/ktest/ktest.pl b/tools/testing/ktest/ktest.pl index 001c4df9f7df66..17bdce9cafac25 100755 --- a/tools/testing/ktest/ktest.pl +++ b/tools/testing/ktest/ktest.pl @@ -100,6 +100,7 @@ my $build_type; my $build_options; my $final_post_ktest; +my $post_ktest_done = 0; my $pre_ktest; my $post_ktest; my $pre_test; @@ -1575,6 +1576,24 @@ () return $name; } +sub run_post_ktest { + my $cmd; + + return if ($post_ktest_done); + + if (defined($final_post_ktest)) { + $cmd = $final_post_ktest; + } elsif (defined($post_ktest)) { + $cmd = $post_ktest; + } else { + return; + } + + my $cp_post_ktest = eval_kernel_version($cmd); + run_command $cp_post_ktest; + $post_ktest_done = 1; +} + sub dodie { # avoid recursion return if ($in_die); @@ -1634,6 +1653,7 @@ sub dodie { if (defined($post_test)) { run_command $post_test; } + run_post_ktest; die @_, "\n"; } @@ -1815,7 +1835,7 @@ sub save_logs { my ($result, $basedir) = @_; my @t = localtime; my $date = sprintf "%04d%02d%02d%02d%02d%02d", - 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0]; + 1900+$t[5],$t[4]+1,$t[3],$t[2],$t[1],$t[0]; my $type = $build_type; if ($type =~ /useconfig/) { @@ -2508,7 +2528,7 @@ sub check_buildlog { my $save_no_reboot = $no_reboot; $no_reboot = 1; - if (-f $warnings_file) { + if (defined($warnings_file) && -f $warnings_file) { open(IN, $warnings_file) or dodie "Error opening $warnings_file"; @@ -4183,7 +4203,8 @@ sub __set_test_option { my $option = "$name\[$i\]"; - if (option_defined($option)) { + if (exists($opt{$option})) { + return undef if (!option_defined($option)); return $opt{$option}; } @@ -4191,7 +4212,8 @@ sub __set_test_option { if ($i >= $test && $i < $test + $repeat_tests{$test}) { $option = "$name\[$test\]"; - if (option_defined($option)) { + if (exists($opt{$option})) { + return undef if (!option_defined($option)); return $opt{$option}; } } @@ -4298,6 +4320,7 @@ sub cancel_test { send_email("KTEST: Your [$name] test was cancelled", "Your test started at $script_start_time was cancelled: sig int"); } + run_post_ktest; die "\nCaught Sig Int, test interrupted: $!\n" } @@ -4659,11 +4682,7 @@ sub die_usage { success $i; } -if (defined($final_post_ktest)) { - - my $cp_final_post_ktest = eval_kernel_version $final_post_ktest; - run_command $cp_final_post_ktest; -} +run_post_ktest; if ($opt{"POWEROFF_ON_SUCCESS"}) { halt; diff --git a/tools/testing/selftests/arm64/gcs/gcs-util.h b/tools/testing/selftests/arm64/gcs/gcs-util.h index c99a6b39ac147b..7a81bb07ed4b8b 100644 --- a/tools/testing/selftests/arm64/gcs/gcs-util.h +++ b/tools/testing/selftests/arm64/gcs/gcs-util.h @@ -18,12 +18,6 @@ #ifndef NT_ARM_GCS #define NT_ARM_GCS 0x410 - -struct user_gcs { - __u64 features_enabled; - __u64 features_locked; - __u64 gcspr_el0; -}; #endif /* Shadow Stack/Guarded Control Stack interface */ diff --git a/tools/testing/selftests/arm64/gcs/libc-gcs.c b/tools/testing/selftests/arm64/gcs/libc-gcs.c index 17b2fabfec386e..72e82bfbecc99e 100644 --- a/tools/testing/selftests/arm64/gcs/libc-gcs.c +++ b/tools/testing/selftests/arm64/gcs/libc-gcs.c @@ -16,6 +16,7 @@ #include #include +#include #include diff --git a/tools/testing/selftests/bpf/prog_tests/access_variable_array.c b/tools/testing/selftests/bpf/prog_tests/access_variable_array.c deleted file mode 100644 index 08131782437c6d..00000000000000 --- a/tools/testing/selftests/bpf/prog_tests/access_variable_array.c +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2022 Bytedance */ - -#include -#include "test_access_variable_array.skel.h" - -void test_access_variable_array(void) -{ - struct test_access_variable_array *skel; - - skel = test_access_variable_array__open_and_load(); - if (!ASSERT_OK_PTR(skel, "test_access_variable_array__open_and_load")) - return; - - test_access_variable_array__destroy(skel); -} diff --git a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c index cb8dd2f63296b8..05fc9a7cf7c926 100644 --- a/tools/testing/selftests/bpf/prog_tests/reg_bounds.c +++ b/tools/testing/selftests/bpf/prog_tests/reg_bounds.c @@ -500,6 +500,39 @@ static struct range range_refine(enum num_t x_t, struct range x, enum num_t y_t, (s64)x.a >= S32_MIN && (s64)x.b <= S32_MAX) return range_intersection(x_t, x, y_cast); + if (y_t == U32 && x_t == U64) { + u64 xmin_swap, xmax_swap, xmin_lower32, xmax_lower32; + + xmin_lower32 = x.a & 0xffffffff; + xmax_lower32 = x.b & 0xffffffff; + if (xmin_lower32 < y.a || xmin_lower32 > y.b) { + /* The 32 lower bits of the umin64 are outside the u32 + * range. Let's update umin64 to match the u32 range. + * We want to *increase* the umin64 to the *minimum* + * value that matches the u32 range. + */ + xmin_swap = swap_low32(x.a, y.a); + /* We should always only increase the minimum, so if + * the new value is lower than before, we need to + * increase the 32 upper bits by 1. + */ + if (xmin_swap < x.a) + xmin_swap += 0x100000000; + if (xmin_swap == x.b) + return range(x_t, x.b, x.b); + } else if (xmax_lower32 < y.a || xmax_lower32 > y.b) { + /* Same for the umax64, but we want to *decrease* + * umax64 to the *maximum* value that matches the u32 + * range. + */ + xmax_swap = swap_low32(x.b, y.b); + if (xmax_swap > x.b) + xmax_swap -= 0x100000000; + if (xmax_swap == x.a) + return range(x_t, x.a, x.a); + } + } + /* the case when new range knowledge, *y*, is a 32-bit subregister * range, while previous range knowledge, *x*, is a full register * 64-bit range, needs special treatment to take into account upper 32 @@ -2129,6 +2162,8 @@ static struct subtest_case crafted_cases[] = { {U64, S64, {0x7fffffff00000001ULL, 0xffffffff00000000ULL}, {0, 0}}, {U64, S64, {0, 0xffffffffULL}, {1, 1}}, {U64, S64, {0, 0xffffffffULL}, {0x7fffffff, 0x7fffffff}}, + {U64, S32, {0xfffffffe00000001, 0xffffffff00000000}, {S64_MIN, S64_MIN}}, + {U64, U32, {0xfffffffe00000000, U64_MAX - 1}, {U64_MAX, U64_MAX}}, {U64, U32, {0, 0x100000000}, {0, 0}}, {U64, U32, {0xfffffffe, 0x300000000}, {0x80000000, 0x80000000}}, diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf.c b/tools/testing/selftests/bpf/prog_tests/snprintf.c index 594441acb7071f..4e4a82d54f799d 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf.c @@ -114,7 +114,8 @@ static void test_snprintf_negative(void) ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5"); ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6"); ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7"); - ASSERT_ERR(load_single_snprintf("\x80"), "non ascii character"); + ASSERT_OK(load_single_snprintf("\x80"), "non ascii plain text"); + ASSERT_ERR(load_single_snprintf("%\x80"), "non ascii in specifier"); ASSERT_ERR(load_single_snprintf("\x1"), "non printable character"); ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8"); ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9"); diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index dd3c757859f6b2..d2846579285f2e 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -1298,10 +1298,23 @@ static void test_sockmap_multi_channels(int sotype) avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); ASSERT_EQ(avail, expected, "ioctl(FIONREAD) full return"); - recvd = recv_timeout(p1, rcv, sizeof(rcv), MSG_DONTWAIT, 1); - if (!ASSERT_EQ(recvd, sizeof(buf), "recv_timeout(p1)") || + recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); + if (!ASSERT_EQ(recvd, expected, "recv_timeout(p1)") || !ASSERT_OK(memcmp(buf, rcv, recvd), "data mismatch")) goto end; + + /* process remaining data for udp if secondary data is available */ + expected = sizeof(buf) - expected; + if (expected) { + avail = wait_for_fionread(p1, expected, IO_TIMEOUT_SEC); + ASSERT_EQ(avail, expected, "second ioctl(FIONREAD) full return"); + + recvd = recv_timeout(p1, rcv, expected, MSG_DONTWAIT, 1); + if (!ASSERT_EQ(recvd, expected, "second recv_timeout(p1)") || + !ASSERT_OK(memcmp(buf + sizeof(buf) - expected, rcv, recvd), + "second data mismatch")) + goto end; + } end: if (c0 >= 0) close(c0); diff --git a/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c b/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c index de22734abc4d2b..40d38280c091e3 100644 --- a/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c +++ b/tools/testing/selftests/bpf/prog_tests/test_bpf_smc.c @@ -131,8 +131,10 @@ static bool get_smc_nl_family_id(void) goto fail; ret = recv(fd, &msg, sizeof(msg), 0); - if (!ASSERT_FALSE(msg.n.nlmsg_type == NLMSG_ERROR || ret < 0 || - !NLMSG_OK(&msg.n, ret), "nl_family response")) + if (msg.n.nlmsg_type == NLMSG_ERROR) + goto fail; + if (!ASSERT_FALSE(ret < 0 || !NLMSG_OK(&msg.n, ret), + "nl_family response")) goto fail; nl = (struct nlattr *)GENLMSG_DATA(&msg); diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index c9bfbe1bafc127..1cd783aec11ad1 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -140,7 +140,7 @@ #define __msg_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_msg_unpriv=" XSTR(__COUNTER__) "=" msg))) #define __not_msg_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_not_msg_unpriv=" XSTR(__COUNTER__) "=" msg))) #define __xlated_unpriv(msg) __attribute__((btf_decl_tag("comment:test_expect_xlated_unpriv=" XSTR(__COUNTER__) "=" msg))) -#define __jited_unpriv(msg) __attribute__((btf_decl_tag("comment:test_jited=" XSTR(__COUNTER__) "=" msg))) +#define __jited_unpriv(msg) __attribute__((btf_decl_tag("comment:test_jited_unpriv=" XSTR(__COUNTER__) "=" msg))) #define __failure_unpriv __attribute__((btf_decl_tag("comment:test_expect_failure_unpriv"))) #define __success_unpriv __attribute__((btf_decl_tag("comment:test_expect_success_unpriv"))) #define __log_level(lvl) __attribute__((btf_decl_tag("comment:test_log_level="#lvl))) diff --git a/tools/testing/selftests/bpf/progs/bpf_smc.c b/tools/testing/selftests/bpf/progs/bpf_smc.c index 70d8b08f591405..6263a45bf0066a 100644 --- a/tools/testing/selftests/bpf/progs/bpf_smc.c +++ b/tools/testing/selftests/bpf/progs/bpf_smc.c @@ -8,6 +8,10 @@ char _license[] SEC("license") = "GPL"; +#ifndef SMC_HS_CTRL_NAME_MAX +#define SMC_HS_CTRL_NAME_MAX 16 +#endif + enum { BPF_SMC_LISTEN = 10, }; @@ -18,6 +22,20 @@ struct smc_sock___local { bool use_fallback; } __attribute__((preserve_access_index)); +struct smc_hs_ctrl___local { + char name[SMC_HS_CTRL_NAME_MAX]; + int (*syn_option)(struct tcp_sock *); + int (*synack_option)(const struct tcp_sock *, struct inet_request_sock *); +} __attribute__((preserve_access_index)); + +struct netns_smc___local { + struct smc_hs_ctrl___local *hs_ctrl; +} __attribute__((preserve_access_index)); + +struct net___local { + struct netns_smc___local smc; +} __attribute__((preserve_access_index)); + int smc_cnt = 0; int fallback_cnt = 0; @@ -88,8 +106,14 @@ int BPF_PROG(smc_run, int family, int type, int protocol) task = bpf_get_current_task_btf(); /* Prevent from affecting other tests */ - if (!task || !task->nsproxy->net_ns->smc.hs_ctrl) + if (!task) { return protocol; + } else { + struct net___local *net = (struct net___local *)task->nsproxy->net_ns; + + if (!bpf_core_field_exists(struct net___local, smc) || !net->smc.hs_ctrl) + return protocol; + } return IPPROTO_SMC; } @@ -110,7 +134,7 @@ int BPF_PROG(bpf_smc_set_tcp_option, struct tcp_sock *tp) } SEC(".struct_ops") -struct smc_hs_ctrl linkcheck = { +struct smc_hs_ctrl___local linkcheck = { .name = "linkcheck", .syn_option = (void *)bpf_smc_set_tcp_option, .synack_option = (void *)bpf_smc_set_tcp_option_cond, diff --git a/tools/testing/selftests/bpf/progs/test_access_variable_array.c b/tools/testing/selftests/bpf/progs/test_access_variable_array.c deleted file mode 100644 index 326b7d1f496ab1..00000000000000 --- a/tools/testing/selftests/bpf/progs/test_access_variable_array.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2023 Bytedance */ - -#include "vmlinux.h" -#include -#include - -unsigned long span = 0; - -SEC("fentry/sched_balance_rq") -int BPF_PROG(fentry_fentry, int this_cpu, struct rq *this_rq, - struct sched_domain *sd) -{ - span = sd->span[0]; - - return 0; -} - -char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index 2fb096a2a9f923..a25eb097b31c1e 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -1280,8 +1280,11 @@ static int tcp_server(const char *cgroup, void *arg) saddr.sin6_port = htons(srv_args->port); sk = socket(AF_INET6, SOCK_STREAM, 0); - if (sk < 0) + if (sk < 0) { + /* Pass back errno to the ctl_fd */ + write(ctl_fd, &errno, sizeof(errno)); return ret; + } if (setsockopt(sk, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0) goto cleanup; @@ -1412,6 +1415,12 @@ static int test_memcg_sock(const char *root) goto cleanup; close(args.ctl[0]); + /* Skip if address family not supported by protocol */ + if (err == EAFNOSUPPORT) { + ret = KSFT_SKIP; + goto cleanup; + } + if (!err) break; if (err != EADDRINUSE) diff --git a/tools/testing/selftests/ftrace/ftracetest b/tools/testing/selftests/ftrace/ftracetest index 3230bd54dba84a..0a56bf209f6c09 100755 --- a/tools/testing/selftests/ftrace/ftracetest +++ b/tools/testing/selftests/ftrace/ftracetest @@ -130,8 +130,7 @@ parse_opts() { # opts shift 1 ;; --logdir|-l) - LOG_DIR=$2 - LINK_PTR= + USER_LOG_DIR=$2 shift 2 ;; --rv) @@ -199,6 +198,7 @@ fi TOP_DIR=`absdir $0` TEST_DIR=$TOP_DIR/test.d TEST_CASES=`find_testcases $TEST_DIR` +USER_LOG_DIR= KEEP_LOG=0 KTAP=0 DEBUG=0 @@ -210,12 +210,18 @@ RV_TEST=0 # Parse command-line options parse_opts $* +[ $DEBUG -ne 0 ] && set -x + +# TOP_DIR can be changed for rv. Setting log directory. LOG_TOP_DIR=$TOP_DIR/logs LOG_DATE=`date +%Y%m%d-%H%M%S` -LOG_DIR=$LOG_TOP_DIR/$LOG_DATE/ -LINK_PTR=$LOG_TOP_DIR/latest - -[ $DEBUG -ne 0 ] && set -x +if [ -n "$USER_LOG_DIR" ]; then + LOG_DIR=$USER_LOG_DIR + LINK_PTR= +else + LOG_DIR=$LOG_TOP_DIR/$LOG_DATE/ + LINK_PTR=$LOG_TOP_DIR/latest +fi if [ $RV_TEST -ne 0 ]; then TRACING_DIR=$TRACING_DIR/rv diff --git a/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc b/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc index a2c42e13f614b4..8e905d4fe6dd22 100644 --- a/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc +++ b/tools/testing/selftests/ftrace/test.d/00basic/trace_marker_raw.tc @@ -4,6 +4,8 @@ # requires: trace_marker_raw # flags: instance +check_awk_strtonum || exit_unresolved + is_little_endian() { if lscpu | grep -q 'Little Endian'; then echo 1; diff --git a/tools/testing/selftests/ftrace/test.d/functions b/tools/testing/selftests/ftrace/test.d/functions index e8e718139294d9..41325f387ee7a9 100644 --- a/tools/testing/selftests/ftrace/test.d/functions +++ b/tools/testing/selftests/ftrace/test.d/functions @@ -173,6 +173,10 @@ check_requires() { # Check required files and tracers done } +check_awk_strtonum() { # strtonum is GNU awk extension + awk 'BEGIN{strtonum("0x1")}' +} + LOCALHOST=127.0.0.1 yield() { diff --git a/tools/testing/selftests/futex/functional/futex_requeue.c b/tools/testing/selftests/futex/functional/futex_requeue.c index 35d4be23db5da6..dcf0d5f2f31227 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue.c +++ b/tools/testing/selftests/futex/functional/futex_requeue.c @@ -34,34 +34,18 @@ TEST(requeue_single) volatile futex_t _f1 = 0; volatile futex_t f2 = 0; pthread_t waiter[10]; - int res; f1 = &_f1; /* * Requeue a waiter from f1 to f2, and wake f2. */ - if (pthread_create(&waiter[0], NULL, waiterfn, NULL)) - ksft_exit_fail_msg("pthread_create failed\n"); + ASSERT_EQ(0, pthread_create(&waiter[0], NULL, waiterfn, NULL)); usleep(WAKE_WAIT_US); - ksft_print_dbg_msg("Requeuing 1 futex from f1 to f2\n"); - res = futex_cmp_requeue(f1, 0, &f2, 0, 1, 0); - if (res != 1) - ksft_test_result_fail("futex_requeue simple returned: %d %s\n", - res ? errno : res, - res ? strerror(errno) : ""); - - ksft_print_dbg_msg("Waking 1 futex at f2\n"); - res = futex_wake(&f2, 1, 0); - if (res != 1) { - ksft_test_result_fail("futex_requeue simple returned: %d %s\n", - res ? errno : res, - res ? strerror(errno) : ""); - } else { - ksft_test_result_pass("futex_requeue simple succeeds\n"); - } + EXPECT_EQ(1, futex_cmp_requeue(f1, 0, &f2, 0, 1, 0)); + EXPECT_EQ(1, futex_wake(&f2, 1, 0)); } TEST(requeue_multiple) @@ -69,7 +53,7 @@ TEST(requeue_multiple) volatile futex_t _f1 = 0; volatile futex_t f2 = 0; pthread_t waiter[10]; - int res, i; + int i; f1 = &_f1; @@ -77,30 +61,13 @@ TEST(requeue_multiple) * Create 10 waiters at f1. At futex_requeue, wake 3 and requeue 7. * At futex_wake, wake INT_MAX (should be exactly 7). */ - for (i = 0; i < 10; i++) { - if (pthread_create(&waiter[i], NULL, waiterfn, NULL)) - ksft_exit_fail_msg("pthread_create failed\n"); - } + for (i = 0; i < 10; i++) + ASSERT_EQ(0, pthread_create(&waiter[i], NULL, waiterfn, NULL)); usleep(WAKE_WAIT_US); - ksft_print_dbg_msg("Waking 3 futexes at f1 and requeuing 7 futexes from f1 to f2\n"); - res = futex_cmp_requeue(f1, 0, &f2, 3, 7, 0); - if (res != 10) { - ksft_test_result_fail("futex_requeue many returned: %d %s\n", - res ? errno : res, - res ? strerror(errno) : ""); - } - - ksft_print_dbg_msg("Waking INT_MAX futexes at f2\n"); - res = futex_wake(&f2, INT_MAX, 0); - if (res != 7) { - ksft_test_result_fail("futex_requeue many returned: %d %s\n", - res ? errno : res, - res ? strerror(errno) : ""); - } else { - ksft_test_result_pass("futex_requeue many succeeds\n"); - } + EXPECT_EQ(10, futex_cmp_requeue(f1, 0, &f2, 3, 7, 0)); + EXPECT_EQ(7, futex_wake(&f2, INT_MAX, 0)); } TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/kvm/x86/msrs_test.c b/tools/testing/selftests/kvm/x86/msrs_test.c index 40d918aedce672..ebd900e713c18d 100644 --- a/tools/testing/selftests/kvm/x86/msrs_test.c +++ b/tools/testing/selftests/kvm/x86/msrs_test.c @@ -175,7 +175,7 @@ void guest_test_reserved_val(const struct kvm_msr *msr) * If the CPU will truncate the written value (e.g. SYSENTER on AMD), * expect success and a truncated value, not #GP. */ - if (!this_cpu_has(msr->feature) || + if ((!this_cpu_has(msr->feature) && !this_cpu_has(msr->feature2)) || msr->rsvd_val == fixup_rdmsr_val(msr->index, msr->rsvd_val)) { u8 vec = wrmsr_safe(msr->index, msr->rsvd_val); diff --git a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c index 0a6dfba3905b68..6b0928e69051d5 100644 --- a/tools/testing/selftests/kvm/x86/sev_migrate_tests.c +++ b/tools/testing/selftests/kvm/x86/sev_migrate_tests.c @@ -36,8 +36,6 @@ static struct kvm_vm *sev_vm_create(bool es) sev_vm_launch(vm, es ? SEV_POLICY_ES : 0); - if (es) - vm_sev_ioctl(vm, KVM_SEV_LAUNCH_UPDATE_VMSA, NULL); return vm; } diff --git a/tools/testing/selftests/landlock/audit.h b/tools/testing/selftests/landlock/audit.h index 44eb433e96661f..33fbbc252c2957 100644 --- a/tools/testing/selftests/landlock/audit.h +++ b/tools/testing/selftests/landlock/audit.h @@ -249,9 +249,9 @@ static __maybe_unused char *regex_escape(const char *const src, char *dst, static int audit_match_record(int audit_fd, const __u16 type, const char *const pattern, __u64 *domain_id) { - struct audit_message msg; + struct audit_message msg, last_mismatch = {}; int ret, err = 0; - bool matches_record = !type; + int num_type_match = 0; regmatch_t matches[2]; regex_t regex; @@ -259,21 +259,35 @@ static int audit_match_record(int audit_fd, const __u16 type, if (ret) return -EINVAL; - do { + /* + * Reads records until one matches both the expected type and the + * pattern. Type-matching records with non-matching content are + * silently consumed, which handles stale domain deallocation records + * from a previous test emitted asynchronously by kworker threads. + */ + while (true) { memset(&msg, 0, sizeof(msg)); err = audit_recv(audit_fd, &msg); - if (err) + if (err) { + if (num_type_match) { + printf("DATA: %s\n", last_mismatch.data); + printf("ERROR: %d record(s) matched type %u" + " but not pattern: %s\n", + num_type_match, type, pattern); + } goto out; + } + + if (type && msg.header.nlmsg_type != type) + continue; - if (msg.header.nlmsg_type == type) - matches_record = true; - } while (!matches_record); + ret = regexec(®ex, msg.data, ARRAY_SIZE(matches), matches, + 0); + if (!ret) + break; - ret = regexec(®ex, msg.data, ARRAY_SIZE(matches), matches, 0); - if (ret) { - printf("DATA: %s\n", msg.data); - printf("ERROR: no match for pattern: %s\n", pattern); - err = -ENOENT; + num_type_match++; + last_mismatch = msg; } if (domain_id) { @@ -309,28 +323,56 @@ static int __maybe_unused matches_log_domain_allocated(int audit_fd, pid_t pid, log_match_len = snprintf(log_match, sizeof(log_match), log_template, pid); - if (log_match_len > sizeof(log_match)) + if (log_match_len >= sizeof(log_match)) return -E2BIG; return audit_match_record(audit_fd, AUDIT_LANDLOCK_DOMAIN, log_match, domain_id); } -static int __maybe_unused matches_log_domain_deallocated( - int audit_fd, unsigned int num_denials, __u64 *domain_id) +/* + * Matches a domain deallocation record. When expected_domain_id is non-zero, + * the pattern includes the specific domain ID so that stale deallocation + * records from a previous test (with a different domain ID) are skipped by + * audit_match_record(), and the socket timeout is temporarily increased to + * audit_tv_dom_drop to wait for the asynchronous kworker deallocation. + */ +static int __maybe_unused +matches_log_domain_deallocated(int audit_fd, unsigned int num_denials, + __u64 expected_domain_id, __u64 *domain_id) { static const char log_template[] = REGEX_LANDLOCK_PREFIX " status=deallocated denials=%u$"; - char log_match[sizeof(log_template) + 10]; - int log_match_len; - - log_match_len = snprintf(log_match, sizeof(log_match), log_template, - num_denials); - if (log_match_len > sizeof(log_match)) + static const char log_template_with_id[] = + "^audit([0-9.:]\\+): domain=\\(%llx\\)" + " status=deallocated denials=%u$"; + char log_match[sizeof(log_template_with_id) + 32]; + int log_match_len, err; + + if (expected_domain_id) + log_match_len = snprintf(log_match, sizeof(log_match), + log_template_with_id, + (unsigned long long)expected_domain_id, + num_denials); + else + log_match_len = snprintf(log_match, sizeof(log_match), + log_template, num_denials); + + if (log_match_len >= sizeof(log_match)) return -E2BIG; - return audit_match_record(audit_fd, AUDIT_LANDLOCK_DOMAIN, log_match, - domain_id); + if (expected_domain_id) + setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, + &audit_tv_dom_drop, sizeof(audit_tv_dom_drop)); + + err = audit_match_record(audit_fd, AUDIT_LANDLOCK_DOMAIN, log_match, + domain_id); + + if (expected_domain_id) + setsockopt(audit_fd, SOL_SOCKET, SO_RCVTIMEO, &audit_tv_default, + sizeof(audit_tv_default)); + + return err; } struct audit_records { @@ -338,6 +380,15 @@ struct audit_records { size_t domain; }; +/* + * WARNING: Do not assert records.domain == 0 without a preceding + * audit_match_record() call. Domain deallocation records are emitted + * asynchronously from kworker threads and can arrive after the drain in + * audit_init(), corrupting the domain count. A preceding audit_match_record() + * call consumes stale records while scanning, making the assertion safe in + * practice because stale deallocation records arrive before the expected access + * records. + */ static int audit_count_records(int audit_fd, struct audit_records *records) { struct audit_message msg; @@ -391,6 +442,16 @@ static int audit_init(void) if (err) return -errno; + /* + * Drains stale audit records that accumulated in the kernel backlog + * while no audit daemon socket was open. This happens when non-audit + * Landlock tests generate records while audit_enabled is non-zero (e.g. + * from boot configuration), or when domain deallocation records arrive + * asynchronously after a previous test's socket was closed. + */ + while (audit_recv(fd, NULL) == 0) + ; + return fd; } diff --git a/tools/testing/selftests/landlock/audit_test.c b/tools/testing/selftests/landlock/audit_test.c index 46d02d49835aae..93ae5bd0dcce0b 100644 --- a/tools/testing/selftests/landlock/audit_test.c +++ b/tools/testing/selftests/landlock/audit_test.c @@ -139,29 +139,31 @@ TEST_F(audit, layers) WEXITSTATUS(status) != EXIT_SUCCESS) _metadata->exit_code = KSFT_FAIL; - /* Purges log from deallocated domains. */ - EXPECT_EQ(0, setsockopt(self->audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_dom_drop, sizeof(audit_tv_dom_drop))); + /* + * Purges log from deallocated domains. Records arrive in LIFO order + * (innermost domain first) because landlock_put_hierarchy() walks the + * chain sequentially in a single kworker context. + */ for (i = ARRAY_SIZE(*domain_stack) - 1; i >= 0; i--) { __u64 deallocated_dom = 2; EXPECT_EQ(0, matches_log_domain_deallocated(self->audit_fd, 1, + (*domain_stack)[i], &deallocated_dom)); EXPECT_EQ((*domain_stack)[i], deallocated_dom) { TH_LOG("Failed to match domain %llx (#%d)", - (*domain_stack)[i], i); + (unsigned long long)(*domain_stack)[i], i); } } EXPECT_EQ(0, munmap(domain_stack, sizeof(*domain_stack))); - EXPECT_EQ(0, setsockopt(self->audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_default, sizeof(audit_tv_default))); EXPECT_EQ(0, close(ruleset_fd)); } struct thread_data { pid_t parent_pid; int ruleset_fd, pipe_child, pipe_parent; + bool mute_subdomains; }; static void *thread_audit_test(void *arg) @@ -270,13 +272,329 @@ TEST_F(audit, thread) EXPECT_EQ(0, close(pipe_parent[1])); ASSERT_EQ(0, pthread_join(thread, NULL)); - EXPECT_EQ(0, setsockopt(self->audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_dom_drop, sizeof(audit_tv_dom_drop))); - EXPECT_EQ(0, matches_log_domain_deallocated(self->audit_fd, 1, - &deallocated_dom)); + EXPECT_EQ(0, matches_log_domain_deallocated( + self->audit_fd, 1, denial_dom, &deallocated_dom)); EXPECT_EQ(denial_dom, deallocated_dom); - EXPECT_EQ(0, setsockopt(self->audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_default, sizeof(audit_tv_default))); +} + +/* + * Verifies that log_subdomains_off set via the ruleset_fd=-1 path (without + * creating a domain) is inherited by children across fork(). This exercises + * the hook_cred_transfer() fix: the Landlock credential blob must be copied + * even when the source credential has no domain. + * + * Phase 1 (baseline): a child without muting creates a domain and triggers a + * denial that IS logged. + * + * Phase 2 (after muting): the parent mutes subdomain logs, forks another child + * who creates a domain and triggers a denial that is NOT logged. + */ +TEST_F(audit, log_subdomains_off_fork) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .scoped = LANDLOCK_SCOPE_SIGNAL, + }; + struct audit_records records; + int ruleset_fd, status; + pid_t child; + + ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, ruleset_fd); + + ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); + + /* + * Phase 1: forks a child that creates a domain and triggers a denial + * before any muting. This proves the audit path works. + */ + child = fork(); + ASSERT_LE(0, child); + if (child == 0) { + ASSERT_EQ(0, landlock_restrict_self(ruleset_fd, 0)); + ASSERT_EQ(-1, kill(getppid(), 0)); + ASSERT_EQ(EPERM, errno); + _exit(0); + return; + } + + ASSERT_EQ(child, waitpid(child, &status, 0)); + ASSERT_EQ(true, WIFEXITED(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); + + /* The denial must be logged (baseline). */ + EXPECT_EQ(0, matches_log_signal(_metadata, self->audit_fd, getpid(), + NULL)); + + /* Drains any remaining records (e.g. domain allocation). */ + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + + /* + * Mutes subdomain logs without creating a domain. The parent's + * credential has domain=NULL and log_subdomains_off=1. + */ + ASSERT_EQ(0, landlock_restrict_self( + -1, LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF)); + + /* + * Phase 2: forks a child that creates a domain and triggers a denial. + * Because log_subdomains_off was inherited via fork(), the child's + * domain has log_status=LANDLOCK_LOG_DISABLED. + */ + child = fork(); + ASSERT_LE(0, child); + if (child == 0) { + ASSERT_EQ(0, landlock_restrict_self(ruleset_fd, 0)); + ASSERT_EQ(-1, kill(getppid(), 0)); + ASSERT_EQ(EPERM, errno); + _exit(0); + return; + } + + ASSERT_EQ(child, waitpid(child, &status, 0)); + ASSERT_EQ(true, WIFEXITED(status)); + ASSERT_EQ(0, WEXITSTATUS(status)); + + /* No denial record should appear. */ + EXPECT_EQ(-EAGAIN, matches_log_signal(_metadata, self->audit_fd, + getpid(), NULL)); + + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + EXPECT_EQ(0, records.access); + + EXPECT_EQ(0, close(ruleset_fd)); +} + +/* + * Thread function: runs two rounds of (create domain, trigger denial, signal + * back), waiting for the main thread before each round. When mute_subdomains + * is set, phase 1 also mutes subdomain logs via the fd=-1 path before creating + * the domain. The ruleset_fd is kept open across both rounds so each + * restrict_self call stacks a new domain layer. + */ +static void *thread_sandbox_deny_twice(void *arg) +{ + const struct thread_data *data = (struct thread_data *)arg; + uintptr_t err = 0; + char buffer; + + /* Phase 1: optionally mutes, creates a domain, and triggers a denial. */ + if (read(data->pipe_parent, &buffer, 1) != 1) { + err = 1; + goto out; + } + + if (data->mute_subdomains && + landlock_restrict_self(-1, + LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF)) { + err = 2; + goto out; + } + + if (landlock_restrict_self(data->ruleset_fd, 0)) { + err = 3; + goto out; + } + + if (kill(data->parent_pid, 0) != -1 || errno != EPERM) { + err = 4; + goto out; + } + + if (write(data->pipe_child, ".", 1) != 1) { + err = 5; + goto out; + } + + /* Phase 2: stacks another domain and triggers a denial. */ + if (read(data->pipe_parent, &buffer, 1) != 1) { + err = 6; + goto out; + } + + if (landlock_restrict_self(data->ruleset_fd, 0)) { + err = 7; + goto out; + } + + if (kill(data->parent_pid, 0) != -1 || errno != EPERM) { + err = 8; + goto out; + } + + if (write(data->pipe_child, ".", 1) != 1) { + err = 9; + goto out; + } + +out: + close(data->ruleset_fd); + close(data->pipe_child); + close(data->pipe_parent); + return (void *)err; +} + +/* + * Verifies that LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF with + * LANDLOCK_RESTRICT_SELF_TSYNC and ruleset_fd=-1 propagates log_subdomains_off + * to a sibling thread, suppressing audit logging on domains it subsequently + * creates. + * + * Phase 1 (before TSYNC) acts as an inline baseline: the sibling creates a + * domain and triggers a denial that IS logged. + * + * Phase 2 (after TSYNC) verifies suppression: the sibling stacks another domain + * and triggers a denial that is NOT logged. + */ +TEST_F(audit, log_subdomains_off_tsync) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .scoped = LANDLOCK_SCOPE_SIGNAL, + }; + struct audit_records records; + struct thread_data child_data = {}; + int pipe_child[2], pipe_parent[2]; + char buffer; + pthread_t thread; + void *thread_ret; + + child_data.parent_pid = getppid(); + ASSERT_EQ(0, pipe2(pipe_child, O_CLOEXEC)); + child_data.pipe_child = pipe_child[1]; + ASSERT_EQ(0, pipe2(pipe_parent, O_CLOEXEC)); + child_data.pipe_parent = pipe_parent[0]; + child_data.ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, child_data.ruleset_fd); + + ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); + + /* Creates the sibling thread. */ + ASSERT_EQ(0, pthread_create(&thread, NULL, thread_sandbox_deny_twice, + &child_data)); + + /* + * Phase 1: the sibling creates a domain and triggers a denial before + * any log muting. This proves the audit path works. + */ + ASSERT_EQ(1, write(pipe_parent[1], ".", 1)); + ASSERT_EQ(1, read(pipe_child[0], &buffer, 1)); + + /* The denial must be logged. */ + EXPECT_EQ(0, matches_log_signal(_metadata, self->audit_fd, + child_data.parent_pid, NULL)); + + /* Drains any remaining records (e.g. domain allocation). */ + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + + /* + * Mutes subdomain logs and propagates to the sibling thread via TSYNC, + * without creating a domain. + */ + ASSERT_EQ(0, landlock_restrict_self( + -1, LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF | + LANDLOCK_RESTRICT_SELF_TSYNC)); + + /* + * Phase 2: the sibling stacks another domain and triggers a denial. + * Because log_subdomains_off was propagated via TSYNC, the new domain + * has log_status=LANDLOCK_LOG_DISABLED. + */ + ASSERT_EQ(1, write(pipe_parent[1], ".", 1)); + ASSERT_EQ(1, read(pipe_child[0], &buffer, 1)); + + /* No denial record should appear. */ + EXPECT_EQ(-EAGAIN, matches_log_signal(_metadata, self->audit_fd, + child_data.parent_pid, NULL)); + + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + EXPECT_EQ(0, records.access); + + EXPECT_EQ(0, close(pipe_child[0])); + EXPECT_EQ(0, close(pipe_parent[1])); + ASSERT_EQ(0, pthread_join(thread, &thread_ret)); + EXPECT_EQ(NULL, thread_ret); +} + +/* + * Verifies that LANDLOCK_RESTRICT_SELF_TSYNC without + * LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF overrides a sibling thread's + * log_subdomains_off, re-enabling audit logging on domains the sibling + * subsequently creates. + * + * Phase 1: the sibling sets log_subdomains_off, creates a muted domain, and + * triggers a denial that is NOT logged. + * + * Phase 2 (after TSYNC without LOG_SUBDOMAINS_OFF): the sibling stacks another + * domain and triggers a denial that IS logged, proving the muting was + * overridden. + */ +TEST_F(audit, tsync_override_log_subdomains_off) +{ + const struct landlock_ruleset_attr ruleset_attr = { + .scoped = LANDLOCK_SCOPE_SIGNAL, + }; + struct audit_records records; + struct thread_data child_data = {}; + int pipe_child[2], pipe_parent[2]; + char buffer; + pthread_t thread; + void *thread_ret; + + child_data.parent_pid = getppid(); + ASSERT_EQ(0, pipe2(pipe_child, O_CLOEXEC)); + child_data.pipe_child = pipe_child[1]; + ASSERT_EQ(0, pipe2(pipe_parent, O_CLOEXEC)); + child_data.pipe_parent = pipe_parent[0]; + child_data.ruleset_fd = + landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0); + ASSERT_LE(0, child_data.ruleset_fd); + + ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); + + child_data.mute_subdomains = true; + + /* Creates the sibling thread. */ + ASSERT_EQ(0, pthread_create(&thread, NULL, thread_sandbox_deny_twice, + &child_data)); + + /* + * Phase 1: the sibling mutes subdomain logs, creates a domain, and + * triggers a denial. The denial must not be logged. + */ + ASSERT_EQ(1, write(pipe_parent[1], ".", 1)); + ASSERT_EQ(1, read(pipe_child[0], &buffer, 1)); + + EXPECT_EQ(-EAGAIN, matches_log_signal(_metadata, self->audit_fd, + child_data.parent_pid, NULL)); + + /* Drains any remaining records. */ + EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); + EXPECT_EQ(0, records.access); + + /* + * Overrides the sibling's log_subdomains_off by calling TSYNC without + * LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF. + */ + ASSERT_EQ(0, landlock_restrict_self(child_data.ruleset_fd, + LANDLOCK_RESTRICT_SELF_TSYNC)); + + /* + * Phase 2: the sibling stacks another domain and triggers a denial. + * Because TSYNC replaced its log_subdomains_off with 0, the new domain + * has log_status=LANDLOCK_LOG_PENDING. + */ + ASSERT_EQ(1, write(pipe_parent[1], ".", 1)); + ASSERT_EQ(1, read(pipe_child[0], &buffer, 1)); + + /* The denial must be logged. */ + EXPECT_EQ(0, matches_log_signal(_metadata, self->audit_fd, + child_data.parent_pid, NULL)); + + EXPECT_EQ(0, close(pipe_child[0])); + EXPECT_EQ(0, close(pipe_parent[1])); + ASSERT_EQ(0, pthread_join(thread, &thread_ret)); + EXPECT_EQ(NULL, thread_ret); } FIXTURE(audit_flags) @@ -412,7 +730,6 @@ TEST_F(audit_flags, signal) } else { EXPECT_EQ(1, records.access); } - EXPECT_EQ(0, records.domain); /* Updates filter rules to match the drop record. */ set_cap(_metadata, CAP_AUDIT_CONTROL); @@ -433,22 +750,21 @@ TEST_F(audit_flags, signal) if (variant->restrict_flags & LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF) { + /* + * No deallocation record: denials=0 never matches a real + * record. + */ EXPECT_EQ(-EAGAIN, - matches_log_domain_deallocated(self->audit_fd, 0, + matches_log_domain_deallocated(self->audit_fd, 0, 0, &deallocated_dom)); EXPECT_EQ(deallocated_dom, 2); } else { - EXPECT_EQ(0, setsockopt(self->audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_dom_drop, - sizeof(audit_tv_dom_drop))); EXPECT_EQ(0, matches_log_domain_deallocated(self->audit_fd, 2, + *self->domain_id, &deallocated_dom)); EXPECT_NE(deallocated_dom, 2); EXPECT_NE(deallocated_dom, 0); EXPECT_EQ(deallocated_dom, *self->domain_id); - EXPECT_EQ(0, setsockopt(self->audit_fd, SOL_SOCKET, SO_RCVTIMEO, - &audit_tv_default, - sizeof(audit_tv_default))); } } @@ -601,7 +917,6 @@ TEST_F(audit_exec, signal_and_open) /* Tests that there was no denial until now. */ EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); EXPECT_EQ(0, records.access); - EXPECT_EQ(0, records.domain); /* * Wait for the child to do a first denied action by layer1 and diff --git a/tools/testing/selftests/landlock/net_test.c b/tools/testing/selftests/landlock/net_test.c index b34b139b3f89c5..4c528154ea92bb 100644 --- a/tools/testing/selftests/landlock/net_test.c +++ b/tools/testing/selftests/landlock/net_test.c @@ -1356,7 +1356,7 @@ TEST_F(mini, network_access_rights) &net_port, 0)) { TH_LOG("Failed to add rule with access 0x%llx: %s", - access, strerror(errno)); + (unsigned long long)access, strerror(errno)); } } EXPECT_EQ(0, close(ruleset_fd)); diff --git a/tools/testing/selftests/landlock/ptrace_test.c b/tools/testing/selftests/landlock/ptrace_test.c index 4f64c90583cd61..1b6c8b53bf33a6 100644 --- a/tools/testing/selftests/landlock/ptrace_test.c +++ b/tools/testing/selftests/landlock/ptrace_test.c @@ -342,7 +342,6 @@ TEST_F(audit, trace) /* Makes sure there is no superfluous logged records. */ EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); EXPECT_EQ(0, records.access); - EXPECT_EQ(0, records.domain); yama_ptrace_scope = get_yama_ptrace_scope(); ASSERT_LE(0, yama_ptrace_scope); diff --git a/tools/testing/selftests/landlock/scoped_abstract_unix_test.c b/tools/testing/selftests/landlock/scoped_abstract_unix_test.c index 72f97648d4a7d4..c47491d2d1c142 100644 --- a/tools/testing/selftests/landlock/scoped_abstract_unix_test.c +++ b/tools/testing/selftests/landlock/scoped_abstract_unix_test.c @@ -312,7 +312,6 @@ TEST_F(scoped_audit, connect_to_child) /* Makes sure there is no superfluous logged records. */ EXPECT_EQ(0, audit_count_records(self->audit_fd, &records)); EXPECT_EQ(0, records.access); - EXPECT_EQ(0, records.domain); ASSERT_EQ(0, pipe2(pipe_child, O_CLOEXEC)); ASSERT_EQ(0, pipe2(pipe_parent, O_CLOEXEC)); diff --git a/tools/testing/selftests/landlock/tsync_test.c b/tools/testing/selftests/landlock/tsync_test.c index 2b9ad4f154f481..9cf1491bbaaff4 100644 --- a/tools/testing/selftests/landlock/tsync_test.c +++ b/tools/testing/selftests/landlock/tsync_test.c @@ -247,4 +247,81 @@ TEST(tsync_interrupt) EXPECT_EQ(0, close(ruleset_fd)); } +/* clang-format off */ +FIXTURE(tsync_without_ruleset) {}; +/* clang-format on */ + +FIXTURE_VARIANT(tsync_without_ruleset) +{ + const __u32 flags; + const int expected_errno; +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tsync_without_ruleset, tsync_only) { + /* clang-format on */ + .flags = LANDLOCK_RESTRICT_SELF_TSYNC, + .expected_errno = EBADF, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tsync_without_ruleset, subdomains_off_same_exec_off) { + /* clang-format on */ + .flags = LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF | + LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF | + LANDLOCK_RESTRICT_SELF_TSYNC, + .expected_errno = EBADF, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tsync_without_ruleset, subdomains_off_new_exec_on) { + /* clang-format on */ + .flags = LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF | + LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON | + LANDLOCK_RESTRICT_SELF_TSYNC, + .expected_errno = EBADF, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tsync_without_ruleset, all_flags) { + /* clang-format on */ + .flags = LANDLOCK_RESTRICT_SELF_LOG_SAME_EXEC_OFF | + LANDLOCK_RESTRICT_SELF_LOG_NEW_EXEC_ON | + LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF | + LANDLOCK_RESTRICT_SELF_TSYNC, + .expected_errno = EBADF, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(tsync_without_ruleset, subdomains_off) { + /* clang-format on */ + .flags = LANDLOCK_RESTRICT_SELF_LOG_SUBDOMAINS_OFF | + LANDLOCK_RESTRICT_SELF_TSYNC, + .expected_errno = 0, +}; + +FIXTURE_SETUP(tsync_without_ruleset) +{ + disable_caps(_metadata); +} + +FIXTURE_TEARDOWN(tsync_without_ruleset) +{ +} + +TEST_F(tsync_without_ruleset, check) +{ + int ret; + + ASSERT_EQ(0, prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)); + + ret = landlock_restrict_self(-1, variant->flags); + if (variant->expected_errno) { + EXPECT_EQ(-1, ret); + EXPECT_EQ(variant->expected_errno, errno); + } else { + EXPECT_EQ(0, ret); + } +} + TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index e8328c89d855ef..788689497e92a4 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -34,6 +34,7 @@ */ #include #include +#include struct hmm_buffer { void *ptr; @@ -548,7 +549,7 @@ TEST_F(hmm, anon_write_child) for (migrate = 0; migrate < 2; ++migrate) { for (use_thp = 0; use_thp < 2; ++use_thp) { - npages = ALIGN(use_thp ? TWOMEG : HMM_BUFFER_SIZE, + npages = ALIGN(use_thp ? read_pmd_pagesize() : HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; @@ -728,7 +729,7 @@ TEST_F(hmm, anon_write_huge) int *ptr; int ret; - size = 2 * TWOMEG; + size = 2 * read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -744,7 +745,7 @@ TEST_F(hmm, anon_write_huge) buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); - size = TWOMEG; + size /= 2; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); @@ -770,54 +771,6 @@ TEST_F(hmm, anon_write_huge) hmm_buffer_free(buffer); } -/* - * Read numeric data from raw and tagged kernel status files. Used to read - * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag). - */ -static long file_read_ulong(char *file, const char *tag) -{ - int fd; - char buf[2048]; - int len; - char *p, *q; - long val; - - fd = open(file, O_RDONLY); - if (fd < 0) { - /* Error opening the file */ - return -1; - } - - len = read(fd, buf, sizeof(buf)); - close(fd); - if (len < 0) { - /* Error in reading the file */ - return -1; - } - if (len == sizeof(buf)) { - /* Error file is too large */ - return -1; - } - buf[len] = '\0'; - - /* Search for a tag if provided */ - if (tag) { - p = strstr(buf, tag); - if (!p) - return -1; /* looks like the line we want isn't there */ - p += strlen(tag); - } else - p = buf; - - val = strtol(p, &q, 0); - if (*q != ' ') { - /* Error parsing the file */ - return -1; - } - - return val; -} - /* * Write huge TLBFS page. */ @@ -826,15 +779,13 @@ TEST_F(hmm, anon_write_hugetlbfs) struct hmm_buffer *buffer; unsigned long npages; unsigned long size; - unsigned long default_hsize; + unsigned long default_hsize = default_huge_page_size(); unsigned long i; int *ptr; int ret; - default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); - if (default_hsize < 0 || default_hsize*1024 < default_hsize) + if (!default_hsize) SKIP(return, "Huge page size could not be determined"); - default_hsize = default_hsize*1024; /* KB to B */ size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -1606,7 +1557,7 @@ TEST_F(hmm, compound) struct hmm_buffer *buffer; unsigned long npages; unsigned long size; - unsigned long default_hsize; + unsigned long default_hsize = default_huge_page_size(); int *ptr; unsigned char *m; int ret; @@ -1614,10 +1565,8 @@ TEST_F(hmm, compound) /* Skip test if we can't allocate a hugetlbfs page. */ - default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); - if (default_hsize < 0 || default_hsize*1024 < default_hsize) + if (!default_hsize) SKIP(return, "Huge page size could not be determined"); - default_hsize = default_hsize*1024; /* KB to B */ size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; @@ -2106,7 +2055,7 @@ TEST_F(hmm, migrate_anon_huge_empty) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2158,7 +2107,7 @@ TEST_F(hmm, migrate_anon_huge_zero) int ret; int val; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2221,7 +2170,7 @@ TEST_F(hmm, migrate_anon_huge_free) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2280,7 +2229,7 @@ TEST_F(hmm, migrate_anon_huge_fault) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2332,7 +2281,7 @@ TEST_F(hmm, migrate_partial_unmap_fault) { struct hmm_buffer *buffer; unsigned long npages; - unsigned long size = TWOMEG; + unsigned long size = read_pmd_pagesize(); unsigned long i; void *old_ptr; void *map; @@ -2398,7 +2347,7 @@ TEST_F(hmm, migrate_remap_fault) { struct hmm_buffer *buffer; unsigned long npages; - unsigned long size = TWOMEG; + unsigned long size = read_pmd_pagesize(); unsigned long i; void *old_ptr, *new_ptr = NULL; void *map; @@ -2498,7 +2447,7 @@ TEST_F(hmm, migrate_anon_huge_err) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); @@ -2593,7 +2542,7 @@ TEST_F(hmm, migrate_anon_huge_zero_err) int *ptr; int ret; - size = TWOMEG; + size = read_pmd_pagesize(); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index ee24b88c2b2483..60e78bbfc0e3e8 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -36,7 +36,8 @@ FIXTURE_SETUP(migration) { int n; - ASSERT_EQ(numa_available(), 0); + if (numa_available() < 0) + SKIP(return, "NUMA not available"); self->nthreads = numa_num_task_cpus() - 1; self->n1 = -1; self->n2 = -1; diff --git a/tools/testing/selftests/mqueue/setting b/tools/testing/selftests/mqueue/settings similarity index 100% rename from tools/testing/selftests/mqueue/setting rename to tools/testing/selftests/mqueue/settings diff --git a/tools/testing/selftests/namespaces/listns_efault_test.c b/tools/testing/selftests/namespaces/listns_efault_test.c index c7ed4023d7a858..b570746e917c11 100644 --- a/tools/testing/selftests/namespaces/listns_efault_test.c +++ b/tools/testing/selftests/namespaces/listns_efault_test.c @@ -19,7 +19,6 @@ #include #include #include "../kselftest_harness.h" -#include "../filesystems/utils.h" #include "../pidfd/pidfd.h" #include "wrappers.h" diff --git a/tools/testing/selftests/net/mptcp/mptcp_lib.sh b/tools/testing/selftests/net/mptcp/mptcp_lib.sh index 5fea7e7df628c8..989a5975dcea62 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_lib.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_lib.sh @@ -474,20 +474,24 @@ mptcp_lib_wait_local_port_listen() { wait_local_port_listen "${@}" "tcp" } +# $1: error file, $2: cmd, $3: expected msg, [$4: expected error] mptcp_lib_check_output() { local err="${1}" local cmd="${2}" local expected="${3}" + local exp_error="${4:-0}" local cmd_ret=0 local out - if ! out=$(${cmd} 2>"${err}"); then - cmd_ret=${?} - fi + out=$(${cmd} 2>"${err}") || cmd_ret=1 - if [ ${cmd_ret} -ne 0 ]; then - mptcp_lib_pr_fail "command execution '${cmd}' stderr" - cat "${err}" + if [ "${cmd_ret}" != "${exp_error}" ]; then + mptcp_lib_pr_fail "unexpected returned code for '${cmd}', info:" + if [ "${exp_error}" = 0 ]; then + cat "${err}" + else + echo "${out}" + fi return 2 elif [ "${out}" = "${expected}" ]; then return 0 diff --git a/tools/testing/selftests/net/mptcp/pm_netlink.sh b/tools/testing/selftests/net/mptcp/pm_netlink.sh index 123d9d7a0278cd..04594dfc22b134 100755 --- a/tools/testing/selftests/net/mptcp/pm_netlink.sh +++ b/tools/testing/selftests/net/mptcp/pm_netlink.sh @@ -122,10 +122,12 @@ check() local cmd="$1" local expected="$2" local msg="$3" + local exp_error="$4" local rc=0 mptcp_lib_print_title "$msg" - mptcp_lib_check_output "${err}" "${cmd}" "${expected}" || rc=${?} + mptcp_lib_check_output "${err}" "${cmd}" "${expected}" "${exp_error}" || + rc=${?} if [ ${rc} -eq 2 ]; then mptcp_lib_result_fail "${msg} # error ${rc}" ret=${KSFT_FAIL} @@ -158,13 +160,13 @@ check "show_endpoints" \ "3,10.0.1.3,signal backup")" "dump addrs" del_endpoint 2 -check "get_endpoint 2" "" "simple del addr" +check "get_endpoint 2" "" "simple del addr" 1 check "show_endpoints" \ "$(format_endpoints "1,10.0.1.1" \ "3,10.0.1.3,signal backup")" "dump addrs after del" add_endpoint 10.0.1.3 2>/dev/null -check "get_endpoint 4" "" "duplicate addr" +check "get_endpoint 4" "" "duplicate addr" 1 add_endpoint 10.0.1.4 flags signal check "get_endpoint 4" "$(format_endpoints "4,10.0.1.4,signal")" "id addr increment" @@ -173,7 +175,7 @@ for i in $(seq 5 9); do add_endpoint "10.0.1.${i}" flags signal >/dev/null 2>&1 done check "get_endpoint 9" "$(format_endpoints "9,10.0.1.9,signal")" "hard addr limit" -check "get_endpoint 10" "" "above hard addr limit" +check "get_endpoint 10" "" "above hard addr limit" 1 del_endpoint 9 for i in $(seq 10 255); do @@ -192,9 +194,13 @@ check "show_endpoints" \ flush_endpoint check "show_endpoints" "" "flush addrs" -add_endpoint 10.0.1.1 flags unknown -check "show_endpoints" "$(format_endpoints "1,10.0.1.1")" "ignore unknown flags" -flush_endpoint +# "unknown" flag is only supported by pm_nl_ctl +if ! mptcp_lib_is_ip_mptcp; then + add_endpoint 10.0.1.1 flags unknown + check "show_endpoints" "$(format_endpoints "1,10.0.1.1")" \ + "ignore unknown flags" + flush_endpoint +fi set_limits 9 1 2>/dev/null check "get_limits" "${default_limits}" "rcv addrs above hard limit" diff --git a/tools/testing/selftests/net/netfilter/nft_tproxy_udp.sh b/tools/testing/selftests/net/netfilter/nft_tproxy_udp.sh index d16de13fe5a752..1dc7b04501459e 100755 --- a/tools/testing/selftests/net/netfilter/nft_tproxy_udp.sh +++ b/tools/testing/selftests/net/netfilter/nft_tproxy_udp.sh @@ -190,13 +190,13 @@ table inet filter { } EOF - timeout "$timeout" ip netns exec "$nsrouter" socat -u "$socat_ipproto" udp-listen:12345,fork,ip-transparent,reuseport udp:"$ns1_ip_port",ip-transparent,reuseport,bind="$ns2_ip_port" 2>/dev/null & + timeout "$timeout" ip netns exec "$nsrouter" socat -u "$socat_ipproto" udp-listen:12345,fork,ip-transparent,reuseport,shut-none udp:"$ns1_ip_port",ip-transparent,reuseport,bind="$ns2_ip_port",shut-none 2>/dev/null & local tproxy_pid=$! - timeout "$timeout" ip netns exec "$ns2" socat "$socat_ipproto" udp-listen:8080,fork SYSTEM:"echo PONG_NS2" 2>/dev/null & + timeout "$timeout" ip netns exec "$ns2" socat "$socat_ipproto" udp-listen:8080,fork,shut-none SYSTEM:"echo PONG_NS2" 2>/dev/null & local server2_pid=$! - timeout "$timeout" ip netns exec "$ns3" socat "$socat_ipproto" udp-listen:8080,fork SYSTEM:"echo PONG_NS3" 2>/dev/null & + timeout "$timeout" ip netns exec "$ns3" socat "$socat_ipproto" udp-listen:8080,fork,shut-none SYSTEM:"echo PONG_NS3" 2>/dev/null & local server3_pid=$! busywait "$BUSYWAIT_TIMEOUT" listener_ready "$nsrouter" 12345 "-u" @@ -205,7 +205,7 @@ EOF local result # request from ns1 to ns2 (forwarded traffic) - result=$(echo I_M_PROXIED | ip netns exec "$ns1" socat -t 2 -T 2 STDIO udp:"$ns2_ip_port",sourceport=18888) + result=$(echo I_M_PROXIED | ip netns exec "$ns1" socat -t 2 -T 2 STDIO udp:"$ns2_ip_port",sourceport=18888,shut-none) if [ "$result" == "$expect_ns1_ns2" ] ;then echo "PASS: tproxy test $testname: ns1 got reply \"$result\" connecting to ns2" else @@ -214,7 +214,7 @@ EOF fi # request from ns1 to ns3 (forwarded traffic) - result=$(echo I_M_PROXIED | ip netns exec "$ns1" socat -t 2 -T 2 STDIO udp:"$ns3_ip_port") + result=$(echo I_M_PROXIED | ip netns exec "$ns1" socat -t 2 -T 2 STDIO udp:"$ns3_ip_port",shut-none) if [ "$result" = "$expect_ns1_ns3" ] ;then echo "PASS: tproxy test $testname: ns1 got reply \"$result\" connecting to ns3" else @@ -223,7 +223,7 @@ EOF fi # request from nsrouter to ns2 (localy originated traffic) - result=$(echo I_M_PROXIED | ip netns exec "$nsrouter" socat -t 2 -T 2 STDIO udp:"$ns2_ip_port") + result=$(echo I_M_PROXIED | ip netns exec "$nsrouter" socat -t 2 -T 2 STDIO udp:"$ns2_ip_port",shut-none) if [ "$result" == "$expect_nsrouter_ns2" ] ;then echo "PASS: tproxy test $testname: nsrouter got reply \"$result\" connecting to ns2" else @@ -232,7 +232,7 @@ EOF fi # request from nsrouter to ns3 (localy originated traffic) - result=$(echo I_M_PROXIED | ip netns exec "$nsrouter" socat -t 2 -T 2 STDIO udp:"$ns3_ip_port") + result=$(echo I_M_PROXIED | ip netns exec "$nsrouter" socat -t 2 -T 2 STDIO udp:"$ns3_ip_port",shut-none) if [ "$result" = "$expect_nsrouter_ns3" ] ;then echo "PASS: tproxy test $testname: nsrouter got reply \"$result\" connecting to ns3" else diff --git a/tools/testing/selftests/net/packetdrill/tcp_ts_recent_invalid_ack.pkt b/tools/testing/selftests/net/packetdrill/tcp_ts_recent_invalid_ack.pkt index 174ce9a1bfc071..ee6baf7c36cfaa 100644 --- a/tools/testing/selftests/net/packetdrill/tcp_ts_recent_invalid_ack.pkt +++ b/tools/testing/selftests/net/packetdrill/tcp_ts_recent_invalid_ack.pkt @@ -19,7 +19,9 @@ // bad packet with high tsval (its ACK sequence is above our sndnxt) +0 < F. 1:1(0) ack 9999 win 20000 - +// Challenge ACK for SEG.ACK > SND.NXT (RFC 5961 5.2 / RFC 793 3.9). +// ecr=200 (not 200000) proves ts_recent was not updated from the bad packet. + +0 > . 1:1(0) ack 1 +0 < . 1:1001(1000) ack 1 win 20000 +0 > . 1:1(0) ack 1001 diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c index 1b9d3b2e2491c9..801b2ad188537e 100644 --- a/tools/testing/selftests/nolibc/nolibc-test.c +++ b/tools/testing/selftests/nolibc/nolibc-test.c @@ -74,6 +74,28 @@ static const int is_nolibc = #endif ; +static const int is_glibc = +#ifdef __GLIBC__ + 1 +#else + 0 +#endif +; + +#if !defined(NOLIBC) +/* Some disabled tests may not compile. */ + +/* strlcat() and strlcpy() may not be in the system headers. */ +#undef strlcat +#undef strlcpy +#define strlcat(d, s, l) 0 +#define strlcpy(d, s, l) 0 + +/* readdir_r() is likely to be marked deprecated */ +#undef readdir_r +#define readdir_r(dir, dirent, result) ((errno = EINVAL), -1) +#endif + /* definition of a series of tests */ struct test { const char *name; /* test name */ @@ -866,7 +888,7 @@ int test_file_stream(void) errno = 0; r = fwrite("foo", 1, 3, f); - if (r != 0 || errno != EBADF) { + if (r != 0 || ((is_nolibc || is_glibc) && errno != EBADF)) { fclose(f); return -1; } @@ -1408,7 +1430,7 @@ int run_syscall(int min, int max) CASE_TEST(fork); EXPECT_SYSZR(1, test_fork(FORK_STANDARD)); break; CASE_TEST(getdents64_root); EXPECT_SYSNE(1, test_getdents64("/"), -1); break; CASE_TEST(getdents64_null); EXPECT_SYSER(1, test_getdents64("/dev/null"), -1, ENOTDIR); break; - CASE_TEST(directories); EXPECT_SYSZR(proc, test_dirent()); break; + CASE_TEST(directories); EXPECT_SYSZR(is_nolibc && proc, test_dirent()); break; CASE_TEST(getrandom); EXPECT_SYSZR(1, test_getrandom()); break; CASE_TEST(gettimeofday_tv); EXPECT_SYSZR(1, gettimeofday(&tv, NULL)); break; CASE_TEST(gettimeofday_tv_tz);EXPECT_SYSZR(1, gettimeofday(&tv, &tz)); break; diff --git a/tools/testing/selftests/powerpc/vphn/Makefile b/tools/testing/selftests/powerpc/vphn/Makefile index 61d519a076c6f5..778fc396340db8 100644 --- a/tools/testing/selftests/powerpc/vphn/Makefile +++ b/tools/testing/selftests/powerpc/vphn/Makefile @@ -5,7 +5,7 @@ top_srcdir = ../../../../.. include ../../lib.mk include ../flags.mk -CFLAGS += -m64 -I$(CURDIR) +CFLAGS += -m64 -I$(CURDIR) -fno-strict-aliasing $(TEST_GEN_PROGS): ../harness.c diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile index 4ef90823b6526f..50d69e22ee7a67 100644 --- a/tools/testing/selftests/rseq/Makefile +++ b/tools/testing/selftests/rseq/Makefile @@ -14,14 +14,20 @@ LDLIBS += -lpthread -ldl # still track changes to header files and depend on shared object. OVERRIDE_TARGETS = 1 -TEST_GEN_PROGS = basic_test basic_percpu_ops_test basic_percpu_ops_mm_cid_test param_test \ - param_test_benchmark param_test_compare_twice param_test_mm_cid \ - param_test_mm_cid_benchmark param_test_mm_cid_compare_twice \ - syscall_errors_test slice_test +TEST_GEN_PROGS = basic_test basic_percpu_ops_test basic_percpu_ops_mm_cid_test \ + param_test_benchmark param_test_mm_cid_benchmark -TEST_GEN_PROGS_EXTENDED = librseq.so +TEST_GEN_PROGS_EXTENDED = librseq.so \ + param_test \ + param_test_compare_twice \ + param_test_mm_cid \ + param_test_mm_cid_compare_twice \ + syscall_errors_test \ + legacy_check \ + slice_test \ + check_optimized -TEST_PROGS = run_param_test.sh run_syscall_errors_test.sh +TEST_PROGS = run_param_test.sh run_syscall_errors_test.sh run_legacy_check.sh run_timeslice_test.sh TEST_FILES := settings @@ -62,3 +68,6 @@ $(OUTPUT)/syscall_errors_test: syscall_errors_test.c $(TEST_GEN_PROGS_EXTENDED) $(OUTPUT)/slice_test: slice_test.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h $(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -o $@ + +$(OUTPUT)/check_optimized: check_optimized.c $(TEST_GEN_PROGS_EXTENDED) rseq.h rseq-*.h + $(CC) $(CFLAGS) $< $(LDLIBS) -lrseq -o $@ diff --git a/tools/testing/selftests/rseq/check_optimized.c b/tools/testing/selftests/rseq/check_optimized.c new file mode 100644 index 00000000000000..a13e3f2c8fc62f --- /dev/null +++ b/tools/testing/selftests/rseq/check_optimized.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: LGPL-2.1 +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include "rseq.h" + +int main(int argc, char **argv) +{ + if (__rseq_register_current_thread(true, false)) + return -1; + return 0; +} diff --git a/tools/testing/selftests/rseq/legacy_check.c b/tools/testing/selftests/rseq/legacy_check.c new file mode 100644 index 00000000000000..3f7de4e2830330 --- /dev/null +++ b/tools/testing/selftests/rseq/legacy_check.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include + +#include "rseq.h" + +#include "../kselftest_harness.h" + +FIXTURE(legacy) +{ +}; + +static int cpu_id_in_sigfn = -1; + +static void sigfn(int sig) +{ + struct rseq_abi *rs = rseq_get_abi(); + + cpu_id_in_sigfn = rs->cpu_id_start; +} + +FIXTURE_SETUP(legacy) +{ + int res = __rseq_register_current_thread(true, true); + + switch (res) { + case -ENOSYS: + SKIP(return, "RSEQ not enabled\n"); + case -EBUSY: + SKIP(return, "GLIBC owns RSEQ. Disable GLIBC RSEQ registration\n"); + default: + ASSERT_EQ(res, 0); + } + + ASSERT_NE(signal(SIGUSR1, sigfn), SIG_ERR); +} + +FIXTURE_TEARDOWN(legacy) +{ +} + +TEST_F(legacy, legacy_test) +{ + struct rseq_abi *rs = rseq_get_abi(); + + ASSERT_NE(rs, NULL); + + /* Overwrite rs::cpu_id_start */ + rs->cpu_id_start = -1; + sleep(1); + ASSERT_NE(rs->cpu_id_start, -1); + + rs->cpu_id_start = -1; + ASSERT_EQ(raise(SIGUSR1), 0); + ASSERT_NE(rs->cpu_id_start, -1); + ASSERT_NE(cpu_id_in_sigfn, -1); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/rseq/param_test.c b/tools/testing/selftests/rseq/param_test.c index 05d03e679e0608..e1e98dbabe4bcf 100644 --- a/tools/testing/selftests/rseq/param_test.c +++ b/tools/testing/selftests/rseq/param_test.c @@ -38,7 +38,7 @@ static int opt_modulo, verbose; static int opt_yield, opt_signal, opt_sleep, opt_disable_rseq, opt_threads = 200, opt_disable_mod = 0, opt_test = 's'; - +static bool opt_rseq_legacy; static long long opt_reps = 5000; static __thread __attribute__((tls_model("initial-exec"))) @@ -281,9 +281,12 @@ unsigned int yield_mod_cnt, nr_abort; } \ } +#define rseq_no_glibc true + #else #define printf_verbose(fmt, ...) +#define rseq_no_glibc false #endif /* BENCHMARK */ @@ -481,7 +484,7 @@ void *test_percpu_spinlock_thread(void *arg) long long i, reps; if (!opt_disable_rseq && thread_data->reg && - rseq_register_current_thread()) + __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = thread_data->reps; for (i = 0; i < reps; i++) { @@ -558,7 +561,7 @@ void *test_percpu_inc_thread(void *arg) long long i, reps; if (!opt_disable_rseq && thread_data->reg && - rseq_register_current_thread()) + __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = thread_data->reps; for (i = 0; i < reps; i++) { @@ -712,7 +715,7 @@ void *test_percpu_list_thread(void *arg) long long i, reps; struct percpu_list *list = (struct percpu_list *)arg; - if (!opt_disable_rseq && rseq_register_current_thread()) + if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = opt_reps; @@ -895,7 +898,7 @@ void *test_percpu_buffer_thread(void *arg) long long i, reps; struct percpu_buffer *buffer = (struct percpu_buffer *)arg; - if (!opt_disable_rseq && rseq_register_current_thread()) + if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = opt_reps; @@ -1105,7 +1108,7 @@ void *test_percpu_memcpy_buffer_thread(void *arg) long long i, reps; struct percpu_memcpy_buffer *buffer = (struct percpu_memcpy_buffer *)arg; - if (!opt_disable_rseq && rseq_register_current_thread()) + if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) abort(); reps = opt_reps; @@ -1258,7 +1261,7 @@ void *test_membarrier_worker_thread(void *arg) const int iters = opt_reps; int i; - if (rseq_register_current_thread()) { + if (__rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); @@ -1323,7 +1326,7 @@ void *test_membarrier_manager_thread(void *arg) intptr_t expect_a = 0, expect_b = 0; int cpu_a = 0, cpu_b = 0; - if (rseq_register_current_thread()) { + if (__rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) { fprintf(stderr, "Error: rseq_register_current_thread(...) failed(%d): %s\n", errno, strerror(errno)); abort(); @@ -1475,6 +1478,7 @@ static void show_usage(int argc, char **argv) printf(" [-D M] Disable rseq for each M threads\n"); printf(" [-T test] Choose test: (s)pinlock, (l)ist, (b)uffer, (m)emcpy, (i)ncrement, membarrie(r)\n"); printf(" [-M] Push into buffer and memcpy buffer with memory barriers.\n"); + printf(" [-O] Test with optimized RSEQ\n"); printf(" [-v] Verbose output.\n"); printf(" [-h] Show this help.\n"); printf("\n"); @@ -1602,6 +1606,9 @@ int main(int argc, char **argv) case 'M': opt_mo = RSEQ_MO_RELEASE; break; + case 'L': + opt_rseq_legacy = true; + break; default: show_usage(argc, argv); goto error; @@ -1618,7 +1625,7 @@ int main(int argc, char **argv) if (set_signal_handler()) goto error; - if (!opt_disable_rseq && rseq_register_current_thread()) + if (!opt_disable_rseq && __rseq_register_current_thread(rseq_no_glibc, opt_rseq_legacy)) goto error; if (!opt_disable_rseq && !rseq_validate_cpu_id()) { fprintf(stderr, "Error: cpu id getter unavailable\n"); diff --git a/tools/testing/selftests/rseq/rseq-abi.h b/tools/testing/selftests/rseq/rseq-abi.h index ecef315204b271..5f4ea2152c2fd8 100644 --- a/tools/testing/selftests/rseq/rseq-abi.h +++ b/tools/testing/selftests/rseq/rseq-abi.h @@ -191,10 +191,15 @@ struct rseq_abi { */ struct rseq_abi_slice_ctrl slice_ctrl; + /* + * Place holder to push the size above 32 bytes. + */ + __u8 __reserved; + /* * Flexible array member at end of structure, after last feature field. */ char end[]; -} __attribute__((aligned(4 * sizeof(__u64)))); +} __attribute__((aligned(256))); #endif /* _RSEQ_ABI_H */ diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index a736727b83c1eb..be0d0a97031ef2 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -56,6 +56,7 @@ ptrdiff_t rseq_offset; * unsuccessful. */ unsigned int rseq_size = -1U; +static unsigned int rseq_alloc_size; /* Flags used during rseq registration. */ unsigned int rseq_flags; @@ -115,29 +116,17 @@ bool rseq_available(void) } } -/* The rseq areas need to be at least 32 bytes. */ -static -unsigned int get_rseq_min_alloc_size(void) -{ - unsigned int alloc_size = rseq_size; - - if (alloc_size < ORIG_RSEQ_ALLOC_SIZE) - alloc_size = ORIG_RSEQ_ALLOC_SIZE; - return alloc_size; -} - /* * Return the feature size supported by the kernel. * * Depending on the value returned by getauxval(AT_RSEQ_FEATURE_SIZE): * - * 0: Return ORIG_RSEQ_FEATURE_SIZE (20) + * 0: Return ORIG_RSEQ_FEATURE_SIZE (20) * > 0: Return the value from getauxval(AT_RSEQ_FEATURE_SIZE). * * It should never return a value below ORIG_RSEQ_FEATURE_SIZE. */ -static -unsigned int get_rseq_kernel_feature_size(void) +static unsigned int get_rseq_kernel_feature_size(void) { unsigned long auxv_rseq_feature_size, auxv_rseq_align; @@ -152,15 +141,24 @@ unsigned int get_rseq_kernel_feature_size(void) return ORIG_RSEQ_FEATURE_SIZE; } -int rseq_register_current_thread(void) +int __rseq_register_current_thread(bool nolibc, bool legacy) { + unsigned int size; int rc; if (!rseq_ownership) { /* Treat libc's ownership as a successful registration. */ - return 0; + return nolibc ? -EBUSY : 0; } - rc = sys_rseq(&__rseq.abi, get_rseq_min_alloc_size(), 0, RSEQ_SIG); + + /* The minimal allocation size is 32, which is the legacy allocation size */ + size = get_rseq_kernel_feature_size(); + if (legacy || size < ORIG_RSEQ_ALLOC_SIZE) + rseq_alloc_size = ORIG_RSEQ_ALLOC_SIZE; + else + rseq_alloc_size = size; + + rc = sys_rseq(&__rseq.abi, rseq_alloc_size, 0, RSEQ_SIG); if (rc) { /* * After at least one thread has registered successfully @@ -179,9 +177,8 @@ int rseq_register_current_thread(void) * The first thread to register sets the rseq_size to mimic the libc * behavior. */ - if (RSEQ_READ_ONCE(rseq_size) == 0) { - RSEQ_WRITE_ONCE(rseq_size, get_rseq_kernel_feature_size()); - } + if (RSEQ_READ_ONCE(rseq_size) == 0) + RSEQ_WRITE_ONCE(rseq_size, size); return 0; } @@ -194,7 +191,7 @@ int rseq_unregister_current_thread(void) /* Treat libc's ownership as a successful unregistration. */ return 0; } - rc = sys_rseq(&__rseq.abi, get_rseq_min_alloc_size(), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); + rc = sys_rseq(&__rseq.abi, rseq_alloc_size, RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); if (rc) return -1; return 0; diff --git a/tools/testing/selftests/rseq/rseq.h b/tools/testing/selftests/rseq/rseq.h index f51a5fdb044431..c62ebb9290c010 100644 --- a/tools/testing/selftests/rseq/rseq.h +++ b/tools/testing/selftests/rseq/rseq.h @@ -8,6 +8,7 @@ #ifndef RSEQ_H #define RSEQ_H +#include #include #include #include @@ -142,7 +143,12 @@ static inline struct rseq_abi *rseq_get_abi(void) * succeed. A restartable sequence executed from a non-registered * thread will always fail. */ -int rseq_register_current_thread(void); +int __rseq_register_current_thread(bool nolibc, bool legacy); + +static inline int rseq_register_current_thread(void) +{ + return __rseq_register_current_thread(false, false); +} /* * Unregister rseq for current thread. diff --git a/tools/testing/selftests/rseq/run_legacy_check.sh b/tools/testing/selftests/rseq/run_legacy_check.sh new file mode 100755 index 00000000000000..5577b46ea09272 --- /dev/null +++ b/tools/testing/selftests/rseq/run_legacy_check.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +GLIBC_TUNABLES="${GLIBC_TUNABLES:-}:glibc.pthread.rseq=0" ./legacy_check diff --git a/tools/testing/selftests/rseq/run_param_test.sh b/tools/testing/selftests/rseq/run_param_test.sh index 8d31426ab41f22..69a3fa049929f3 100755 --- a/tools/testing/selftests/rseq/run_param_test.sh +++ b/tools/testing/selftests/rseq/run_param_test.sh @@ -34,6 +34,11 @@ REPS=1000 SLOW_REPS=100 NR_THREADS=$((6*${NR_CPUS})) +# Prevent GLIBC from registering RSEQ so the selftest can run in legacy and +# performance optimized mode. +GLIBC_TUNABLES="${GLIBC_TUNABLES:-}:glibc.pthread.rseq=0" +export GLIBC_TUNABLES + function do_tests() { local i=0 @@ -103,6 +108,40 @@ function inject_blocking() NR_LOOPS= } +echo "Testing in legacy RSEQ mode" +echo "Yield injection (25%)" +inject_blocking -m 4 -y -L + +echo "Yield injection (50%)" +inject_blocking -m 2 -y -L + +echo "Yield injection (100%)" +inject_blocking -m 1 -y -L + +echo "Kill injection (25%)" +inject_blocking -m 4 -k -L + +echo "Kill injection (50%)" +inject_blocking -m 2 -k -L + +echo "Kill injection (100%)" +inject_blocking -m 1 -k -L + +echo "Sleep injection (1ms, 25%)" +inject_blocking -m 4 -s 1 -L + +echo "Sleep injection (1ms, 50%)" +inject_blocking -m 2 -s 1 -L + +echo "Sleep injection (1ms, 100%)" +inject_blocking -m 1 -s 1 -L + +./check_optimized || { + echo "Skipping optimized RSEQ mode test. Not supported"; + exit 0 +} + +echo "Testing in optimized RSEQ mode" echo "Yield injection (25%)" inject_blocking -m 4 -y diff --git a/tools/testing/selftests/rseq/run_timeslice_test.sh b/tools/testing/selftests/rseq/run_timeslice_test.sh new file mode 100755 index 00000000000000..551ebed71ec61c --- /dev/null +++ b/tools/testing/selftests/rseq/run_timeslice_test.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0+ + +# Prevent GLIBC from registering RSEQ so the selftest can run in legacy +# and performance optimized mode. +GLIBC_TUNABLES="${GLIBC_TUNABLES:-}:glibc.pthread.rseq=0" +export GLIBC_TUNABLES + +./check_optimized || { + echo "Skipping optimized RSEQ mode test. Not supported"; + exit 0 +} + +./slice_test diff --git a/tools/testing/selftests/rseq/slice_test.c b/tools/testing/selftests/rseq/slice_test.c index 357122dcb48703..e402d4440bc27f 100644 --- a/tools/testing/selftests/rseq/slice_test.c +++ b/tools/testing/selftests/rseq/slice_test.c @@ -124,6 +124,13 @@ FIXTURE_SETUP(slice_ext) { cpu_set_t affinity; + if (__rseq_register_current_thread(true, false)) + SKIP(return, "RSEQ not supported\n"); + + if (prctl(PR_RSEQ_SLICE_EXTENSION, PR_RSEQ_SLICE_EXTENSION_SET, + PR_RSEQ_SLICE_EXT_ENABLE, 0, 0)) + SKIP(return, "Time slice extension not supported\n"); + ASSERT_EQ(sched_getaffinity(0, sizeof(affinity), &affinity), 0); /* Pin it on a single CPU. Avoid CPU 0 */ @@ -137,11 +144,6 @@ FIXTURE_SETUP(slice_ext) break; } - ASSERT_EQ(rseq_register_current_thread(), 0); - - ASSERT_EQ(prctl(PR_RSEQ_SLICE_EXTENSION, PR_RSEQ_SLICE_EXTENSION_SET, - PR_RSEQ_SLICE_EXT_ENABLE, 0, 0), 0); - self->noise_params.noise_nsecs = variant->noise_nsecs; self->noise_params.sleep_nsecs = variant->sleep_nsecs; self->noise_params.run = 1; diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index ee25824b1cbe6d..b987611789d16e 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -33,7 +33,7 @@ static enum scx_test_status run(void *ctx) skel = exit__open(); SCX_ENUM_INIT(skel); skel->rodata->exit_point = tc; - exit__load(skel); + SCX_FAIL_IF(exit__load(skel), "Failed to load skel"); link = bpf_map__attach_struct_ops(skel->maps.exit_ops); if (!link) { SCX_ERR("Failed to attach scheduler"); diff --git a/tools/testing/selftests/vfio/Makefile b/tools/testing/selftests/vfio/Makefile index 8e90e409e91d8d..0684932d91bfcb 100644 --- a/tools/testing/selftests/vfio/Makefile +++ b/tools/testing/selftests/vfio/Makefile @@ -1,6 +1,6 @@ ARCH ?= $(shell uname -m) -ifeq (,$(filter $(ARCH),arm64 x86_64)) +ifeq (,$(filter $(ARCH),aarch64 arm64 x86_64)) # Do nothing on unsupported architectures include ../lib.mk else diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index 4e5871f1ebc3bc..fc75e04ef010cf 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -30,10 +30,12 @@ static void vfio_pci_irq_set(struct vfio_pci_device *device, u32 index, u32 vector, u32 count, int *fds) { - u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count] = {}; + u8 buf[sizeof(struct vfio_irq_set) + sizeof(int) * count]; struct vfio_irq_set *irq = (void *)&buf; int *irq_fds = (void *)&irq->data; + memset(buf, 0, sizeof(buf)); + irq->argsz = sizeof(buf); irq->flags = VFIO_IRQ_SET_ACTION_TRIGGER; irq->index = index; diff --git a/tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c b/tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c index 957a89ce7b3a02..d7f25ef7767158 100644 --- a/tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c +++ b/tools/testing/selftests/vfio/vfio_dma_mapping_mmio_test.c @@ -100,7 +100,6 @@ static void do_mmio_map_test(struct iommu *iommu, iommu_unmap(iommu, ®ion); } else { VFIO_ASSERT_NE(__iommu_map(iommu, ®ion), 0); - VFIO_ASSERT_NE(__iommu_unmap(iommu, ®ion, NULL), 0); } } diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 3078ff1487d3fc..79d34f44821713 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -1071,8 +1071,17 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) static inline void set_vma_from_desc(struct vm_area_struct *vma, struct vm_area_desc *desc); -static inline int __compat_vma_mmap(const struct file_operations *f_op, - struct file *file, struct vm_area_struct *vma) +static inline unsigned long vma_pages(struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; +} + +static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) +{ + return file->f_op->mmap_prepare(desc); +} + +static inline int compat_vma_mmap(struct file *file, struct vm_area_struct *vma) { struct vm_area_desc desc = { .mm = vma->vm_mm, @@ -1082,40 +1091,38 @@ static inline int __compat_vma_mmap(const struct file_operations *f_op, .pgoff = vma->vm_pgoff, .vm_file = vma->vm_file, - .vm_flags = vma->vm_flags, + .vma_flags = vma->flags, .page_prot = vma->vm_page_prot, .action.type = MMAP_NOTHING, /* Default */ }; int err; - err = f_op->mmap_prepare(&desc); + err = vfs_mmap_prepare(file, &desc); + if (err) + return err; + + err = mmap_action_prepare(&desc); if (err) return err; - mmap_action_prepare(&desc.action, &desc); set_vma_from_desc(vma, &desc); - return mmap_action_complete(&desc.action, vma); -} + err = mmap_action_complete(vma, &desc.action, + /*is_compat=*/true); + if (err) { + const size_t len = vma_pages(vma) << PAGE_SHIFT; -static inline int compat_vma_mmap(struct file *file, - struct vm_area_struct *vma) -{ - return __compat_vma_mmap(file->f_op, file, vma); + do_munmap(current->mm, vma->vm_start, len, NULL); + } + return err; } - static inline void vma_iter_init(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long addr) { mas_init(&vmi->mas, &mm->mm_mt, addr); } -static inline unsigned long vma_pages(struct vm_area_struct *vma) -{ - return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; -} - static inline void mmap_assert_locked(struct mm_struct *); static inline struct vm_area_struct *find_vma_intersection(struct mm_struct *mm, unsigned long start_addr, @@ -1306,11 +1313,6 @@ static inline int vfs_mmap(struct file *file, struct vm_area_struct *vma) return file->f_op->mmap(file, vma); } -static inline int vfs_mmap_prepare(struct file *file, struct vm_area_desc *desc) -{ - return file->f_op->mmap_prepare(desc); -} - static inline void vma_set_file(struct vm_area_struct *vma, struct file *file) { /* Changing an anonymous vma with this is illegal */ diff --git a/tools/testing/vma/include/stubs.h b/tools/testing/vma/include/stubs.h index 947a3a0c256658..c56d96979d4d21 100644 --- a/tools/testing/vma/include/stubs.h +++ b/tools/testing/vma/include/stubs.h @@ -81,13 +81,14 @@ static inline void free_anon_vma_name(struct vm_area_struct *vma) { } -static inline void mmap_action_prepare(struct mmap_action *action, - struct vm_area_desc *desc) +static inline int mmap_action_prepare(struct vm_area_desc *desc) { + return 0; } -static inline int mmap_action_complete(struct mmap_action *action, - struct vm_area_struct *vma) +static inline int mmap_action_complete(struct vm_area_struct *vma, + struct mmap_action *action, + bool is_compat) { return 0; } @@ -426,3 +427,8 @@ static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, } static inline void hugetlb_split(struct vm_area_struct *, unsigned long) {} + +static inline bool vma_supports_mlock(const struct vm_area_struct *vma) +{ + return false; +} diff --git a/tools/tracing/rtla/src/common.c b/tools/tracing/rtla/src/common.c index ceff76a62a30b8..839c78c065e12f 100644 --- a/tools/tracing/rtla/src/common.c +++ b/tools/tracing/rtla/src/common.c @@ -39,6 +39,48 @@ static void set_signals(struct common_params *params) } } +/* + * unset_signals - unsets the signals to stop the tool + */ +static void unset_signals(struct common_params *params) +{ + signal(SIGINT, SIG_DFL); + if (params->duration) { + alarm(0); + signal(SIGALRM, SIG_DFL); + } +} + +/* + * getopt_auto - auto-generates optstring from long_options + */ +int getopt_auto(int argc, char **argv, const struct option *long_opts) +{ + char opts[256]; + int n = 0; + + for (int i = 0; long_opts[i].name; i++) { + if (long_opts[i].val < 32 || long_opts[i].val > 127) + continue; + + if (n + 4 >= sizeof(opts)) + fatal("optstring buffer overflow"); + + opts[n++] = long_opts[i].val; + + if (long_opts[i].has_arg == required_argument) + opts[n++] = ':'; + else if (long_opts[i].has_arg == optional_argument) { + opts[n++] = ':'; + opts[n++] = ':'; + } + } + + opts[n] = '\0'; + + return getopt_long(argc, argv, opts, long_opts, NULL); +} + /* * common_parse_options - parse common command line options * @@ -69,7 +111,7 @@ int common_parse_options(int argc, char **argv, struct common_params *common) }; opterr = 0; - c = getopt_long(argc, argv, "c:C::Dd:e:H:P:", long_options, NULL); + c = getopt_auto(argc, argv, long_options); opterr = 1; switch (c) { @@ -284,7 +326,7 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[]) retval = ops->main(tool); if (retval) - goto out_trace; + goto out_signals; if (params->user_workload && !params->user.stopped_running) { params->user.should_run = 0; @@ -306,6 +348,8 @@ int run_tool(struct tool_ops *ops, int argc, char *argv[]) if (ops->analyze) ops->analyze(tool, stopped); +out_signals: + unset_signals(params); out_trace: trace_events_destroy(&tool->record->trace, params->events); params->events = NULL; diff --git a/tools/tracing/rtla/src/common.h b/tools/tracing/rtla/src/common.h index 7602c5593ef5d9..d4b3715700be73 100644 --- a/tools/tracing/rtla/src/common.h +++ b/tools/tracing/rtla/src/common.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ #pragma once +#include #include "actions.h" #include "timerlat_u.h" #include "trace.h" @@ -156,6 +157,7 @@ int osnoise_set_stop_us(struct osnoise_context *context, long long stop_us); int osnoise_set_stop_total_us(struct osnoise_context *context, long long stop_total_us); +int getopt_auto(int argc, char **argv, const struct option *long_opts); int common_parse_options(int argc, char **argv, struct common_params *common); int common_apply_config(struct osnoise_tool *tool, struct common_params *params); int top_main_loop(struct osnoise_tool *tool); diff --git a/tools/tracing/rtla/src/osnoise_hist.c b/tools/tracing/rtla/src/osnoise_hist.c index 9d70ea34807ffb..5c863e7aad28b4 100644 --- a/tools/tracing/rtla/src/osnoise_hist.c +++ b/tools/tracing/rtla/src/osnoise_hist.c @@ -506,8 +506,7 @@ static struct common_params if (common_parse_options(argc, argv, ¶ms->common)) continue; - c = getopt_long(argc, argv, "a:b:E:hp:r:s:S:t::T:01234:5:6:7:", - long_options, NULL); + c = getopt_auto(argc, argv, long_options); /* detect the end of the options. */ if (c == -1) diff --git a/tools/tracing/rtla/src/osnoise_top.c b/tools/tracing/rtla/src/osnoise_top.c index d54d47947fb440..b7aed40fd2164f 100644 --- a/tools/tracing/rtla/src/osnoise_top.c +++ b/tools/tracing/rtla/src/osnoise_top.c @@ -358,8 +358,7 @@ struct common_params *osnoise_top_parse_args(int argc, char **argv) if (common_parse_options(argc, argv, ¶ms->common)) continue; - c = getopt_long(argc, argv, "a:hp:qr:s:S:t::T:0:1:2:3:", - long_options, NULL); + c = getopt_auto(argc, argv, long_options); /* Detect the end of the options. */ if (c == -1) diff --git a/tools/tracing/rtla/src/timerlat_hist.c b/tools/tracing/rtla/src/timerlat_hist.c index 4e8c38a61197c3..096de8ba3efbb6 100644 --- a/tools/tracing/rtla/src/timerlat_hist.c +++ b/tools/tracing/rtla/src/timerlat_hist.c @@ -825,8 +825,7 @@ static struct common_params if (common_parse_options(argc, argv, ¶ms->common)) continue; - c = getopt_long(argc, argv, "a:b:E:hi:knp:s:t::T:uU0123456:7:8:9\1\2:\3:", - long_options, NULL); + c = getopt_auto(argc, argv, long_options); /* detect the end of the options. */ if (c == -1) diff --git a/tools/tracing/rtla/src/timerlat_top.c b/tools/tracing/rtla/src/timerlat_top.c index 284b74773c2b5b..27c14aa71a8bc8 100644 --- a/tools/tracing/rtla/src/timerlat_top.c +++ b/tools/tracing/rtla/src/timerlat_top.c @@ -588,8 +588,7 @@ static struct common_params if (common_parse_options(argc, argv, ¶ms->common)) continue; - c = getopt_long(argc, argv, "a:hi:knp:qs:t::T:uU0:1:2:345:6:7:", - long_options, NULL); + c = getopt_auto(argc, argv, long_options); /* detect the end of the options. */ if (c == -1) diff --git a/tools/tracing/rtla/src/utils.c b/tools/tracing/rtla/src/utils.c index 0da3b2470c3179..8d5e3b9724b917 100644 --- a/tools/tracing/rtla/src/utils.c +++ b/tools/tracing/rtla/src/utils.c @@ -361,22 +361,23 @@ int set_comm_sched_attr(const char *comm_prefix, struct sched_attr *attr) if (strtoi(proc_entry->d_name, &pid)) { err_msg("'%s' is not a valid pid", proc_entry->d_name); - goto out_err; + retval = 1; + goto out; } /* procfs_is_workload_pid confirmed it is a pid */ retval = __set_sched_attr(pid, attr); if (retval) { err_msg("Error setting sched attributes for pid:%s\n", proc_entry->d_name); - goto out_err; + goto out; } debug_msg("Set sched attributes for pid:%s\n", proc_entry->d_name); } - return 0; -out_err: + retval = 0; +out: closedir(procfs); - return 1; + return retval; } #define INVALID_VAL (~0L) diff --git a/virt/kvm/dirty_ring.c b/virt/kvm/dirty_ring.c index 02bc6b00d76cbd..572b854edf740d 100644 --- a/virt/kvm/dirty_ring.c +++ b/virt/kvm/dirty_ring.c @@ -63,7 +63,8 @@ static void kvm_reset_dirty_gfn(struct kvm *kvm, u32 slot, u64 offset, u64 mask) memslot = id_to_memslot(__kvm_memslots(kvm, as_id), id); - if (!memslot || (offset + __fls(mask)) >= memslot->npages) + if (!memslot || offset >= memslot->npages || + offset + __fls(mask) >= memslot->npages) return; KVM_MMU_LOCK(kvm);