diff options
941 files changed, 19915 insertions, 13128 deletions
@@ -1864,10 +1864,11 @@ S: The Netherlands N: Martin Kepplinger E: martink@posteo.de -E: martin.kepplinger@theobroma-systems.com +E: martin.kepplinger@ginzinger.com W: http://www.martinkepplinger.com D: mma8452 accelerators iio driver -D: Kernel cleanups +D: pegasus_notetaker input driver +D: Kernel fixes and cleanups S: Garnisonstraße 26 S: 4020 Linz S: Austria diff --git a/Documentation/device-mapper/dm-raid.txt b/Documentation/device-mapper/dm-raid.txt index e5b6497116f4..c75b64a85859 100644 --- a/Documentation/device-mapper/dm-raid.txt +++ b/Documentation/device-mapper/dm-raid.txt @@ -309,3 +309,4 @@ Version History with a reshape in progress. 1.9.0 Add support for RAID level takeover/reshape/region size and set size reduction. +1.9.1 Fix activation of existing RAID 4/10 mapped devices diff --git a/Documentation/devicetree/bindings/clock/uniphier-clock.txt b/Documentation/devicetree/bindings/clock/uniphier-clock.txt index c7179d3b5c33..812163060fa3 100644 --- a/Documentation/devicetree/bindings/clock/uniphier-clock.txt +++ b/Documentation/devicetree/bindings/clock/uniphier-clock.txt @@ -24,7 +24,7 @@ Example: reg = <0x61840000 0x4000>; clock { - compatible = "socionext,uniphier-ld20-clock"; + compatible = "socionext,uniphier-ld11-clock"; #clock-cells = <1>; }; @@ -43,8 +43,8 @@ Provided clocks: 21: USB3 ch1 PHY1 -Media I/O (MIO) clock ---------------------- +Media I/O (MIO) clock, SD clock +------------------------------- Required properties: - compatible: should be one of the following: @@ -52,10 +52,10 @@ Required properties: "socionext,uniphier-ld4-mio-clock" - for LD4 SoC. "socionext,uniphier-pro4-mio-clock" - for Pro4 SoC. "socionext,uniphier-sld8-mio-clock" - for sLD8 SoC. - "socionext,uniphier-pro5-mio-clock" - for Pro5 SoC. - "socionext,uniphier-pxs2-mio-clock" - for PXs2/LD6b SoC. + "socionext,uniphier-pro5-sd-clock" - for Pro5 SoC. + "socionext,uniphier-pxs2-sd-clock" - for PXs2/LD6b SoC. "socionext,uniphier-ld11-mio-clock" - for LD11 SoC. - "socionext,uniphier-ld20-mio-clock" - for LD20 SoC. + "socionext,uniphier-ld20-sd-clock" - for LD20 SoC. - #clock-cells: should be 1. Example: @@ -66,7 +66,7 @@ Example: reg = <0x59810000 0x800>; clock { - compatible = "socionext,uniphier-ld20-mio-clock"; + compatible = "socionext,uniphier-ld11-mio-clock"; #clock-cells = <1>; }; @@ -112,7 +112,7 @@ Example: reg = <0x59820000 0x200>; clock { - compatible = "socionext,uniphier-ld20-peri-clock"; + compatible = "socionext,uniphier-ld11-peri-clock"; #clock-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/display/panel/display-timing.txt b/Documentation/devicetree/bindings/display/panel/display-timing.txt index e1d4a0b59612..81a75893d1b8 100644 --- a/Documentation/devicetree/bindings/display/panel/display-timing.txt +++ b/Documentation/devicetree/bindings/display/panel/display-timing.txt @@ -32,6 +32,14 @@ optional properties: - active low = drive pixel data on falling edge/ sample data on rising edge - ignored = ignored + - syncclk-active: with + - active high = drive sync on rising edge/ + sample sync on falling edge of pixel + clock + - active low = drive sync on falling edge/ + sample sync on rising edge of pixel + clock + - omitted = same configuration as pixelclk-active - interlaced (bool): boolean to enable interlaced mode - doublescan (bool): boolean to enable doublescan mode - doubleclk (bool): boolean to enable doubleclock mode diff --git a/Documentation/devicetree/bindings/net/marvell-orion-net.txt b/Documentation/devicetree/bindings/net/marvell-orion-net.txt index bce52b2ec55e..6fd988c84c4f 100644 --- a/Documentation/devicetree/bindings/net/marvell-orion-net.txt +++ b/Documentation/devicetree/bindings/net/marvell-orion-net.txt @@ -49,6 +49,7 @@ Optional port properties: and - phy-handle: See ethernet.txt file in the same directory. + - phy-mode: See ethernet.txt file in the same directory. or diff --git a/Documentation/devicetree/bindings/reset/uniphier-reset.txt b/Documentation/devicetree/bindings/reset/uniphier-reset.txt index e6bbfccd56c3..5020524cddeb 100644 --- a/Documentation/devicetree/bindings/reset/uniphier-reset.txt +++ b/Documentation/devicetree/bindings/reset/uniphier-reset.txt @@ -6,25 +6,25 @@ System reset Required properties: - compatible: should be one of the following: - "socionext,uniphier-sld3-reset" - for PH1-sLD3 SoC. - "socionext,uniphier-ld4-reset" - for PH1-LD4 SoC. - "socionext,uniphier-pro4-reset" - for PH1-Pro4 SoC. - "socionext,uniphier-sld8-reset" - for PH1-sLD8 SoC. - "socionext,uniphier-pro5-reset" - for PH1-Pro5 SoC. - "socionext,uniphier-pxs2-reset" - for ProXstream2/PH1-LD6b SoC. - "socionext,uniphier-ld11-reset" - for PH1-LD11 SoC. - "socionext,uniphier-ld20-reset" - for PH1-LD20 SoC. + "socionext,uniphier-sld3-reset" - for sLD3 SoC. + "socionext,uniphier-ld4-reset" - for LD4 SoC. + "socionext,uniphier-pro4-reset" - for Pro4 SoC. + "socionext,uniphier-sld8-reset" - for sLD8 SoC. + "socionext,uniphier-pro5-reset" - for Pro5 SoC. + "socionext,uniphier-pxs2-reset" - for PXs2/LD6b SoC. + "socionext,uniphier-ld11-reset" - for LD11 SoC. + "socionext,uniphier-ld20-reset" - for LD20 SoC. - #reset-cells: should be 1. Example: sysctrl@61840000 { - compatible = "socionext,uniphier-ld20-sysctrl", + compatible = "socionext,uniphier-ld11-sysctrl", "simple-mfd", "syscon"; reg = <0x61840000 0x4000>; reset { - compatible = "socionext,uniphier-ld20-reset"; + compatible = "socionext,uniphier-ld11-reset"; #reset-cells = <1>; }; @@ -32,30 +32,30 @@ Example: }; -Media I/O (MIO) reset ---------------------- +Media I/O (MIO) reset, SD reset +------------------------------- Required properties: - compatible: should be one of the following: - "socionext,uniphier-sld3-mio-reset" - for PH1-sLD3 SoC. - "socionext,uniphier-ld4-mio-reset" - for PH1-LD4 SoC. - "socionext,uniphier-pro4-mio-reset" - for PH1-Pro4 SoC. - "socionext,uniphier-sld8-mio-reset" - for PH1-sLD8 SoC. - "socionext,uniphier-pro5-mio-reset" - for PH1-Pro5 SoC. - "socionext,uniphier-pxs2-mio-reset" - for ProXstream2/PH1-LD6b SoC. - "socionext,uniphier-ld11-mio-reset" - for PH1-LD11 SoC. - "socionext,uniphier-ld20-mio-reset" - for PH1-LD20 SoC. + "socionext,uniphier-sld3-mio-reset" - for sLD3 SoC. + "socionext,uniphier-ld4-mio-reset" - for LD4 SoC. + "socionext,uniphier-pro4-mio-reset" - for Pro4 SoC. + "socionext,uniphier-sld8-mio-reset" - for sLD8 SoC. + "socionext,uniphier-pro5-sd-reset" - for Pro5 SoC. + "socionext,uniphier-pxs2-sd-reset" - for PXs2/LD6b SoC. + "socionext,uniphier-ld11-mio-reset" - for LD11 SoC. + "socionext,uniphier-ld20-sd-reset" - for LD20 SoC. - #reset-cells: should be 1. Example: mioctrl@59810000 { - compatible = "socionext,uniphier-ld20-mioctrl", + compatible = "socionext,uniphier-ld11-mioctrl", "simple-mfd", "syscon"; reg = <0x59810000 0x800>; reset { - compatible = "socionext,uniphier-ld20-mio-reset"; + compatible = "socionext,uniphier-ld11-mio-reset"; #reset-cells = <1>; }; @@ -68,24 +68,24 @@ Peripheral reset Required properties: - compatible: should be one of the following: - "socionext,uniphier-ld4-peri-reset" - for PH1-LD4 SoC. - "socionext,uniphier-pro4-peri-reset" - for PH1-Pro4 SoC. - "socionext,uniphier-sld8-peri-reset" - for PH1-sLD8 SoC. - "socionext,uniphier-pro5-peri-reset" - for PH1-Pro5 SoC. - "socionext,uniphier-pxs2-peri-reset" - for ProXstream2/PH1-LD6b SoC. - "socionext,uniphier-ld11-peri-reset" - for PH1-LD11 SoC. - "socionext,uniphier-ld20-peri-reset" - for PH1-LD20 SoC. + "socionext,uniphier-ld4-peri-reset" - for LD4 SoC. + "socionext,uniphier-pro4-peri-reset" - for Pro4 SoC. + "socionext,uniphier-sld8-peri-reset" - for sLD8 SoC. + "socionext,uniphier-pro5-peri-reset" - for Pro5 SoC. + "socionext,uniphier-pxs2-peri-reset" - for PXs2/LD6b SoC. + "socionext,uniphier-ld11-peri-reset" - for LD11 SoC. + "socionext,uniphier-ld20-peri-reset" - for LD20 SoC. - #reset-cells: should be 1. Example: perictrl@59820000 { - compatible = "socionext,uniphier-ld20-perictrl", + compatible = "socionext,uniphier-ld11-perictrl", "simple-mfd", "syscon"; reg = <0x59820000 0x200>; reset { - compatible = "socionext,uniphier-ld20-peri-reset"; + compatible = "socionext,uniphier-ld11-peri-reset"; #reset-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/serial/cdns,uart.txt b/Documentation/devicetree/bindings/serial/cdns,uart.txt index a3eb154c32ca..227bb770b027 100644 --- a/Documentation/devicetree/bindings/serial/cdns,uart.txt +++ b/Documentation/devicetree/bindings/serial/cdns,uart.txt @@ -1,7 +1,9 @@ Binding for Cadence UART Controller Required properties: -- compatible : should be "cdns,uart-r1p8", or "xlnx,xuartps" +- compatible : + Use "xlnx,xuartps","cdns,uart-r1p8" for Zynq-7xxx SoC. + Use "xlnx,zynqmp-uart","cdns,uart-r1p12" for Zynq Ultrascale+ MPSoC. - reg: Should contain UART controller registers location and length. - interrupts: Should contain UART controller interrupts. - clocks: Must contain phandles to the UART clocks diff --git a/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt b/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt index 1e4000d83aee..8d27d1a603e7 100644 --- a/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt +++ b/Documentation/devicetree/bindings/serial/renesas,sci-serial.txt @@ -9,6 +9,14 @@ Required properties: - "renesas,scifb-r8a73a4" for R8A73A4 (R-Mobile APE6) SCIFB compatible UART. - "renesas,scifa-r8a7740" for R8A7740 (R-Mobile A1) SCIFA compatible UART. - "renesas,scifb-r8a7740" for R8A7740 (R-Mobile A1) SCIFB compatible UART. + - "renesas,scif-r8a7743" for R8A7743 (RZ/G1M) SCIF compatible UART. + - "renesas,scifa-r8a7743" for R8A7743 (RZ/G1M) SCIFA compatible UART. + - "renesas,scifb-r8a7743" for R8A7743 (RZ/G1M) SCIFB compatible UART. + - "renesas,hscif-r8a7743" for R8A7743 (RZ/G1M) HSCIF compatible UART. + - "renesas,scif-r8a7745" for R8A7745 (RZ/G1E) SCIF compatible UART. + - "renesas,scifa-r8a7745" for R8A7745 (RZ/G1E) SCIFA compatible UART. + - "renesas,scifb-r8a7745" for R8A7745 (RZ/G1E) SCIFB compatible UART. + - "renesas,hscif-r8a7745" for R8A7745 (RZ/G1E) HSCIF compatible UART. - "renesas,scif-r8a7778" for R8A7778 (R-Car M1) SCIF compatible UART. - "renesas,scif-r8a7779" for R8A7779 (R-Car H1) SCIF compatible UART. - "renesas,scif-r8a7790" for R8A7790 (R-Car H2) SCIF compatible UART. diff --git a/Documentation/devicetree/bindings/usb/dwc2.txt b/Documentation/devicetree/bindings/usb/dwc2.txt index 455f2c310a1b..2c30a5479069 100644 --- a/Documentation/devicetree/bindings/usb/dwc2.txt +++ b/Documentation/devicetree/bindings/usb/dwc2.txt @@ -28,10 +28,7 @@ Refer to phy/phy-bindings.txt for generic phy consumer properties - g-use-dma: enable dma usage in gadget driver. - g-rx-fifo-size: size of rx fifo size in gadget mode. - g-np-tx-fifo-size: size of non-periodic tx fifo size in gadget mode. - -Deprecated properties: -- g-tx-fifo-size: size of periodic tx fifo per endpoint (except ep0) - in gadget mode. +- g-tx-fifo-size: size of periodic tx fifo per endpoint (except ep0) in gadget mode. Example: diff --git a/Documentation/gpio/board.txt b/Documentation/gpio/board.txt index 40884c4fe40c..a0f61898d493 100644 --- a/Documentation/gpio/board.txt +++ b/Documentation/gpio/board.txt @@ -6,7 +6,7 @@ Note that it only applies to the new descriptor-based interface. For a description of the deprecated integer-based GPIO interface please refer to gpio-legacy.txt (actually, there is no real mapping possible with the old interface; you just fetch an integer from somewhere and request the -corresponding GPIO. +corresponding GPIO). All platforms can enable the GPIO library, but if the platform strictly requires GPIO functionality to be present, it needs to select GPIOLIB from its @@ -162,6 +162,9 @@ The driver controlling "foo.0" will then be able to obtain its GPIOs as follows: Since the "led" GPIOs are mapped as active-high, this example will switch their signals to 1, i.e. enabling the LEDs. And for the "power" GPIO, which is mapped -as active-low, its actual signal will be 0 after this code. Contrary to the legacy -integer GPIO interface, the active-low property is handled during mapping and is -thus transparent to GPIO consumers. +as active-low, its actual signal will be 0 after this code. Contrary to the +legacy integer GPIO interface, the active-low property is handled during +mapping and is thus transparent to GPIO consumers. + +A set of functions such as gpiod_set_value() is available to work with +the new descriptor-oriented interface. diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst index 95ce77ff4342..ba83b7d88f1f 100644 --- a/Documentation/gpu/i915.rst +++ b/Documentation/gpu/i915.rst @@ -189,7 +189,7 @@ Display Refresh Rate Switching (DRRS) DPIO ---- -.. kernel-doc:: drivers/gpu/drm/i915/i915_reg.h +.. kernel-doc:: drivers/gpu/drm/i915/intel_dpio_phy.c :doc: DPIO CSR firmware support for DMC diff --git a/Documentation/networking/netdev-FAQ.txt b/Documentation/networking/netdev-FAQ.txt index 0fe1c6e0dbcd..a20b2fae942b 100644 --- a/Documentation/networking/netdev-FAQ.txt +++ b/Documentation/networking/netdev-FAQ.txt @@ -29,8 +29,8 @@ A: There are always two trees (git repositories) in play. Both are driven Linus, and net-next is where the new code goes for the future release. You can find the trees here: - http://git.kernel.org/?p=linux/kernel/git/davem/net.git - http://git.kernel.org/?p=linux/kernel/git/davem/net-next.git + https://git.kernel.org/pub/scm/linux/kernel/git/davem/net.git + https://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next.git Q: How often do changes from these trees make it to the mainline Linus tree? @@ -76,7 +76,7 @@ Q: So where are we now in this cycle? A: Load the mainline (Linus) page here: - http://git.kernel.org/?p=linux/kernel/git/torvalds/linux.git + https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git and note the top of the "tags" section. If it is rc1, it is early in the dev cycle. If it was tagged rc7 a week ago, then a release @@ -123,7 +123,7 @@ A: Normally Greg Kroah-Hartman collects stable commits himself, but It contains the patches which Dave has selected, but not yet handed off to Greg. If Greg already has the patch, then it will be here: - http://git.kernel.org/cgit/linux/kernel/git/stable/stable-queue.git + https://git.kernel.org/pub/scm/linux/kernel/git/stable/stable-queue.git A quick way to find whether the patch is in this stable-queue is to simply clone the repo, and then git grep the mainline commit ID, e.g. diff --git a/Documentation/networking/nf_conntrack-sysctl.txt b/Documentation/networking/nf_conntrack-sysctl.txt index 4fb51d32fccc..399e4e866a9c 100644 --- a/Documentation/networking/nf_conntrack-sysctl.txt +++ b/Documentation/networking/nf_conntrack-sysctl.txt @@ -33,24 +33,6 @@ nf_conntrack_events - BOOLEAN If this option is enabled, the connection tracking code will provide userspace with connection tracking events via ctnetlink. -nf_conntrack_events_retry_timeout - INTEGER (seconds) - default 15 - - This option is only relevant when "reliable connection tracking - events" are used. Normally, ctnetlink is "lossy", that is, - events are normally dropped when userspace listeners can't keep up. - - Userspace can request "reliable event mode". When this mode is - active, the conntrack will only be destroyed after the event was - delivered. If event delivery fails, the kernel periodically - re-tries to send the event to userspace. - - This is the maximum interval the kernel should use when re-trying - to deliver the destroy event. - - A higher number means there will be fewer delivery retries and it - will take longer for a backlog to be processed. - nf_conntrack_expect_max - INTEGER Maximum size of expectation table. Default value is nf_conntrack_buckets / 256. Minimum is 1. diff --git a/Documentation/virtual/kvm/locking.txt b/Documentation/virtual/kvm/locking.txt index f2491a8c68b4..e5dd9f4d6100 100644 --- a/Documentation/virtual/kvm/locking.txt +++ b/Documentation/virtual/kvm/locking.txt @@ -4,7 +4,17 @@ KVM Lock Overview 1. Acquisition Orders --------------------- -(to be written) +The acquisition orders for mutexes are as follows: + +- kvm->lock is taken outside vcpu->mutex + +- kvm->lock is taken outside kvm->slots_lock and kvm->irq_lock + +- kvm->slots_lock is taken outside kvm->irq_lock, though acquiring + them together is quite rare. + +For spinlocks, kvm_lock is taken outside kvm->mmu_lock. Everything +else is a leaf: no other lock is taken inside the critical sections. 2: Exception ------------ diff --git a/MAINTAINERS b/MAINTAINERS index 10f1bc05e4e9..561d4174cdfe 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1442,6 +1442,7 @@ F: drivers/cpufreq/mvebu-cpufreq.c F: arch/arm/configs/mvebu_*_defconfig ARM/Marvell Berlin SoC support +M: Jisheng Zhang <jszhang@marvell.com> M: Sebastian Hesselbarth <sebastian.hesselbarth@gmail.com> L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained @@ -2551,15 +2552,18 @@ S: Supported F: drivers/net/ethernet/broadcom/genet/ BROADCOM BNX2 GIGABIT ETHERNET DRIVER -M: Sony Chacko <sony.chacko@qlogic.com> -M: Dept-HSGLinuxNICDev@qlogic.com +M: Rasesh Mody <rasesh.mody@cavium.com> +M: Harish Patil <harish.patil@cavium.com> +M: Dept-GELinuxNICDev@cavium.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/bnx2.* F: drivers/net/ethernet/broadcom/bnx2_* BROADCOM BNX2X 10 GIGABIT ETHERNET DRIVER -M: Ariel Elior <ariel.elior@qlogic.com> +M: Yuval Mintz <Yuval.Mintz@cavium.com> +M: Ariel Elior <ariel.elior@cavium.com> +M: everest-linux-l2@cavium.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/broadcom/bnx2x/ @@ -2766,7 +2770,9 @@ S: Supported F: drivers/scsi/bfa/ BROCADE BNA 10 GIGABIT ETHERNET DRIVER -M: Rasesh Mody <rasesh.mody@qlogic.com> +M: Rasesh Mody <rasesh.mody@cavium.com> +M: Sudarsana Kalluru <sudarsana.kalluru@cavium.com> +M: Dept-GELinuxNICDev@cavium.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/brocade/bna/ @@ -4056,7 +4062,6 @@ INTEL DRM DRIVERS (excluding Poulsbo, Moorestown and derivative chipsets) M: Daniel Vetter <daniel.vetter@intel.com> M: Jani Nikula <jani.nikula@linux.intel.com> L: intel-gfx@lists.freedesktop.org -L: dri-devel@lists.freedesktop.org W: https://01.org/linuxgraphics/ Q: http://patchwork.freedesktop.org/project/intel-gfx/ T: git git://anongit.freedesktop.org/drm-intel @@ -5299,6 +5304,12 @@ M: Joe Perches <joe@perches.com> S: Maintained F: scripts/get_maintainer.pl +GENWQE (IBM Generic Workqueue Card) +M: Frank Haverkamp <haver@linux.vnet.ibm.com> +M: Gabriel Krisman Bertazi <krisman@linux.vnet.ibm.com> +S: Supported +F: drivers/misc/genwqe/ + GFS2 FILE SYSTEM M: Steven Whitehouse <swhiteho@redhat.com> M: Bob Peterson <rpeterso@redhat.com> @@ -7925,6 +7936,10 @@ F: mm/ MEMORY TECHNOLOGY DEVICES (MTD) M: David Woodhouse <dwmw2@infradead.org> M: Brian Norris <computersforpeace@gmail.com> +M: Boris Brezillon <boris.brezillon@free-electrons.com> +M: Marek Vasut <marek.vasut@gmail.com> +M: Richard Weinberger <richard@nod.at> +M: Cyrille Pitchen <cyrille.pitchen@atmel.com> L: linux-mtd@lists.infradead.org W: http://www.linux-mtd.infradead.org/ Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ @@ -8112,6 +8127,7 @@ S: Maintained F: drivers/media/dvb-frontends/mn88473* MODULE SUPPORT +M: Jessica Yu <jeyu@redhat.com> M: Rusty Russell <rusty@rustcorp.com.au> S: Maintained F: include/linux/module.h @@ -8521,11 +8537,10 @@ F: Documentation/devicetree/bindings/net/wireless/ F: drivers/net/wireless/ NETXEN (1/10) GbE SUPPORT -M: Manish Chopra <manish.chopra@qlogic.com> -M: Sony Chacko <sony.chacko@qlogic.com> -M: Rajesh Borundia <rajesh.borundia@qlogic.com> +M: Manish Chopra <manish.chopra@cavium.com> +M: Rahul Verma <rahul.verma@cavium.com> +M: Dept-GELinuxNICDev@cavium.com L: netdev@vger.kernel.org -W: http://www.qlogic.com S: Supported F: drivers/net/ethernet/qlogic/netxen/ @@ -9901,33 +9916,32 @@ F: Documentation/scsi/LICENSE.qla4xxx F: drivers/scsi/qla4xxx/ QLOGIC QLA3XXX NETWORK DRIVER -M: Jitendra Kalsaria <jitendra.kalsaria@qlogic.com> -M: Ron Mercer <ron.mercer@qlogic.com> -M: linux-driver@qlogic.com +M: Dept-GELinuxNICDev@cavium.com L: netdev@vger.kernel.org S: Supported F: Documentation/networking/LICENSE.qla3xxx F: drivers/net/ethernet/qlogic/qla3xxx.* QLOGIC QLCNIC (1/10)Gb ETHERNET DRIVER -M: Dept-GELinuxNICDev@qlogic.com +M: Harish Patil <harish.patil@cavium.com> +M: Manish Chopra <manish.chopra@cavium.com> +M: Dept-GELinuxNICDev@cavium.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/qlogic/qlcnic/ QLOGIC QLGE 10Gb ETHERNET DRIVER -M: Harish Patil <harish.patil@qlogic.com> -M: Sudarsana Kalluru <sudarsana.kalluru@qlogic.com> -M: Dept-GELinuxNICDev@qlogic.com -M: linux-driver@qlogic.com +M: Harish Patil <harish.patil@cavium.com> +M: Manish Chopra <manish.chopra@cavium.com> +M: Dept-GELinuxNICDev@cavium.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/qlogic/qlge/ QLOGIC QL4xxx ETHERNET DRIVER -M: Yuval Mintz <Yuval.Mintz@qlogic.com> -M: Ariel Elior <Ariel.Elior@qlogic.com> -M: everest-linux-l2@qlogic.com +M: Yuval Mintz <Yuval.Mintz@cavium.com> +M: Ariel Elior <Ariel.Elior@cavium.com> +M: everest-linux-l2@cavium.com L: netdev@vger.kernel.org S: Supported F: drivers/net/ethernet/qlogic/qed/ @@ -11405,6 +11419,17 @@ W: http://www.st.com/spear S: Maintained F: drivers/clk/spear/ +SPI NOR SUBSYSTEM +M: Cyrille Pitchen <cyrille.pitchen@atmel.com> +M: Marek Vasut <marek.vasut@gmail.com> +L: linux-mtd@lists.infradead.org +W: http://www.linux-mtd.infradead.org/ +Q: http://patchwork.ozlabs.org/project/linux-mtd/list/ +T: git git://github.com/spi-nor/linux.git +S: Maintained +F: drivers/mtd/spi-nor/ +F: include/linux/mtd/spi-nor.h + SPI SUBSYSTEM M: Mark Brown <broonie@kernel.org> L: linux-spi@vger.kernel.org @@ -12784,6 +12809,7 @@ F: include/uapi/linux/virtio_console.h VIRTIO CORE, NET AND BLOCK DRIVERS M: "Michael S. Tsirkin" <mst@redhat.com> +M: Jason Wang <jasowang@redhat.com> L: virtualization@lists.linux-foundation.org S: Maintained F: Documentation/devicetree/bindings/virtio/ @@ -12814,6 +12840,7 @@ F: include/uapi/linux/virtio_gpu.h VIRTIO HOST (VHOST) M: "Michael S. Tsirkin" <mst@redhat.com> +M: Jason Wang <jasowang@redhat.com> L: kvm@vger.kernel.org L: virtualization@lists.linux-foundation.org L: netdev@vger.kernel.org @@ -1,7 +1,7 @@ VERSION = 4 PATCHLEVEL = 9 SUBLEVEL = 0 -EXTRAVERSION = -rc2 +EXTRAVERSION = -rc4 NAME = Psychotic Stoned Sheep # *DOCUMENTATION* diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index ecd12379e2cd..bd204bfa29ed 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -41,6 +41,8 @@ config ARC select PERF_USE_VMALLOC select HAVE_DEBUG_STACKOVERFLOW select HAVE_GENERIC_DMA_COHERENT + select HAVE_KERNEL_GZIP + select HAVE_KERNEL_LZMA config MIGHT_HAVE_PCI bool @@ -186,14 +188,6 @@ if SMP config ARC_HAS_COH_CACHES def_bool n -config ARC_MCIP - bool "ARConnect Multicore IP (MCIP) Support " - depends on ISA_ARCV2 - help - This IP block enables SMP in ARC-HS38 cores. - It provides for cross-core interrupts, multi-core debug - hardware semaphores, shared memory,.... - config NR_CPUS int "Maximum number of CPUs (2-4096)" range 2 4096 @@ -211,6 +205,15 @@ config ARC_SMP_HALT_ON_RESET endif #SMP +config ARC_MCIP + bool "ARConnect Multicore IP (MCIP) Support " + depends on ISA_ARCV2 + default y if SMP + help + This IP block enables SMP in ARC-HS38 cores. + It provides for cross-core interrupts, multi-core debug + hardware semaphores, shared memory,.... + menuconfig ARC_CACHE bool "Enable Cache Support" default y @@ -537,14 +540,6 @@ config ARC_DBG_TLB_PARANOIA bool "Paranoia Checks in Low Level TLB Handlers" default n -config ARC_DBG_TLB_MISS_COUNT - bool "Profile TLB Misses" - default n - select DEBUG_FS - help - Counts number of I and D TLB Misses and exports them via Debugfs - The counters can be cleared via Debugfs as well - endif config ARC_UBOOT_SUPPORT diff --git a/arch/arc/Makefile b/arch/arc/Makefile index aa82d13d4213..864adad52280 100644 --- a/arch/arc/Makefile +++ b/arch/arc/Makefile @@ -50,9 +50,6 @@ atleast_gcc44 := $(call cc-ifversion, -ge, 0404, y) cflags-$(atleast_gcc44) += -fsection-anchors -cflags-$(CONFIG_ARC_HAS_LLSC) += -mlock -cflags-$(CONFIG_ARC_HAS_SWAPE) += -mswape - ifdef CONFIG_ISA_ARCV2 ifndef CONFIG_ARC_HAS_LL64 diff --git a/arch/arc/boot/Makefile b/arch/arc/boot/Makefile index e597cb34c16a..f94cf151e06a 100644 --- a/arch/arc/boot/Makefile +++ b/arch/arc/boot/Makefile @@ -14,9 +14,15 @@ UIMAGE_ENTRYADDR = $(LINUX_START_TEXT) suffix-y := bin suffix-$(CONFIG_KERNEL_GZIP) := gz +suffix-$(CONFIG_KERNEL_LZMA) := lzma -targets += uImage uImage.bin uImage.gz -extra-y += vmlinux.bin vmlinux.bin.gz +targets += uImage +targets += uImage.bin +targets += uImage.gz +targets += uImage.lzma +extra-y += vmlinux.bin +extra-y += vmlinux.bin.gz +extra-y += vmlinux.bin.lzma $(obj)/vmlinux.bin: vmlinux FORCE $(call if_changed,objcopy) @@ -24,12 +30,18 @@ $(obj)/vmlinux.bin: vmlinux FORCE $(obj)/vmlinux.bin.gz: $(obj)/vmlinux.bin FORCE $(call if_changed,gzip) +$(obj)/vmlinux.bin.lzma: $(obj)/vmlinux.bin FORCE + $(call if_changed,lzma) + $(obj)/uImage.bin: $(obj)/vmlinux.bin FORCE $(call if_changed,uimage,none) $(obj)/uImage.gz: $(obj)/vmlinux.bin.gz FORCE $(call if_changed,uimage,gzip) +$(obj)/uImage.lzma: $(obj)/vmlinux.bin.lzma FORCE + $(call if_changed,uimage,lzma) + $(obj)/uImage: $(obj)/uImage.$(suffix-y) @ln -sf $(notdir $<) $@ @echo ' Image $@ is ready' diff --git a/arch/arc/include/asm/arcregs.h b/arch/arc/include/asm/arcregs.h index db25c65155cb..7f3f9f63708c 100644 --- a/arch/arc/include/asm/arcregs.h +++ b/arch/arc/include/asm/arcregs.h @@ -349,10 +349,11 @@ struct cpuinfo_arc { struct cpuinfo_arc_bpu bpu; struct bcr_identity core; struct bcr_isa isa; + const char *details, *name; unsigned int vec_base; struct cpuinfo_arc_ccm iccm, dccm; struct { - unsigned int swap:1, norm:1, minmax:1, barrel:1, crc:1, pad1:3, + unsigned int swap:1, norm:1, minmax:1, barrel:1, crc:1, swape:1, pad1:2, fpu_sp:1, fpu_dp:1, pad2:6, debug:1, ap:1, smart:1, rtt:1, pad3:4, timer0:1, timer1:1, rtc:1, gfrc:1, pad4:4; diff --git a/arch/arc/include/asm/cache.h b/arch/arc/include/asm/cache.h index fb781e34f322..b3410ff6a62d 100644 --- a/arch/arc/include/asm/cache.h +++ b/arch/arc/include/asm/cache.h @@ -53,7 +53,7 @@ extern void arc_cache_init(void); extern char *arc_cache_mumbojumbo(int cpu_id, char *buf, int len); extern void read_decode_cache_bcr(void); -extern int ioc_exists; +extern int ioc_enable; extern unsigned long perip_base, perip_end; #endif /* !__ASSEMBLY__ */ diff --git a/arch/arc/include/asm/elf.h b/arch/arc/include/asm/elf.h index 7096f97a1434..aa2d6da9d187 100644 --- a/arch/arc/include/asm/elf.h +++ b/arch/arc/include/asm/elf.h @@ -54,7 +54,7 @@ extern int elf_check_arch(const struct elf32_hdr *); * the loader. We need to make sure that it is out of the way of the program * that it will "exec", and that there is sufficient room for the brk. */ -#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3) +#define ELF_ET_DYN_BASE (2UL * TASK_SIZE / 3) /* * When the program starts, a1 contains a pointer to a function to be diff --git a/arch/arc/include/asm/mcip.h b/arch/arc/include/asm/mcip.h index 847e3bbe387f..c8fbe4114bad 100644 --- a/arch/arc/include/asm/mcip.h +++ b/arch/arc/include/asm/mcip.h @@ -55,6 +55,22 @@ struct mcip_cmd { #define IDU_M_DISTRI_DEST 0x2 }; +struct mcip_bcr { +#ifdef CONFIG_CPU_BIG_ENDIAN + unsigned int pad3:8, + idu:1, llm:1, num_cores:6, + iocoh:1, gfrc:1, dbg:1, pad2:1, + msg:1, sem:1, ipi:1, pad:1, + ver:8; +#else + unsigned int ver:8, + pad:1, ipi:1, sem:1, msg:1, + pad2:1, dbg:1, gfrc:1, iocoh:1, + num_cores:6, llm:1, idu:1, + pad3:8; +#endif +}; + /* * MCIP programming model * diff --git a/arch/arc/include/asm/module.h b/arch/arc/include/asm/module.h index 518222bb3f8e..6e91d8b339c3 100644 --- a/arch/arc/include/asm/module.h +++ b/arch/arc/include/asm/module.h @@ -18,6 +18,7 @@ struct mod_arch_specific { void *unw_info; int unw_sec_idx; + const char *secstr; }; #endif diff --git a/arch/arc/include/asm/setup.h b/arch/arc/include/asm/setup.h index 48b37c693db3..cb954cdab070 100644 --- a/arch/arc/include/asm/setup.h +++ b/arch/arc/include/asm/setup.h @@ -27,11 +27,6 @@ struct id_to_str { const char *str; }; -struct cpuinfo_data { - struct id_to_str info; - int up_range; -}; - extern int root_mountflags, end_mem; void setup_processor(void); @@ -43,5 +38,6 @@ void __init setup_arch_memory(void); #define IS_USED_RUN(v) ((v) ? "" : "(not used) ") #define IS_USED_CFG(cfg) IS_USED_RUN(IS_ENABLED(cfg)) #define IS_AVAIL2(v, s, cfg) IS_AVAIL1(v, s), IS_AVAIL1(v, IS_USED_CFG(cfg)) +#define IS_AVAIL3(v, v2, s) IS_AVAIL1(v, s), IS_AVAIL1(v, IS_DISABLED_RUN(v2)) #endif /* __ASMARC_SETUP_H */ diff --git a/arch/arc/include/asm/syscalls.h b/arch/arc/include/asm/syscalls.h index e56f9fcc5581..772b67ca56e7 100644 --- a/arch/arc/include/asm/syscalls.h +++ b/arch/arc/include/asm/syscalls.h @@ -17,6 +17,7 @@ int sys_clone_wrapper(int, int, int, int, int); int sys_cacheflush(uint32_t, uint32_t uint32_t); int sys_arc_settls(void *); int sys_arc_gettls(void); +int sys_arc_usr_cmpxchg(int *, int, int); #include <asm-generic/syscalls.h> diff --git a/arch/arc/include/uapi/asm/unistd.h b/arch/arc/include/uapi/asm/unistd.h index 41fa2ec9e02c..9a34136d84b2 100644 --- a/arch/arc/include/uapi/asm/unistd.h +++ b/arch/arc/include/uapi/asm/unistd.h @@ -27,18 +27,19 @@ #define NR_syscalls __NR_syscalls +/* Generic syscall (fs/filesystems.c - lost in asm-generic/unistd.h */ +#define __NR_sysfs (__NR_arch_specific_syscall + 3) + /* ARC specific syscall */ #define __NR_cacheflush (__NR_arch_specific_syscall + 0) #define __NR_arc_settls (__NR_arch_specific_syscall + 1) #define __NR_arc_gettls (__NR_arch_specific_syscall + 2) +#define __NR_arc_usr_cmpxchg (__NR_arch_specific_syscall + 4) __SYSCALL(__NR_cacheflush, sys_cacheflush) __SYSCALL(__NR_arc_settls, sys_arc_settls) __SYSCALL(__NR_arc_gettls, sys_arc_gettls) - - -/* Generic syscall (fs/filesystems.c - lost in asm-generic/unistd.h */ -#define __NR_sysfs (__NR_arch_specific_syscall + 3) +__SYSCALL(__NR_arc_usr_cmpxchg, sys_arc_usr_cmpxchg) __SYSCALL(__NR_sysfs, sys_sysfs) #undef __SYSCALL diff --git a/arch/arc/kernel/mcip.c b/arch/arc/kernel/mcip.c index 72f9179b1a24..c424d5abc318 100644 --- a/arch/arc/kernel/mcip.c +++ b/arch/arc/kernel/mcip.c @@ -15,11 +15,12 @@ #include <asm/mcip.h> #include <asm/setup.h> -static char smp_cpuinfo_buf[128]; -static int idu_detected; - static DEFINE_RAW_SPINLOCK(mcip_lock); +#ifdef CONFIG_SMP + +static char smp_cpuinfo_buf[128]; + static void mcip_setup_per_cpu(int cpu) { smp_ipi_irq_setup(cpu, IPI_IRQ); @@ -86,21 +87,7 @@ static void mcip_ipi_clear(int irq) static void mcip_probe_n_setup(void) { - struct mcip_bcr { -#ifdef CONFIG_CPU_BIG_ENDIAN - unsigned int pad3:8, - idu:1, llm:1, num_cores:6, - iocoh:1, gfrc:1, dbg:1, pad2:1, - msg:1, sem:1, ipi:1, pad:1, - ver:8; -#else - unsigned int ver:8, - pad:1, ipi:1, sem:1, msg:1, - pad2:1, dbg:1, gfrc:1, iocoh:1, - num_cores:6, llm:1, idu:1, - pad3:8; -#endif - } mp; + struct mcip_bcr mp; READ_BCR(ARC_REG_MCIP_BCR, mp); @@ -114,7 +101,6 @@ static void mcip_probe_n_setup(void) IS_AVAIL1(mp.gfrc, "GFRC")); cpuinfo_arc700[0].extn.gfrc = mp.gfrc; - idu_detected = mp.idu; if (mp.dbg) { __mcip_cmd_data(CMD_DEBUG_SET_SELECT, 0, 0xf); @@ -130,6 +116,8 @@ struct plat_smp_ops plat_smp_ops = { .ipi_clear = mcip_ipi_clear, }; +#endif + /*************************************************************************** * ARCv2 Interrupt Distribution Unit (IDU) * @@ -295,8 +283,11 @@ idu_of_init(struct device_node *intc, struct device_node *parent) /* Read IDU BCR to confirm nr_irqs */ int nr_irqs = of_irq_count(intc); int i, irq; + struct mcip_bcr mp; + + READ_BCR(ARC_REG_MCIP_BCR, mp); - if (!idu_detected) + if (!mp.idu) panic("IDU not detected, but DeviceTree using it"); pr_info("MCIP: IDU referenced from Devicetree %d irqs\n", nr_irqs); diff --git a/arch/arc/kernel/module.c b/arch/arc/kernel/module.c index 9a2849756022..42e964db2967 100644 --- a/arch/arc/kernel/module.c +++ b/arch/arc/kernel/module.c @@ -30,17 +30,9 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, char *secstr, struct module *mod) { #ifdef CONFIG_ARC_DW2_UNWIND - int i; - mod->arch.unw_sec_idx = 0; mod->arch.unw_info = NULL; - - for (i = 1; i < hdr->e_shnum; i++) { - if (strcmp(secstr+sechdrs[i].sh_name, ".eh_frame") == 0) { - mod->arch.unw_sec_idx = i; - break; - } - } + mod->arch.secstr = secstr; #endif return 0; } @@ -59,29 +51,33 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, unsigned int relsec, /* sec index for relo sec */ struct module *module) { - int i, n; + int i, n, relo_type; Elf32_Rela *rel_entry = (void *)sechdrs[relsec].sh_addr; Elf32_Sym *sym_entry, *sym_sec; - Elf32_Addr relocation; - Elf32_Addr location; - Elf32_Addr sec_to_patch; - int relo_type; - - sec_to_patch = sechdrs[sechdrs[relsec].sh_info].sh_addr; + Elf32_Addr relocation, location, tgt_addr; + unsigned int tgtsec; + + /* + * @relsec has relocations e.g. .rela.init.text + * @tgtsec is section to patch e.g. .init.text + */ + tgtsec = sechdrs[relsec].sh_info; + tgt_addr = sechdrs[tgtsec].sh_addr; sym_sec = (Elf32_Sym *) sechdrs[symindex].sh_addr; n = sechdrs[relsec].sh_size / sizeof(*rel_entry); - pr_debug("\n========== Module Sym reloc ===========================\n"); - pr_debug("Section to fixup %x\n", sec_to_patch); + pr_debug("\nSection to fixup %s @%x\n", + module->arch.secstr + sechdrs[tgtsec].sh_name, tgt_addr); pr_debug("=========================================================\n"); - pr_debug("rela->r_off | rela->addend | sym->st_value | ADDR | VALUE\n"); + pr_debug("r_off\tr_add\tst_value ADDRESS VALUE\n"); pr_debug("=========================================================\n"); /* Loop thru entries in relocation section */ for (i = 0; i < n; i++) { + const char *s; /* This is where to make the change */ - location = sec_to_patch + rel_entry[i].r_offset; + location = tgt_addr + rel_entry[i].r_offset; /* This is the symbol it is referring to. Note that all undefined symbols have been resolved. */ @@ -89,10 +85,15 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, relocation = sym_entry->st_value + rel_entry[i].r_addend; - pr_debug("\t%x\t\t%x\t\t%x %x %x [%s]\n", - rel_entry[i].r_offset, rel_entry[i].r_addend, - sym_entry->st_value, location, relocation, - strtab + sym_entry->st_name); + if (sym_entry->st_name == 0 && ELF_ST_TYPE (sym_entry->st_info) == STT_SECTION) { + s = module->arch.secstr + sechdrs[sym_entry->st_shndx].sh_name; + } else { + s = strtab + sym_entry->st_name; + } + + pr_debug(" %x\t%x\t%x %x %x [%s]\n", + rel_entry[i].r_offset, rel_entry[i].r_addend, + sym_entry->st_value, location, relocation, s); /* This assumes modules are built with -mlong-calls * so any branches/jumps are absolute 32 bit jmps @@ -111,6 +112,10 @@ int apply_relocate_add(Elf32_Shdr *sechdrs, goto relo_err; } + + if (strcmp(module->arch.secstr+sechdrs[tgtsec].sh_name, ".eh_frame") == 0) + module->arch.unw_sec_idx = tgtsec; + return 0; relo_err: diff --git a/arch/arc/kernel/process.c b/arch/arc/kernel/process.c index be1972bd2729..59aa43cb146e 100644 --- a/arch/arc/kernel/process.c +++ b/arch/arc/kernel/process.c @@ -41,6 +41,39 @@ SYSCALL_DEFINE0(arc_gettls) return task_thread_info(current)->thr_ptr; } +SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new) +{ + int uval; + int ret; + + /* + * This is only for old cores lacking LLOCK/SCOND, which by defintion + * can't possibly be SMP. Thus doesn't need to be SMP safe. + * And this also helps reduce the overhead for serializing in + * the UP case + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_SMP)); + + if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) + return -EFAULT; + + preempt_disable(); + + ret = __get_user(uval, uaddr); + if (ret) + goto done; + + if (uval != expected) + ret = -EAGAIN; + else + ret = __put_user(new, uaddr); + +done: + preempt_enable(); + + return ret; +} + void arch_cpu_idle(void) { /* sleep, but enable all interrupts before committing */ diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c index 3df7f9c72f42..0385df77a697 100644 --- a/arch/arc/kernel/setup.c +++ b/arch/arc/kernel/setup.c @@ -40,6 +40,29 @@ struct task_struct *_current_task[NR_CPUS]; /* For stack switching */ struct cpuinfo_arc cpuinfo_arc700[NR_CPUS]; +static const struct id_to_str arc_cpu_rel[] = { +#ifdef CONFIG_ISA_ARCOMPACT + { 0x34, "R4.10"}, + { 0x35, "R4.11"}, +#else + { 0x51, "R2.0" }, + { 0x52, "R2.1" }, + { 0x53, "R3.0" }, +#endif + { 0x00, NULL } +}; + +static const struct id_to_str arc_cpu_nm[] = { +#ifdef CONFIG_ISA_ARCOMPACT + { 0x20, "ARC 600" }, + { 0x30, "ARC 770" }, /* 750 identified seperately */ +#else + { 0x40, "ARC EM" }, + { 0x50, "ARC HS38" }, +#endif + { 0x00, "Unknown" } +}; + static void read_decode_ccm_bcr(struct cpuinfo_arc *cpu) { if (is_isa_arcompact()) { @@ -92,11 +115,26 @@ static void read_arc_build_cfg_regs(void) struct bcr_timer timer; struct bcr_generic bcr; struct cpuinfo_arc *cpu = &cpuinfo_arc700[smp_processor_id()]; + const struct id_to_str *tbl; + FIX_PTR(cpu); READ_BCR(AUX_IDENTITY, cpu->core); READ_BCR(ARC_REG_ISA_CFG_BCR, cpu->isa); + for (tbl = &arc_cpu_rel[0]; tbl->id != 0; tbl++) { + if (cpu->core.family == tbl->id) { + cpu->details = tbl->str; + break; + } + } + + for (tbl = &arc_cpu_nm[0]; tbl->id != 0; tbl++) { + if ((cpu->core.family & 0xF0) == tbl->id) + break; + } + cpu->name = tbl->str; + READ_BCR(ARC_REG_TIMERS_BCR, timer); cpu->extn.timer0 = timer.t0; cpu->extn.timer1 = timer.t1; @@ -111,6 +149,9 @@ static void read_arc_build_cfg_regs(void) cpu->extn.swap = read_aux_reg(ARC_REG_SWAP_BCR) ? 1 : 0; /* 1,3 */ cpu->extn.crc = read_aux_reg(ARC_REG_CRC_BCR) ? 1 : 0; cpu->extn.minmax = read_aux_reg(ARC_REG_MIXMAX_BCR) > 1 ? 1 : 0; /* 2 */ + cpu->extn.swape = (cpu->core.family >= 0x34) ? 1 : + IS_ENABLED(CONFIG_ARC_HAS_SWAPE); + READ_BCR(ARC_REG_XY_MEM_BCR, cpu->extn_xymem); /* Read CCM BCRs for boot reporting even if not enabled in Kconfig */ @@ -160,64 +201,38 @@ static void read_arc_build_cfg_regs(void) cpu->extn.rtt = bcr.ver ? 1 : 0; cpu->extn.debug = cpu->extn.ap | cpu->extn.smart | cpu->extn.rtt; -} -static const struct cpuinfo_data arc_cpu_tbl[] = { -#ifdef CONFIG_ISA_ARCOMPACT - { {0x20, "ARC 600" }, 0x2F}, - { {0x30, "ARC 700" }, 0x33}, - { {0x34, "ARC 700 R4.10"}, 0x34}, - { {0x35, "ARC 700 R4.11"}, 0x35}, -#else - { {0x50, "ARC HS38 R2.0"}, 0x51}, - { {0x52, "ARC HS38 R2.1"}, 0x52}, - { {0x53, "ARC HS38 R3.0"}, 0x53}, -#endif - { {0x00, NULL } } -}; + /* some hacks for lack of feature BCR info in old ARC700 cores */ + if (is_isa_arcompact()) { + if (!cpu->isa.ver) /* ISA BCR absent, use Kconfig info */ + cpu->isa.atomic = IS_ENABLED(CONFIG_ARC_HAS_LLSC); + else + cpu->isa.atomic = cpu->isa.atomic1; + cpu->isa.be = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); + + /* there's no direct way to distinguish 750 vs. 770 */ + if (unlikely(cpu->core.family < 0x34 || cpu->mmu.ver < 3)) + cpu->name = "ARC750"; + } +} static char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len) { struct cpuinfo_arc *cpu = &cpuinfo_arc700[cpu_id]; struct bcr_identity *core = &cpu->core; - const struct cpuinfo_data *tbl; - char *isa_nm; - int i, be, atomic; - int n = 0; + int i, n = 0; FIX_PTR(cpu); - if (is_isa_arcompact()) { - isa_nm = "ARCompact"; - be = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN); - - atomic = cpu->isa.atomic1; - if (!cpu->isa.ver) /* ISA BCR absent, use Kconfig info */ - atomic = IS_ENABLED(CONFIG_ARC_HAS_LLSC); - } else { - isa_nm = "ARCv2"; - be = cpu->isa.be; - atomic = cpu->isa.atomic; - } - n += scnprintf(buf + n, len - n, "\nIDENTITY\t: ARCVER [%#02x] ARCNUM [%#02x] CHIPID [%#4x]\n", core->family, core->cpu_id, core->chip_id); - for (tbl = &arc_cpu_tbl[0]; tbl->info.id != 0; tbl++) { - if ((core->family >= tbl->info.id) && - (core->family <= tbl->up_range)) { - n += scnprintf(buf + n, len - n, - "processor [%d]\t: %s (%s ISA) %s\n", - cpu_id, tbl->info.str, isa_nm, - IS_AVAIL1(be, "[Big-Endian]")); - break; - } - } - - if (tbl->info.id == 0) - n += scnprintf(buf + n, len - n, "UNKNOWN ARC Processor\n"); + n += scnprintf(buf + n, len - n, "processor [%d]\t: %s %s (%s ISA) %s\n", + cpu_id, cpu->name, cpu->details, + is_isa_arcompact() ? "ARCompact" : "ARCv2", + IS_AVAIL1(cpu->isa.be, "[Big-Endian]")); n += scnprintf(buf + n, len - n, "Timers\t\t: %s%s%s%s\nISA Extn\t: ", IS_AVAIL1(cpu->extn.timer0, "Timer0 "), @@ -226,7 +241,7 @@ static char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len) CONFIG_ARC_HAS_RTC)); n += i = scnprintf(buf + n, len - n, "%s%s%s%s%s", - IS_AVAIL2(atomic, "atomic ", CONFIG_ARC_HAS_LLSC), + IS_AVAIL2(cpu->isa.atomic, "atomic ", CONFIG_ARC_HAS_LLSC), IS_AVAIL2(cpu->isa.ldd, "ll64 ", CONFIG_ARC_HAS_LL64), IS_AVAIL1(cpu->isa.unalign, "unalign (not used)")); @@ -253,7 +268,7 @@ static char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len) IS_AVAIL1(cpu->extn.swap, "swap "), IS_AVAIL1(cpu->extn.minmax, "minmax "), IS_AVAIL1(cpu->extn.crc, "crc "), - IS_AVAIL2(1, "swape", CONFIG_ARC_HAS_SWAPE)); + IS_AVAIL2(cpu->extn.swape, "swape", CONFIG_ARC_HAS_SWAPE)); if (cpu->bpu.ver) n += scnprintf(buf + n, len - n, @@ -272,9 +287,7 @@ static char *arc_extn_mumbojumbo(int cpu_id, char *buf, int len) FIX_PTR(cpu); - n += scnprintf(buf + n, len - n, - "Vector Table\t: %#x\nPeripherals\t: %#lx:%#lx\n", - cpu->vec_base, perip_base, perip_end); + n += scnprintf(buf + n, len - n, "Vector Table\t: %#x\n", cpu->vec_base); if (cpu->extn.fpu_sp || cpu->extn.fpu_dp) n += scnprintf(buf + n, len - n, "FPU\t\t: %s%s\n", @@ -507,7 +520,7 @@ static void *c_start(struct seq_file *m, loff_t *pos) * way to pass it w/o having to kmalloc/free a 2 byte string. * Encode cpu-id as 0xFFcccc, which is decoded by show routine. */ - return *pos < num_possible_cpus() ? cpu_to_ptr(*pos) : NULL; + return *pos < nr_cpu_ids ? cpu_to_ptr(*pos) : NULL; } static void *c_next(struct seq_file *m, void *v, loff_t *pos) diff --git a/arch/arc/kernel/troubleshoot.c b/arch/arc/kernel/troubleshoot.c index 934150e7ac48..82f9bc819f4a 100644 --- a/arch/arc/kernel/troubleshoot.c +++ b/arch/arc/kernel/troubleshoot.c @@ -237,113 +237,3 @@ void show_kernel_fault_diag(const char *str, struct pt_regs *regs, if (!user_mode(regs)) show_stacktrace(current, regs); } - -#ifdef CONFIG_DEBUG_FS - -#include <linux/module.h> -#include <linux/fs.h> -#include <linux/mount.h> -#include <linux/pagemap.h> -#include <linux/init.h> -#include <linux/namei.h> -#include <linux/debugfs.h> - -static struct dentry *test_dentry; -static struct dentry *test_dir; -static struct dentry *test_u32_dentry; - -static u32 clr_on_read = 1; - -#ifdef CONFIG_ARC_DBG_TLB_MISS_COUNT -u32 numitlb, numdtlb, num_pte_not_present; - -static int fill_display_data(char *kbuf) -{ - size_t num = 0; - num += sprintf(kbuf + num, "I-TLB Miss %x\n", numitlb); - num += sprintf(kbuf + num, "D-TLB Miss %x\n", numdtlb); - num += sprintf(kbuf + num, "PTE not present %x\n", num_pte_not_present); - - if (clr_on_read) - numitlb = numdtlb = num_pte_not_present = 0; - - return num; -} - -static int tlb_stats_open(struct inode *inode, struct file *file) -{ - file->private_data = (void *)__get_free_page(GFP_KERNEL); - return 0; -} - -/* called on user read(): display the counters */ -static ssize_t tlb_stats_output(struct file *file, /* file descriptor */ - char __user *user_buf, /* user buffer */ - size_t len, /* length of buffer */ - loff_t *offset) /* offset in the file */ -{ - size_t num; - char *kbuf = (char *)file->private_data; - - /* All of the data can he shoved in one iteration */ - if (*offset != 0) - return 0; - - num = fill_display_data(kbuf); - - /* simple_read_from_buffer() is helper for copy to user space - It copies up to @2 (num) bytes from kernel buffer @4 (kbuf) at offset - @3 (offset) into the user space address starting at @1 (user_buf). - @5 (len) is max size of user buffer - */ - return simple_read_from_buffer(user_buf, num, offset, kbuf, len); -} - -/* called on user write : clears the counters */ -static ssize_t tlb_stats_clear(struct file *file, const char __user *user_buf, - size_t length, loff_t *offset) -{ - numitlb = numdtlb = num_pte_not_present = 0; - return length; -} - -static int tlb_stats_close(struct inode *inode, struct file *file) -{ - free_page((unsigned long)(file->private_data)); - return 0; -} - -static const struct file_operations tlb_stats_file_ops = { - .read = tlb_stats_output, - .write = tlb_stats_clear, - .open = tlb_stats_open, - .release = tlb_stats_close -}; -#endif - -static int __init arc_debugfs_init(void) -{ - test_dir = debugfs_create_dir("arc", NULL); - -#ifdef CONFIG_ARC_DBG_TLB_MISS_COUNT - test_dentry = debugfs_create_file("tlb_stats", 0444, test_dir, NULL, - &tlb_stats_file_ops); -#endif - - test_u32_dentry = - debugfs_create_u32("clr_on_read", 0444, test_dir, &clr_on_read); - - return 0; -} - -module_init(arc_debugfs_init); - -static void __exit arc_debugfs_exit(void) -{ - debugfs_remove(test_u32_dentry); - debugfs_remove(test_dentry); - debugfs_remove(test_dir); -} -module_exit(arc_debugfs_exit); - -#endif diff --git a/arch/arc/mm/cache.c b/arch/arc/mm/cache.c index 97dddbefb86a..2b96cfc3be75 100644 --- a/arch/arc/mm/cache.c +++ b/arch/arc/mm/cache.c @@ -22,8 +22,8 @@ #include <asm/setup.h> static int l2_line_sz; -int ioc_exists; -volatile int slc_enable = 1, ioc_enable = 1; +static int ioc_exists; +int slc_enable = 1, ioc_enable = 1; unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ @@ -53,18 +53,15 @@ char *arc_cache_mumbojumbo(int c, char *buf, int len) PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); - if (!is_isa_arcv2()) - return buf; - p = &cpuinfo_arc700[c].slc; if (p->ver) n += scnprintf(buf + n, len - n, "SLC\t\t: %uK, %uB Line%s\n", p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); - if (ioc_exists) - n += scnprintf(buf + n, len - n, "IOC\t\t:%s\n", - IS_DISABLED_RUN(ioc_enable)); + n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n", + perip_base, + IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency ")); return buf; } @@ -113,8 +110,10 @@ static void read_decode_cache_bcr_arcv2(int cpu) } READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); - if (cbcr.c && ioc_enable) + if (cbcr.c) ioc_exists = 1; + else + ioc_enable = 0; /* HS 2.0 didn't have AUX_VOL */ if (cpuinfo_arc700[cpu].core.family > 0x51) { @@ -1002,7 +1001,7 @@ void arc_cache_init(void) read_aux_reg(ARC_REG_SLC_CTRL) | SLC_CTRL_DISABLE); } - if (is_isa_arcv2() && ioc_exists) { + if (is_isa_arcv2() && ioc_enable) { /* IO coherency base - 0x8z */ write_aux_reg(ARC_REG_IO_COH_AP0_BASE, 0x80000); /* IO coherency aperture size - 512Mb: 0x8z-0xAz */ diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c index 20afc65e22dc..60aab5a7522b 100644 --- a/arch/arc/mm/dma.c +++ b/arch/arc/mm/dma.c @@ -45,7 +45,7 @@ static void *arc_dma_alloc(struct device *dev, size_t size, * -For coherent data, Read/Write to buffers terminate early in cache * (vs. always going to memory - thus are faster) */ - if ((is_isa_arcv2() && ioc_exists) || + if ((is_isa_arcv2() && ioc_enable) || (attrs & DMA_ATTR_NON_CONSISTENT)) need_coh = 0; @@ -97,7 +97,7 @@ static void arc_dma_free(struct device *dev, size_t size, void *vaddr, int is_non_coh = 1; is_non_coh = (attrs & DMA_ATTR_NON_CONSISTENT) || - (is_isa_arcv2() && ioc_exists); + (is_isa_arcv2() && ioc_enable); if (PageHighMem(page) || !is_non_coh) iounmap((void __force __iomem *)vaddr); diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index ec868a9081a1..bdb295e09160 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -793,16 +793,16 @@ char *arc_mmu_mumbojumbo(int cpu_id, char *buf, int len) char super_pg[64] = ""; if (p_mmu->s_pg_sz_m) - scnprintf(super_pg, 64, "%dM Super Page%s, ", + scnprintf(super_pg, 64, "%dM Super Page %s", p_mmu->s_pg_sz_m, IS_USED_CFG(CONFIG_TRANSPARENT_HUGEPAGE)); n += scnprintf(buf + n, len - n, - "MMU [v%x]\t: %dk PAGE, %sJTLB %d (%dx%d), uDTLB %d, uITLB %d %s%s\n", + "MMU [v%x]\t: %dk PAGE, %sJTLB %d (%dx%d), uDTLB %d, uITLB %d%s%s\n", p_mmu->ver, p_mmu->pg_sz_k, super_pg, p_mmu->sets * p_mmu->ways, p_mmu->sets, p_mmu->ways, p_mmu->u_dtlb, p_mmu->u_itlb, - IS_AVAIL2(p_mmu->pae, "PAE40 ", CONFIG_ARC_HAS_PAE40)); + IS_AVAIL2(p_mmu->pae, ", PAE40 ", CONFIG_ARC_HAS_PAE40)); return buf; } diff --git a/arch/arc/mm/tlbex.S b/arch/arc/mm/tlbex.S index f1967eeb32e7..b30e4e36bb00 100644 --- a/arch/arc/mm/tlbex.S +++ b/arch/arc/mm/tlbex.S @@ -237,15 +237,6 @@ ex_saved_reg1: 2: -#ifdef CONFIG_ARC_DBG_TLB_MISS_COUNT - and.f 0, r0, _PAGE_PRESENT - bz 1f - ld r3, [num_pte_not_present] - add r3, r3, 1 - st r3, [num_pte_not_present] -1: -#endif - .endm ;----------------------------------------------------------------- @@ -309,12 +300,6 @@ ENTRY(EV_TLBMissI) TLBMISS_FREEUP_REGS -#ifdef CONFIG_ARC_DBG_TLB_MISS_COUNT - ld r0, [@numitlb] - add r0, r0, 1 - st r0, [@numitlb] -#endif - ;---------------------------------------------------------------- ; Get the PTE corresponding to V-addr accessed, r2 is setup with EFA LOAD_FAULT_PTE @@ -349,12 +334,6 @@ ENTRY(EV_TLBMissD) TLBMISS_FREEUP_REGS -#ifdef CONFIG_ARC_DBG_TLB_MISS_COUNT - ld r0, [@numdtlb] - add r0, r0, 1 - st r0, [@numdtlb] -#endif - ;---------------------------------------------------------------- ; Get the PTE corresponding to V-addr accessed ; If PTE exists, it will setup, r0 = PTE, r1 = Ptr to PTE, r2 = EFA diff --git a/arch/arm/boot/dts/ste-snowball.dts b/arch/arm/boot/dts/ste-snowball.dts index b3df1c60d465..386eee6de232 100644 --- a/arch/arm/boot/dts/ste-snowball.dts +++ b/arch/arm/boot/dts/ste-snowball.dts @@ -239,14 +239,25 @@ arm,primecell-periphid = <0x10480180>; max-frequency = <100000000>; bus-width = <4>; + cap-sd-highspeed; cap-mmc-highspeed; + sd-uhs-sdr12; + sd-uhs-sdr25; + /* All direction control is used */ + st,sig-dir-cmd; + st,sig-dir-dat0; + st,sig-dir-dat2; + st,sig-dir-dat31; + st,sig-pin-fbclk; + full-pwr-cycle; vmmc-supply = <&ab8500_ldo_aux3_reg>; vqmmc-supply = <&vmmci>; pinctrl-names = "default", "sleep"; pinctrl-0 = <&sdi0_default_mode>; pinctrl-1 = <&sdi0_sleep_mode>; - cd-gpios = <&gpio6 26 GPIO_ACTIVE_LOW>; // 218 + /* GPIO218 MMC_CD */ + cd-gpios = <&gpio6 26 GPIO_ACTIVE_LOW>; status = "okay"; }; @@ -549,7 +560,7 @@ /* VMMCI level-shifter enable */ snowball_cfg3 { pins = "GPIO217_AH12"; - ste,config = <&gpio_out_lo>; + ste,config = <&gpio_out_hi>; }; /* VMMCI level-shifter voltage select */ snowball_cfg4 { diff --git a/arch/arm/boot/dts/uniphier-pro5.dtsi b/arch/arm/boot/dts/uniphier-pro5.dtsi index 2c49c3614bda..5357ea9c14b1 100644 --- a/arch/arm/boot/dts/uniphier-pro5.dtsi +++ b/arch/arm/boot/dts/uniphier-pro5.dtsi @@ -184,11 +184,11 @@ }; &mio_clk { - compatible = "socionext,uniphier-pro5-mio-clock"; + compatible = "socionext,uniphier-pro5-sd-clock"; }; &mio_rst { - compatible = "socionext,uniphier-pro5-mio-reset"; + compatible = "socionext,uniphier-pro5-sd-reset"; }; &peri_clk { diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi index 8789cd518933..950f07ba0337 100644 --- a/arch/arm/boot/dts/uniphier-pxs2.dtsi +++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi @@ -197,11 +197,11 @@ }; &mio_clk { - compatible = "socionext,uniphier-pxs2-mio-clock"; + compatible = "socionext,uniphier-pxs2-sd-clock"; }; &mio_rst { - compatible = "socionext,uniphier-pxs2-mio-reset"; + compatible = "socionext,uniphier-pxs2-sd-reset"; }; &peri_clk { diff --git a/arch/arm/boot/dts/vf500.dtsi b/arch/arm/boot/dts/vf500.dtsi index a3824e61bd72..d7fdb2a7d97b 100644 --- a/arch/arm/boot/dts/vf500.dtsi +++ b/arch/arm/boot/dts/vf500.dtsi @@ -70,7 +70,7 @@ global_timer: timer@40002200 { compatible = "arm,cortex-a9-global-timer"; reg = <0x40002200 0x20>; - interrupts = <GIC_PPI 11 IRQ_TYPE_LEVEL_HIGH>; + interrupts = <GIC_PPI 11 IRQ_TYPE_EDGE_RISING>; interrupt-parent = <&intc>; clocks = <&clks VF610_CLK_PLATFORM_BUS>; }; diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index 437d0740dec6..11f37ed1dbff 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -850,6 +850,7 @@ CONFIG_PWM_SUN4I=y CONFIG_PWM_TEGRA=y CONFIG_PWM_VT8500=y CONFIG_PHY_HIX5HD2_SATA=y +CONFIG_E1000E=y CONFIG_PWM_STI=y CONFIG_PWM_BCM2835=y CONFIG_PWM_BRCMSTB=m diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h index 194b69923389..ada0d29a660f 100644 --- a/arch/arm/include/asm/unistd.h +++ b/arch/arm/include/asm/unistd.h @@ -19,7 +19,7 @@ * This may need to be greater than __NR_last_syscall+1 in order to * account for the padding in the syscall table */ -#define __NR_syscalls (396) +#define __NR_syscalls (400) #define __ARCH_WANT_STAT64 #define __ARCH_WANT_SYS_GETHOSTNAME diff --git a/arch/arm/include/uapi/asm/unistd.h b/arch/arm/include/uapi/asm/unistd.h index 2cb9dc770e1d..314100a06ccb 100644 --- a/arch/arm/include/uapi/asm/unistd.h +++ b/arch/arm/include/uapi/asm/unistd.h @@ -420,6 +420,9 @@ #define __NR_copy_file_range (__NR_SYSCALL_BASE+391) #define __NR_preadv2 (__NR_SYSCALL_BASE+392) #define __NR_pwritev2 (__NR_SYSCALL_BASE+393) +#define __NR_pkey_mprotect (__NR_SYSCALL_BASE+394) +#define __NR_pkey_alloc (__NR_SYSCALL_BASE+395) +#define __NR_pkey_free (__NR_SYSCALL_BASE+396) /* * The following SWIs are ARM private. diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S index 703fa0f3cd8f..08030b18f10a 100644 --- a/arch/arm/kernel/calls.S +++ b/arch/arm/kernel/calls.S @@ -403,6 +403,9 @@ CALL(sys_copy_file_range) CALL(sys_preadv2) CALL(sys_pwritev2) + CALL(sys_pkey_mprotect) +/* 395 */ CALL(sys_pkey_alloc) + CALL(sys_pkey_free) #ifndef syscalls_counted .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls #define syscalls_counted diff --git a/arch/arm/mach-imx/gpc.c b/arch/arm/mach-imx/gpc.c index 0df062d8b2c9..b54db47f6f32 100644 --- a/arch/arm/mach-imx/gpc.c +++ b/arch/arm/mach-imx/gpc.c @@ -408,7 +408,7 @@ static struct genpd_onecell_data imx_gpc_onecell_data = { static int imx_gpc_genpd_init(struct device *dev, struct regulator *pu_reg) { struct clk *clk; - int i; + int i, ret; imx6q_pu_domain.reg = pu_reg; @@ -430,13 +430,22 @@ static int imx_gpc_genpd_init(struct device *dev, struct regulator *pu_reg) if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS)) return 0; - pm_genpd_init(&imx6q_pu_domain.base, NULL, false); - return of_genpd_add_provider_onecell(dev->of_node, + for (i = 0; i < ARRAY_SIZE(imx_gpc_domains); i++) + pm_genpd_init(imx_gpc_domains[i], NULL, false); + + ret = of_genpd_add_provider_onecell(dev->of_node, &imx_gpc_onecell_data); + if (ret) + goto power_off; + + return 0; +power_off: + imx6q_pm_pu_power_off(&imx6q_pu_domain.base); clk_err: while (i--) clk_put(imx6q_pu_domain.clk[i]); + imx6q_pu_domain.reg = NULL; return -EINVAL; } diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c index 97fd25105e2c..45801b27ee5c 100644 --- a/arch/arm/mach-imx/mach-imx6q.c +++ b/arch/arm/mach-imx/mach-imx6q.c @@ -173,7 +173,7 @@ static void __init imx6q_enet_phy_init(void) ksz9021rn_phy_fixup); phy_register_fixup_for_uid(PHY_ID_KSZ9031, MICREL_PHY_ID_MASK, ksz9031rn_phy_fixup); - phy_register_fixup_for_uid(PHY_ID_AR8031, 0xffffffff, + phy_register_fixup_for_uid(PHY_ID_AR8031, 0xffffffef, ar8031_phy_fixup); phy_register_fixup_for_uid(PHY_ID_AR8035, 0xffffffef, ar8035_phy_fixup); diff --git a/arch/arm/mach-mvebu/Kconfig b/arch/arm/mach-mvebu/Kconfig index f9b6bd306cfe..541647f57192 100644 --- a/arch/arm/mach-mvebu/Kconfig +++ b/arch/arm/mach-mvebu/Kconfig @@ -23,6 +23,7 @@ config MACH_MVEBU_V7 select CACHE_L2X0 select ARM_CPU_SUSPEND select MACH_MVEBU_ANY + select MVEBU_CLK_COREDIV config MACH_ARMADA_370 bool "Marvell Armada 370 boards" @@ -32,7 +33,6 @@ config MACH_ARMADA_370 select CPU_PJ4B select MACH_MVEBU_V7 select PINCTRL_ARMADA_370 - select MVEBU_CLK_COREDIV help Say 'Y' here if you want your kernel to support boards based on the Marvell Armada 370 SoC with device tree. @@ -50,7 +50,6 @@ config MACH_ARMADA_375 select HAVE_SMP select MACH_MVEBU_V7 select PINCTRL_ARMADA_375 - select MVEBU_CLK_COREDIV help Say 'Y' here if you want your kernel to support boards based on the Marvell Armada 375 SoC with device tree. @@ -68,7 +67,6 @@ config MACH_ARMADA_38X select HAVE_SMP select MACH_MVEBU_V7 select PINCTRL_ARMADA_38X - select MVEBU_CLK_COREDIV help Say 'Y' here if you want your kernel to support boards based on the Marvell Armada 380/385 SoC with device tree. diff --git a/arch/arm/mach-uniphier/Kconfig b/arch/arm/mach-uniphier/Kconfig index 82dddee3a469..3930fbba30b4 100644 --- a/arch/arm/mach-uniphier/Kconfig +++ b/arch/arm/mach-uniphier/Kconfig @@ -1,6 +1,7 @@ config ARCH_UNIPHIER bool "Socionext UniPhier SoCs" depends on ARCH_MULTI_V7 + select ARCH_HAS_RESET_CONTROLLER select ARM_AMBA select ARM_GLOBAL_TIMER select ARM_GIC diff --git a/arch/arm/mm/abort-lv4t.S b/arch/arm/mm/abort-lv4t.S index 6d8e8e3365d1..4cdfab31a0b6 100644 --- a/arch/arm/mm/abort-lv4t.S +++ b/arch/arm/mm/abort-lv4t.S @@ -7,7 +7,7 @@ * : r4 = aborted context pc * : r5 = aborted context psr * - * Returns : r4-r5, r10-r11, r13 preserved + * Returns : r4-r5, r9-r11, r13 preserved * * Purpose : obtain information about current aborted instruction. * Note: we read user space. This means we might cause a data @@ -48,7 +48,10 @@ ENTRY(v4t_late_abort) /* c */ b do_DataAbort @ ldc rd, [rn], #m @ Same as ldr rd, [rn], #m /* d */ b do_DataAbort @ ldc rd, [rn, #m] /* e */ b .data_unknown -/* f */ +/* f */ b .data_unknown + +.data_unknown_r9: + ldr r9, [sp], #4 .data_unknown: @ Part of jumptable mov r0, r4 mov r1, r8 @@ -57,6 +60,7 @@ ENTRY(v4t_late_abort) .data_arm_ldmstm: tst r8, #1 << 21 @ check writeback bit beq do_DataAbort @ no writeback -> no fixup + str r9, [sp, #-4]! mov r7, #0x11 orr r7, r7, #0x1100 and r6, r8, r7 @@ -75,12 +79,14 @@ ENTRY(v4t_late_abort) subne r7, r7, r6, lsl #2 @ Undo increment addeq r7, r7, r6, lsl #2 @ Undo decrement str r7, [r2, r9, lsr #14] @ Put register 'Rn' + ldr r9, [sp], #4 b do_DataAbort .data_arm_lateldrhpre: tst r8, #1 << 21 @ Check writeback bit beq do_DataAbort @ No writeback -> no fixup .data_arm_lateldrhpost: + str r9, [sp, #-4]! and r9, r8, #0x00f @ get Rm / low nibble of immediate value tst r8, #1 << 22 @ if (immediate offset) andne r6, r8, #0xf00 @ { immediate high nibble @@ -93,6 +99,7 @@ ENTRY(v4t_late_abort) subne r7, r7, r6 @ Undo incrmenet addeq r7, r7, r6 @ Undo decrement str r7, [r2, r9, lsr #14] @ Put register 'Rn' + ldr r9, [sp], #4 b do_DataAbort .data_arm_lateldrpreconst: @@ -101,12 +108,14 @@ ENTRY(v4t_late_abort) .data_arm_lateldrpostconst: movs r6, r8, lsl #20 @ Get offset beq do_DataAbort @ zero -> no fixup + str r9, [sp, #-4]! and r9, r8, #15 << 16 @ Extract 'n' from instruction ldr r7, [r2, r9, lsr #14] @ Get register 'Rn' tst r8, #1 << 23 @ Check U bit subne r7, r7, r6, lsr #20 @ Undo increment addeq r7, r7, r6, lsr #20 @ Undo decrement str r7, [r2, r9, lsr #14] @ Put register 'Rn' + ldr r9, [sp], #4 b do_DataAbort .data_arm_lateldrprereg: @@ -115,6 +124,7 @@ ENTRY(v4t_late_abort) .data_arm_lateldrpostreg: and r7, r8, #15 @ Extract 'm' from instruction ldr r6, [r2, r7, lsl #2] @ Get register 'Rm' + str r9, [sp, #-4]! mov r9, r8, lsr #7 @ get shift count ands r9, r9, #31 and r7, r8, #0x70 @ get shift type @@ -126,33 +136,33 @@ ENTRY(v4t_late_abort) b .data_arm_apply_r6_and_rn b .data_arm_apply_r6_and_rn @ 1: LSL #0 nop - b .data_unknown @ 2: MUL? + b .data_unknown_r9 @ 2: MUL? nop - b .data_unknown @ 3: MUL? + b .data_unknown_r9 @ 3: MUL? nop mov r6, r6, lsr r9 @ 4: LSR #!0 b .data_arm_apply_r6_and_rn mov r6, r6, lsr #32 @ 5: LSR #32 b .data_arm_apply_r6_and_rn - b .data_unknown @ 6: MUL? + b .data_unknown_r9 @ 6: MUL? nop - b .data_unknown @ 7: MUL? + b .data_unknown_r9 @ 7: MUL? nop mov r6, r6, asr r9 @ 8: ASR #!0 b .data_arm_apply_r6_and_rn mov r6, r6, asr #32 @ 9: ASR #32 b .data_arm_apply_r6_and_rn - b .data_unknown @ A: MUL? + b .data_unknown_r9 @ A: MUL? nop - b .data_unknown @ B: MUL? + b .data_unknown_r9 @ B: MUL? nop mov r6, r6, ror r9 @ C: ROR #!0 b .data_arm_apply_r6_and_rn mov r6, r6, rrx @ D: RRX b .data_arm_apply_r6_and_rn - b .data_unknown @ E: MUL? + b .data_unknown_r9 @ E: MUL? nop - b .data_unknown @ F: MUL? + b .data_unknown_r9 @ F: MUL? .data_thumb_abort: ldrh r8, [r4] @ read instruction @@ -190,6 +200,7 @@ ENTRY(v4t_late_abort) .data_thumb_pushpop: tst r8, #1 << 10 beq .data_unknown + str r9, [sp, #-4]! and r6, r8, #0x55 @ hweight8(r8) + R bit and r9, r8, #0xaa add r6, r6, r9, lsr #1 @@ -204,9 +215,11 @@ ENTRY(v4t_late_abort) addeq r7, r7, r6, lsl #2 @ increment SP if PUSH subne r7, r7, r6, lsl #2 @ decrement SP if POP str r7, [r2, #13 << 2] + ldr r9, [sp], #4 b do_DataAbort .data_thumb_ldmstm: + str r9, [sp, #-4]! and r6, r8, #0x55 @ hweight8(r8) and r9, r8, #0xaa add r6, r6, r9, lsr #1 @@ -219,4 +232,5 @@ ENTRY(v4t_late_abort) and r6, r6, #15 @ number of regs to transfer sub r7, r7, r6, lsl #2 @ always decrement str r7, [r2, r9, lsr #6] + ldr r9, [sp], #4 b do_DataAbort diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms index cfbdf02ef566..101794f5ce10 100644 --- a/arch/arm64/Kconfig.platforms +++ b/arch/arm64/Kconfig.platforms @@ -190,6 +190,7 @@ config ARCH_THUNDER config ARCH_UNIPHIER bool "Socionext UniPhier SoC Family" + select ARCH_HAS_RESET_CONTROLLER select PINCTRL help This enables support for Socionext UniPhier SoC family. diff --git a/arch/arm64/boot/dts/broadcom/ns2-svk.dts b/arch/arm64/boot/dts/broadcom/ns2-svk.dts index 2d7872a36b91..b09f3bc5c6c1 100644 --- a/arch/arm64/boot/dts/broadcom/ns2-svk.dts +++ b/arch/arm64/boot/dts/broadcom/ns2-svk.dts @@ -164,6 +164,8 @@ nand-ecc-mode = "hw"; nand-ecc-strength = <8>; nand-ecc-step-size = <512>; + nand-bus-width = <16>; + brcm,nand-oob-sector-size = <16>; #address-cells = <1>; #size-cells = <1>; }; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi index 220ac7057d12..97d331ec2500 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi @@ -123,6 +123,7 @@ <1 14 0xf08>, /* Physical Non-Secure PPI */ <1 11 0xf08>, /* Virtual PPI */ <1 10 0xf08>; /* Hypervisor PPI */ + fsl,erratum-a008585; }; pmu { diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi index 337da90bd7da..7f0dc13b4087 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi @@ -195,6 +195,7 @@ <1 14 4>, /* Physical Non-Secure PPI, active-low */ <1 11 4>, /* Virtual PPI, active-low */ <1 10 4>; /* Hypervisor PPI, active-low */ + fsl,erratum-a008585; }; pmu { diff --git a/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi b/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi index e5e3ed678b6f..602e2c2e9a4d 100644 --- a/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi +++ b/arch/arm64/boot/dts/marvell/armada-cp110-master.dtsi @@ -131,7 +131,7 @@ #address-cells = <0x1>; #size-cells = <0x0>; cell-index = <1>; - clocks = <&cpm_syscon0 0 3>; + clocks = <&cpm_syscon0 1 21>; status = "disabled"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-geekbox.dts b/arch/arm64/boot/dts/rockchip/rk3368-geekbox.dts index 46cdddfcea6c..e5eeca2c2456 100644 --- a/arch/arm64/boot/dts/rockchip/rk3368-geekbox.dts +++ b/arch/arm64/boot/dts/rockchip/rk3368-geekbox.dts @@ -116,7 +116,6 @@ cap-mmc-highspeed; clock-frequency = <150000000>; disable-wp; - keep-power-in-suspend; non-removable; num-slots = <1>; vmmc-supply = <&vcc_io>; @@ -258,8 +257,6 @@ }; vcc_sd: SWITCH_REG1 { - regulator-always-on; - regulator-boot-on; regulator-name = "vcc_sd"; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts b/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts index 5797933ef80e..ea0a8eceefd4 100644 --- a/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts +++ b/arch/arm64/boot/dts/rockchip/rk3368-orion-r68-meta.dts @@ -152,8 +152,6 @@ gpio = <&gpio3 11 GPIO_ACTIVE_LOW>; regulator-min-microvolt = <1800000>; regulator-max-microvolt = <3300000>; - regulator-always-on; - regulator-boot-on; vin-supply = <&vcc_io>; }; @@ -201,7 +199,6 @@ bus-width = <8>; cap-mmc-highspeed; disable-wp; - keep-power-in-suspend; mmc-pwrseq = <&emmc_pwrseq>; mmc-hs200-1_2v; mmc-hs200-1_8v; @@ -350,7 +347,6 @@ clock-freq-min-max = <400000 50000000>; cap-sd-highspeed; card-detect-delay = <200>; - keep-power-in-suspend; num-slots = <1>; pinctrl-names = "default"; pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>; diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi index 08fd7cf7769c..56a1b2e92cf3 100644 --- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi +++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi @@ -257,18 +257,18 @@ reg = <0x59801000 0x400>; }; - mioctrl@59810000 { - compatible = "socionext,uniphier-mioctrl", + sdctrl@59810000 { + compatible = "socionext,uniphier-ld20-sdctrl", "simple-mfd", "syscon"; reg = <0x59810000 0x800>; - mio_clk: clock { - compatible = "socionext,uniphier-ld20-mio-clock"; + sd_clk: clock { + compatible = "socionext,uniphier-ld20-sd-clock"; #clock-cells = <1>; }; - mio_rst: reset { - compatible = "socionext,uniphier-ld20-mio-reset"; + sd_rst: reset { + compatible = "socionext,uniphier-ld20-sd-reset"; #reset-cells = <1>; }; }; diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index ba62df8c6e35..b71086d25195 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -217,7 +217,7 @@ static inline void *phys_to_virt(phys_addr_t x) #define _virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) #else #define __virt_to_pgoff(kaddr) (((u64)(kaddr) & ~PAGE_OFFSET) / PAGE_SIZE * sizeof(struct page)) -#define __page_to_voff(kaddr) (((u64)(page) & ~VMEMMAP_START) * PAGE_SIZE / sizeof(struct page)) +#define __page_to_voff(page) (((u64)(page) & ~VMEMMAP_START) * PAGE_SIZE / sizeof(struct page)) #define page_to_virt(page) ((void *)((__page_to_voff(page)) | PAGE_OFFSET)) #define virt_to_page(vaddr) ((struct page *)((__virt_to_pgoff(vaddr)) | VMEMMAP_START)) diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c index 778a985c8a70..4b32168cf91a 100644 --- a/arch/arm64/mm/numa.c +++ b/arch/arm64/mm/numa.c @@ -147,7 +147,7 @@ static int __init early_cpu_to_node(int cpu) static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) { - return node_distance(from, to); + return node_distance(early_cpu_to_node(from), early_cpu_to_node(to)); } static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, @@ -223,8 +223,11 @@ static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn) void *nd; int tnid; - pr_info("Initmem setup node %d [mem %#010Lx-%#010Lx]\n", - nid, start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1); + if (start_pfn < end_pfn) + pr_info("Initmem setup node %d [mem %#010Lx-%#010Lx]\n", nid, + start_pfn << PAGE_SHIFT, (end_pfn << PAGE_SHIFT) - 1); + else + pr_info("Initmem setup node %d [<memory-less node>]\n", nid); nd_pa = memblock_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid); nd = __va(nd_pa); diff --git a/arch/cris/arch-v32/drivers/cryptocop.c b/arch/cris/arch-v32/drivers/cryptocop.c index 099e170a93ee..0068fd411a84 100644 --- a/arch/cris/arch-v32/drivers/cryptocop.c +++ b/arch/cris/arch-v32/drivers/cryptocop.c @@ -3149,7 +3149,7 @@ static void print_dma_descriptors(struct cryptocop_int_operation *iop) printk("print_dma_descriptors start\n"); printk("iop:\n"); - printk("\tsid: 0x%lld\n", iop->sid); + printk("\tsid: 0x%llx\n", iop->sid); printk("\tcdesc_out: 0x%p\n", iop->cdesc_out); printk("\tcdesc_in: 0x%p\n", iop->cdesc_in); diff --git a/arch/h8300/include/asm/thread_info.h b/arch/h8300/include/asm/thread_info.h index b408fe660cf8..3cef06875f5c 100644 --- a/arch/h8300/include/asm/thread_info.h +++ b/arch/h8300/include/asm/thread_info.h @@ -31,7 +31,6 @@ struct thread_info { int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, <0 => BUG */ mm_segment_t addr_limit; - struct restart_block restart_block; }; /* @@ -44,9 +43,6 @@ struct thread_info { .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ } #define init_thread_info (init_thread_union.thread_info) diff --git a/arch/h8300/kernel/signal.c b/arch/h8300/kernel/signal.c index ad1f81f574e5..7138303cbbf2 100644 --- a/arch/h8300/kernel/signal.c +++ b/arch/h8300/kernel/signal.c @@ -79,7 +79,7 @@ restore_sigcontext(struct sigcontext *usc, int *pd0) unsigned int er0; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; /* restore passed registers */ #define COPY(r) do { err |= get_user(regs->r, &usc->sc_##r); } while (0) diff --git a/arch/mips/Makefile b/arch/mips/Makefile index fbf40d3c8123..1a6bac7b076f 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -263,7 +263,7 @@ KBUILD_CPPFLAGS += -DDATAOFFSET=$(if $(dataoffset-y),$(dataoffset-y),0) bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) \ VMLINUX_ENTRY_ADDRESS=$(entry-y) \ - PLATFORM=$(platform-y) + PLATFORM="$(platform-y)" ifdef CONFIG_32BIT bootvars-y += ADDR_BITS=32 endif diff --git a/arch/mips/boot/dts/mti/malta.dts b/arch/mips/boot/dts/mti/malta.dts index f604a272d91d..ffe3a1508e72 100644 --- a/arch/mips/boot/dts/mti/malta.dts +++ b/arch/mips/boot/dts/mti/malta.dts @@ -84,12 +84,13 @@ fpga_regs: system-controller@1f000000 { compatible = "mti,malta-fpga", "syscon", "simple-mfd"; reg = <0x1f000000 0x1000>; + native-endian; reboot { compatible = "syscon-reboot"; regmap = <&fpga_regs>; offset = <0x500>; - mask = <0x4d>; + mask = <0x42>; }; }; diff --git a/arch/mips/generic/init.c b/arch/mips/generic/init.c index 0ea73e845440..d493ccbf274a 100644 --- a/arch/mips/generic/init.c +++ b/arch/mips/generic/init.c @@ -30,9 +30,19 @@ static __initdata const void *mach_match_data; void __init prom_init(void) { + plat_get_fdt(); + BUG_ON(!fdt); +} + +void __init *plat_get_fdt(void) +{ const struct mips_machine *check_mach; const struct of_device_id *match; + if (fdt) + /* Already set up */ + return (void *)fdt; + if ((fw_arg0 == -2) && !fdt_check_header((void *)fw_arg1)) { /* * We booted using the UHI boot protocol, so we have been @@ -75,12 +85,6 @@ void __init prom_init(void) /* Retrieve the machine's FDT */ fdt = mach->fdt; } - - BUG_ON(!fdt); -} - -void __init *plat_get_fdt(void) -{ return (void *)fdt; } diff --git a/arch/mips/include/asm/fpu_emulator.h b/arch/mips/include/asm/fpu_emulator.h index 355dc25172e7..c05369e0b8d6 100644 --- a/arch/mips/include/asm/fpu_emulator.h +++ b/arch/mips/include/asm/fpu_emulator.h @@ -63,6 +63,8 @@ do { \ extern int fpu_emulator_cop1Handler(struct pt_regs *xcp, struct mips_fpu_struct *ctx, int has_fpu, void *__user *fault_addr); +void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr, + struct task_struct *tsk); int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31); int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, @@ -81,4 +83,15 @@ static inline void fpu_emulator_init_fpu(void) set_fpr64(&t->thread.fpu.fpr[i], 0, SIGNALLING_NAN); } +/* + * Mask the FCSR Cause bits according to the Enable bits, observing + * that Unimplemented is always enabled. + */ +static inline unsigned long mask_fcr31_x(unsigned long fcr31) +{ + return fcr31 & (FPU_CSR_UNI_X | + ((fcr31 & FPU_CSR_ALL_E) << + (ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E)))); +} + #endif /* _ASM_FPU_EMULATOR_H */ diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index 07f58cfc1ab9..bebec370324f 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -293,7 +293,10 @@ struct kvm_vcpu_arch { /* Host KSEG0 address of the EI/DI offset */ void *kseg0_commpage; - u32 io_gpr; /* GPR used as IO source/target */ + /* Resume PC after MMIO completion */ + unsigned long io_pc; + /* GPR used as IO source/target */ + u32 io_gpr; struct hrtimer comparecount_timer; /* Count timer control KVM register */ @@ -315,8 +318,6 @@ struct kvm_vcpu_arch { /* Bitmask of pending exceptions to be cleared */ unsigned long pending_exceptions_clr; - u32 pending_load_cause; - /* Save/Restore the entryhi register when are are preempted/scheduled back in */ unsigned long preempt_entryhi; diff --git a/arch/mips/include/asm/switch_to.h b/arch/mips/include/asm/switch_to.h index ebb5c0f2f90d..c0ae27971e31 100644 --- a/arch/mips/include/asm/switch_to.h +++ b/arch/mips/include/asm/switch_to.h @@ -76,6 +76,22 @@ do { if (cpu_has_rw_llb) { \ } while (0) /* + * Check FCSR for any unmasked exceptions pending set with `ptrace', + * clear them and send a signal. + */ +#define __sanitize_fcr31(next) \ +do { \ + unsigned long fcr31 = mask_fcr31_x(next->thread.fpu.fcr31); \ + void __user *pc; \ + \ + if (unlikely(fcr31)) { \ + pc = (void __user *)task_pt_regs(next)->cp0_epc; \ + next->thread.fpu.fcr31 &= ~fcr31; \ + force_fcr31_sig(fcr31, pc, next); \ + } \ +} while (0) + +/* * For newly created kernel threads switch_to() will return to * ret_from_kernel_thread, newly created user threads to ret_from_fork. * That is, everything following resume() will be skipped for new threads. @@ -85,6 +101,8 @@ do { if (cpu_has_rw_llb) { \ do { \ __mips_mt_fpaff_switch_to(prev); \ lose_fpu_inatomic(1, prev); \ + if (tsk_used_math(next)) \ + __sanitize_fcr31(next); \ if (cpu_has_dsp) { \ __save_dsp(prev); \ __restore_dsp(next); \ diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c index 2a45867d3b4f..a4964c334cab 100644 --- a/arch/mips/kernel/mips-cpc.c +++ b/arch/mips/kernel/mips-cpc.c @@ -21,6 +21,11 @@ static DEFINE_PER_CPU_ALIGNED(spinlock_t, cpc_core_lock); static DEFINE_PER_CPU_ALIGNED(unsigned long, cpc_core_lock_flags); +phys_addr_t __weak mips_cpc_default_phys_base(void) +{ + return 0; +} + /** * mips_cpc_phys_base - retrieve the physical base address of the CPC * @@ -43,8 +48,12 @@ static phys_addr_t mips_cpc_phys_base(void) if (cpc_base & CM_GCR_CPC_BASE_CPCEN_MSK) return cpc_base & CM_GCR_CPC_BASE_CPCBASE_MSK; - /* Otherwise, give it the default address & enable it */ + /* Otherwise, use the default address */ cpc_base = mips_cpc_default_phys_base(); + if (!cpc_base) + return cpc_base; + + /* Enable the CPC, mapped at the default address */ write_gcr_cpc_base(cpc_base | CM_GCR_CPC_BASE_CPCEN_MSK); return cpc_base; } diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c index 22dedd62818a..bd09853aecdf 100644 --- a/arch/mips/kernel/mips-r2-to-r6-emul.c +++ b/arch/mips/kernel/mips-r2-to-r6-emul.c @@ -899,7 +899,7 @@ static inline int mipsr2_find_op_func(struct pt_regs *regs, u32 inst, * mipsr2_decoder: Decode and emulate a MIPS R2 instruction * @regs: Process register set * @inst: Instruction to decode and emulate - * @fcr31: Floating Point Control and Status Register returned + * @fcr31: Floating Point Control and Status Register Cause bits returned */ int mipsr2_decoder(struct pt_regs *regs, u32 inst, unsigned long *fcr31) { @@ -1172,13 +1172,13 @@ fpu_emul: err = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 0, &fault_addr); - *fcr31 = current->thread.fpu.fcr31; /* - * We can't allow the emulated instruction to leave any of - * the cause bits set in $fcr31. + * We can't allow the emulated instruction to leave any + * enabled Cause bits set in $fcr31. */ - current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; + *fcr31 = res = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~res; /* * this is a tricky issue - lose_fpu() uses LL/SC atomics diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 6103b24d1bfc..a92994d60e91 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -79,16 +79,15 @@ void ptrace_disable(struct task_struct *child) } /* - * Poke at FCSR according to its mask. Don't set the cause bits as - * this is currently not handled correctly in FP context restoration - * and will cause an oops if a corresponding enable bit is set. + * Poke at FCSR according to its mask. Set the Cause bits even + * if a corresponding Enable bit is set. This will be noticed at + * the time the thread is switched to and SIGFPE thrown accordingly. */ static void ptrace_setfcr31(struct task_struct *child, u32 value) { u32 fcr31; u32 mask; - value &= ~FPU_CSR_ALL_X; fcr31 = child->thread.fpu.fcr31; mask = boot_cpu_data.fpu_msk31; child->thread.fpu.fcr31 = (value & ~mask) | (fcr31 & mask); @@ -817,6 +816,7 @@ long arch_ptrace(struct task_struct *child, long request, break; #endif case FPC_CSR: + init_fp_ctx(child); ptrace_setfcr31(child, data); break; case DSP_BASE ... DSP_BASE + 5: { diff --git a/arch/mips/kernel/r2300_fpu.S b/arch/mips/kernel/r2300_fpu.S index b4ac6374a38f..918f2f6d3861 100644 --- a/arch/mips/kernel/r2300_fpu.S +++ b/arch/mips/kernel/r2300_fpu.S @@ -21,106 +21,84 @@ #define EX(a,b) \ 9: a,##b; \ .section __ex_table,"a"; \ + PTR 9b,fault; \ + .previous + +#define EX2(a,b) \ +9: a,##b; \ + .section __ex_table,"a"; \ PTR 9b,bad_stack; \ + PTR 9b+4,bad_stack; \ .previous .set noreorder .set mips1 - /* Save floating point context */ + +/** + * _save_fp_context() - save FP context from the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Save FP context, including the 32 FP data registers and the FP + * control & status register, from the FPU to signal context. + */ LEAF(_save_fp_context) .set push SET_HARDFLOAT li v0, 0 # assume success - cfc1 t1,fcr31 - EX(swc1 $f0,(SC_FPREGS+0)(a0)) - EX(swc1 $f1,(SC_FPREGS+8)(a0)) - EX(swc1 $f2,(SC_FPREGS+16)(a0)) - EX(swc1 $f3,(SC_FPREGS+24)(a0)) - EX(swc1 $f4,(SC_FPREGS+32)(a0)) - EX(swc1 $f5,(SC_FPREGS+40)(a0)) - EX(swc1 $f6,(SC_FPREGS+48)(a0)) - EX(swc1 $f7,(SC_FPREGS+56)(a0)) - EX(swc1 $f8,(SC_FPREGS+64)(a0)) - EX(swc1 $f9,(SC_FPREGS+72)(a0)) - EX(swc1 $f10,(SC_FPREGS+80)(a0)) - EX(swc1 $f11,(SC_FPREGS+88)(a0)) - EX(swc1 $f12,(SC_FPREGS+96)(a0)) - EX(swc1 $f13,(SC_FPREGS+104)(a0)) - EX(swc1 $f14,(SC_FPREGS+112)(a0)) - EX(swc1 $f15,(SC_FPREGS+120)(a0)) - EX(swc1 $f16,(SC_FPREGS+128)(a0)) - EX(swc1 $f17,(SC_FPREGS+136)(a0)) - EX(swc1 $f18,(SC_FPREGS+144)(a0)) - EX(swc1 $f19,(SC_FPREGS+152)(a0)) - EX(swc1 $f20,(SC_FPREGS+160)(a0)) - EX(swc1 $f21,(SC_FPREGS+168)(a0)) - EX(swc1 $f22,(SC_FPREGS+176)(a0)) - EX(swc1 $f23,(SC_FPREGS+184)(a0)) - EX(swc1 $f24,(SC_FPREGS+192)(a0)) - EX(swc1 $f25,(SC_FPREGS+200)(a0)) - EX(swc1 $f26,(SC_FPREGS+208)(a0)) - EX(swc1 $f27,(SC_FPREGS+216)(a0)) - EX(swc1 $f28,(SC_FPREGS+224)(a0)) - EX(swc1 $f29,(SC_FPREGS+232)(a0)) - EX(swc1 $f30,(SC_FPREGS+240)(a0)) - EX(swc1 $f31,(SC_FPREGS+248)(a0)) - EX(sw t1,(SC_FPC_CSR)(a0)) - cfc1 t0,$0 # implementation/version + cfc1 t1, fcr31 + EX2(s.d $f0, 0(a0)) + EX2(s.d $f2, 16(a0)) + EX2(s.d $f4, 32(a0)) + EX2(s.d $f6, 48(a0)) + EX2(s.d $f8, 64(a0)) + EX2(s.d $f10, 80(a0)) + EX2(s.d $f12, 96(a0)) + EX2(s.d $f14, 112(a0)) + EX2(s.d $f16, 128(a0)) + EX2(s.d $f18, 144(a0)) + EX2(s.d $f20, 160(a0)) + EX2(s.d $f22, 176(a0)) + EX2(s.d $f24, 192(a0)) + EX2(s.d $f26, 208(a0)) + EX2(s.d $f28, 224(a0)) + EX2(s.d $f30, 240(a0)) jr ra + EX(sw t1, (a1)) .set pop - .set nomacro - EX(sw t0,(SC_FPC_EIR)(a0)) - .set macro END(_save_fp_context) -/* - * Restore FPU state: - * - fp gp registers - * - cp1 status/control register +/** + * _restore_fp_context() - restore FP context to the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext * - * We base the decision which registers to restore from the signal stack - * frame on the current content of c0_status, not on the content of the - * stack frame which might have been changed by the user. + * Restore FP context, including the 32 FP data registers and the FP + * control & status register, from signal context to the FPU. */ LEAF(_restore_fp_context) .set push SET_HARDFLOAT li v0, 0 # assume success - EX(lw t0,(SC_FPC_CSR)(a0)) - EX(lwc1 $f0,(SC_FPREGS+0)(a0)) - EX(lwc1 $f1,(SC_FPREGS+8)(a0)) - EX(lwc1 $f2,(SC_FPREGS+16)(a0)) - EX(lwc1 $f3,(SC_FPREGS+24)(a0)) - EX(lwc1 $f4,(SC_FPREGS+32)(a0)) - EX(lwc1 $f5,(SC_FPREGS+40)(a0)) - EX(lwc1 $f6,(SC_FPREGS+48)(a0)) - EX(lwc1 $f7,(SC_FPREGS+56)(a0)) - EX(lwc1 $f8,(SC_FPREGS+64)(a0)) - EX(lwc1 $f9,(SC_FPREGS+72)(a0)) - EX(lwc1 $f10,(SC_FPREGS+80)(a0)) - EX(lwc1 $f11,(SC_FPREGS+88)(a0)) - EX(lwc1 $f12,(SC_FPREGS+96)(a0)) - EX(lwc1 $f13,(SC_FPREGS+104)(a0)) - EX(lwc1 $f14,(SC_FPREGS+112)(a0)) - EX(lwc1 $f15,(SC_FPREGS+120)(a0)) - EX(lwc1 $f16,(SC_FPREGS+128)(a0)) - EX(lwc1 $f17,(SC_FPREGS+136)(a0)) - EX(lwc1 $f18,(SC_FPREGS+144)(a0)) - EX(lwc1 $f19,(SC_FPREGS+152)(a0)) - EX(lwc1 $f20,(SC_FPREGS+160)(a0)) - EX(lwc1 $f21,(SC_FPREGS+168)(a0)) - EX(lwc1 $f22,(SC_FPREGS+176)(a0)) - EX(lwc1 $f23,(SC_FPREGS+184)(a0)) - EX(lwc1 $f24,(SC_FPREGS+192)(a0)) - EX(lwc1 $f25,(SC_FPREGS+200)(a0)) - EX(lwc1 $f26,(SC_FPREGS+208)(a0)) - EX(lwc1 $f27,(SC_FPREGS+216)(a0)) - EX(lwc1 $f28,(SC_FPREGS+224)(a0)) - EX(lwc1 $f29,(SC_FPREGS+232)(a0)) - EX(lwc1 $f30,(SC_FPREGS+240)(a0)) - EX(lwc1 $f31,(SC_FPREGS+248)(a0)) + EX(lw t0, (a1)) + EX2(l.d $f0, 0(a0)) + EX2(l.d $f2, 16(a0)) + EX2(l.d $f4, 32(a0)) + EX2(l.d $f6, 48(a0)) + EX2(l.d $f8, 64(a0)) + EX2(l.d $f10, 80(a0)) + EX2(l.d $f12, 96(a0)) + EX2(l.d $f14, 112(a0)) + EX2(l.d $f16, 128(a0)) + EX2(l.d $f18, 144(a0)) + EX2(l.d $f20, 160(a0)) + EX2(l.d $f22, 176(a0)) + EX2(l.d $f24, 192(a0)) + EX2(l.d $f26, 208(a0)) + EX2(l.d $f28, 224(a0)) + EX2(l.d $f30, 240(a0)) jr ra - ctc1 t0,fcr31 + ctc1 t0, fcr31 .set pop END(_restore_fp_context) .set reorder diff --git a/arch/mips/kernel/r6000_fpu.S b/arch/mips/kernel/r6000_fpu.S index 47077380c15c..9cc7bfab3419 100644 --- a/arch/mips/kernel/r6000_fpu.S +++ b/arch/mips/kernel/r6000_fpu.S @@ -21,7 +21,14 @@ .set push SET_HARDFLOAT - /* Save floating point context */ +/** + * _save_fp_context() - save FP context from the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Save FP context, including the 32 FP data registers and the FP + * control & status register, from the FPU to signal context. + */ LEAF(_save_fp_context) mfc0 t0,CP0_STATUS sll t0,t0,2 @@ -30,59 +37,59 @@ cfc1 t1,fcr31 /* Store the 16 double precision registers */ - sdc1 $f0,(SC_FPREGS+0)(a0) - sdc1 $f2,(SC_FPREGS+16)(a0) - sdc1 $f4,(SC_FPREGS+32)(a0) - sdc1 $f6,(SC_FPREGS+48)(a0) - sdc1 $f8,(SC_FPREGS+64)(a0) - sdc1 $f10,(SC_FPREGS+80)(a0) - sdc1 $f12,(SC_FPREGS+96)(a0) - sdc1 $f14,(SC_FPREGS+112)(a0) - sdc1 $f16,(SC_FPREGS+128)(a0) - sdc1 $f18,(SC_FPREGS+144)(a0) - sdc1 $f20,(SC_FPREGS+160)(a0) - sdc1 $f22,(SC_FPREGS+176)(a0) - sdc1 $f24,(SC_FPREGS+192)(a0) - sdc1 $f26,(SC_FPREGS+208)(a0) - sdc1 $f28,(SC_FPREGS+224)(a0) - sdc1 $f30,(SC_FPREGS+240)(a0) + sdc1 $f0,0(a0) + sdc1 $f2,16(a0) + sdc1 $f4,32(a0) + sdc1 $f6,48(a0) + sdc1 $f8,64(a0) + sdc1 $f10,80(a0) + sdc1 $f12,96(a0) + sdc1 $f14,112(a0) + sdc1 $f16,128(a0) + sdc1 $f18,144(a0) + sdc1 $f20,160(a0) + sdc1 $f22,176(a0) + sdc1 $f24,192(a0) + sdc1 $f26,208(a0) + sdc1 $f28,224(a0) + sdc1 $f30,240(a0) jr ra - sw t0,SC_FPC_CSR(a0) + sw t0,(a1) 1: jr ra nop END(_save_fp_context) -/* Restore FPU state: - * - fp gp registers - * - cp1 status/control register +/** + * _restore_fp_context() - restore FP context to the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext * - * We base the decision which registers to restore from the signal stack - * frame on the current content of c0_status, not on the content of the - * stack frame which might have been changed by the user. + * Restore FP context, including the 32 FP data registers and the FP + * control & status register, from signal context to the FPU. */ LEAF(_restore_fp_context) mfc0 t0,CP0_STATUS sll t0,t0,2 bgez t0,1f - lw t0,SC_FPC_CSR(a0) + lw t0,(a1) /* Restore the 16 double precision registers */ - ldc1 $f0,(SC_FPREGS+0)(a0) - ldc1 $f2,(SC_FPREGS+16)(a0) - ldc1 $f4,(SC_FPREGS+32)(a0) - ldc1 $f6,(SC_FPREGS+48)(a0) - ldc1 $f8,(SC_FPREGS+64)(a0) - ldc1 $f10,(SC_FPREGS+80)(a0) - ldc1 $f12,(SC_FPREGS+96)(a0) - ldc1 $f14,(SC_FPREGS+112)(a0) - ldc1 $f16,(SC_FPREGS+128)(a0) - ldc1 $f18,(SC_FPREGS+144)(a0) - ldc1 $f20,(SC_FPREGS+160)(a0) - ldc1 $f22,(SC_FPREGS+176)(a0) - ldc1 $f24,(SC_FPREGS+192)(a0) - ldc1 $f26,(SC_FPREGS+208)(a0) - ldc1 $f28,(SC_FPREGS+224)(a0) - ldc1 $f30,(SC_FPREGS+240)(a0) + ldc1 $f0,0(a0) + ldc1 $f2,16(a0) + ldc1 $f4,32(a0) + ldc1 $f6,48(a0) + ldc1 $f8,64(a0) + ldc1 $f10,80(a0) + ldc1 $f12,96(a0) + ldc1 $f14,112(a0) + ldc1 $f16,128(a0) + ldc1 $f18,144(a0) + ldc1 $f20,160(a0) + ldc1 $f22,176(a0) + ldc1 $f24,192(a0) + ldc1 $f26,208(a0) + ldc1 $f28,224(a0) + ldc1 $f30,240(a0) jr ra ctc1 t0,fcr31 1: jr ra diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c index ca1cc30c0891..1958910b75c0 100644 --- a/arch/mips/kernel/relocate.c +++ b/arch/mips/kernel/relocate.c @@ -200,7 +200,7 @@ static inline __init unsigned long get_random_boot(void) #if defined(CONFIG_USE_OF) /* Get any additional entropy passed in device tree */ - { + if (initial_boot_params) { int node, len; u64 *prop; diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 0d57909d9026..f66e5ce505b2 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -368,6 +368,19 @@ static void __init bootmem_init(void) end = PFN_DOWN(boot_mem_map.map[i].addr + boot_mem_map.map[i].size); +#ifndef CONFIG_HIGHMEM + /* + * Skip highmem here so we get an accurate max_low_pfn if low + * memory stops short of high memory. + * If the region overlaps HIGHMEM_START, end is clipped so + * max_pfn excludes the highmem portion. + */ + if (start >= PFN_DOWN(HIGHMEM_START)) + continue; + if (end > PFN_DOWN(HIGHMEM_START)) + end = PFN_DOWN(HIGHMEM_START); +#endif + if (end > max_low_pfn) max_low_pfn = end; if (start < min_low_pfn) diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 1f5fdee1dfc3..3905003dfe2b 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -156,7 +156,7 @@ static void show_backtrace(struct task_struct *task, const struct pt_regs *regs) print_ip_sym(pc); pc = unwind_stack(task, &sp, pc, &ra); } while (pc); - printk("\n"); + pr_cont("\n"); } /* @@ -174,22 +174,24 @@ static void show_stacktrace(struct task_struct *task, printk("Stack :"); i = 0; while ((unsigned long) sp & (PAGE_SIZE - 1)) { - if (i && ((i % (64 / field)) == 0)) - printk("\n "); + if (i && ((i % (64 / field)) == 0)) { + pr_cont("\n"); + printk(" "); + } if (i > 39) { - printk(" ..."); + pr_cont(" ..."); break; } if (__get_user(stackdata, sp++)) { - printk(" (Bad stack address)"); + pr_cont(" (Bad stack address)"); break; } - printk(" %0*lx", field, stackdata); + pr_cont(" %0*lx", field, stackdata); i++; } - printk("\n"); + pr_cont("\n"); show_backtrace(task, regs); } @@ -229,18 +231,19 @@ static void show_code(unsigned int __user *pc) long i; unsigned short __user *pc16 = NULL; - printk("\nCode:"); + printk("Code:"); if ((unsigned long)pc & 1) pc16 = (unsigned short __user *)((unsigned long)pc & ~1); for(i = -3 ; i < 6 ; i++) { unsigned int insn; if (pc16 ? __get_user(insn, pc16 + i) : __get_user(insn, pc + i)) { - printk(" (Bad address in epc)\n"); + pr_cont(" (Bad address in epc)\n"); break; } - printk("%c%0*x%c", (i?' ':'<'), pc16 ? 4 : 8, insn, (i?' ':'>')); + pr_cont("%c%0*x%c", (i?' ':'<'), pc16 ? 4 : 8, insn, (i?' ':'>')); } + pr_cont("\n"); } static void __show_regs(const struct pt_regs *regs) @@ -259,15 +262,15 @@ static void __show_regs(const struct pt_regs *regs) if ((i % 4) == 0) printk("$%2d :", i); if (i == 0) - printk(" %0*lx", field, 0UL); + pr_cont(" %0*lx", field, 0UL); else if (i == 26 || i == 27) - printk(" %*s", field, ""); + pr_cont(" %*s", field, ""); else - printk(" %0*lx", field, regs->regs[i]); + pr_cont(" %0*lx", field, regs->regs[i]); i++; if ((i % 4) == 0) - printk("\n"); + pr_cont("\n"); } #ifdef CONFIG_CPU_HAS_SMARTMIPS @@ -288,46 +291,46 @@ static void __show_regs(const struct pt_regs *regs) if (cpu_has_3kex) { if (regs->cp0_status & ST0_KUO) - printk("KUo "); + pr_cont("KUo "); if (regs->cp0_status & ST0_IEO) - printk("IEo "); + pr_cont("IEo "); if (regs->cp0_status & ST0_KUP) - printk("KUp "); + pr_cont("KUp "); if (regs->cp0_status & ST0_IEP) - printk("IEp "); + pr_cont("IEp "); if (regs->cp0_status & ST0_KUC) - printk("KUc "); + pr_cont("KUc "); if (regs->cp0_status & ST0_IEC) - printk("IEc "); + pr_cont("IEc "); } else if (cpu_has_4kex) { if (regs->cp0_status & ST0_KX) - printk("KX "); + pr_cont("KX "); if (regs->cp0_status & ST0_SX) - printk("SX "); + pr_cont("SX "); if (regs->cp0_status & ST0_UX) - printk("UX "); + pr_cont("UX "); switch (regs->cp0_status & ST0_KSU) { case KSU_USER: - printk("USER "); + pr_cont("USER "); break; case KSU_SUPERVISOR: - printk("SUPERVISOR "); + pr_cont("SUPERVISOR "); break; case KSU_KERNEL: - printk("KERNEL "); + pr_cont("KERNEL "); break; default: - printk("BAD_MODE "); + pr_cont("BAD_MODE "); break; } if (regs->cp0_status & ST0_ERL) - printk("ERL "); + pr_cont("ERL "); if (regs->cp0_status & ST0_EXL) - printk("EXL "); + pr_cont("EXL "); if (regs->cp0_status & ST0_IE) - printk("IE "); + pr_cont("IE "); } - printk("\n"); + pr_cont("\n"); exccode = (cause & CAUSEF_EXCCODE) >> CAUSEB_EXCCODE; printk("Cause : %08x (ExcCode %02x)\n", cause, exccode); @@ -705,6 +708,32 @@ asmlinkage void do_ov(struct pt_regs *regs) exception_exit(prev_state); } +/* + * Send SIGFPE according to FCSR Cause bits, which must have already + * been masked against Enable bits. This is impotant as Inexact can + * happen together with Overflow or Underflow, and `ptrace' can set + * any bits. + */ +void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr, + struct task_struct *tsk) +{ + struct siginfo si = { .si_addr = fault_addr, .si_signo = SIGFPE }; + + if (fcr31 & FPU_CSR_INV_X) + si.si_code = FPE_FLTINV; + else if (fcr31 & FPU_CSR_DIV_X) + si.si_code = FPE_FLTDIV; + else if (fcr31 & FPU_CSR_OVF_X) + si.si_code = FPE_FLTOVF; + else if (fcr31 & FPU_CSR_UDF_X) + si.si_code = FPE_FLTUND; + else if (fcr31 & FPU_CSR_INE_X) + si.si_code = FPE_FLTRES; + else + si.si_code = __SI_FAULT; + force_sig_info(SIGFPE, &si, tsk); +} + int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31) { struct siginfo si = { 0 }; @@ -715,27 +744,7 @@ int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31) return 0; case SIGFPE: - si.si_addr = fault_addr; - si.si_signo = sig; - /* - * Inexact can happen together with Overflow or Underflow. - * Respect the mask to deliver the correct exception. - */ - fcr31 &= (fcr31 & FPU_CSR_ALL_E) << - (ffs(FPU_CSR_ALL_X) - ffs(FPU_CSR_ALL_E)); - if (fcr31 & FPU_CSR_INV_X) - si.si_code = FPE_FLTINV; - else if (fcr31 & FPU_CSR_DIV_X) - si.si_code = FPE_FLTDIV; - else if (fcr31 & FPU_CSR_OVF_X) - si.si_code = FPE_FLTOVF; - else if (fcr31 & FPU_CSR_UDF_X) - si.si_code = FPE_FLTUND; - else if (fcr31 & FPU_CSR_INE_X) - si.si_code = FPE_FLTRES; - else - si.si_code = __SI_FAULT; - force_sig_info(sig, &si, current); + force_fcr31_sig(fcr31, fault_addr, current); return 1; case SIGBUS: @@ -799,13 +808,13 @@ static int simulate_fp(struct pt_regs *regs, unsigned int opcode, /* Run the emulator */ sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); - fcr31 = current->thread.fpu.fcr31; /* - * We can't allow the emulated instruction to leave any of - * the cause bits set in $fcr31. + * We can't allow the emulated instruction to leave any + * enabled Cause bits set in $fcr31. */ - current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; + fcr31 = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~fcr31; /* Restore the hardware register state */ own_fpu(1); @@ -831,7 +840,7 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) goto out; /* Clear FCSR.Cause before enabling interrupts */ - write_32bit_cp1_register(CP1_STATUS, fcr31 & ~FPU_CSR_ALL_X); + write_32bit_cp1_register(CP1_STATUS, fcr31 & ~mask_fcr31_x(fcr31)); local_irq_enable(); die_if_kernel("FP exception in kernel code", regs); @@ -853,13 +862,13 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) /* Run the emulator */ sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, &fault_addr); - fcr31 = current->thread.fpu.fcr31; /* - * We can't allow the emulated instruction to leave any of - * the cause bits set in $fcr31. + * We can't allow the emulated instruction to leave any + * enabled Cause bits set in $fcr31. */ - current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; + fcr31 = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~fcr31; /* Restore the hardware register state */ own_fpu(1); /* Using the FPU again. */ @@ -1424,13 +1433,13 @@ asmlinkage void do_cpu(struct pt_regs *regs) sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 0, &fault_addr); - fcr31 = current->thread.fpu.fcr31; /* * We can't allow the emulated instruction to leave - * any of the cause bits set in $fcr31. + * any enabled Cause bits set in $fcr31. */ - current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; + fcr31 = mask_fcr31_x(current->thread.fpu.fcr31); + current->thread.fpu.fcr31 &= ~fcr31; /* Send a signal if required. */ if (!process_fpemu_return(sig, fault_addr, fcr31) && !err) diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index 8770f32c9e0b..aa0937423e28 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -790,15 +790,15 @@ enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu) struct mips_coproc *cop0 = vcpu->arch.cop0; enum emulation_result er = EMULATE_DONE; - if (kvm_read_c0_guest_status(cop0) & ST0_EXL) { + if (kvm_read_c0_guest_status(cop0) & ST0_ERL) { + kvm_clear_c0_guest_status(cop0, ST0_ERL); + vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0); + } else if (kvm_read_c0_guest_status(cop0) & ST0_EXL) { kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc, kvm_read_c0_guest_epc(cop0)); kvm_clear_c0_guest_status(cop0, ST0_EXL); vcpu->arch.pc = kvm_read_c0_guest_epc(cop0); - } else if (kvm_read_c0_guest_status(cop0) & ST0_ERL) { - kvm_clear_c0_guest_status(cop0, ST0_ERL); - vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0); } else { kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n", vcpu->arch.pc); @@ -1528,13 +1528,25 @@ enum emulation_result kvm_mips_emulate_load(union mips_instruction inst, struct kvm_vcpu *vcpu) { enum emulation_result er = EMULATE_DO_MMIO; + unsigned long curr_pc; u32 op, rt; u32 bytes; rt = inst.i_format.rt; op = inst.i_format.opcode; - vcpu->arch.pending_load_cause = cause; + /* + * Find the resume PC now while we have safe and easy access to the + * prior branch instruction, and save it for + * kvm_mips_complete_mmio_load() to restore later. + */ + curr_pc = vcpu->arch.pc; + er = update_pc(vcpu, cause); + if (er == EMULATE_FAIL) + return er; + vcpu->arch.io_pc = vcpu->arch.pc; + vcpu->arch.pc = curr_pc; + vcpu->arch.io_gpr = rt; switch (op) { @@ -2494,9 +2506,8 @@ enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, goto done; } - er = update_pc(vcpu, vcpu->arch.pending_load_cause); - if (er == EMULATE_FAIL) - return er; + /* Restore saved resume PC */ + vcpu->arch.pc = vcpu->arch.io_pc; switch (run->mmio.len) { case 4: @@ -2518,11 +2529,6 @@ enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, break; } - if (vcpu->arch.pending_load_cause & CAUSEF_BD) - kvm_debug("[%#lx] Completing %d byte BD Load to gpr %d (0x%08lx) type %d\n", - vcpu->arch.pc, run->mmio.len, vcpu->arch.io_gpr, *gpr, - vcpu->mmio_needed); - done: return er; } diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 622037d851a3..06a60b19acfb 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -426,7 +426,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, static void kvm_mips_check_asids(struct kvm_vcpu *vcpu) { struct mips_coproc *cop0 = vcpu->arch.cop0; - int cpu = smp_processor_id(); + int i, cpu = smp_processor_id(); unsigned int gasid; /* @@ -442,6 +442,9 @@ static void kvm_mips_check_asids(struct kvm_vcpu *vcpu) vcpu); vcpu->arch.guest_user_asid[cpu] = vcpu->arch.guest_user_mm.context.asid[cpu]; + for_each_possible_cpu(i) + if (i != cpu) + vcpu->arch.guest_user_asid[cpu] = 0; vcpu->arch.last_user_gasid = gasid; } } diff --git a/arch/mips/kvm/mmu.c b/arch/mips/kvm/mmu.c index 03883ba806e2..3b677c851be0 100644 --- a/arch/mips/kvm/mmu.c +++ b/arch/mips/kvm/mmu.c @@ -260,13 +260,9 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) if ((vcpu->arch.guest_user_asid[cpu] ^ asid_cache(cpu)) & asid_version_mask(cpu)) { - u32 gasid = kvm_read_c0_guest_entryhi(vcpu->arch.cop0) & - KVM_ENTRYHI_ASID; - kvm_get_new_mmu_context(&vcpu->arch.guest_user_mm, cpu, vcpu); vcpu->arch.guest_user_asid[cpu] = vcpu->arch.guest_user_mm.context.asid[cpu]; - vcpu->arch.last_user_gasid = gasid; newasid++; kvm_debug("[%d]: cpu_context: %#lx\n", cpu, diff --git a/arch/mips/lib/dump_tlb.c b/arch/mips/lib/dump_tlb.c index 0f80b936e75e..6eb50a7137db 100644 --- a/arch/mips/lib/dump_tlb.c +++ b/arch/mips/lib/dump_tlb.c @@ -135,42 +135,42 @@ static void dump_tlb(int first, int last) c0 = (entrylo0 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; c1 = (entrylo1 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; - printk("va=%0*lx asid=%0*lx", - vwidth, (entryhi & ~0x1fffUL), - asidwidth, entryhi & asidmask); + pr_cont("va=%0*lx asid=%0*lx", + vwidth, (entryhi & ~0x1fffUL), + asidwidth, entryhi & asidmask); if (cpu_has_guestid) - printk(" gid=%02lx", - (guestctl1 & MIPS_GCTL1_RID) + pr_cont(" gid=%02lx", + (guestctl1 & MIPS_GCTL1_RID) >> MIPS_GCTL1_RID_SHIFT); /* RI/XI are in awkward places, so mask them off separately */ pa = entrylo0 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); if (xpa) pa |= (unsigned long long)readx_c0_entrylo0() << 30; pa = (pa << 6) & PAGE_MASK; - printk("\n\t["); + pr_cont("\n\t["); if (cpu_has_rixi) - printk("ri=%d xi=%d ", - (entrylo0 & MIPS_ENTRYLO_RI) ? 1 : 0, - (entrylo0 & MIPS_ENTRYLO_XI) ? 1 : 0); - printk("pa=%0*llx c=%d d=%d v=%d g=%d] [", - pwidth, pa, c0, - (entrylo0 & ENTRYLO_D) ? 1 : 0, - (entrylo0 & ENTRYLO_V) ? 1 : 0, - (entrylo0 & ENTRYLO_G) ? 1 : 0); + pr_cont("ri=%d xi=%d ", + (entrylo0 & MIPS_ENTRYLO_RI) ? 1 : 0, + (entrylo0 & MIPS_ENTRYLO_XI) ? 1 : 0); + pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d] [", + pwidth, pa, c0, + (entrylo0 & ENTRYLO_D) ? 1 : 0, + (entrylo0 & ENTRYLO_V) ? 1 : 0, + (entrylo0 & ENTRYLO_G) ? 1 : 0); /* RI/XI are in awkward places, so mask them off separately */ pa = entrylo1 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); if (xpa) pa |= (unsigned long long)readx_c0_entrylo1() << 30; pa = (pa << 6) & PAGE_MASK; if (cpu_has_rixi) - printk("ri=%d xi=%d ", - (entrylo1 & MIPS_ENTRYLO_RI) ? 1 : 0, - (entrylo1 & MIPS_ENTRYLO_XI) ? 1 : 0); - printk("pa=%0*llx c=%d d=%d v=%d g=%d]\n", - pwidth, pa, c1, - (entrylo1 & ENTRYLO_D) ? 1 : 0, - (entrylo1 & ENTRYLO_V) ? 1 : 0, - (entrylo1 & ENTRYLO_G) ? 1 : 0); + pr_cont("ri=%d xi=%d ", + (entrylo1 & MIPS_ENTRYLO_RI) ? 1 : 0, + (entrylo1 & MIPS_ENTRYLO_XI) ? 1 : 0); + pr_cont("pa=%0*llx c=%d d=%d v=%d g=%d]\n", + pwidth, pa, c1, + (entrylo1 & ENTRYLO_D) ? 1 : 0, + (entrylo1 & ENTRYLO_V) ? 1 : 0, + (entrylo1 & ENTRYLO_G) ? 1 : 0); } printk("\n"); diff --git a/arch/mips/lib/r3k_dump_tlb.c b/arch/mips/lib/r3k_dump_tlb.c index 744f4a7bc49d..85b4086e553e 100644 --- a/arch/mips/lib/r3k_dump_tlb.c +++ b/arch/mips/lib/r3k_dump_tlb.c @@ -53,15 +53,15 @@ static void dump_tlb(int first, int last) */ printk("Index: %2d ", i); - printk("va=%08lx asid=%08lx" - " [pa=%06lx n=%d d=%d v=%d g=%d]", - entryhi & PAGE_MASK, - entryhi & asid_mask, - entrylo0 & PAGE_MASK, - (entrylo0 & R3K_ENTRYLO_N) ? 1 : 0, - (entrylo0 & R3K_ENTRYLO_D) ? 1 : 0, - (entrylo0 & R3K_ENTRYLO_V) ? 1 : 0, - (entrylo0 & R3K_ENTRYLO_G) ? 1 : 0); + pr_cont("va=%08lx asid=%08lx" + " [pa=%06lx n=%d d=%d v=%d g=%d]", + entryhi & PAGE_MASK, + entryhi & asid_mask, + entrylo0 & PAGE_MASK, + (entrylo0 & R3K_ENTRYLO_N) ? 1 : 0, + (entrylo0 & R3K_ENTRYLO_D) ? 1 : 0, + (entrylo0 & R3K_ENTRYLO_V) ? 1 : 0, + (entrylo0 & R3K_ENTRYLO_G) ? 1 : 0); } } printk("\n"); diff --git a/arch/parisc/include/uapi/asm/unistd.h b/arch/parisc/include/uapi/asm/unistd.h index a9b9407f38f7..6b0741e7a7ed 100644 --- a/arch/parisc/include/uapi/asm/unistd.h +++ b/arch/parisc/include/uapi/asm/unistd.h @@ -368,7 +368,9 @@ #define __IGNORE_select /* newselect */ #define __IGNORE_fadvise64 /* fadvise64_64 */ - +#define __IGNORE_pkey_mprotect +#define __IGNORE_pkey_alloc +#define __IGNORE_pkey_free #define LINUX_GATEWAY_ADDR 0x100 diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index f8150669b8c6..700e2d2da096 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -873,11 +873,11 @@ static void print_parisc_device(struct parisc_device *dev) if (dev->num_addrs) { int k; - printk(", additional addresses: "); + pr_cont(", additional addresses: "); for (k = 0; k < dev->num_addrs; k++) - printk("0x%lx ", dev->addr[k]); + pr_cont("0x%lx ", dev->addr[k]); } - printk("\n"); + pr_cont("\n"); } /** diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S index d03422e5f188..23de307c3052 100644 --- a/arch/parisc/kernel/syscall.S +++ b/arch/parisc/kernel/syscall.S @@ -100,14 +100,12 @@ set_thread_pointer: .endr /* This address must remain fixed at 0x100 for glibc's syscalls to work */ - .align 256 + .align LINUX_GATEWAY_ADDR linux_gateway_entry: gate .+8, %r0 /* become privileged */ mtsp %r0,%sr4 /* get kernel space into sr4 */ mtsp %r0,%sr5 /* get kernel space into sr5 */ mtsp %r0,%sr6 /* get kernel space into sr6 */ - mfsp %sr7,%r1 /* save user sr7 */ - mtsp %r1,%sr3 /* and store it in sr3 */ #ifdef CONFIG_64BIT /* for now we can *always* set the W bit on entry to the syscall @@ -133,6 +131,14 @@ linux_gateway_entry: depdi 0, 31, 32, %r21 1: #endif + + /* We use a rsm/ssm pair to prevent sr3 from being clobbered + * by external interrupts. + */ + mfsp %sr7,%r1 /* save user sr7 */ + rsm PSW_SM_I, %r0 /* disable interrupts */ + mtsp %r1,%sr3 /* and store it in sr3 */ + mfctl %cr30,%r1 xor %r1,%r30,%r30 /* ye olde xor trick */ xor %r1,%r30,%r1 @@ -147,6 +153,7 @@ linux_gateway_entry: */ mtsp %r0,%sr7 /* get kernel space into sr7 */ + ssm PSW_SM_I, %r0 /* enable interrupts */ STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ mfctl %cr30,%r1 /* get task ptr in %r1 */ LDREG TI_TASK(%r1),%r1 @@ -474,11 +481,6 @@ lws_start: comiclr,>> __NR_lws_entries, %r20, %r0 b,n lws_exit_nosys - /* WARNING: Trashing sr2 and sr3 */ - mfsp %sr7,%r1 /* get userspace into sr3 */ - mtsp %r1,%sr3 - mtsp %r0,%sr2 /* get kernel space into sr2 */ - /* Load table start */ ldil L%lws_table, %r1 ldo R%lws_table(%r1), %r28 /* Scratch use of r28 */ @@ -627,9 +629,9 @@ cas_action: stw %r1, 4(%sr2,%r20) #endif /* The load and store could fail */ -1: ldw,ma 0(%sr3,%r26), %r28 +1: ldw,ma 0(%r26), %r28 sub,<> %r28, %r25, %r0 -2: stw,ma %r24, 0(%sr3,%r26) +2: stw,ma %r24, 0(%r26) /* Free lock */ stw,ma %r20, 0(%sr2,%r20) #if ENABLE_LWS_DEBUG @@ -706,9 +708,9 @@ lws_compare_and_swap_2: nop /* 8bit load */ -4: ldb 0(%sr3,%r25), %r25 +4: ldb 0(%r25), %r25 b cas2_lock_start -5: ldb 0(%sr3,%r24), %r24 +5: ldb 0(%r24), %r24 nop nop nop @@ -716,9 +718,9 @@ lws_compare_and_swap_2: nop /* 16bit load */ -6: ldh 0(%sr3,%r25), %r25 +6: ldh 0(%r25), %r25 b cas2_lock_start -7: ldh 0(%sr3,%r24), %r24 +7: ldh 0(%r24), %r24 nop nop nop @@ -726,9 +728,9 @@ lws_compare_and_swap_2: nop /* 32bit load */ -8: ldw 0(%sr3,%r25), %r25 +8: ldw 0(%r25), %r25 b cas2_lock_start -9: ldw 0(%sr3,%r24), %r24 +9: ldw 0(%r24), %r24 nop nop nop @@ -737,14 +739,14 @@ lws_compare_and_swap_2: /* 64bit load */ #ifdef CONFIG_64BIT -10: ldd 0(%sr3,%r25), %r25 -11: ldd 0(%sr3,%r24), %r24 +10: ldd 0(%r25), %r25 +11: ldd 0(%r24), %r24 #else /* Load new value into r22/r23 - high/low */ -10: ldw 0(%sr3,%r25), %r22 -11: ldw 4(%sr3,%r25), %r23 +10: ldw 0(%r25), %r22 +11: ldw 4(%r25), %r23 /* Load new value into fr4 for atomic store later */ -12: flddx 0(%sr3,%r24), %fr4 +12: flddx 0(%r24), %fr4 #endif cas2_lock_start: @@ -794,30 +796,30 @@ cas2_action: ldo 1(%r0),%r28 /* 8bit CAS */ -13: ldb,ma 0(%sr3,%r26), %r29 +13: ldb,ma 0(%r26), %r29 sub,= %r29, %r25, %r0 b,n cas2_end -14: stb,ma %r24, 0(%sr3,%r26) +14: stb,ma %r24, 0(%r26) b cas2_end copy %r0, %r28 nop nop /* 16bit CAS */ -15: ldh,ma 0(%sr3,%r26), %r29 +15: ldh,ma 0(%r26), %r29 sub,= %r29, %r25, %r0 b,n cas2_end -16: sth,ma %r24, 0(%sr3,%r26) +16: sth,ma %r24, 0(%r26) b cas2_end copy %r0, %r28 nop nop /* 32bit CAS */ -17: ldw,ma 0(%sr3,%r26), %r29 +17: ldw,ma 0(%r26), %r29 sub,= %r29, %r25, %r0 b,n cas2_end -18: stw,ma %r24, 0(%sr3,%r26) +18: stw,ma %r24, 0(%r26) b cas2_end copy %r0, %r28 nop @@ -825,22 +827,22 @@ cas2_action: /* 64bit CAS */ #ifdef CONFIG_64BIT -19: ldd,ma 0(%sr3,%r26), %r29 +19: ldd,ma 0(%r26), %r29 sub,*= %r29, %r25, %r0 b,n cas2_end -20: std,ma %r24, 0(%sr3,%r26) +20: std,ma %r24, 0(%r26) copy %r0, %r28 #else /* Compare first word */ -19: ldw,ma 0(%sr3,%r26), %r29 +19: ldw,ma 0(%r26), %r29 sub,= %r29, %r22, %r0 b,n cas2_end /* Compare second word */ -20: ldw,ma 4(%sr3,%r26), %r29 +20: ldw,ma 4(%r26), %r29 sub,= %r29, %r23, %r0 b,n cas2_end /* Perform the store */ -21: fstdx %fr4, 0(%sr3,%r26) +21: fstdx %fr4, 0(%r26) copy %r0, %r28 #endif diff --git a/arch/powerpc/include/asm/checksum.h b/arch/powerpc/include/asm/checksum.h index ee655ed1ff1b..1e8fceb308a5 100644 --- a/arch/powerpc/include/asm/checksum.h +++ b/arch/powerpc/include/asm/checksum.h @@ -53,10 +53,8 @@ static inline __sum16 csum_fold(__wsum sum) return (__force __sum16)(~((__force u32)sum + tmp) >> 16); } -static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) +static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, __u32 len, + __u8 proto, __wsum sum) { #ifdef __powerpc64__ unsigned long s = (__force u32)sum; @@ -83,10 +81,8 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, * computes the checksum of the TCP/UDP pseudo-header * returns a 16-bit checksum, already complemented */ -static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) +static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, + __u8 proto, __wsum sum) { return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); } diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h index 01b8a13f0224..3919332965af 100644 --- a/arch/powerpc/include/asm/cpuidle.h +++ b/arch/powerpc/include/asm/cpuidle.h @@ -26,7 +26,7 @@ extern u64 pnv_first_deep_stop_state; std r0,0(r1); \ ptesync; \ ld r0,0(r1); \ -1: cmp cr0,r0,r0; \ +1: cmpd cr0,r0,r0; \ bne 1b; \ IDLE_INST; \ b . diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 2e4e7d878c8e..84d49b197c32 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -93,6 +93,10 @@ ld reg,PACAKBASE(r13); /* get high part of &label */ \ ori reg,reg,(FIXED_SYMBOL_ABS_ADDR(label))@l; +#define __LOAD_HANDLER(reg, label) \ + ld reg,PACAKBASE(r13); \ + ori reg,reg,(ABS_ADDR(label))@l; + /* Exception register prefixes */ #define EXC_HV H #define EXC_STD @@ -208,6 +212,18 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define kvmppc_interrupt kvmppc_interrupt_pr #endif +#ifdef CONFIG_RELOCATABLE +#define BRANCH_TO_COMMON(reg, label) \ + __LOAD_HANDLER(reg, label); \ + mtctr reg; \ + bctr + +#else +#define BRANCH_TO_COMMON(reg, label) \ + b label + +#endif + #define __KVM_HANDLER_PROLOG(area, n) \ BEGIN_FTR_SECTION_NESTED(947) \ ld r10,area+EX_CFAR(r13); \ diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h index f6f68f73e858..99e1397b71da 100644 --- a/arch/powerpc/include/asm/tlb.h +++ b/arch/powerpc/include/asm/tlb.h @@ -52,11 +52,23 @@ static inline int mm_is_core_local(struct mm_struct *mm) return cpumask_subset(mm_cpumask(mm), topology_sibling_cpumask(smp_processor_id())); } + +static inline int mm_is_thread_local(struct mm_struct *mm) +{ + return cpumask_equal(mm_cpumask(mm), + cpumask_of(smp_processor_id())); +} + #else static inline int mm_is_core_local(struct mm_struct *mm) { return 1; } + +static inline int mm_is_thread_local(struct mm_struct *mm) +{ + return 1; +} #endif #endif /* __KERNEL__ */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index f129408c6022..08ba447a4b3d 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -95,19 +95,35 @@ __start_interrupts: /* No virt vectors corresponding with 0x0..0x100 */ EXC_VIRT_NONE(0x4000, 0x4100) -EXC_REAL_BEGIN(system_reset, 0x100, 0x200) - SET_SCRATCH0(r13) + #ifdef CONFIG_PPC_P7_NAP -BEGIN_FTR_SECTION - /* Running native on arch 2.06 or later, check if we are - * waking up from nap/sleep/winkle. + /* + * If running native on arch 2.06 or later, check if we are waking up + * from nap/sleep/winkle, and branch to idle handler. */ - mfspr r13,SPRN_SRR1 - rlwinm. r13,r13,47-31,30,31 - beq 9f +#define IDLETEST(n) \ + BEGIN_FTR_SECTION ; \ + mfspr r10,SPRN_SRR1 ; \ + rlwinm. r10,r10,47-31,30,31 ; \ + beq- 1f ; \ + cmpwi cr3,r10,2 ; \ + BRANCH_TO_COMMON(r10, system_reset_idle_common) ; \ +1: \ + END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) +#else +#define IDLETEST NOTEST +#endif - cmpwi cr3,r13,2 - GET_PACA(r13) +EXC_REAL_BEGIN(system_reset, 0x100, 0x200) + SET_SCRATCH0(r13) + EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD, + IDLETEST, 0x100) + +EXC_REAL_END(system_reset, 0x100, 0x200) +EXC_VIRT_NONE(0x4100, 0x4200) + +#ifdef CONFIG_PPC_P7_NAP +EXC_COMMON_BEGIN(system_reset_idle_common) bl pnv_restore_hyp_resource li r0,PNV_THREAD_RUNNING @@ -130,14 +146,8 @@ BEGIN_FTR_SECTION blt cr3,2f b pnv_wakeup_loss 2: b pnv_wakeup_noloss +#endif -9: -END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) -#endif /* CONFIG_PPC_P7_NAP */ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common, EXC_STD, - NOTEST, 0x100) -EXC_REAL_END(system_reset, 0x100, 0x200) -EXC_VIRT_NONE(0x4100, 0x4200) EXC_COMMON(system_reset_common, 0x100, system_reset_exception) #ifdef CONFIG_PPC_PSERIES @@ -817,10 +827,8 @@ EXC_VIRT(trap_0b, 0x4b00, 0x4c00, 0xb00) TRAMP_KVM(PACA_EXGEN, 0xb00) EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) - -#define LOAD_SYSCALL_HANDLER(reg) \ - ld reg,PACAKBASE(r13); \ - ori reg,reg,(ABS_ADDR(system_call_common))@l; +#define LOAD_SYSCALL_HANDLER(reg) \ + __LOAD_HANDLER(reg, system_call_common) /* Syscall routine is used twice, in reloc-off and reloc-on paths */ #define SYSCALL_PSERIES_1 \ diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c index 9781c69eae57..03d089b3ed72 100644 --- a/arch/powerpc/kernel/hw_breakpoint.c +++ b/arch/powerpc/kernel/hw_breakpoint.c @@ -275,7 +275,7 @@ int hw_breakpoint_handler(struct die_args *args) if (!stepped) { WARN(1, "Unable to handle hardware breakpoint. Breakpoint at " "0x%lx will be disabled.", info->address); - perf_event_disable(bp); + perf_event_disable_inatomic(bp); goto out; } /* diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S index bd739fed26e3..72dac0b58061 100644 --- a/arch/powerpc/kernel/idle_book3s.S +++ b/arch/powerpc/kernel/idle_book3s.S @@ -90,6 +90,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) * Threads will spin in HMT_LOW until the lock bit is cleared. * r14 - pointer to core_idle_state * r15 - used to load contents of core_idle_state + * r9 - used as a temporary variable */ core_idle_lock_held: @@ -99,6 +100,8 @@ core_idle_lock_held: bne 3b HMT_MEDIUM lwarx r15,0,r14 + andi. r9,r15,PNV_CORE_IDLE_LOCK_BIT + bne core_idle_lock_held blr /* @@ -163,12 +166,6 @@ _GLOBAL(pnv_powersave_common) std r9,_MSR(r1) std r1,PACAR1(r13) -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE - /* Tell KVM we're entering idle */ - li r4,KVM_HWTHREAD_IN_IDLE - stb r4,HSTATE_HWTHREAD_STATE(r13) -#endif - /* * Go to real mode to do the nap, as required by the architecture. * Also, we need to be in real mode before setting hwthread_state, @@ -185,6 +182,26 @@ _GLOBAL(pnv_powersave_common) .globl pnv_enter_arch207_idle_mode pnv_enter_arch207_idle_mode: +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + /* Tell KVM we're entering idle */ + li r4,KVM_HWTHREAD_IN_IDLE + /******************************************************/ + /* N O T E W E L L ! ! ! N O T E W E L L */ + /* The following store to HSTATE_HWTHREAD_STATE(r13) */ + /* MUST occur in real mode, i.e. with the MMU off, */ + /* and the MMU must stay off until we clear this flag */ + /* and test HSTATE_HWTHREAD_REQ(r13) in the system */ + /* reset interrupt vector in exceptions-64s.S. */ + /* The reason is that another thread can switch the */ + /* MMU to a guest context whenever this flag is set */ + /* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on, */ + /* that would potentially cause this thread to start */ + /* executing instructions from guest memory in */ + /* hypervisor mode, leading to a host crash or data */ + /* corruption, or worse. */ + /******************************************************/ + stb r4,HSTATE_HWTHREAD_STATE(r13) +#endif stb r3,PACA_THREAD_IDLE_STATE(r13) cmpwi cr3,r3,PNV_THREAD_SLEEP bge cr3,2f @@ -250,6 +267,12 @@ enter_winkle: * r3 - requested stop state */ power_enter_stop: +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + /* Tell KVM we're entering idle */ + li r4,KVM_HWTHREAD_IN_IDLE + /* DO THIS IN REAL MODE! See comment above. */ + stb r4,HSTATE_HWTHREAD_STATE(r13) +#endif /* * Check if the requested state is a deep idle state. */ diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 9e7c10fe205f..ce6dc61b15b2 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1012,7 +1012,7 @@ void restore_tm_state(struct pt_regs *regs) /* Ensure that restore_math() will restore */ if (msr_diff & MSR_FP) current->thread.load_fp = 1; -#ifdef CONFIG_ALIVEC +#ifdef CONFIG_ALTIVEC if (cpu_has_feature(CPU_FTR_ALTIVEC) && msr_diff & MSR_VEC) current->thread.load_vec = 1; #endif diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index 82ff5de8b1e7..a0ea63ac2b52 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -23,6 +23,7 @@ #include <asm/ppc-opcode.h> #include <asm/pnv-pci.h> #include <asm/opal.h> +#include <asm/smp.h> #include "book3s_xics.h" diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c index 0e49ec541ab5..bda8c43be78a 100644 --- a/arch/powerpc/mm/tlb-radix.c +++ b/arch/powerpc/mm/tlb-radix.c @@ -175,7 +175,7 @@ void radix__flush_tlb_mm(struct mm_struct *mm) if (unlikely(pid == MMU_NO_CONTEXT)) goto no_context; - if (!mm_is_core_local(mm)) { + if (!mm_is_thread_local(mm)) { int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); if (lock_tlbie) @@ -201,7 +201,7 @@ void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr) if (unlikely(pid == MMU_NO_CONTEXT)) goto no_context; - if (!mm_is_core_local(mm)) { + if (!mm_is_thread_local(mm)) { int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); if (lock_tlbie) @@ -226,7 +226,7 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, pid = mm ? mm->context.id : 0; if (unlikely(pid == MMU_NO_CONTEXT)) goto bail; - if (!mm_is_core_local(mm)) { + if (!mm_is_thread_local(mm)) { int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); if (lock_tlbie) @@ -321,7 +321,7 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, { unsigned long pid; unsigned long addr; - int local = mm_is_core_local(mm); + int local = mm_is_thread_local(mm); unsigned long ap = mmu_get_ap(psize); int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); unsigned long page_size = 1UL << mmu_psize_defs[psize].shift; diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 64053d9ac3f2..836c56290499 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -12,9 +12,7 @@ #ifndef __ASSEMBLY__ -unsigned long return_address(int depth); - -#define ftrace_return_address(n) return_address(n) +#define ftrace_return_address(n) __builtin_return_address(n) void _mcount(void); void ftrace_caller(void); diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 03323175de30..602af692efdc 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -192,7 +192,7 @@ struct task_struct; struct mm_struct; struct seq_file; -typedef int (*dump_trace_func_t)(void *data, unsigned long address); +typedef int (*dump_trace_func_t)(void *data, unsigned long address, int reliable); void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, unsigned long sp); diff --git a/arch/s390/include/asm/unistd.h b/arch/s390/include/asm/unistd.h index 02613bad8bbb..3066031a73fe 100644 --- a/arch/s390/include/asm/unistd.h +++ b/arch/s390/include/asm/unistd.h @@ -9,6 +9,9 @@ #include <uapi/asm/unistd.h> #define __IGNORE_time +#define __IGNORE_pkey_mprotect +#define __IGNORE_pkey_alloc +#define __IGNORE_pkey_free #define __ARCH_WANT_OLD_READDIR #define __ARCH_WANT_SYS_ALARM diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 43446fa2a4e5..c74c59236f44 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -2014,12 +2014,12 @@ void show_code(struct pt_regs *regs) *ptr++ = '\t'; ptr += print_insn(ptr, code + start, addr); start += opsize; - printk("%s", buffer); + pr_cont("%s", buffer); ptr = buffer; ptr += sprintf(ptr, "\n "); hops++; } - printk("\n"); + pr_cont("\n"); } void print_fn_code(unsigned char *code, unsigned long len) diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index 6693383bc01b..55d4fe174fd9 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -38,10 +38,10 @@ __dump_trace(dump_trace_func_t func, void *data, unsigned long sp, if (sp < low || sp > high - sizeof(*sf)) return sp; sf = (struct stack_frame *) sp; + if (func(data, sf->gprs[8], 0)) + return sp; /* Follow the backchain. */ while (1) { - if (func(data, sf->gprs[8])) - return sp; low = sp; sp = sf->back_chain; if (!sp) @@ -49,6 +49,8 @@ __dump_trace(dump_trace_func_t func, void *data, unsigned long sp, if (sp <= low || sp > high - sizeof(*sf)) return sp; sf = (struct stack_frame *) sp; + if (func(data, sf->gprs[8], 1)) + return sp; } /* Zero backchain detected, check for interrupt frame. */ sp = (unsigned long) (sf + 1); @@ -56,7 +58,7 @@ __dump_trace(dump_trace_func_t func, void *data, unsigned long sp, return sp; regs = (struct pt_regs *) sp; if (!user_mode(regs)) { - if (func(data, regs->psw.addr)) + if (func(data, regs->psw.addr, 1)) return sp; } low = sp; @@ -85,33 +87,12 @@ void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, } EXPORT_SYMBOL_GPL(dump_trace); -struct return_address_data { - unsigned long address; - int depth; -}; - -static int __return_address(void *data, unsigned long address) -{ - struct return_address_data *rd = data; - - if (rd->depth--) - return 0; - rd->address = address; - return 1; -} - -unsigned long return_address(int depth) -{ - struct return_address_data rd = { .depth = depth + 2 }; - - dump_trace(__return_address, &rd, NULL, current_stack_pointer()); - return rd.address; -} -EXPORT_SYMBOL_GPL(return_address); - -static int show_address(void *data, unsigned long address) +static int show_address(void *data, unsigned long address, int reliable) { - printk("([<%016lx>] %pSR)\n", address, (void *)address); + if (reliable) + printk(" [<%016lx>] %pSR \n", address, (void *)address); + else + printk("([<%016lx>] %pSR)\n", address, (void *)address); return 0; } @@ -138,14 +119,14 @@ void show_stack(struct task_struct *task, unsigned long *sp) else stack = (unsigned long *)task->thread.ksp; } + printk(KERN_DEFAULT "Stack:\n"); for (i = 0; i < 20; i++) { if (((addr_t) stack & (THREAD_SIZE-1)) == 0) break; - if ((i * sizeof(long) % 32) == 0) - printk("%s ", i == 0 ? "" : "\n"); - printk("%016lx ", *stack++); + if (i % 4 == 0) + printk(KERN_DEFAULT " "); + pr_cont("%016lx%c", *stack++, i % 4 == 3 ? '\n' : ' '); } - printk("\n"); show_trace(task, (unsigned long)sp); } @@ -163,13 +144,13 @@ void show_registers(struct pt_regs *regs) mode = user_mode(regs) ? "User" : "Krnl"; printk("%s PSW : %p %p", mode, (void *)regs->psw.mask, (void *)regs->psw.addr); if (!user_mode(regs)) - printk(" (%pSR)", (void *)regs->psw.addr); - printk("\n"); + pr_cont(" (%pSR)", (void *)regs->psw.addr); + pr_cont("\n"); printk(" R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x " "P:%x AS:%x CC:%x PM:%x", psw->r, psw->t, psw->i, psw->e, psw->key, psw->m, psw->w, psw->p, psw->as, psw->cc, psw->pm); - printk(" RI:%x EA:%x", psw->ri, psw->eaba); - printk("\n%s GPRS: %016lx %016lx %016lx %016lx\n", mode, + pr_cont(" RI:%x EA:%x\n", psw->ri, psw->eaba); + printk("%s GPRS: %016lx %016lx %016lx %016lx\n", mode, regs->gprs[0], regs->gprs[1], regs->gprs[2], regs->gprs[3]); printk(" %016lx %016lx %016lx %016lx\n", regs->gprs[4], regs->gprs[5], regs->gprs[6], regs->gprs[7]); @@ -205,14 +186,14 @@ void die(struct pt_regs *regs, const char *str) printk("%s: %04x ilc:%d [#%d] ", str, regs->int_code & 0xffff, regs->int_code >> 17, ++die_counter); #ifdef CONFIG_PREEMPT - printk("PREEMPT "); + pr_cont("PREEMPT "); #endif #ifdef CONFIG_SMP - printk("SMP "); + pr_cont("SMP "); #endif if (debug_pagealloc_enabled()) - printk("DEBUG_PAGEALLOC"); - printk("\n"); + pr_cont("DEBUG_PAGEALLOC"); + pr_cont("\n"); notify_die(DIE_OOPS, str, regs, 0, regs->int_code & 0xffff, SIGSEGV); print_modules(); show_regs(regs); diff --git a/arch/s390/kernel/perf_event.c b/arch/s390/kernel/perf_event.c index 17431f63de00..955a7b6fa0a4 100644 --- a/arch/s390/kernel/perf_event.c +++ b/arch/s390/kernel/perf_event.c @@ -222,7 +222,7 @@ static int __init service_level_perf_register(void) } arch_initcall(service_level_perf_register); -static int __perf_callchain_kernel(void *data, unsigned long address) +static int __perf_callchain_kernel(void *data, unsigned long address, int reliable) { struct perf_callchain_entry_ctx *entry = data; diff --git a/arch/s390/kernel/stacktrace.c b/arch/s390/kernel/stacktrace.c index 44f84b23d4e5..355db9db8210 100644 --- a/arch/s390/kernel/stacktrace.c +++ b/arch/s390/kernel/stacktrace.c @@ -27,12 +27,12 @@ static int __save_address(void *data, unsigned long address, int nosched) return 1; } -static int save_address(void *data, unsigned long address) +static int save_address(void *data, unsigned long address, int reliable) { return __save_address(data, address, 0); } -static int save_address_nosched(void *data, unsigned long address) +static int save_address_nosched(void *data, unsigned long address, int reliable) { return __save_address(data, address, 1); } diff --git a/arch/s390/kvm/sthyi.c b/arch/s390/kvm/sthyi.c index bd98b7d25200..05c98bb853cf 100644 --- a/arch/s390/kvm/sthyi.c +++ b/arch/s390/kvm/sthyi.c @@ -315,7 +315,7 @@ static void fill_diag(struct sthyi_sctns *sctns) if (r < 0) goto out; - diag224_buf = kmalloc(PAGE_SIZE, GFP_KERNEL | GFP_DMA); + diag224_buf = (void *)__get_free_page(GFP_KERNEL | GFP_DMA); if (!diag224_buf || diag224(diag224_buf)) goto out; @@ -378,7 +378,7 @@ static void fill_diag(struct sthyi_sctns *sctns) sctns->par.infpval1 |= PAR_WGHT_VLD; out: - kfree(diag224_buf); + free_page((unsigned long)diag224_buf); vfree(diag204_buf); } diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index cd404aa3931c..4a0c5bce3552 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -217,6 +217,7 @@ static __init int setup_hugepagesz(char *opt) } else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) { hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT); } else { + hugetlb_bad_size(); pr_err("hugepagesz= specifies an unsupported page size %s\n", string); return 0; diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index f56a39bd8ba6..b3e9d18f2ec6 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -151,36 +151,40 @@ void __init free_initrd_mem(unsigned long start, unsigned long end) #ifdef CONFIG_MEMORY_HOTPLUG int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { - unsigned long normal_end_pfn = PFN_DOWN(memblock_end_of_DRAM()); - unsigned long dma_end_pfn = PFN_DOWN(MAX_DMA_ADDRESS); + unsigned long zone_start_pfn, zone_end_pfn, nr_pages; unsigned long start_pfn = PFN_DOWN(start); unsigned long size_pages = PFN_DOWN(size); - unsigned long nr_pages; - int rc, zone_enum; + pg_data_t *pgdat = NODE_DATA(nid); + struct zone *zone; + int rc, i; rc = vmem_add_mapping(start, size); if (rc) return rc; - while (size_pages > 0) { - if (start_pfn < dma_end_pfn) { - nr_pages = (start_pfn + size_pages > dma_end_pfn) ? - dma_end_pfn - start_pfn : size_pages; - zone_enum = ZONE_DMA; - } else if (start_pfn < normal_end_pfn) { - nr_pages = (start_pfn + size_pages > normal_end_pfn) ? - normal_end_pfn - start_pfn : size_pages; - zone_enum = ZONE_NORMAL; + for (i = 0; i < MAX_NR_ZONES; i++) { + zone = pgdat->node_zones + i; + if (zone_idx(zone) != ZONE_MOVABLE) { + /* Add range within existing zone limits, if possible */ + zone_start_pfn = zone->zone_start_pfn; + zone_end_pfn = zone->zone_start_pfn + + zone->spanned_pages; } else { - nr_pages = size_pages; - zone_enum = ZONE_MOVABLE; + /* Add remaining range to ZONE_MOVABLE */ + zone_start_pfn = start_pfn; + zone_end_pfn = start_pfn + size_pages; } - rc = __add_pages(nid, NODE_DATA(nid)->node_zones + zone_enum, - start_pfn, size_pages); + if (start_pfn < zone_start_pfn || start_pfn >= zone_end_pfn) + continue; + nr_pages = (start_pfn + size_pages > zone_end_pfn) ? + zone_end_pfn - start_pfn : size_pages; + rc = __add_pages(nid, zone, start_pfn, nr_pages); if (rc) break; start_pfn += nr_pages; size_pages -= nr_pages; + if (!size_pages) + break; } if (rc) vmem_remove_mapping(start, size); diff --git a/arch/s390/oprofile/init.c b/arch/s390/oprofile/init.c index 16f4c3960b87..9a4de4599c7b 100644 --- a/arch/s390/oprofile/init.c +++ b/arch/s390/oprofile/init.c @@ -13,7 +13,7 @@ #include <linux/init.h> #include <asm/processor.h> -static int __s390_backtrace(void *data, unsigned long address) +static int __s390_backtrace(void *data, unsigned long address, int reliable) { unsigned int *depth = data; diff --git a/arch/sparc/include/asm/cpudata_64.h b/arch/sparc/include/asm/cpudata_64.h index a6cfdabb6054..5b0ed48e5b0c 100644 --- a/arch/sparc/include/asm/cpudata_64.h +++ b/arch/sparc/include/asm/cpudata_64.h @@ -24,9 +24,10 @@ typedef struct { unsigned int icache_line_size; unsigned int ecache_size; unsigned int ecache_line_size; - unsigned short sock_id; + unsigned short sock_id; /* physical package */ unsigned short core_id; - int proc_id; + unsigned short max_cache_id; /* groupings of highest shared cache */ + unsigned short proc_id; /* strand (aka HW thread) id */ } cpuinfo_sparc; DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h index d9c5876c6121..8011e79f59c9 100644 --- a/arch/sparc/include/asm/spinlock_32.h +++ b/arch/sparc/include/asm/spinlock_32.h @@ -134,7 +134,7 @@ static inline void arch_write_lock(arch_rwlock_t *rw) *(volatile __u32 *)&lp->lock = ~0U; } -static void inline arch_write_unlock(arch_rwlock_t *lock) +static inline void arch_write_unlock(arch_rwlock_t *lock) { __asm__ __volatile__( " st %%g0, [%0]" diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h index 87990b7c6b0d..07c9f2e9bf57 100644 --- a/arch/sparc/include/asm/spinlock_64.h +++ b/arch/sparc/include/asm/spinlock_64.h @@ -96,7 +96,7 @@ static inline void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long fla /* Multi-reader locks, these are much saner than the 32-bit Sparc ones... */ -static void inline arch_read_lock(arch_rwlock_t *lock) +static inline void arch_read_lock(arch_rwlock_t *lock) { unsigned long tmp1, tmp2; @@ -119,7 +119,7 @@ static void inline arch_read_lock(arch_rwlock_t *lock) : "memory"); } -static int inline arch_read_trylock(arch_rwlock_t *lock) +static inline int arch_read_trylock(arch_rwlock_t *lock) { int tmp1, tmp2; @@ -140,7 +140,7 @@ static int inline arch_read_trylock(arch_rwlock_t *lock) return tmp1; } -static void inline arch_read_unlock(arch_rwlock_t *lock) +static inline void arch_read_unlock(arch_rwlock_t *lock) { unsigned long tmp1, tmp2; @@ -156,7 +156,7 @@ static void inline arch_read_unlock(arch_rwlock_t *lock) : "memory"); } -static void inline arch_write_lock(arch_rwlock_t *lock) +static inline void arch_write_lock(arch_rwlock_t *lock) { unsigned long mask, tmp1, tmp2; @@ -181,7 +181,7 @@ static void inline arch_write_lock(arch_rwlock_t *lock) : "memory"); } -static void inline arch_write_unlock(arch_rwlock_t *lock) +static inline void arch_write_unlock(arch_rwlock_t *lock) { __asm__ __volatile__( " stw %%g0, [%0]" @@ -190,7 +190,7 @@ static void inline arch_write_unlock(arch_rwlock_t *lock) : "memory"); } -static int inline arch_write_trylock(arch_rwlock_t *lock) +static inline int arch_write_trylock(arch_rwlock_t *lock) { unsigned long mask, tmp1, tmp2, result; diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h index bec481aaca16..7b4898a36eee 100644 --- a/arch/sparc/include/asm/topology_64.h +++ b/arch/sparc/include/asm/topology_64.h @@ -44,14 +44,20 @@ int __node_distance(int, int); #define topology_physical_package_id(cpu) (cpu_data(cpu).proc_id) #define topology_core_id(cpu) (cpu_data(cpu).core_id) #define topology_core_cpumask(cpu) (&cpu_core_sib_map[cpu]) +#define topology_core_cache_cpumask(cpu) (&cpu_core_sib_cache_map[cpu]) #define topology_sibling_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) #endif /* CONFIG_SMP */ extern cpumask_t cpu_core_map[NR_CPUS]; extern cpumask_t cpu_core_sib_map[NR_CPUS]; +extern cpumask_t cpu_core_sib_cache_map[NR_CPUS]; + +/** + * Return cores that shares the last level cache. + */ static inline const struct cpumask *cpu_coregroup_mask(int cpu) { - return &cpu_core_map[cpu]; + return &cpu_core_sib_cache_map[cpu]; } #endif /* _ASM_SPARC64_TOPOLOGY_H */ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index b68acc563235..5373136c412b 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -82,7 +82,6 @@ static inline int access_ok(int type, const void __user * addr, unsigned long si return 1; } -void __ret_efault(void); void __retl_efault(void); /* Uh, these should become the main single-value transfer routines.. @@ -189,55 +188,34 @@ int __get_user_bad(void); unsigned long __must_check ___copy_from_user(void *to, const void __user *from, unsigned long size); -unsigned long copy_from_user_fixup(void *to, const void __user *from, - unsigned long size); static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long size) { - unsigned long ret; - check_object_size(to, size, false); - ret = ___copy_from_user(to, from, size); - if (unlikely(ret)) - ret = copy_from_user_fixup(to, from, size); - - return ret; + return ___copy_from_user(to, from, size); } #define __copy_from_user copy_from_user unsigned long __must_check ___copy_to_user(void __user *to, const void *from, unsigned long size); -unsigned long copy_to_user_fixup(void __user *to, const void *from, - unsigned long size); static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long size) { - unsigned long ret; - check_object_size(from, size, true); - ret = ___copy_to_user(to, from, size); - if (unlikely(ret)) - ret = copy_to_user_fixup(to, from, size); - return ret; + return ___copy_to_user(to, from, size); } #define __copy_to_user copy_to_user unsigned long __must_check ___copy_in_user(void __user *to, const void __user *from, unsigned long size); -unsigned long copy_in_user_fixup(void __user *to, void __user *from, - unsigned long size); static inline unsigned long __must_check copy_in_user(void __user *to, void __user *from, unsigned long size) { - unsigned long ret = ___copy_in_user(to, from, size); - - if (unlikely(ret)) - ret = copy_in_user_fixup(to, from, size); - return ret; + return ___copy_in_user(to, from, size); } #define __copy_in_user copy_in_user diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S index beba6c11554c..6aa3da152c20 100644 --- a/arch/sparc/kernel/head_64.S +++ b/arch/sparc/kernel/head_64.S @@ -926,48 +926,11 @@ tlb_type: .word 0 /* Must NOT end up in BSS */ EXPORT_SYMBOL(tlb_type) .section ".fixup",#alloc,#execinstr - .globl __ret_efault, __retl_efault, __ret_one, __retl_one -ENTRY(__ret_efault) - ret - restore %g0, -EFAULT, %o0 -ENDPROC(__ret_efault) -EXPORT_SYMBOL(__ret_efault) - ENTRY(__retl_efault) retl mov -EFAULT, %o0 ENDPROC(__retl_efault) -ENTRY(__retl_one) - retl - mov 1, %o0 -ENDPROC(__retl_one) - -ENTRY(__retl_one_fp) - VISExitHalf - retl - mov 1, %o0 -ENDPROC(__retl_one_fp) - -ENTRY(__ret_one_asi) - wr %g0, ASI_AIUS, %asi - ret - restore %g0, 1, %o0 -ENDPROC(__ret_one_asi) - -ENTRY(__retl_one_asi) - wr %g0, ASI_AIUS, %asi - retl - mov 1, %o0 -ENDPROC(__retl_one_asi) - -ENTRY(__retl_one_asi_fp) - wr %g0, ASI_AIUS, %asi - VISExitHalf - retl - mov 1, %o0 -ENDPROC(__retl_one_asi_fp) - ENTRY(__retl_o1) retl mov %o1, %o0 diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index 59bbeff55024..07933b9e9ce0 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -13,19 +13,30 @@ void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { - u32 val; u32 *insn = (u32 *) (unsigned long) entry->code; + u32 val; if (type == JUMP_LABEL_JMP) { s32 off = (s32)entry->target - (s32)entry->code; + bool use_v9_branch = false; + + BUG_ON(off & 3); #ifdef CONFIG_SPARC64 - /* ba,pt %xcc, . + (off << 2) */ - val = 0x10680000 | ((u32) off >> 2); -#else - /* ba . + (off << 2) */ - val = 0x10800000 | ((u32) off >> 2); + if (off <= 0xfffff && off >= -0x100000) + use_v9_branch = true; #endif + if (use_v9_branch) { + /* WDISP19 - target is . + immed << 2 */ + /* ba,pt %xcc, . + off */ + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff); + } else { + /* WDISP22 - target is . + immed << 2 */ + BUG_ON(off > 0x7fffff); + BUG_ON(off < -0x800000); + /* ba . + off */ + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff); + } } else { val = 0x01000000; } diff --git a/arch/sparc/kernel/mdesc.c b/arch/sparc/kernel/mdesc.c index 11228861d9b4..8a6982dfd733 100644 --- a/arch/sparc/kernel/mdesc.c +++ b/arch/sparc/kernel/mdesc.c @@ -645,13 +645,20 @@ static void __mark_core_id(struct mdesc_handle *hp, u64 node, cpu_data(*id).core_id = core_id; } -static void __mark_sock_id(struct mdesc_handle *hp, u64 node, - int sock_id) +static void __mark_max_cache_id(struct mdesc_handle *hp, u64 node, + int max_cache_id) { const u64 *id = mdesc_get_property(hp, node, "id", NULL); - if (*id < num_possible_cpus()) - cpu_data(*id).sock_id = sock_id; + if (*id < num_possible_cpus()) { + cpu_data(*id).max_cache_id = max_cache_id; + + /** + * On systems without explicit socket descriptions socket + * is max_cache_id + */ + cpu_data(*id).sock_id = max_cache_id; + } } static void mark_core_ids(struct mdesc_handle *hp, u64 mp, @@ -660,10 +667,11 @@ static void mark_core_ids(struct mdesc_handle *hp, u64 mp, find_back_node_value(hp, mp, "cpu", __mark_core_id, core_id, 10); } -static void mark_sock_ids(struct mdesc_handle *hp, u64 mp, - int sock_id) +static void mark_max_cache_ids(struct mdesc_handle *hp, u64 mp, + int max_cache_id) { - find_back_node_value(hp, mp, "cpu", __mark_sock_id, sock_id, 10); + find_back_node_value(hp, mp, "cpu", __mark_max_cache_id, + max_cache_id, 10); } static void set_core_ids(struct mdesc_handle *hp) @@ -694,14 +702,15 @@ static void set_core_ids(struct mdesc_handle *hp) } } -static int set_sock_ids_by_cache(struct mdesc_handle *hp, int level) +static int set_max_cache_ids_by_cache(struct mdesc_handle *hp, int level) { u64 mp; int idx = 1; int fnd = 0; - /* Identify unique sockets by looking for cpus backpointed to by - * shared level n caches. + /** + * Identify unique highest level of shared cache by looking for cpus + * backpointed to by shared level N caches. */ mdesc_for_each_node_by_name(hp, mp, "cache") { const u64 *cur_lvl; @@ -709,8 +718,7 @@ static int set_sock_ids_by_cache(struct mdesc_handle *hp, int level) cur_lvl = mdesc_get_property(hp, mp, "level", NULL); if (*cur_lvl != level) continue; - - mark_sock_ids(hp, mp, idx); + mark_max_cache_ids(hp, mp, idx); idx++; fnd = 1; } @@ -745,15 +753,17 @@ static void set_sock_ids(struct mdesc_handle *hp) { u64 mp; - /* If machine description exposes sockets data use it. - * Otherwise fallback to use shared L3 or L2 caches. + /** + * Find the highest level of shared cache which pre-T7 is also + * the socket. */ + if (!set_max_cache_ids_by_cache(hp, 3)) + set_max_cache_ids_by_cache(hp, 2); + + /* If machine description exposes sockets data use it.*/ mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "sockets"); if (mp != MDESC_NODE_NULL) - return set_sock_ids_by_socket(hp, mp); - - if (!set_sock_ids_by_cache(hp, 3)) - set_sock_ids_by_cache(hp, 2); + set_sock_ids_by_socket(hp, mp); } static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index d3035ba6cd31..8182f7caf5b1 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -63,9 +63,13 @@ cpumask_t cpu_core_map[NR_CPUS] __read_mostly = cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = CPU_MASK_NONE }; +cpumask_t cpu_core_sib_cache_map[NR_CPUS] __read_mostly = { + [0 ... NR_CPUS - 1] = CPU_MASK_NONE }; + EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); EXPORT_SYMBOL(cpu_core_map); EXPORT_SYMBOL(cpu_core_sib_map); +EXPORT_SYMBOL(cpu_core_sib_cache_map); static cpumask_t smp_commenced_mask; @@ -1265,6 +1269,10 @@ void smp_fill_in_sib_core_maps(void) unsigned int j; for_each_present_cpu(j) { + if (cpu_data(i).max_cache_id == + cpu_data(j).max_cache_id) + cpumask_set_cpu(j, &cpu_core_sib_cache_map[i]); + if (cpu_data(i).sock_id == cpu_data(j).sock_id) cpumask_set_cpu(j, &cpu_core_sib_map[i]); } diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S index b7d0bd6b1406..69a439fa2fc1 100644 --- a/arch/sparc/lib/GENcopy_from_user.S +++ b/arch/sparc/lib/GENcopy_from_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S index 780550e1afc7..9947427ce354 100644 --- a/arch/sparc/lib/GENcopy_to_user.S +++ b/arch/sparc/lib/GENcopy_to_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S index 89358ee94851..059ea24ad73d 100644 --- a/arch/sparc/lib/GENmemcpy.S +++ b/arch/sparc/lib/GENmemcpy.S @@ -4,21 +4,18 @@ */ #ifdef __KERNEL__ +#include <linux/linkage.h> #define GLOBAL_SPARE %g7 #else #define GLOBAL_SPARE %g5 #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST(x,y) x #endif #ifndef LOAD @@ -45,6 +42,29 @@ .register %g3,#scratch .text + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(GEN_retl_o4_1) + add %o4, %o2, %o4 + retl + add %o4, 1, %o0 +ENDPROC(GEN_retl_o4_1) +ENTRY(GEN_retl_g1_8) + add %g1, %o2, %g1 + retl + add %g1, 8, %o0 +ENDPROC(GEN_retl_g1_8) +ENTRY(GEN_retl_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(GEN_retl_o2_4) +ENTRY(GEN_retl_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(GEN_retl_o2_1) +#endif + .align 64 .globl FUNC_NAME @@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %o4, %o4 sub %o2, %o4, %o2 1: subcc %o4, 1, %o4 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1) + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1) add %o1, 1, %o1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x7, %g1 sub %o2, %g1, %o2 1: subcc %g1, 0x8, %g1 - EX_LD(LOAD(ldx, %o1, %g2)) - EX_ST(STORE(stx, %g2, %o0)) + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8) + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8) add %o1, 0x8, %o1 bne,pt %XCC, 1b add %o0, 0x8, %o0 @@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4) + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1) + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 885f00e81d1a..69912d2f8b54 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o -lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o obj-$(CONFIG_SPARC64) += iomap.o diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S index d5242b8c4f94..b79a6998d87c 100644 --- a/arch/sparc/lib/NG2copy_from_user.S +++ b/arch/sparc/lib/NG2copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S index 4e962d993b10..dcec55f254ab 100644 --- a/arch/sparc/lib/NG2copy_to_user.S +++ b/arch/sparc/lib/NG2copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S index d5f585df2f3f..c629dbd121b6 100644 --- a/arch/sparc/lib/NG2memcpy.S +++ b/arch/sparc/lib/NG2memcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE %g7 @@ -32,21 +33,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -140,45 +137,110 @@ fsrc2 %x6, %f12; \ fsrc2 %x7, %f14; #define FREG_LOAD_1(base, x0) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)) + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1) #define FREG_LOAD_2(base, x0, x1) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); #define FREG_LOAD_3(base, x0, x1, x2) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); #define FREG_LOAD_4(base, x0, x1, x2, x3) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ - EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ - EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ - EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ - EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ - EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ - EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \ - EX_LD_FP(LOAD(ldd, base + 0x30, %x6)); + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1); .register %g2,#scratch .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf +__restore_asi: + retl + wr %g0, ASI_AIUS, %asi +ENTRY(NG2_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(NG2_retl_o2) +ENTRY(NG2_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(NG2_retl_o2_plus_1) +ENTRY(NG2_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(NG2_retl_o2_plus_4) +ENTRY(NG2_retl_o2_plus_8) + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(NG2_retl_o2_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_1) + add %o4, 1, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_1) +ENTRY(NG2_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_16) +ENTRY(NG2_retl_o2_plus_g1_fp) + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp) + add %g1, 64, %g1 + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_1) +ENTRY(NG2_retl_o2_and_7_plus_o4) + and %o2, 7, %o2 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4) +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8) + and %o2, 7, %o2 + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8) +#endif + .align 64 .globl FUNC_NAME @@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %o4, %o4 ! bytes to align dst sub %o2, %o4, %o2 1: subcc %o4, 1, %o4 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1) add %o1, 1, %o1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop /* fall through for 0 < low bits < 8 */ 110: sub %o4, 64, %g2 - EX_LD_FP(LOAD_BLK(%g2, %f0)) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 120: sub %o4, 56, %g2 FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 130: sub %o4, 48, %g2 FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_6(f20, f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 140: sub %o4, 40, %g2 FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_5(f22, f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 150: sub %o4, 32, %g2 FREG_LOAD_4(%g2, f0, f2, f4, f6) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_4(f24, f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 160: sub %o4, 24, %g2 FREG_LOAD_3(%g2, f0, f2, f4) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_3(f26, f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 170: sub %o4, 16, %g2 FREG_LOAD_2(%g2, f0, f2) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_2(f28, f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 180: sub %o4, 8, %g2 FREG_LOAD_1(%g2, f0) -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) - EX_LD_FP(LOAD_BLK(%o4, %f16)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) FREG_MOVE_1(f30) subcc %g1, 64, %g1 add %o4, 64, %o4 @@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop 190: -1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) subcc %g1, 64, %g1 - EX_LD_FP(LOAD_BLK(%o4, %f0)) - EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64) add %o4, 64, %o4 bne,pt %xcc, 1b LOAD(prefetch, %o4 + 64, #one_read) @@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0xf, %o4 and %o2, 0xf, %o2 1: subcc %o4, 0x10, %o4 - EX_LD(LOAD(ldx, %o1, %o5)) + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16) add %o1, 0x08, %o1 - EX_LD(LOAD(ldx, %o1, %g1)) + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16) sub %o1, 0x08, %o1 - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop sub %o2, 0x8, %o2 - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g1, %o2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, GLOBAL_SPARE andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2) sub GLOBAL_SPARE, %g1, GLOBAL_SPARE andn %o2, 0x7, %o4 sllx %g2, %g1, %g2 1: add %o1, 0x8, %o1 - EX_LD(LOAD(ldx, %o1, %g3)) + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4) subcc %o4, 0x8, %o4 srlx %g3, GLOBAL_SPARE, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S index 2e8ee7ad07a9..16a286c1a528 100644 --- a/arch/sparc/lib/NG4copy_from_user.S +++ b/arch/sparc/lib/NG4copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x, y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S index be0bf4590df8..6b0276ffc858 100644 --- a/arch/sparc/lib/NG4copy_to_user.S +++ b/arch/sparc/lib/NG4copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 2012 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi;\ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_asi_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S index 8e13ee1f4454..75bb93b1437f 100644 --- a/arch/sparc/lib/NG4memcpy.S +++ b/arch/sparc/lib/NG4memcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE %g7 @@ -46,22 +47,19 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x +#define EX_ST_FP(x,y) x #endif -#ifndef EX_RETVAL -#define EX_RETVAL(x) x -#endif #ifndef LOAD #define LOAD(type,addr,dest) type [addr], dest @@ -94,6 +92,158 @@ .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_asi_fp: + VISExitHalf +__restore_asi: + retl + wr %g0, ASI_AIUS, %asi + +ENTRY(NG4_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(NG4_retl_o2) +ENTRY(NG4_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(NG4_retl_o2_plus_1) +ENTRY(NG4_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(NG4_retl_o2_plus_4) +ENTRY(NG4_retl_o2_plus_o5) + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5) +ENTRY(NG4_retl_o2_plus_o5_plus_4) + add %o5, 4, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_4) +ENTRY(NG4_retl_o2_plus_o5_plus_8) + add %o5, 8, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_8) +ENTRY(NG4_retl_o2_plus_o5_plus_16) + add %o5, 16, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_16) +ENTRY(NG4_retl_o2_plus_o5_plus_24) + add %o5, 24, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_24) +ENTRY(NG4_retl_o2_plus_o5_plus_32) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(NG4_retl_o2_plus_o5_plus_32) +ENTRY(NG4_retl_o2_plus_g1) + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG4_retl_o2_plus_g1) +ENTRY(NG4_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG4_retl_o2_plus_g1_plus_1) +ENTRY(NG4_retl_o2_plus_g1_plus_8) + add %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG4_retl_o2_plus_g1_plus_8) +ENTRY(NG4_retl_o2_plus_o4) + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4) +ENTRY(NG4_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_8) +ENTRY(NG4_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_16) +ENTRY(NG4_retl_o2_plus_o4_plus_24) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_24) +ENTRY(NG4_retl_o2_plus_o4_plus_32) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_32) +ENTRY(NG4_retl_o2_plus_o4_plus_40) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_40) +ENTRY(NG4_retl_o2_plus_o4_plus_48) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_48) +ENTRY(NG4_retl_o2_plus_o4_plus_56) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_56) +ENTRY(NG4_retl_o2_plus_o4_plus_64) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_64) +ENTRY(NG4_retl_o2_plus_o4_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_8_fp) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_16_fp) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_24_fp) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_32_fp) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_40_fp) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_48_fp) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_56_fp) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp) +ENTRY(NG4_retl_o2_plus_o4_plus_64_fp) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp) +#endif .align 64 .globl FUNC_NAME @@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, 51f sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) add %o1, 1, %o1 subcc %g1, 1, %g1 add %o0, 1, %o0 bne,pt %icc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01)) + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) LOAD(prefetch, %o1 + 0x080, #n_reads_strong) @@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, .Llarge_aligned sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) add %o1, 8, %o1 subcc %g1, 8, %g1 add %o0, 8, %o0 bne,pt %icc, 1b - EX_ST(STORE(stx, %g2, %o0 - 0x08)) + EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8) .Llarge_aligned: /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ andn %o2, 0x3f, %o4 sub %o2, %o4, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4) add %o1, 0x40, %o1 - EX_LD(LOAD(ldx, %o1 - 0x38, %g2)) + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4) subcc %o4, 0x40, %o4 - EX_LD(LOAD(ldx, %o1 - 0x30, %g3)) - EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE)) - EX_LD(LOAD(ldx, %o1 - 0x20, %o5)) - EX_ST(STORE_INIT(%g1, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64) + EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g2, %o0)) + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x18, %g2)) - EX_ST(STORE_INIT(%g3, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48) + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x10, %g3)) - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40) add %o0, 0x08, %o0 - EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE)) - EX_ST(STORE_INIT(%o5, %o0)) + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32) + EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g2, %o0)) + EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(%g3, %o0)) + EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16) add %o0, 0x08, %o0 - EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8) add %o0, 0x08, %o0 bne,pt %icc, 1b LOAD(prefetch, %o1 + 0x200, #n_reads_strong) @@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %o4, %o2 alignaddr %o1, %g0, %g1 add %o1, %o4, %o1 - EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0)) -1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2)) + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4) +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4) subcc %o4, 0x40, %o4 - EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4)) - EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6)) - EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8)) - EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10)) - EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12)) - EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14)) + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64) faligndata %f0, %f2, %f16 - EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0)) + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64) faligndata %f2, %f4, %f18 add %g1, 0x40, %g1 faligndata %f4, %f6, %f20 @@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ faligndata %f10, %f12, %f26 faligndata %f12, %f14, %f28 faligndata %f14, %f0, %f30 - EX_ST_FP(STORE(std, %f16, %o0 + 0x00)) - EX_ST_FP(STORE(std, %f18, %o0 + 0x08)) - EX_ST_FP(STORE(std, %f20, %o0 + 0x10)) - EX_ST_FP(STORE(std, %f22, %o0 + 0x18)) - EX_ST_FP(STORE(std, %f24, %o0 + 0x20)) - EX_ST_FP(STORE(std, %f26, %o0 + 0x28)) - EX_ST_FP(STORE(std, %f28, %o0 + 0x30)) - EX_ST_FP(STORE(std, %f30, %o0 + 0x38)) + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64) + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56) + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48) + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40) + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32) + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24) + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16) + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8) add %o0, 0x40, %o0 bne,pt %icc, 1b LOAD(prefetch, %g1 + 0x200, #n_reads_strong) @@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andncc %o2, 0x20 - 1, %o5 be,pn %icc, 2f sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g2)) - EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE)) - EX_LD(LOAD(ldx, %o1 + 0x18, %o4)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5) add %o1, 0x20, %o1 subcc %o5, 0x20, %o5 - EX_ST(STORE(stx, %g1, %o0 + 0x00)) - EX_ST(STORE(stx, %g2, %o0 + 0x08)) - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10)) - EX_ST(STORE(stx, %o4, %o0 + 0x18)) + EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32) + EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8) bne,pt %icc, 1b add %o0, 0x20, %o0 2: andcc %o2, 0x18, %o5 be,pt %icc, 3f sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) add %o1, 0x08, %o1 add %o0, 0x08, %o0 subcc %o5, 0x08, %o5 bne,pt %icc, 1b - EX_ST(STORE(stx, %g1, %o0 - 0x08)) + EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8) 3: brz,pt %o2, .Lexit cmp %o2, 0x04 bl,pn %icc, .Ltiny nop - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2) add %o1, 0x04, %o1 add %o0, 0x04, %o0 subcc %o2, 0x04, %o2 bne,pn %icc, .Ltiny - EX_ST(STORE(stw, %g1, %o0 - 0x04)) + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4) ba,a,pt %icc, .Lexit .Lmedium_unaligned: /* First get dest 8 byte aligned. */ @@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ brz,pt %g1, 2f sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) add %o1, 1, %o1 subcc %g1, 1, %g1 add %o0, 1, %o0 bne,pt %icc, 1b - EX_ST(STORE(stb, %g2, %o0 - 0x01)) + EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) 2: and %o1, 0x7, %g1 brz,pn %g1, .Lmedium_noprefetch @@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov 64, %g2 sub %g2, %g1, %g2 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1 + 0x00, %o4)) + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2) sllx %o4, %g1, %o4 andn %o2, 0x08 - 1, %o5 sub %o2, %o5, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5) add %o1, 0x08, %o1 subcc %o5, 0x08, %o5 srlx %g3, %g2, GLOBAL_SPARE or GLOBAL_SPARE, %o4, GLOBAL_SPARE - EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00)) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8) add %o0, 0x08, %o0 bne,pt %icc, 1b sllx %g3, %g1, %o4 @@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ ba,pt %icc, .Lsmall_unaligned .Ltiny: - EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) subcc %o2, 1, %o2 be,pn %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x00)) - EX_LD(LOAD(ldub, %o1 + 0x01, %g1)) + EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2) subcc %o2, 1, %o2 be,pn %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x01)) - EX_LD(LOAD(ldub, %o1 + 0x02, %g1)) + EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2) ba,pt %icc, .Lexit - EX_ST(STORE(stb, %g1, %o0 + 0x02)) + EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2) .Lsmall: andcc %g2, 0x3, %g0 @@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x4 - 1, %o5 sub %o2, %o5, %o2 1: - EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) add %o1, 0x04, %o1 subcc %o5, 0x04, %o5 add %o0, 0x04, %o0 bne,pt %icc, 1b - EX_ST(STORE(stw, %g1, %o0 - 0x04)) + EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4) brz,pt %o2, .Lexit nop ba,a,pt %icc, .Ltiny .Lsmall_unaligned: -1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) add %o1, 1, %o1 add %o0, 1, %o0 subcc %o2, 1, %o2 bne,pt %icc, 1b - EX_ST(STORE(stb, %g1, %o0 - 0x01)) + EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1) ba,a,pt %icc, .Lexit .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S index 5d1e4d1ac21e..9cd42fcbc781 100644 --- a/arch/sparc/lib/NGcopy_from_user.S +++ b/arch/sparc/lib/NGcopy_from_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __ret_one_asi;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S index ff630dcb273c..5c358afd464e 100644 --- a/arch/sparc/lib/NGcopy_to_user.S +++ b/arch/sparc/lib/NGcopy_to_user.S @@ -3,11 +3,11 @@ * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __ret_one_asi;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S index 96a14caf6966..d88c4ed50a00 100644 --- a/arch/sparc/lib/NGmemcpy.S +++ b/arch/sparc/lib/NGmemcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/asi.h> #include <asm/thread_info.h> #define GLOBAL_SPARE %g7 @@ -27,15 +28,11 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST(x,y) x #endif #ifndef LOAD @@ -79,6 +76,92 @@ .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_asi: + ret + wr %g0, ASI_AIUS, %asi + restore +ENTRY(NG_ret_i2_plus_i4_plus_1) + ba,pt %xcc, __restore_asi + add %i2, %i5, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_1) +ENTRY(NG_ret_i2_plus_g1) + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1) +ENTRY(NG_ret_i2_plus_g1_minus_8) + sub %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_8) +ENTRY(NG_ret_i2_plus_g1_minus_16) + sub %g1, 16, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_16) +ENTRY(NG_ret_i2_plus_g1_minus_24) + sub %g1, 24, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_24) +ENTRY(NG_ret_i2_plus_g1_minus_32) + sub %g1, 32, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_32) +ENTRY(NG_ret_i2_plus_g1_minus_40) + sub %g1, 40, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_40) +ENTRY(NG_ret_i2_plus_g1_minus_48) + sub %g1, 48, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_48) +ENTRY(NG_ret_i2_plus_g1_minus_56) + sub %g1, 56, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_56) +ENTRY(NG_ret_i2_plus_i4) + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4) +ENTRY(NG_ret_i2_plus_i4_minus_8) + sub %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4_minus_8) +ENTRY(NG_ret_i2_plus_8) + ba,pt %xcc, __restore_asi + add %i2, 8, %i0 +ENDPROC(NG_ret_i2_plus_8) +ENTRY(NG_ret_i2_plus_4) + ba,pt %xcc, __restore_asi + add %i2, 4, %i0 +ENDPROC(NG_ret_i2_plus_4) +ENTRY(NG_ret_i2_plus_1) + ba,pt %xcc, __restore_asi + add %i2, 1, %i0 +ENDPROC(NG_ret_i2_plus_1) +ENTRY(NG_ret_i2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_plus_1) +ENTRY(NG_ret_i2) + ba,pt %xcc, __restore_asi + mov %i2, %i0 +ENDPROC(NG_ret_i2) +ENTRY(NG_ret_i2_and_7_plus_i4) + and %i2, 7, %i2 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) +#endif + .align 64 .globl FUNC_NAME @@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ sub %g0, %i4, %i4 ! bytes to align dst sub %i2, %i4, %i2 1: subcc %i4, 1, %i4 - EX_LD(LOAD(ldub, %i1, %g1)) - EX_ST(STORE(stb, %g1, %o0)) + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1) add %i1, 1, %i1 bne,pt %XCC, 1b add %o0, 1, %o0 @@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ and %i4, 0x7, GLOBAL_SPARE sll GLOBAL_SPARE, 3, GLOBAL_SPARE mov 64, %i5 - EX_LD(LOAD_TWIN(%i1, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1) sub %i5, GLOBAL_SPARE, %i5 mov 16, %o4 mov 32, %o5 @@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ srlx WORD3, PRE_SHIFT, TMP; \ or WORD2, TMP, WORD2; -8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) LOAD(prefetch, %i1 + %i3, #one_read) - EX_ST(STORE_INIT(%g2, %o0 + 0x00)) - EX_ST(STORE_INIT(%g3, %o0 + 0x08)) + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%g2, %o0 + 0x20)) - EX_ST(STORE_INIT(%g3, %o0 + 0x28)) + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 8b @@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ ba,pt %XCC, 60f add %i1, %i4, %i1 -9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) LOAD(prefetch, %i1 + %i3, #one_read) - EX_ST(STORE_INIT(%g3, %o0 + 0x00)) - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) - EX_ST(STORE_INIT(%g2, %o0 + 0x18)) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%g3, %o0 + 0x20)) - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) - EX_ST(STORE_INIT(%g2, %o0 + 0x38)) + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 9b @@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ * one twin load ahead, then add 8 back into source when * we finish the loop. */ - EX_LD(LOAD_TWIN(%i1, %o4, %o5)) + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1) mov 16, %o7 mov 32, %g2 mov 48, %g3 mov 64, %o1 -1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) LOAD(prefetch, %i1 + %o1, #one_read) - EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line - EX_ST(STORE_INIT(%o2, %o0 + 0x08)) - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) - EX_ST(STORE_INIT(%o3, %o0 + 0x10)) - EX_ST(STORE_INIT(%o4, %o0 + 0x18)) - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) - EX_ST(STORE_INIT(%o5, %o0 + 0x20)) - EX_ST(STORE_INIT(%o2, %o0 + 0x28)) - EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5)) + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48) add %i1, 64, %i1 - EX_ST(STORE_INIT(%o3, %o0 + 0x30)) - EX_ST(STORE_INIT(%o4, %o0 + 0x38)) + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 1b add %o0, 64, %o0 @@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ mov 32, %g2 mov 48, %g3 mov 64, %o1 -1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5)) - EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) LOAD(prefetch, %i1 + %o1, #one_read) - EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line - EX_ST(STORE_INIT(%o5, %o0 + 0x08)) - EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) - EX_ST(STORE_INIT(%o2, %o0 + 0x10)) - EX_ST(STORE_INIT(%o3, %o0 + 0x18)) - EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) add %i1, 64, %i1 - EX_ST(STORE_INIT(%o4, %o0 + 0x20)) - EX_ST(STORE_INIT(%o5, %o0 + 0x28)) - EX_ST(STORE_INIT(%o2, %o0 + 0x30)) - EX_ST(STORE_INIT(%o3, %o0 + 0x38)) + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) subcc %g1, 64, %g1 bne,pt %XCC, 1b add %o0, 64, %o0 @@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ andn %i2, 0xf, %i4 and %i2, 0xf, %i2 1: subcc %i4, 0x10, %i4 - EX_LD(LOAD(ldx, %i1, %o4)) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4) add %i1, 0x08, %i1 - EX_LD(LOAD(ldx, %i1, %g1)) + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4) sub %i1, 0x08, %i1 - EX_ST(STORE(stx, %o4, %i1 + %i3)) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4) add %i1, 0x8, %i1 - EX_ST(STORE(stx, %g1, %i1 + %i3)) + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8) bgu,pt %XCC, 1b add %i1, 0x8, %i1 73: andcc %i2, 0x8, %g0 be,pt %XCC, 1f nop sub %i2, 0x8, %i2 - EX_LD(LOAD(ldx, %i1, %o4)) - EX_ST(STORE(stx, %o4, %i1 + %i3)) + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8) add %i1, 0x8, %i1 1: andcc %i2, 0x4, %g0 be,pt %XCC, 1f nop sub %i2, 0x4, %i2 - EX_LD(LOAD(lduw, %i1, %i5)) - EX_ST(STORE(stw, %i5, %i1 + %i3)) + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4) add %i1, 0x4, %i1 1: cmp %i2, 0 be,pt %XCC, 85f @@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ sub %i2, %g1, %i2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %i1, %i5)) - EX_ST(STORE(stb, %i5, %i1 + %i3)) + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1) + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1) bgu,pt %icc, 1b add %i1, 1, %i1 @@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ 8: mov 64, %i3 andn %i1, 0x7, %i1 - EX_LD(LOAD(ldx, %i1, %g2)) + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2) sub %i3, %g1, %i3 andn %i2, 0x7, %i4 sllx %g2, %g1, %g2 1: add %i1, 0x8, %i1 - EX_LD(LOAD(ldx, %i1, %g3)) + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4) subcc %i4, 0x8, %i4 srlx %g3, %i3, %i5 or %i5, %g2, %i5 - EX_ST(STORE(stx, %i5, %o0)) + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ 1: subcc %i2, 4, %i2 - EX_LD(LOAD(lduw, %i1, %g1)) - EX_ST(STORE(stw, %g1, %i1 + %i3)) + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4) bgu,pt %XCC, 1b add %i1, 4, %i1 @@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ .align 32 90: subcc %i2, 1, %i2 - EX_LD(LOAD(ldub, %i1, %g1)) - EX_ST(STORE(stb, %g1, %i1 + %i3)) + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1) + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1) bgu,pt %XCC, 90b add %i1, 1, %i1 ret diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S index ecc5692fa2b4..bb6ff73229e3 100644 --- a/arch/sparc/lib/U1copy_from_user.S +++ b/arch/sparc/lib/U1copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S index 9eea392e44d4..ed92ce739558 100644 --- a/arch/sparc/lib/U1copy_to_user.S +++ b/arch/sparc/lib/U1copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y; \ .text; \ .align 4; diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S index 97e1b211090c..4f0d50b33a72 100644 --- a/arch/sparc/lib/U1memcpy.S +++ b/arch/sparc/lib/U1memcpy.S @@ -5,6 +5,7 @@ */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #include <asm/export.h> @@ -24,21 +25,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -79,53 +76,169 @@ faligndata %f7, %f8, %f60; \ faligndata %f8, %f9, %f62; -#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ - EX_LD_FP(LOAD_BLK(%src, %fdest)); \ - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ - add %src, 0x40, %src; \ - subcc %len, 0x40, %len; \ - be,pn %xcc, jmptgt; \ - add %dest, 0x40, %dest; \ - -#define LOOP_CHUNK1(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) -#define LOOP_CHUNK2(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) -#define LOOP_CHUNK3(src, dest, len, branch_dest) \ - MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \ + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ + add %src, 0x40, %src; \ + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \ + be,pn %xcc, jmptgt; \ + add %dest, 0x40, %dest; \ + +#define LOOP_CHUNK1(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest) +#define LOOP_CHUNK2(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest) +#define LOOP_CHUNK3(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest) #define DO_SYNC membar #Sync; #define STORE_SYNC(dest, fsrc) \ - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ add %dest, 0x40, %dest; \ DO_SYNC #define STORE_JUMP(dest, fsrc, target) \ - EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \ add %dest, 0x40, %dest; \ ba,pt %xcc, target; \ nop; -#define FINISH_VISCHUNK(dest, f0, f1, left) \ - subcc %left, 8, %left;\ - bl,pn %xcc, 95f; \ - faligndata %f0, %f1, %f48; \ - EX_ST_FP(STORE(std, %f48, %dest)); \ +#define FINISH_VISCHUNK(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ + faligndata %f0, %f1, %f48; \ + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \ add %dest, 8, %dest; -#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ - subcc %left, 8, %left; \ - bl,pn %xcc, 95f; \ +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ fsrc2 %f0, %f1; -#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ - UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ +#define UNEVEN_VISCHUNK(dest, f0, f1) \ + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ ba,a,pt %xcc, 93f; .register %g2,#scratch .register %g3,#scratch .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(U1_g1_1_fp) + VISExitHalf + add %g1, 1, %g1 + add %g1, %g2, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1_fp) +ENTRY(U1_g2_0_fp) + VISExitHalf + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_0_fp) +ENTRY(U1_g2_8_fp) + VISExitHalf + add %g2, 8, %g2 + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_8_fp) +ENTRY(U1_gs_0_fp) + VISExitHalf + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_0_fp) +ENTRY(U1_gs_80_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_80_fp) +ENTRY(U1_gs_40_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_40_fp) +ENTRY(U1_g3_0_fp) + VISExitHalf + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_0_fp) +ENTRY(U1_g3_8_fp) + VISExitHalf + add %g3, 8, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_8_fp) +ENTRY(U1_o2_0_fp) + VISExitHalf + retl + mov %o2, %o0 +ENDPROC(U1_o2_0_fp) +ENTRY(U1_o2_1_fp) + VISExitHalf + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1_fp) +ENTRY(U1_gs_0) + VISExitHalf + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0) +ENTRY(U1_gs_8) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x8, %o0 +ENDPROC(U1_gs_8) +ENTRY(U1_gs_10) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x10, %o0 +ENDPROC(U1_gs_10) +ENTRY(U1_o2_0) + retl + mov %o2, %o0 +ENDPROC(U1_o2_0) +ENTRY(U1_o2_8) + retl + add %o2, 8, %o0 +ENDPROC(U1_o2_8) +ENTRY(U1_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(U1_o2_4) +ENTRY(U1_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1) +ENTRY(U1_g1_0) + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_0) +ENTRY(U1_g1_1) + add %g1, 1, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1) +ENTRY(U1_gs_0_o2_adj) + and %o2, 7, %o2 + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0_o2_adj) +ENTRY(U1_gs_8_o2_adj) + and %o2, 7, %o2 + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_8_o2_adj) +#endif + .align 64 .globl FUNC_NAME @@ -167,8 +280,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ and %g2, 0x38, %g2 1: subcc %g1, 0x1, %g1 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) - EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp) + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp) bgu,pt %XCC, 1b add %o1, 0x1, %o1 @@ -179,20 +292,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pt %icc, 3f alignaddr %o1, %g0, %o1 - EX_LD_FP(LOAD(ldd, %o1, %f4)) -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f4, %f6, %f0 - EX_ST_FP(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) be,pn %icc, 3f add %o0, 0x8, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f6, %f4, %f0 - EX_ST_FP(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) bne,pt %icc, 1b add %o0, 0x8, %o0 @@ -215,13 +328,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ add %g1, %GLOBAL_SPARE, %g1 subcc %o2, %g3, %o2 - EX_LD_FP(LOAD_BLK(%o1, %f0)) + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp) add %o1, 0x40, %o1 add %g1, %g3, %g1 - EX_LD_FP(LOAD_BLK(%o1, %f16)) + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp) add %o1, 0x40, %o1 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE - EX_LD_FP(LOAD_BLK(%o1, %f32)) + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp) add %o1, 0x40, %o1 /* There are 8 instances of the unrolled loop, @@ -241,11 +354,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 64 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f0, %f2, %f48 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) @@ -262,11 +375,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 56f) 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f2, %f4, %f48 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) @@ -283,11 +396,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 57f) 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f4, %f6, %f48 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) @@ -304,11 +417,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 58f) 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f6, %f8, %f48 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) @@ -325,11 +438,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 59f) 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f8, %f10, %f48 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) @@ -346,11 +459,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 60f) 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f10, %f12, %f48 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) @@ -367,11 +480,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 61f) 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f12, %f14, %f48 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) @@ -388,11 +501,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ STORE_JUMP(o0, f48, 62f) 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) - LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) + LOOP_CHUNK1(o1, o0, 1f) FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) - LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) + LOOP_CHUNK2(o1, o0, 2f) FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) - LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) + LOOP_CHUNK3(o1, o0, 3f) ba,pt %xcc, 1b+4 faligndata %f14, %f16, %f48 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) @@ -408,53 +521,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) STORE_JUMP(o0, f48, 63f) -40: FINISH_VISCHUNK(o0, f0, f2, g3) -41: FINISH_VISCHUNK(o0, f2, f4, g3) -42: FINISH_VISCHUNK(o0, f4, f6, g3) -43: FINISH_VISCHUNK(o0, f6, f8, g3) -44: FINISH_VISCHUNK(o0, f8, f10, g3) -45: FINISH_VISCHUNK(o0, f10, f12, g3) -46: FINISH_VISCHUNK(o0, f12, f14, g3) -47: UNEVEN_VISCHUNK(o0, f14, f0, g3) -48: FINISH_VISCHUNK(o0, f16, f18, g3) -49: FINISH_VISCHUNK(o0, f18, f20, g3) -50: FINISH_VISCHUNK(o0, f20, f22, g3) -51: FINISH_VISCHUNK(o0, f22, f24, g3) -52: FINISH_VISCHUNK(o0, f24, f26, g3) -53: FINISH_VISCHUNK(o0, f26, f28, g3) -54: FINISH_VISCHUNK(o0, f28, f30, g3) -55: UNEVEN_VISCHUNK(o0, f30, f0, g3) -56: FINISH_VISCHUNK(o0, f32, f34, g3) -57: FINISH_VISCHUNK(o0, f34, f36, g3) -58: FINISH_VISCHUNK(o0, f36, f38, g3) -59: FINISH_VISCHUNK(o0, f38, f40, g3) -60: FINISH_VISCHUNK(o0, f40, f42, g3) -61: FINISH_VISCHUNK(o0, f42, f44, g3) -62: FINISH_VISCHUNK(o0, f44, f46, g3) -63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) - -93: EX_LD_FP(LOAD(ldd, %o1, %f2)) +40: FINISH_VISCHUNK(o0, f0, f2) +41: FINISH_VISCHUNK(o0, f2, f4) +42: FINISH_VISCHUNK(o0, f4, f6) +43: FINISH_VISCHUNK(o0, f6, f8) +44: FINISH_VISCHUNK(o0, f8, f10) +45: FINISH_VISCHUNK(o0, f10, f12) +46: FINISH_VISCHUNK(o0, f12, f14) +47: UNEVEN_VISCHUNK(o0, f14, f0) +48: FINISH_VISCHUNK(o0, f16, f18) +49: FINISH_VISCHUNK(o0, f18, f20) +50: FINISH_VISCHUNK(o0, f20, f22) +51: FINISH_VISCHUNK(o0, f22, f24) +52: FINISH_VISCHUNK(o0, f24, f26) +53: FINISH_VISCHUNK(o0, f26, f28) +54: FINISH_VISCHUNK(o0, f28, f30) +55: UNEVEN_VISCHUNK(o0, f30, f0) +56: FINISH_VISCHUNK(o0, f32, f34) +57: FINISH_VISCHUNK(o0, f34, f36) +58: FINISH_VISCHUNK(o0, f36, f38) +59: FINISH_VISCHUNK(o0, f38, f40) +60: FINISH_VISCHUNK(o0, f40, f42) +61: FINISH_VISCHUNK(o0, f42, f44) +62: FINISH_VISCHUNK(o0, f44, f46) +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) + +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f0, %f2, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) bl,pn %xcc, 95f add %o0, 8, %o0 - EX_LD_FP(LOAD(ldd, %o1, %f0)) + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp) add %o1, 8, %o1 subcc %g3, 8, %g3 faligndata %f2, %f0, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) bge,pt %xcc, 93b add %o0, 8, %o0 95: brz,pt %o2, 2f mov %g1, %o1 -1: EX_LD_FP(LOAD(ldub, %o1, %o3)) +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp) add %o1, 1, %o1 subcc %o2, 1, %o2 - EX_ST_FP(STORE(stb, %o3, %o0)) + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp) bne,pt %xcc, 1b add %o0, 1, %o0 @@ -470,27 +583,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 72: andn %o2, 0xf, %GLOBAL_SPARE and %o2, 0xf, %o2 -1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0) subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop - EX_LD(LOAD(ldx, %o1, %o5)) + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) sub %o2, 0x8, %o2 - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop - EX_LD(LOAD(lduw, %o1, %o5)) + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) sub %o2, 0x4, %o2 - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -504,9 +617,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %g0, %g1, %g1 sub %o2, %g1, %o2 -1: EX_LD(LOAD(ldub, %o1, %o5)) +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) subcc %g1, 1, %g1 - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -522,16 +635,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, %o3 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) sub %o3, %g1, %o3 andn %o2, 0x7, %GLOBAL_SPARE sllx %g2, %g1, %g2 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj) subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE add %o1, 0x8, %o1 srlx %g3, %o3, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -549,9 +662,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ bne,pn %XCC, 90f sub %o0, %o1, %o3 -1: EX_LD(LOAD(lduw, %o1, %g1)) +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) subcc %o2, 4, %o2 - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -559,9 +672,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov EX_RETVAL(%o4), %o0 .align 32 -90: EX_LD(LOAD(ldub, %o1, %g1)) +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) subcc %o2, 1, %o2 - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S index 88ad73d86fe4..db73010a1af8 100644 --- a/arch/sparc/lib/U3copy_from_user.S +++ b/arch/sparc/lib/U3copy_from_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_LD(x) \ +#define EX_LD(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_LD_FP(x) \ +#define EX_LD_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S index 845139d75537..c4ee858e352a 100644 --- a/arch/sparc/lib/U3copy_to_user.S +++ b/arch/sparc/lib/U3copy_to_user.S @@ -3,19 +3,19 @@ * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) */ -#define EX_ST(x) \ +#define EX_ST(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, y; \ .text; \ .align 4; -#define EX_ST_FP(x) \ +#define EX_ST_FP(x,y) \ 98: x; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one_fp;\ + .word 98b, y##_fp; \ .text; \ .align 4; diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S index 491ee69e4995..54f98706b03b 100644 --- a/arch/sparc/lib/U3memcpy.S +++ b/arch/sparc/lib/U3memcpy.S @@ -4,6 +4,7 @@ */ #ifdef __KERNEL__ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #define GLOBAL_SPARE %g7 @@ -22,21 +23,17 @@ #endif #ifndef EX_LD -#define EX_LD(x) x +#define EX_LD(x,y) x #endif #ifndef EX_LD_FP -#define EX_LD_FP(x) x +#define EX_LD_FP(x,y) x #endif #ifndef EX_ST -#define EX_ST(x) x +#define EX_ST(x,y) x #endif #ifndef EX_ST_FP -#define EX_ST_FP(x) x -#endif - -#ifndef EX_RETVAL -#define EX_RETVAL(x) x +#define EX_ST_FP(x,y) x #endif #ifndef LOAD @@ -77,6 +74,87 @@ */ .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf + retl + nop +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) + add %g1, 1, %g1 + add %g2, %g1, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) +ENTRY(U3_retl_o2_plus_g2_fp) + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_fp) +ENTRY(U3_retl_o2_plus_g2_plus_8_fp) + add %g2, 8, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp) +ENTRY(U3_retl_o2) + retl + mov %o2, %o0 +ENDPROC(U3_retl_o2) +ENTRY(U3_retl_o2_plus_1) + retl + add %o2, 1, %o0 +ENDPROC(U3_retl_o2_plus_1) +ENTRY(U3_retl_o2_plus_4) + retl + add %o2, 4, %o0 +ENDPROC(U3_retl_o2_plus_4) +ENTRY(U3_retl_o2_plus_8) + retl + add %o2, 8, %o0 +ENDPROC(U3_retl_o2_plus_8) +ENTRY(U3_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + retl + add %o2, %g1, %o0 +ENDPROC(U3_retl_o2_plus_g1_plus_1) +ENTRY(U3_retl_o2_fp) + ba,pt %xcc, __restore_fp + mov %o2, %o0 +ENDPROC(U3_retl_o2_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) + sll %o3, 6, %o3 + add %o3, 0x80, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) + sll %o3, 6, %o3 + add %o3, 0x40, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) +ENTRY(U3_retl_o2_plus_GS_plus_0x10) + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x10) +ENTRY(U3_retl_o2_plus_GS_plus_0x08) + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x08) +ENTRY(U3_retl_o2_and_7_plus_GS) + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o2 +ENDPROC(U3_retl_o2_and_7_plus_GS) +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8) + add GLOBAL_SPARE, 8, GLOBAL_SPARE + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o2 +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8) +#endif + .align 64 /* The cheetah's flexible spine, oversized liver, enlarged heart, @@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ and %g2, 0x38, %g2 1: subcc %g1, 0x1, %g1 - EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) - EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE)) + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1) + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1) bgu,pt %XCC, 1b add %o1, 0x1, %o1 @@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pt %icc, 3f alignaddr %o1, %g0, %o1 - EX_LD_FP(LOAD(ldd, %o1, %f4)) -1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f4, %f6, %f0 - EX_ST_FP(STORE(std, %f0, %o0)) + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8) be,pn %icc, 3f add %o0, 0x8, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f6, %f4, %f2 - EX_ST_FP(STORE(std, %f2, %o0)) + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8) bne,pt %icc, 1b add %o0, 0x8, %o0 @@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ LOAD(prefetch, %o1 + 0x080, #one_read) LOAD(prefetch, %o1 + 0x0c0, #one_read) LOAD(prefetch, %o1 + 0x100, #one_read) - EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2) LOAD(prefetch, %o1 + 0x140, #one_read) - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2) LOAD(prefetch, %o1 + 0x180, #one_read) - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2) LOAD(prefetch, %o1 + 0x1c0, #one_read) faligndata %f0, %f2, %f16 - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2) faligndata %f2, %f4, %f18 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2) faligndata %f4, %f6, %f20 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2) faligndata %f6, %f8, %f22 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2) faligndata %f8, %f10, %f24 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2) faligndata %f10, %f12, %f26 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE add %o1, 0x40, %o1 @@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 64 1: - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f12, %f14, %f28 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f14, %f0, %f30 - EX_ST_FP(STORE_BLK(%f16, %o0)) - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f0, %f2, %f16 add %o0, 0x40, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f2, %f4, %f18 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f4, %f6, %f20 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) subcc %o3, 0x01, %o3 faligndata %f6, %f8, %f22 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f8, %f10, %f24 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80) LOAD(prefetch, %o1 + 0x1c0, #one_read) faligndata %f10, %f12, %f26 bg,pt %XCC, 1b @@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ /* Finally we copy the last full 64-byte block. */ 2: - EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f12, %f14, %f28 - EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) faligndata %f14, %f0, %f30 - EX_ST_FP(STORE_BLK(%f16, %o0)) - EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f0, %f2, %f16 - EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f2, %f4, %f18 - EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f4, %f6, %f20 - EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f6, %f8, %f22 - EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40) faligndata %f8, %f10, %f24 cmp %g1, 0 be,pt %XCC, 1f add %o0, 0x40, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40) 1: faligndata %f10, %f12, %f26 faligndata %f12, %f14, %f28 faligndata %f14, %f0, %f30 - EX_ST_FP(STORE_BLK(%f16, %o0)) + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40) add %o0, 0x40, %o0 add %o1, 0x40, %o1 membar #Sync @@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g2, %o2 be,a,pt %XCC, 1f - EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2) -1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2)) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f0, %f2, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) be,pn %XCC, 2f add %o0, 0x8, %o0 - EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0)) + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2) add %o1, 0x8, %o1 subcc %g2, 0x8, %g2 faligndata %f2, %f0, %f8 - EX_ST_FP(STORE(std, %f8, %o0)) + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) bne,pn %XCC, 1b add %o0, 0x8, %o0 @@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andcc %o2, 0x8, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x8, %o1 + sub %o2, 8, %o2 1: andcc %o2, 0x4, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x4, %o1 + sub %o2, 4, %o2 1: andcc %o2, 0x2, %g0 be,pt %icc, 1f nop - EX_LD(LOAD(lduh, %o1, %o5)) - EX_ST(STORE(sth, %o5, %o1 + %o3)) + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2) + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2) add %o1, 0x2, %o1 + sub %o2, 2, %o2 1: andcc %o2, 0x1, %g0 be,pt %icc, 85f nop - EX_LD(LOAD(ldub, %o1, %o5)) + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2) ba,pt %xcc, 85f - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2) .align 64 70: /* 16 < len <= 64 */ @@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0xf, GLOBAL_SPARE and %o2, 0xf, %o2 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE - EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) - EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 be,pt %XCC, 1f nop sub %o2, 0x8, %o2 - EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ sub %o2, %g1, %o2 1: subcc %g1, 1, %g1 - EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1) bgu,pt %icc, 1b add %o1, 1, %o1 @@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 8: mov 64, %o3 andn %o1, 0x7, %o1 - EX_LD(LOAD(ldx, %o1, %g2)) + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2) sub %o3, %g1, %o3 andn %o2, 0x7, GLOBAL_SPARE sllx %g2, %g1, %g2 -1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS) subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE add %o1, 0x8, %o1 srlx %g3, %o3, %o5 or %o5, %g2, %o5 - EX_ST(STORE(stx, %o5, %o0)) + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8) add %o0, 0x8, %o0 bgu,pt %icc, 1b sllx %g3, %g1, %g2 @@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %o2, 4, %o2 - EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4) bgu,pt %XCC, 1b add %o1, 4, %o1 @@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1) bgu,pt %XCC, 90b add %o1, 1, %o1 retl diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 482de093bdae..0252b218de45 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S @@ -9,18 +9,33 @@ #define XCC xcc -#define EX(x,y) \ +#define EX(x,y,z) \ 98: x,y; \ .section __ex_table,"a";\ .align 4; \ - .word 98b, __retl_one; \ + .word 98b, z; \ .text; \ .align 4; +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8) +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4) +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1) + .register %g2,#scratch .register %g3,#scratch .text +__retl_o4_plus_8: + add %o4, %o2, %o4 + retl + add %o4, 8, %o0 +__retl_o2_plus_4: + retl + add %o2, 4, %o0 +__retl_o2_plus_1: + retl + add %o2, 1, %o0 + .align 32 /* Don't try to get too fancy here, just nice and @@ -45,8 +60,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ andn %o2, 0x7, %o4 and %o2, 0x7, %o2 1: subcc %o4, 0x8, %o4 - EX(ldxa [%o1] %asi, %o5) - EX(stxa %o5, [%o0] %asi) + EX_O4(ldxa [%o1] %asi, %o5) + EX_O4(stxa %o5, [%o0] %asi) add %o1, 0x8, %o1 bgu,pt %XCC, 1b add %o0, 0x8, %o0 @@ -54,8 +69,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ be,pt %XCC, 1f nop sub %o2, 0x4, %o2 - EX(lduwa [%o1] %asi, %o5) - EX(stwa %o5, [%o0] %asi) + EX_O2_4(lduwa [%o1] %asi, %o5) + EX_O2_4(stwa %o5, [%o0] %asi) add %o1, 0x4, %o1 add %o0, 0x4, %o0 1: cmp %o2, 0 @@ -71,8 +86,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ 82: subcc %o2, 4, %o2 - EX(lduwa [%o1] %asi, %g1) - EX(stwa %g1, [%o0] %asi) + EX_O2_4(lduwa [%o1] %asi, %g1) + EX_O2_4(stwa %g1, [%o0] %asi) add %o1, 4, %o1 bgu,pt %XCC, 82b add %o0, 4, %o0 @@ -83,8 +98,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ .align 32 90: subcc %o2, 1, %o2 - EX(lduba [%o1] %asi, %g1) - EX(stba %g1, [%o0] %asi) + EX_O2_1(lduba [%o1] %asi, %g1) + EX_O2_1(stba %g1, [%o0] %asi) add %o1, 1, %o1 bgu,pt %XCC, 90b add %o0, 1, %o0 diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c deleted file mode 100644 index ac96ae236709..000000000000 --- a/arch/sparc/lib/user_fixup.c +++ /dev/null @@ -1,71 +0,0 @@ -/* user_fixup.c: Fix up user copy faults. - * - * Copyright (C) 2004 David S. Miller <davem@redhat.com> - */ - -#include <linux/compiler.h> -#include <linux/kernel.h> -#include <linux/string.h> -#include <linux/errno.h> -#include <linux/module.h> - -#include <asm/uaccess.h> - -/* Calculating the exact fault address when using - * block loads and stores can be very complicated. - * - * Instead of trying to be clever and handling all - * of the cases, just fix things up simply here. - */ - -static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset) -{ - unsigned long fault_addr = current_thread_info()->fault_address; - unsigned long end = start + size; - - if (fault_addr < start || fault_addr >= end) { - *offset = 0; - } else { - *offset = fault_addr - start; - size = end - fault_addr; - } - return size; -} - -unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size) -{ - unsigned long offset; - - size = compute_size((unsigned long) from, size, &offset); - if (likely(size)) - memset(to + offset, 0, size); - - return size; -} -EXPORT_SYMBOL(copy_from_user_fixup); - -unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size) -{ - unsigned long offset; - - return compute_size((unsigned long) to, size, &offset); -} -EXPORT_SYMBOL(copy_to_user_fixup); - -unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size) -{ - unsigned long fault_addr = current_thread_info()->fault_address; - unsigned long start = (unsigned long) to; - unsigned long end = start + size; - - if (fault_addr >= start && fault_addr < end) - return end - fault_addr; - - start = (unsigned long) from; - end = start + size; - if (fault_addr >= start && fault_addr < end) - return end - fault_addr; - - return size; -} -EXPORT_SYMBOL(copy_in_user_fixup); diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index f2b77112e9d8..e20fbbafb0b0 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr) return (tag == (vaddr >> 22)); } +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end) +{ + unsigned long idx; + + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) { + struct tsb *ent = &swapper_tsb[idx]; + unsigned long match = idx << 13; + + match |= (ent->tag << 22); + if (match >= start && match < end) + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +} + /* TSB flushes need only occur on the processor initiating the address * space modification, not on each cpu the address space has run on. * Only the TLB flush needs that treatment. @@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) { unsigned long v; + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES) + return flush_tsb_kernel_range_scan(start, end); + for (v = start; v < end; v += PAGE_SIZE) { unsigned long hash = tsb_hash(v, PAGE_SHIFT, KERNEL_TSB_NENTRIES); diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S index b4f4733abc6e..5d2fd6cd3189 100644 --- a/arch/sparc/mm/ultra.S +++ b/arch/sparc/mm/ultra.S @@ -30,7 +30,7 @@ .text .align 32 .globl __flush_tlb_mm -__flush_tlb_mm: /* 18 insns */ +__flush_tlb_mm: /* 19 insns */ /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ ldxa [%o1] ASI_DMMU, %g2 cmp %g2, %o0 @@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */ .align 32 .globl __flush_tlb_pending -__flush_tlb_pending: /* 26 insns */ +__flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ rdpr %pstate, %g7 sllx %o1, 3, %o1 @@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */ .align 32 .globl __flush_tlb_kernel_range -__flush_tlb_kernel_range: /* 16 insns */ +__flush_tlb_kernel_range: /* 31 insns */ /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow sethi %hi(PAGE_SIZE), %o4 - sub %o1, %o0, %o3 sub %o3, %o4, %o3 or %o0, 0x20, %o0 ! Nucleus 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP @@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */ retl nop nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + +__spitfire_flush_tlb_kernel_range_slow: + mov 63 * 8, %o4 +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_IMMU + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_DMMU + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %o4, 8, %o4 + brgez,pt %o4, 1b + nop + retl + nop __spitfire_flush_tlb_mm_slow: rdpr %pstate, %g1 @@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */ retl wrpr %g7, 0x0, %pstate +__cheetah_flush_tlb_kernel_range: /* 31 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, 3f + sethi %hi(PAGE_SIZE), %o4 + sub %o3, %o4, %o3 + or %o0, 0x20, %o0 ! Nucleus +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %o3, 1b + sub %o3, %o4, %o3 +2: sethi %hi(KERNBASE), %o3 + flush %o3 + retl + nop +3: mov 0x80, %o4 + stxa %g0, [%o4] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%o4] ASI_IMMU_DEMAP + membar #Sync + retl + nop + nop + nop + nop + nop + nop + nop + nop + #ifdef DCACHE_ALIASING_POSSIBLE __cheetah_flush_dcache_page: /* 11 insns */ sethi %hi(PAGE_OFFSET), %g1 @@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error: ret restore -__hypervisor_flush_tlb_mm: /* 10 insns */ +__hypervisor_flush_tlb_mm: /* 19 insns */ mov %o0, %o2 /* ARG2: mmu context */ mov 0, %o0 /* ARG0: CPU lists unimplemented */ mov 0, %o1 /* ARG1: CPU lists unimplemented */ mov HV_MMU_ALL, %o3 /* ARG3: flags */ mov HV_FAST_MMU_DEMAP_CTX, %o5 ta HV_FAST_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_FAST_MMU_DEMAP_CTX, %o1 retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_page: /* 11 insns */ +__hypervisor_flush_tlb_page: /* 22 insns */ /* %o0 = context, %o1 = vaddr */ mov %o0, %g2 mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ @@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */ srlx %o0, PAGE_SHIFT, %o0 sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_pending: /* 16 insns */ +__hypervisor_flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ sllx %o1, 3, %g1 mov %o2, %g2 @@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */ srlx %o0, PAGE_SHIFT, %o0 sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 1f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 brnz,pt %g1, 1b nop retl nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop -__hypervisor_flush_tlb_kernel_range: /* 16 insns */ +__hypervisor_flush_tlb_kernel_range: /* 31 insns */ /* %o0=start, %o1=end */ cmp %o0, %o1 be,pn %xcc, 2f - sethi %hi(PAGE_SIZE), %g3 - mov %o0, %g1 - sub %o1, %g1, %g2 + sub %o1, %o0, %g2 + srlx %g2, 18, %g3 + brnz,pn %g3, 4f + mov %o0, %g1 + sethi %hi(PAGE_SIZE), %g3 sub %g2, %g3, %g2 1: add %g1, %g2, %o0 /* ARG0: virtual address */ mov 0, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ ta HV_MMU_UNMAP_ADDR_TRAP - brnz,pn %o0, __hypervisor_tlb_tl0_error + brnz,pn %o0, 3f mov HV_MMU_UNMAP_ADDR_TRAP, %o1 brnz,pt %g2, 1b sub %g2, %g3, %g2 2: retl nop +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 3b + mov HV_FAST_MMU_DEMAP_CTX, %o1 + retl + nop #ifdef DCACHE_ALIASING_POSSIBLE /* XXX Niagara and friends have an 8K cache, so no aliasing is @@ -394,43 +511,6 @@ tlb_patch_one: retl nop - .globl cheetah_patch_cachetlbops -cheetah_patch_cachetlbops: - save %sp, -128, %sp - - sethi %hi(__flush_tlb_mm), %o0 - or %o0, %lo(__flush_tlb_mm), %o0 - sethi %hi(__cheetah_flush_tlb_mm), %o1 - or %o1, %lo(__cheetah_flush_tlb_mm), %o1 - call tlb_patch_one - mov 19, %o2 - - sethi %hi(__flush_tlb_page), %o0 - or %o0, %lo(__flush_tlb_page), %o0 - sethi %hi(__cheetah_flush_tlb_page), %o1 - or %o1, %lo(__cheetah_flush_tlb_page), %o1 - call tlb_patch_one - mov 22, %o2 - - sethi %hi(__flush_tlb_pending), %o0 - or %o0, %lo(__flush_tlb_pending), %o0 - sethi %hi(__cheetah_flush_tlb_pending), %o1 - or %o1, %lo(__cheetah_flush_tlb_pending), %o1 - call tlb_patch_one - mov 27, %o2 - -#ifdef DCACHE_ALIASING_POSSIBLE - sethi %hi(__flush_dcache_page), %o0 - or %o0, %lo(__flush_dcache_page), %o0 - sethi %hi(__cheetah_flush_dcache_page), %o1 - or %o1, %lo(__cheetah_flush_dcache_page), %o1 - call tlb_patch_one - mov 11, %o2 -#endif /* DCACHE_ALIASING_POSSIBLE */ - - ret - restore - #ifdef CONFIG_SMP /* These are all called by the slaves of a cross call, at * trap level 1, with interrupts fully disabled. @@ -447,7 +527,7 @@ cheetah_patch_cachetlbops: */ .align 32 .globl xcall_flush_tlb_mm -xcall_flush_tlb_mm: /* 21 insns */ +xcall_flush_tlb_mm: /* 24 insns */ mov PRIMARY_CONTEXT, %g2 ldxa [%g2] ASI_DMMU, %g3 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 @@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */ nop nop nop + nop + nop + nop .globl xcall_flush_tlb_page -xcall_flush_tlb_page: /* 17 insns */ +xcall_flush_tlb_page: /* 20 insns */ /* %g5=context, %g1=vaddr */ mov PRIMARY_CONTEXT, %g4 ldxa [%g4] ASI_DMMU, %g2 @@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */ retry nop nop + nop + nop + nop .globl xcall_flush_tlb_kernel_range -xcall_flush_tlb_kernel_range: /* 25 insns */ +xcall_flush_tlb_kernel_range: /* 44 insns */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 andn %g7, %g2, %g7 sub %g7, %g1, %g3 - add %g2, 1, %g2 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + add %g2, 1, %g2 sub %g3, %g2, %g3 or %g1, 0x20, %g1 ! Nucleus 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP @@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */ brnz,pt %g3, 1b sub %g3, %g2, %g3 retry - nop - nop +2: mov 63 * 8, %g1 +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_IMMU + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_DMMU + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %g1, 8, %g1 + brgez,pt %g1, 1b + nop + retry nop nop nop @@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4: retry +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + add %g2, 1, %g2 + sub %g3, %g2, %g3 + or %g1, 0x20, %g1 ! Nucleus +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %g3, 1b + sub %g3, %g2, %g3 + retry +2: mov 0x80, %g2 + stxa %g0, [%g2] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%g2] ASI_IMMU_DEMAP + membar #Sync + retry + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + #ifdef DCACHE_ALIASING_POSSIBLE .align 32 .globl xcall_flush_dcache_page_cheetah @@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error: ba,a,pt %xcc, rtrap .globl __hypervisor_xcall_flush_tlb_mm -__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ mov %o0, %g2 mov %o1, %g3 @@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ mov HV_FAST_MMU_DEMAP_CTX, %o5 ta HV_FAST_TRAP mov HV_FAST_MMU_DEMAP_CTX, %g6 - brnz,pn %o0, __hypervisor_tlb_xcall_error + brnz,pn %o0, 1f mov %o0, %g5 mov %g2, %o0 mov %g3, %o1 @@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ mov %g7, %o5 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop .globl __hypervisor_xcall_flush_tlb_page -__hypervisor_xcall_flush_tlb_page: /* 17 insns */ +__hypervisor_xcall_flush_tlb_page: /* 20 insns */ /* %g5=ctx, %g1=vaddr */ mov %o0, %g2 mov %o1, %g3 @@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */ sllx %o0, PAGE_SHIFT, %o0 ta HV_MMU_UNMAP_ADDR_TRAP mov HV_MMU_UNMAP_ADDR_TRAP, %g6 - brnz,a,pn %o0, __hypervisor_tlb_xcall_error + brnz,a,pn %o0, 1f mov %o0, %g5 mov %g2, %o0 mov %g3, %o1 mov %g4, %o2 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop .globl __hypervisor_xcall_flush_tlb_kernel_range -__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ sethi %hi(PAGE_SIZE - 1), %g2 or %g2, %lo(PAGE_SIZE - 1), %g2 andn %g1, %g2, %g1 andn %g7, %g2, %g7 sub %g7, %g1, %g3 + srlx %g3, 18, %g7 add %g2, 1, %g2 sub %g3, %g2, %g3 mov %o0, %g2 mov %o1, %g4 - mov %o2, %g7 + brnz,pn %g7, 2f + mov %o2, %g7 1: add %g1, %g3, %o0 /* ARG0: virtual address */ mov 0, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ ta HV_MMU_UNMAP_ADDR_TRAP mov HV_MMU_UNMAP_ADDR_TRAP, %g6 - brnz,pn %o0, __hypervisor_tlb_xcall_error + brnz,pn %o0, 1f mov %o0, %g5 sethi %hi(PAGE_SIZE), %o2 brnz,pt %g3, 1b sub %g3, %o2, %g3 - mov %g2, %o0 +5: mov %g2, %o0 mov %g4, %o1 mov %g7, %o2 membar #Sync retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop +2: mov %o3, %g1 + mov %o5, %g3 + mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + mov %g1, %o3 + brz,pt %o0, 5b + mov %g3, %o5 + mov HV_FAST_MMU_DEMAP_CTX, %g6 + ba,pt %xcc, 1b + clr %g5 /* These just get rescheduled to PIL vectors. */ .globl xcall_call_function @@ -809,6 +985,58 @@ xcall_kgdb_capture: #endif /* CONFIG_SMP */ + .globl cheetah_patch_cachetlbops +cheetah_patch_cachetlbops: + save %sp, -128, %sp + + sethi %hi(__flush_tlb_mm), %o0 + or %o0, %lo(__flush_tlb_mm), %o0 + sethi %hi(__cheetah_flush_tlb_mm), %o1 + or %o1, %lo(__cheetah_flush_tlb_mm), %o1 + call tlb_patch_one + mov 19, %o2 + + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__cheetah_flush_tlb_pending), %o1 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1 + call tlb_patch_one + mov 27, %o2 + + sethi %hi(__flush_tlb_kernel_range), %o0 + or %o0, %lo(__flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 31, %o2 + +#ifdef DCACHE_ALIASING_POSSIBLE + sethi %hi(__flush_dcache_page), %o0 + or %o0, %lo(__flush_dcache_page), %o0 + sethi %hi(__cheetah_flush_dcache_page), %o1 + or %o1, %lo(__cheetah_flush_dcache_page), %o1 + call tlb_patch_one + mov 11, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ + +#ifdef CONFIG_SMP + sethi %hi(xcall_flush_tlb_kernel_range), %o0 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 44, %o2 +#endif /* CONFIG_SMP */ + + ret + restore .globl hypervisor_patch_cachetlbops hypervisor_patch_cachetlbops: @@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops: sethi %hi(__hypervisor_flush_tlb_mm), %o1 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 call tlb_patch_one - mov 10, %o2 + mov 19, %o2 sethi %hi(__flush_tlb_page), %o0 or %o0, %lo(__flush_tlb_page), %o0 sethi %hi(__hypervisor_flush_tlb_page), %o1 or %o1, %lo(__hypervisor_flush_tlb_page), %o1 call tlb_patch_one - mov 11, %o2 + mov 22, %o2 sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__hypervisor_flush_tlb_pending), %o1 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 call tlb_patch_one - mov 16, %o2 + mov 27, %o2 sethi %hi(__flush_tlb_kernel_range), %o0 or %o0, %lo(__flush_tlb_kernel_range), %o0 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 call tlb_patch_one - mov 16, %o2 + mov 31, %o2 #ifdef DCACHE_ALIASING_POSSIBLE sethi %hi(__flush_dcache_page), %o0 @@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops: sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 call tlb_patch_one - mov 21, %o2 + mov 24, %o2 sethi %hi(xcall_flush_tlb_page), %o0 or %o0, %lo(xcall_flush_tlb_page), %o0 sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 call tlb_patch_one - mov 17, %o2 + mov 20, %o2 sethi %hi(xcall_flush_tlb_kernel_range), %o0 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 call tlb_patch_one - mov 25, %o2 + mov 44, %o2 #endif /* CONFIG_SMP */ ret diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile index 77f28ce9c646..9976fcecd17e 100644 --- a/arch/x86/entry/Makefile +++ b/arch/x86/entry/Makefile @@ -5,8 +5,8 @@ OBJECT_FILES_NON_STANDARD_entry_$(BITS).o := y OBJECT_FILES_NON_STANDARD_entry_64_compat.o := y -CFLAGS_syscall_64.o += -Wno-override-init -CFLAGS_syscall_32.o += -Wno-override-init +CFLAGS_syscall_64.o += $(call cc-option,-Wno-override-init,) +CFLAGS_syscall_32.o += $(call cc-option,-Wno-override-init,) obj-y := entry_$(BITS).o thunk_$(BITS).o syscall_$(BITS).o obj-y += common.o diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index eab0915f5995..a74a2dbc0180 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3607,10 +3607,14 @@ __init int intel_pmu_init(void) /* * Quirk: v2 perfmon does not report fixed-purpose events, so - * assume at least 3 events: + * assume at least 3 events, when not running in a hypervisor: */ - if (version > 1) - x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3); + if (version > 1) { + int assume = 3 * !boot_cpu_has(X86_FEATURE_HYPERVISOR); + + x86_pmu.num_counters_fixed = + max((int)edx.split.num_counters_fixed, assume); + } if (boot_cpu_has(X86_FEATURE_PDCM)) { u64 capabilities; diff --git a/arch/x86/events/intel/cstate.c b/arch/x86/events/intel/cstate.c index 3ca87b5a8677..4f5ac726335f 100644 --- a/arch/x86/events/intel/cstate.c +++ b/arch/x86/events/intel/cstate.c @@ -48,7 +48,8 @@ * Scope: Core * MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter * perf code: 0x02 - * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL + * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW + * SKL,KNL * Scope: Core * MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter * perf code: 0x03 @@ -56,15 +57,16 @@ * Scope: Core * MSR_PKG_C2_RESIDENCY: Package C2 Residency Counter. * perf code: 0x00 - * Available model: SNB,IVB,HSW,BDW,SKL + * Available model: SNB,IVB,HSW,BDW,SKL,KNL * Scope: Package (physical package) * MSR_PKG_C3_RESIDENCY: Package C3 Residency Counter. * perf code: 0x01 - * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL + * Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL,KNL * Scope: Package (physical package) * MSR_PKG_C6_RESIDENCY: Package C6 Residency Counter. * perf code: 0x02 - * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL + * Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW + * SKL,KNL * Scope: Package (physical package) * MSR_PKG_C7_RESIDENCY: Package C7 Residency Counter. * perf code: 0x03 @@ -118,6 +120,7 @@ struct cstate_model { /* Quirk flags */ #define SLM_PKG_C6_USE_C7_MSR (1UL << 0) +#define KNL_CORE_C6_MSR (1UL << 1) struct perf_cstate_msr { u64 msr; @@ -488,6 +491,18 @@ static const struct cstate_model slm_cstates __initconst = { .quirks = SLM_PKG_C6_USE_C7_MSR, }; + +static const struct cstate_model knl_cstates __initconst = { + .core_events = BIT(PERF_CSTATE_CORE_C6_RES), + + .pkg_events = BIT(PERF_CSTATE_PKG_C2_RES) | + BIT(PERF_CSTATE_PKG_C3_RES) | + BIT(PERF_CSTATE_PKG_C6_RES), + .quirks = KNL_CORE_C6_MSR, +}; + + + #define X86_CSTATES_MODEL(model, states) \ { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long) &(states) } @@ -523,6 +538,8 @@ static const struct x86_cpu_id intel_cstates_match[] __initconst = { X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_MOBILE, snb_cstates), X86_CSTATES_MODEL(INTEL_FAM6_SKYLAKE_DESKTOP, snb_cstates), + + X86_CSTATES_MODEL(INTEL_FAM6_XEON_PHI_KNL, knl_cstates), { }, }; MODULE_DEVICE_TABLE(x86cpu, intel_cstates_match); @@ -558,6 +575,11 @@ static int __init cstate_probe(const struct cstate_model *cm) if (cm->quirks & SLM_PKG_C6_USE_C7_MSR) pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY; + /* KNL has different MSR for CORE C6 */ + if (cm->quirks & KNL_CORE_C6_MSR) + pkg_msr[PERF_CSTATE_CORE_C6_RES].msr = MSR_KNL_CORE_C6_RESIDENCY; + + has_cstate_core = cstate_probe_msr(cm->core_events, PERF_CSTATE_CORE_EVENT_MAX, core_msr, core_events_attrs); diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index de25aad07853..d34bd370074b 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -351,4 +351,10 @@ extern void arch_phys_wc_del(int handle); #define arch_phys_wc_add arch_phys_wc_add #endif +#ifdef CONFIG_X86_PAT +extern int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size); +extern void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size); +#define arch_io_reserve_memtype_wc arch_io_reserve_memtype_wc +#endif + #endif /* _ASM_X86_IO_H */ diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 4b20f7304b9c..bdde80731f49 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -948,7 +948,6 @@ struct kvm_x86_ops { int (*get_lpage_level)(void); bool (*rdtscp_supported)(void); bool (*invpcid_supported)(void); - void (*adjust_tsc_offset_guest)(struct kvm_vcpu *vcpu, s64 adjustment); void (*set_tdp_cr3)(struct kvm_vcpu *vcpu, unsigned long cr3); @@ -958,8 +957,6 @@ struct kvm_x86_ops { void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); - u64 (*read_l1_tsc)(struct kvm_vcpu *vcpu, u64 host_tsc); - void (*get_exit_info)(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2); int (*check_intercept)(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 8a5abaa7d453..931ced8ca345 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -454,6 +454,7 @@ static void __init acpi_sci_ioapic_setup(u8 bus_irq, u16 polarity, u16 trigger, polarity = acpi_sci_flags & ACPI_MADT_POLARITY_MASK; mp_override_legacy_irq(bus_irq, polarity, trigger, gsi); + acpi_penalize_sci_irq(bus_irq, trigger, polarity); /* * stash over-ride to indicate we've been here diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 620ab06bcf45..017bda12caae 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -429,7 +429,7 @@ int __init save_microcode_in_initrd_amd(void) * We need the physical address of the container for both bitness since * boot_params.hdr.ramdisk_image is a physical address. */ - cont = __pa(container); + cont = __pa_nodebug(container); cont_va = container; #endif diff --git a/arch/x86/kernel/mcount_64.S b/arch/x86/kernel/mcount_64.S index efe73aacf966..7b0d3da52fb4 100644 --- a/arch/x86/kernel/mcount_64.S +++ b/arch/x86/kernel/mcount_64.S @@ -18,8 +18,10 @@ #ifdef CC_USING_FENTRY # define function_hook __fentry__ +EXPORT_SYMBOL(__fentry__) #else # define function_hook mcount +EXPORT_SYMBOL(mcount) #endif /* All cases save the original rbp (8 bytes) */ @@ -295,7 +297,6 @@ trace: jmp fgraph_trace END(function_hook) #endif /* CONFIG_DYNAMIC_FTRACE */ -EXPORT_SYMBOL(function_hook) #endif /* CONFIG_FUNCTION_TRACER */ #ifdef CONFIG_FUNCTION_GRAPH_TRACER diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c index 51402a7e4ca6..0bee04d41bed 100644 --- a/arch/x86/kernel/quirks.c +++ b/arch/x86/kernel/quirks.c @@ -625,8 +625,6 @@ static void amd_disable_seq_and_redirect_scrub(struct pci_dev *dev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_16H_NB_F3, amd_disable_seq_and_redirect_scrub); -#endif - #if defined(CONFIG_X86_64) && defined(CONFIG_X86_MCE) #include <linux/jump_label.h> #include <asm/string_64.h> @@ -657,3 +655,4 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2fc0, quirk_intel_brickland_xeon_ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x6fc0, quirk_intel_brickland_xeon_ras_cap); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x2083, quirk_intel_purley_xeon_ras_cap); #endif +#endif diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index bbfbca5fea0c..9c337b0e8ba7 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -1221,11 +1221,16 @@ void __init setup_arch(char **cmdline_p) */ get_smp_config(); + /* + * Systems w/o ACPI and mptables might not have it mapped the local + * APIC yet, but prefill_possible_map() might need to access it. + */ + init_apic_mappings(); + prefill_possible_map(); init_cpu_to_node(); - init_apic_mappings(); io_apic_init_mappings(); kvm_guest_init(); diff --git a/arch/x86/kernel/unwind_guess.c b/arch/x86/kernel/unwind_guess.c index 9298993dc8b7..2d721e533cf4 100644 --- a/arch/x86/kernel/unwind_guess.c +++ b/arch/x86/kernel/unwind_guess.c @@ -47,7 +47,14 @@ void __unwind_start(struct unwind_state *state, struct task_struct *task, get_stack_info(first_frame, state->task, &state->stack_info, &state->stack_mask); - if (!__kernel_text_address(*first_frame)) + /* + * The caller can provide the address of the first frame directly + * (first_frame) or indirectly (regs->sp) to indicate which stack frame + * to start unwinding at. Skip ahead until we reach it. + */ + if (!unwind_done(state) && + (!on_stack(&state->stack_info, first_frame, sizeof(long)) || + !__kernel_text_address(*first_frame))) unwind_next_frame(state); } EXPORT_SYMBOL_GPL(__unwind_start); diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 4e95d3eb2955..cbd7b92585bb 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -5045,7 +5045,7 @@ done_prefixes: /* Decode and fetch the destination operand: register or memory. */ rc = decode_operand(ctxt, &ctxt->dst, (ctxt->d >> DstShift) & OpMask); - if (ctxt->rip_relative) + if (ctxt->rip_relative && likely(ctxt->memopp)) ctxt->memopp->addr.mem.ea = address_mask(ctxt, ctxt->memopp->addr.mem.ea + ctxt->_eip); diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index f8157a36ab09..8ca1eca5038d 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c @@ -1138,21 +1138,6 @@ static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) mark_dirty(svm->vmcb, VMCB_INTERCEPTS); } -static void svm_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - svm->vmcb->control.tsc_offset += adjustment; - if (is_guest_mode(vcpu)) - svm->nested.hsave->control.tsc_offset += adjustment; - else - trace_kvm_write_tsc_offset(vcpu->vcpu_id, - svm->vmcb->control.tsc_offset - adjustment, - svm->vmcb->control.tsc_offset); - - mark_dirty(svm->vmcb, VMCB_INTERCEPTS); -} - static void avic_init_vmcb(struct vcpu_svm *svm) { struct vmcb *vmcb = svm->vmcb; @@ -3449,12 +3434,6 @@ static int cr8_write_interception(struct vcpu_svm *svm) return 0; } -static u64 svm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) -{ - struct vmcb *vmcb = get_host_vmcb(to_svm(vcpu)); - return vmcb->control.tsc_offset + host_tsc; -} - static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { struct vcpu_svm *svm = to_svm(vcpu); @@ -5422,8 +5401,6 @@ static struct kvm_x86_ops svm_x86_ops __ro_after_init = { .has_wbinvd_exit = svm_has_wbinvd_exit, .write_tsc_offset = svm_write_tsc_offset, - .adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest, - .read_l1_tsc = svm_read_l1_tsc, .set_tdp_cr3 = set_tdp_cr3, diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index cf1b16dbc98a..5382b82462fc 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -187,6 +187,7 @@ struct vmcs { */ struct loaded_vmcs { struct vmcs *vmcs; + struct vmcs *shadow_vmcs; int cpu; int launched; struct list_head loaded_vmcss_on_cpu_link; @@ -411,7 +412,6 @@ struct nested_vmx { * memory during VMXOFF, VMCLEAR, VMPTRLD. */ struct vmcs12 *cached_vmcs12; - struct vmcs *current_shadow_vmcs; /* * Indicates if the shadow vmcs must be updated with the * data hold by vmcs12 @@ -421,7 +421,6 @@ struct nested_vmx { /* vmcs02_list cache of VMCSs recently used to run L2 guests */ struct list_head vmcs02_pool; int vmcs02_num; - u64 vmcs01_tsc_offset; bool change_vmcs01_virtual_x2apic_mode; /* L2 must run next, and mustn't decide to exit to L1. */ bool nested_run_pending; @@ -1419,6 +1418,8 @@ static void vmcs_clear(struct vmcs *vmcs) static inline void loaded_vmcs_init(struct loaded_vmcs *loaded_vmcs) { vmcs_clear(loaded_vmcs->vmcs); + if (loaded_vmcs->shadow_vmcs && loaded_vmcs->launched) + vmcs_clear(loaded_vmcs->shadow_vmcs); loaded_vmcs->cpu = -1; loaded_vmcs->launched = 0; } @@ -2605,20 +2606,6 @@ static u64 guest_read_tsc(struct kvm_vcpu *vcpu) } /* - * Like guest_read_tsc, but always returns L1's notion of the timestamp - * counter, even if a nested guest (L2) is currently running. - */ -static u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) -{ - u64 tsc_offset; - - tsc_offset = is_guest_mode(vcpu) ? - to_vmx(vcpu)->nested.vmcs01_tsc_offset : - vmcs_read64(TSC_OFFSET); - return host_tsc + tsc_offset; -} - -/* * writes 'offset' into guest's timestamp counter offset register */ static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) @@ -2631,7 +2618,6 @@ static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) * to the newly set TSC to get L2's TSC. */ struct vmcs12 *vmcs12; - to_vmx(vcpu)->nested.vmcs01_tsc_offset = offset; /* recalculate vmcs02.TSC_OFFSET: */ vmcs12 = get_vmcs12(vcpu); vmcs_write64(TSC_OFFSET, offset + @@ -2644,19 +2630,6 @@ static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) } } -static void vmx_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment) -{ - u64 offset = vmcs_read64(TSC_OFFSET); - - vmcs_write64(TSC_OFFSET, offset + adjustment); - if (is_guest_mode(vcpu)) { - /* Even when running L2, the adjustment needs to apply to L1 */ - to_vmx(vcpu)->nested.vmcs01_tsc_offset += adjustment; - } else - trace_kvm_write_tsc_offset(vcpu->vcpu_id, offset, - offset + adjustment); -} - static bool guest_cpuid_has_vmx(struct kvm_vcpu *vcpu) { struct kvm_cpuid_entry2 *best = kvm_find_cpuid_entry(vcpu, 1, 0); @@ -3562,6 +3535,7 @@ static void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) loaded_vmcs_clear(loaded_vmcs); free_vmcs(loaded_vmcs->vmcs); loaded_vmcs->vmcs = NULL; + WARN_ON(loaded_vmcs->shadow_vmcs != NULL); } static void free_kvm_area(void) @@ -6696,6 +6670,7 @@ static struct loaded_vmcs *nested_get_current_vmcs02(struct vcpu_vmx *vmx) if (!item) return NULL; item->vmcs02.vmcs = alloc_vmcs(); + item->vmcs02.shadow_vmcs = NULL; if (!item->vmcs02.vmcs) { kfree(item); return NULL; @@ -7072,7 +7047,7 @@ static int handle_vmon(struct kvm_vcpu *vcpu) shadow_vmcs->revision_id |= (1u << 31); /* init shadow vmcs */ vmcs_clear(shadow_vmcs); - vmx->nested.current_shadow_vmcs = shadow_vmcs; + vmx->vmcs01.shadow_vmcs = shadow_vmcs; } INIT_LIST_HEAD(&(vmx->nested.vmcs02_pool)); @@ -7174,8 +7149,11 @@ static void free_nested(struct vcpu_vmx *vmx) free_page((unsigned long)vmx->nested.msr_bitmap); vmx->nested.msr_bitmap = NULL; } - if (enable_shadow_vmcs) - free_vmcs(vmx->nested.current_shadow_vmcs); + if (enable_shadow_vmcs) { + vmcs_clear(vmx->vmcs01.shadow_vmcs); + free_vmcs(vmx->vmcs01.shadow_vmcs); + vmx->vmcs01.shadow_vmcs = NULL; + } kfree(vmx->nested.cached_vmcs12); /* Unpin physical memory we referred to in current vmcs02 */ if (vmx->nested.apic_access_page) { @@ -7352,7 +7330,7 @@ static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx) int i; unsigned long field; u64 field_value; - struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs; + struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; const unsigned long *fields = shadow_read_write_fields; const int num_fields = max_shadow_read_write_fields; @@ -7401,7 +7379,7 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) int i, q; unsigned long field; u64 field_value = 0; - struct vmcs *shadow_vmcs = vmx->nested.current_shadow_vmcs; + struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs; vmcs_load(shadow_vmcs); @@ -7591,7 +7569,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) vmcs_set_bits(SECONDARY_VM_EXEC_CONTROL, SECONDARY_EXEC_SHADOW_VMCS); vmcs_write64(VMCS_LINK_POINTER, - __pa(vmx->nested.current_shadow_vmcs)); + __pa(vmx->vmcs01.shadow_vmcs)); vmx->nested.sync_shadow_vmcs = true; } } @@ -7659,7 +7637,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) types = (vmx->nested.nested_vmx_ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6; - if (!(types & (1UL << type))) { + if (type >= 32 || !(types & (1 << type))) { nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); skip_emulated_instruction(vcpu); @@ -7722,7 +7700,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) types = (vmx->nested.nested_vmx_vpid_caps >> 8) & 0x7; - if (!(types & (1UL << type))) { + if (type >= 32 || !(types & (1 << type))) { nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); skip_emulated_instruction(vcpu); @@ -9156,6 +9134,7 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id) vmx->loaded_vmcs = &vmx->vmcs01; vmx->loaded_vmcs->vmcs = alloc_vmcs(); + vmx->loaded_vmcs->shadow_vmcs = NULL; if (!vmx->loaded_vmcs->vmcs) goto free_msrs; if (!vmm_exclusive) @@ -10061,9 +10040,9 @@ static void prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETING) vmcs_write64(TSC_OFFSET, - vmx->nested.vmcs01_tsc_offset + vmcs12->tsc_offset); + vcpu->arch.tsc_offset + vmcs12->tsc_offset); else - vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset); + vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); if (kvm_has_tsc_control) decache_tsc_multiplier(vmx); @@ -10293,8 +10272,6 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) enter_guest_mode(vcpu); - vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET); - if (!(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL); @@ -10818,7 +10795,7 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, load_vmcs12_host_state(vcpu, vmcs12); /* Update any VMCS fields that might have changed while L2 ran */ - vmcs_write64(TSC_OFFSET, vmx->nested.vmcs01_tsc_offset); + vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); if (vmx->hv_deadline_tsc == -1) vmcs_clear_bits(PIN_BASED_VM_EXEC_CONTROL, PIN_BASED_VMX_PREEMPTION_TIMER); @@ -11339,8 +11316,6 @@ static struct kvm_x86_ops vmx_x86_ops __ro_after_init = { .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, .write_tsc_offset = vmx_write_tsc_offset, - .adjust_tsc_offset_guest = vmx_adjust_tsc_offset_guest, - .read_l1_tsc = vmx_read_l1_tsc, .set_tdp_cr3 = vmx_set_cr3, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e375235d81c9..3017de0431bd 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1409,7 +1409,7 @@ static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) { - return kvm_x86_ops->read_l1_tsc(vcpu, kvm_scale_tsc(vcpu, host_tsc)); + return vcpu->arch.tsc_offset + kvm_scale_tsc(vcpu, host_tsc); } EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); @@ -1547,7 +1547,7 @@ EXPORT_SYMBOL_GPL(kvm_write_tsc); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment) { - kvm_x86_ops->adjust_tsc_offset_guest(vcpu, adjustment); + kvm_vcpu_write_tsc_offset(vcpu, vcpu->arch.tsc_offset + adjustment); } static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) @@ -1555,7 +1555,7 @@ static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio) WARN_ON(adjustment < 0); adjustment = kvm_scale_tsc(vcpu, (u64) adjustment); - kvm_x86_ops->adjust_tsc_offset_guest(vcpu, adjustment); + adjust_tsc_offset_guest(vcpu, adjustment); } #ifdef CONFIG_X86_64 @@ -2262,7 +2262,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) /* Drop writes to this legacy MSR -- see rdmsr * counterpart for further detail. */ - vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", msr, data); + vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n", msr, data); break; case MSR_AMD64_OSVW_ID_LENGTH: if (!guest_cpuid_has_osvw(vcpu)) @@ -2280,11 +2280,11 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (kvm_pmu_is_valid_msr(vcpu, msr)) return kvm_pmu_set_msr(vcpu, msr_info); if (!ignore_msrs) { - vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data %llx\n", + vcpu_unimpl(vcpu, "unhandled wrmsr: 0x%x data 0x%llx\n", msr, data); return 1; } else { - vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data %llx\n", + vcpu_unimpl(vcpu, "ignored wrmsr: 0x%x data 0x%llx\n", msr, data); break; } @@ -7410,10 +7410,12 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) { + void *wbinvd_dirty_mask = vcpu->arch.wbinvd_dirty_mask; + kvmclock_reset(vcpu); - free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); kvm_x86_ops->vcpu_free(vcpu); + free_cpumask_var(wbinvd_dirty_mask); } struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, diff --git a/arch/x86/mm/kaslr.c b/arch/x86/mm/kaslr.c index ddd2661c4502..887e57182716 100644 --- a/arch/x86/mm/kaslr.c +++ b/arch/x86/mm/kaslr.c @@ -104,10 +104,10 @@ void __init kernel_randomize_memory(void) * consistent with the vaddr_start/vaddr_end variables. */ BUILD_BUG_ON(vaddr_start >= vaddr_end); - BUILD_BUG_ON(config_enabled(CONFIG_X86_ESPFIX64) && + BUILD_BUG_ON(IS_ENABLED(CONFIG_X86_ESPFIX64) && vaddr_end >= EFI_VA_START); - BUILD_BUG_ON((config_enabled(CONFIG_X86_ESPFIX64) || - config_enabled(CONFIG_EFI)) && + BUILD_BUG_ON((IS_ENABLED(CONFIG_X86_ESPFIX64) || + IS_ENABLED(CONFIG_EFI)) && vaddr_end >= __START_KERNEL_map); BUILD_BUG_ON(vaddr_end > __START_KERNEL_map); diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c index 170cc4ff057b..83e701f160a9 100644 --- a/arch/x86/mm/pat.c +++ b/arch/x86/mm/pat.c @@ -730,6 +730,20 @@ void io_free_memtype(resource_size_t start, resource_size_t end) free_memtype(start, end); } +int arch_io_reserve_memtype_wc(resource_size_t start, resource_size_t size) +{ + enum page_cache_mode type = _PAGE_CACHE_MODE_WC; + + return io_reserve_memtype(start, start + size, &type); +} +EXPORT_SYMBOL(arch_io_reserve_memtype_wc); + +void arch_io_free_memtype_wc(resource_size_t start, resource_size_t size) +{ + io_free_memtype(start, start + size); +} +EXPORT_SYMBOL(arch_io_free_memtype_wc); + pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot) { diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index c0fdd57da7aa..bdd855685403 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -1837,6 +1837,7 @@ static void __init init_hvm_pv_info(void) xen_domain_type = XEN_HVM_DOMAIN; } +#endif static int xen_cpu_up_prepare(unsigned int cpu) { @@ -1887,6 +1888,7 @@ static int xen_cpu_up_online(unsigned int cpu) return 0; } +#ifdef CONFIG_XEN_PVHVM #ifdef CONFIG_KEXEC_CORE static void xen_hvm_shutdown(void) { diff --git a/block/badblocks.c b/block/badblocks.c index 6610e282a03e..6ebcef282314 100644 --- a/block/badblocks.c +++ b/block/badblocks.c @@ -133,6 +133,26 @@ retry: } EXPORT_SYMBOL_GPL(badblocks_check); +static void badblocks_update_acked(struct badblocks *bb) +{ + u64 *p = bb->page; + int i; + bool unacked = false; + + if (!bb->unacked_exist) + return; + + for (i = 0; i < bb->count ; i++) { + if (!BB_ACK(p[i])) { + unacked = true; + break; + } + } + + if (!unacked) + bb->unacked_exist = 0; +} + /** * badblocks_set() - Add a range of bad blocks to the table. * @bb: the badblocks structure that holds all badblock information @@ -294,6 +314,8 @@ int badblocks_set(struct badblocks *bb, sector_t s, int sectors, bb->changed = 1; if (!acknowledged) bb->unacked_exist = 1; + else + badblocks_update_acked(bb); write_sequnlock_irqrestore(&bb->lock, flags); return rv; @@ -401,6 +423,7 @@ int badblocks_clear(struct badblocks *bb, sector_t s, int sectors) } } + badblocks_update_acked(bb); bb->changed = 1; out: write_sequnlock_irq(&bb->lock); diff --git a/block/blk-flush.c b/block/blk-flush.c index 6a14b68b9135..3c882cbc7541 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -343,6 +343,34 @@ static void flush_data_end_io(struct request *rq, int error) struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL); /* + * Updating q->in_flight[] here for making this tag usable + * early. Because in blk_queue_start_tag(), + * q->in_flight[BLK_RW_ASYNC] is used to limit async I/O and + * reserve tags for sync I/O. + * + * More importantly this way can avoid the following I/O + * deadlock: + * + * - suppose there are 40 fua requests comming to flush queue + * and queue depth is 31 + * - 30 rqs are scheduled then blk_queue_start_tag() can't alloc + * tag for async I/O any more + * - all the 30 rqs are completed before FLUSH_PENDING_TIMEOUT + * and flush_data_end_io() is called + * - the other rqs still can't go ahead if not updating + * q->in_flight[BLK_RW_ASYNC] here, meantime these rqs + * are held in flush data queue and make no progress of + * handling post flush rq + * - only after the post flush rq is handled, all these rqs + * can be completed + */ + + elv_completed_request(q, rq); + + /* for avoiding double accounting */ + rq->cmd_flags &= ~REQ_STARTED; + + /* * After populating an empty queue, kick it to avoid stall. Read * the comment in flush_end_io(). */ diff --git a/block/blk-mq.c b/block/blk-mq.c index ddc2eed64771..f3d27a6dee09 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1217,9 +1217,9 @@ static struct request *blk_mq_map_request(struct request_queue *q, blk_mq_set_alloc_data(&alloc_data, q, 0, ctx, hctx); rq = __blk_mq_alloc_request(&alloc_data, op, op_flags); - hctx->queued++; - data->hctx = hctx; - data->ctx = ctx; + data->hctx = alloc_data.hctx; + data->ctx = alloc_data.ctx; + data->hctx->queued++; return rq; } diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c index f1e6dcc7a827..54d48b90de2c 100644 --- a/drivers/acpi/acpica/dsinit.c +++ b/drivers/acpi/acpica/dsinit.c @@ -46,6 +46,7 @@ #include "acdispat.h" #include "acnamesp.h" #include "actables.h" +#include "acinterp.h" #define _COMPONENT ACPI_DISPATCHER ACPI_MODULE_NAME("dsinit") @@ -214,23 +215,17 @@ acpi_ds_initialize_objects(u32 table_index, /* Walk entire namespace from the supplied root */ - status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE); - if (ACPI_FAILURE(status)) { - return_ACPI_STATUS(status); - } - /* * We don't use acpi_walk_namespace since we do not want to acquire * the namespace reader lock. */ status = acpi_ns_walk_namespace(ACPI_TYPE_ANY, start_node, ACPI_UINT32_MAX, - ACPI_NS_WALK_UNLOCK, acpi_ds_init_one_object, - NULL, &info, NULL); + 0, acpi_ds_init_one_object, NULL, &info, + NULL); if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "During WalkNamespace")); } - (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); status = acpi_get_table_by_index(table_index, &table); if (ACPI_FAILURE(status)) { diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c index 32e9ddc0cf2b..2b3210f42a46 100644 --- a/drivers/acpi/acpica/dsmethod.c +++ b/drivers/acpi/acpica/dsmethod.c @@ -99,14 +99,11 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node, "Method auto-serialization parse [%4.4s] %p\n", acpi_ut_get_node_name(node), node)); - acpi_ex_enter_interpreter(); - /* Create/Init a root op for the method parse tree */ op = acpi_ps_alloc_op(AML_METHOD_OP, obj_desc->method.aml_start); if (!op) { - status = AE_NO_MEMORY; - goto unlock; + return_ACPI_STATUS(AE_NO_MEMORY); } acpi_ps_set_name(op, node->name.integer); @@ -118,8 +115,7 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node, acpi_ds_create_walk_state(node->owner_id, NULL, NULL, NULL); if (!walk_state) { acpi_ps_free_op(op); - status = AE_NO_MEMORY; - goto unlock; + return_ACPI_STATUS(AE_NO_MEMORY); } status = acpi_ds_init_aml_walk(walk_state, op, node, @@ -138,8 +134,6 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node, status = acpi_ps_parse_aml(walk_state); acpi_ps_delete_parse_tree(op); -unlock: - acpi_ex_exit_interpreter(); return_ACPI_STATUS(status); } @@ -731,26 +725,6 @@ acpi_ds_terminate_control_method(union acpi_operand_object *method_desc, acpi_ds_method_data_delete_all(walk_state); /* - * If method is serialized, release the mutex and restore the - * current sync level for this thread - */ - if (method_desc->method.mutex) { - - /* Acquisition Depth handles recursive calls */ - - method_desc->method.mutex->mutex.acquisition_depth--; - if (!method_desc->method.mutex->mutex.acquisition_depth) { - walk_state->thread->current_sync_level = - method_desc->method.mutex->mutex. - original_sync_level; - - acpi_os_release_mutex(method_desc->method. - mutex->mutex.os_mutex); - method_desc->method.mutex->mutex.thread_id = 0; - } - } - - /* * Delete any namespace objects created anywhere within the * namespace by the execution of this method. Unless: * 1) This method is a module-level executable code method, in which @@ -786,6 +760,26 @@ acpi_ds_terminate_control_method(union acpi_operand_object *method_desc, ~ACPI_METHOD_MODIFIED_NAMESPACE; } } + + /* + * If method is serialized, release the mutex and restore the + * current sync level for this thread + */ + if (method_desc->method.mutex) { + + /* Acquisition Depth handles recursive calls */ + + method_desc->method.mutex->mutex.acquisition_depth--; + if (!method_desc->method.mutex->mutex.acquisition_depth) { + walk_state->thread->current_sync_level = + method_desc->method.mutex->mutex. + original_sync_level; + + acpi_os_release_mutex(method_desc->method. + mutex->mutex.os_mutex); + method_desc->method.mutex->mutex.thread_id = 0; + } + } } /* Decrement the thread count on the method */ diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c index 028b22a3154e..e36218206bb0 100644 --- a/drivers/acpi/acpica/dswload2.c +++ b/drivers/acpi/acpica/dswload2.c @@ -607,11 +607,9 @@ acpi_status acpi_ds_load2_end_op(struct acpi_walk_state *walk_state) } } - acpi_ex_exit_interpreter(); status = acpi_ev_initialize_region (acpi_ns_get_attached_object(node), FALSE); - acpi_ex_enter_interpreter(); if (ACPI_FAILURE(status)) { /* diff --git a/drivers/acpi/acpica/evrgnini.c b/drivers/acpi/acpica/evrgnini.c index 3843f1fc5dbb..75ddd160a716 100644 --- a/drivers/acpi/acpica/evrgnini.c +++ b/drivers/acpi/acpica/evrgnini.c @@ -45,6 +45,7 @@ #include "accommon.h" #include "acevents.h" #include "acnamesp.h" +#include "acinterp.h" #define _COMPONENT ACPI_EVENTS ACPI_MODULE_NAME("evrgnini") @@ -597,9 +598,11 @@ acpi_ev_initialize_region(union acpi_operand_object *region_obj, } } + acpi_ex_exit_interpreter(); status = acpi_ev_execute_reg_method(region_obj, ACPI_REG_CONNECT); + acpi_ex_enter_interpreter(); if (acpi_ns_locked) { status = diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c index 334d3c5ba617..d1f20143bb11 100644 --- a/drivers/acpi/acpica/nsload.c +++ b/drivers/acpi/acpica/nsload.c @@ -137,7 +137,9 @@ unlock: ACPI_DEBUG_PRINT((ACPI_DB_INFO, "**** Begin Table Object Initialization\n")); + acpi_ex_enter_interpreter(); status = acpi_ds_initialize_objects(table_index, node); + acpi_ex_exit_interpreter(); ACPI_DEBUG_PRINT((ACPI_DB_INFO, "**** Completed Table Object Initialization\n")); diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index f0a029e68d3e..0d099a24f776 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -662,7 +662,7 @@ static int ghes_proc(struct ghes *ghes) ghes_do_proc(ghes, ghes->estatus); out: ghes_clear_estatus(ghes); - return 0; + return rc; } static void ghes_add_timer(struct ghes *ghes) diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c index c983bf733ad3..bc3d914dfc3e 100644 --- a/drivers/acpi/pci_link.c +++ b/drivers/acpi/pci_link.c @@ -87,6 +87,7 @@ struct acpi_pci_link { static LIST_HEAD(acpi_link_list); static DEFINE_MUTEX(acpi_link_lock); +static int sci_irq = -1, sci_penalty; /* -------------------------------------------------------------------------- PCI Link Device Management @@ -496,25 +497,13 @@ static int acpi_irq_get_penalty(int irq) { int penalty = 0; - /* - * Penalize IRQ used by ACPI SCI. If ACPI SCI pin attributes conflict - * with PCI IRQ attributes, mark ACPI SCI as ISA_ALWAYS so it won't be - * use for PCI IRQs. - */ - if (irq == acpi_gbl_FADT.sci_interrupt) { - u32 type = irq_get_trigger_type(irq) & IRQ_TYPE_SENSE_MASK; - - if (type != IRQ_TYPE_LEVEL_LOW) - penalty += PIRQ_PENALTY_ISA_ALWAYS; - else - penalty += PIRQ_PENALTY_PCI_USING; - } + if (irq == sci_irq) + penalty += sci_penalty; if (irq < ACPI_MAX_ISA_IRQS) return penalty + acpi_isa_irq_penalty[irq]; - penalty += acpi_irq_pci_sharing_penalty(irq); - return penalty; + return penalty + acpi_irq_pci_sharing_penalty(irq); } int __init acpi_irq_penalty_init(void) @@ -619,6 +608,10 @@ static int acpi_pci_link_allocate(struct acpi_pci_link *link) acpi_device_bid(link->device)); return -ENODEV; } else { + if (link->irq.active < ACPI_MAX_ISA_IRQS) + acpi_isa_irq_penalty[link->irq.active] += + PIRQ_PENALTY_PCI_USING; + printk(KERN_WARNING PREFIX "%s [%s] enabled at IRQ %d\n", acpi_device_name(link->device), acpi_device_bid(link->device), link->irq.active); @@ -849,7 +842,7 @@ static int __init acpi_irq_penalty_update(char *str, int used) continue; if (used) - new_penalty = acpi_irq_get_penalty(irq) + + new_penalty = acpi_isa_irq_penalty[irq] + PIRQ_PENALTY_ISA_USED; else new_penalty = 0; @@ -871,7 +864,7 @@ static int __init acpi_irq_penalty_update(char *str, int used) void acpi_penalize_isa_irq(int irq, int active) { if ((irq >= 0) && (irq < ARRAY_SIZE(acpi_isa_irq_penalty))) - acpi_isa_irq_penalty[irq] = acpi_irq_get_penalty(irq) + + acpi_isa_irq_penalty[irq] += (active ? PIRQ_PENALTY_ISA_USED : PIRQ_PENALTY_PCI_USING); } @@ -881,6 +874,17 @@ bool acpi_isa_irq_available(int irq) acpi_irq_get_penalty(irq) < PIRQ_PENALTY_ISA_ALWAYS); } +void acpi_penalize_sci_irq(int irq, int trigger, int polarity) +{ + sci_irq = irq; + + if (trigger == ACPI_MADT_TRIGGER_LEVEL && + polarity == ACPI_MADT_POLARITY_ACTIVE_LOW) + sci_penalty = PIRQ_PENALTY_PCI_USING; + else + sci_penalty = PIRQ_PENALTY_ISA_ALWAYS; +} + /* * Over-ride default table to reserve additional IRQs for use by ISA * e.g. acpi_irq_isa=5 diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 562af94bec35..3c71b982bf2a 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -1002,7 +1002,7 @@ static int binder_dec_node(struct binder_node *node, int strong, int internal) static struct binder_ref *binder_get_ref(struct binder_proc *proc, - uint32_t desc) + u32 desc, bool need_strong_ref) { struct rb_node *n = proc->refs_by_desc.rb_node; struct binder_ref *ref; @@ -1010,12 +1010,16 @@ static struct binder_ref *binder_get_ref(struct binder_proc *proc, while (n) { ref = rb_entry(n, struct binder_ref, rb_node_desc); - if (desc < ref->desc) + if (desc < ref->desc) { n = n->rb_left; - else if (desc > ref->desc) + } else if (desc > ref->desc) { n = n->rb_right; - else + } else if (need_strong_ref && !ref->strong) { + binder_user_error("tried to use weak ref as strong ref\n"); + return NULL; + } else { return ref; + } } return NULL; } @@ -1285,7 +1289,10 @@ static void binder_transaction_buffer_release(struct binder_proc *proc, } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); + struct binder_ref *ref; + + ref = binder_get_ref(proc, fp->handle, + fp->type == BINDER_TYPE_HANDLE); if (ref == NULL) { pr_err("transaction release %d bad handle %d\n", @@ -1380,7 +1387,7 @@ static void binder_transaction(struct binder_proc *proc, if (tr->target.handle) { struct binder_ref *ref; - ref = binder_get_ref(proc, tr->target.handle); + ref = binder_get_ref(proc, tr->target.handle, true); if (ref == NULL) { binder_user_error("%d:%d got transaction to invalid handle\n", proc->pid, thread->pid); @@ -1577,7 +1584,9 @@ static void binder_transaction(struct binder_proc *proc, fp->type = BINDER_TYPE_HANDLE; else fp->type = BINDER_TYPE_WEAK_HANDLE; + fp->binder = 0; fp->handle = ref->desc; + fp->cookie = 0; binder_inc_ref(ref, fp->type == BINDER_TYPE_HANDLE, &thread->todo); @@ -1589,7 +1598,10 @@ static void binder_transaction(struct binder_proc *proc, } break; case BINDER_TYPE_HANDLE: case BINDER_TYPE_WEAK_HANDLE: { - struct binder_ref *ref = binder_get_ref(proc, fp->handle); + struct binder_ref *ref; + + ref = binder_get_ref(proc, fp->handle, + fp->type == BINDER_TYPE_HANDLE); if (ref == NULL) { binder_user_error("%d:%d got transaction with invalid handle, %d\n", @@ -1624,7 +1636,9 @@ static void binder_transaction(struct binder_proc *proc, return_error = BR_FAILED_REPLY; goto err_binder_get_ref_for_node_failed; } + fp->binder = 0; fp->handle = new_ref->desc; + fp->cookie = 0; binder_inc_ref(new_ref, fp->type == BINDER_TYPE_HANDLE, NULL); trace_binder_transaction_ref_to_ref(t, ref, new_ref); @@ -1678,6 +1692,7 @@ static void binder_transaction(struct binder_proc *proc, binder_debug(BINDER_DEBUG_TRANSACTION, " fd %d -> %d\n", fp->handle, target_fd); /* TODO: fput? */ + fp->binder = 0; fp->handle = target_fd; } break; @@ -1800,7 +1815,9 @@ static int binder_thread_write(struct binder_proc *proc, ref->desc); } } else - ref = binder_get_ref(proc, target); + ref = binder_get_ref(proc, target, + cmd == BC_ACQUIRE || + cmd == BC_RELEASE); if (ref == NULL) { binder_user_error("%d:%d refcount change on invalid ref %d\n", proc->pid, thread->pid, target); @@ -1996,7 +2013,7 @@ static int binder_thread_write(struct binder_proc *proc, if (get_user(cookie, (binder_uintptr_t __user *)ptr)) return -EFAULT; ptr += sizeof(binder_uintptr_t); - ref = binder_get_ref(proc, target); + ref = binder_get_ref(proc, target, false); if (ref == NULL) { binder_user_error("%d:%d %s invalid ref %d\n", proc->pid, thread->pid, diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index ba5f11cebee2..9669fc7c19df 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1418,31 +1418,34 @@ static int ahci_init_msi(struct pci_dev *pdev, unsigned int n_ports, * Message mode could be enforced. In this case assume that advantage * of multipe MSIs is negated and use single MSI mode instead. */ - nvec = pci_alloc_irq_vectors(pdev, n_ports, INT_MAX, - PCI_IRQ_MSIX | PCI_IRQ_MSI); - if (nvec > 0) { - if (!(readl(hpriv->mmio + HOST_CTL) & HOST_MRSM)) { - hpriv->get_irq_vector = ahci_get_irq_vector; - hpriv->flags |= AHCI_HFLAG_MULTI_MSI; - return nvec; + if (n_ports > 1) { + nvec = pci_alloc_irq_vectors(pdev, n_ports, INT_MAX, + PCI_IRQ_MSIX | PCI_IRQ_MSI); + if (nvec > 0) { + if (!(readl(hpriv->mmio + HOST_CTL) & HOST_MRSM)) { + hpriv->get_irq_vector = ahci_get_irq_vector; + hpriv->flags |= AHCI_HFLAG_MULTI_MSI; + return nvec; + } + + /* + * Fallback to single MSI mode if the controller + * enforced MRSM mode. + */ + printk(KERN_INFO + "ahci: MRSM is on, fallback to single MSI\n"); + pci_free_irq_vectors(pdev); } /* - * Fallback to single MSI mode if the controller enforced MRSM - * mode. + * -ENOSPC indicated we don't have enough vectors. Don't bother + * trying a single vectors for any other error: */ - printk(KERN_INFO "ahci: MRSM is on, fallback to single MSI\n"); - pci_free_irq_vectors(pdev); + if (nvec < 0 && nvec != -ENOSPC) + return nvec; } /* - * -ENOSPC indicated we don't have enough vectors. Don't bother trying - * a single vectors for any other error: - */ - if (nvec < 0 && nvec != -ENOSPC) - return nvec; - - /* * If the host is not capable of supporting per-port vectors, fall * back to single MSI before finally attempting single MSI-X. */ @@ -1617,7 +1620,7 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) /* legacy intx interrupts */ pci_intx(pdev, 1); } - hpriv->irq = pdev->irq; + hpriv->irq = pci_irq_vector(pdev, 0); if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss) host->flags |= ATA_HOST_PARALLEL_SCAN; diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 37bf25c6b4a6..0e40967a5d6e 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -213,14 +213,16 @@ config DEBUG_DEVRES If you are unsure about this, Say N here. config DEBUG_TEST_DRIVER_REMOVE - bool "Test driver remove calls during probe" + bool "Test driver remove calls during probe (UNSTABLE)" depends on DEBUG_KERNEL help Say Y here if you want the Driver core to test driver remove functions by calling probe, remove, probe. This tests the remove path without having to unbind the driver or unload the driver module. - If you are unsure about this, say N here. + This option is expected to find errors and may render your system + unusable. You should say N here unless you are explicitly looking to + test this functionality. config SYS_HYPERVISOR bool diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c index 811e11c82f32..0809cda93cc0 100644 --- a/drivers/block/DAC960.c +++ b/drivers/block/DAC960.c @@ -2954,7 +2954,7 @@ DAC960_DetectController(struct pci_dev *PCI_Device, case DAC960_PD_Controller: if (!request_region(Controller->IO_Address, 0x80, Controller->FullModelName)) { - DAC960_Error("IO port 0x%d busy for Controller at\n", + DAC960_Error("IO port 0x%lx busy for Controller at\n", Controller, Controller->IO_Address); goto Failure; } @@ -2990,7 +2990,7 @@ DAC960_DetectController(struct pci_dev *PCI_Device, case DAC960_P_Controller: if (!request_region(Controller->IO_Address, 0x80, Controller->FullModelName)){ - DAC960_Error("IO port 0x%d busy for Controller at\n", + DAC960_Error("IO port 0x%lx busy for Controller at\n", Controller, Controller->IO_Address); goto Failure; } diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index ba405b55329f..19a16b2dbb91 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -164,7 +164,7 @@ static void sock_shutdown(struct nbd_device *nbd) spin_lock(&nbd->sock_lock); if (!nbd->sock) { - spin_unlock_irq(&nbd->sock_lock); + spin_unlock(&nbd->sock_lock); return; } diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 2dc5c96c186a..5545a679abd8 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -376,7 +376,7 @@ static void virtblk_config_changed(struct virtio_device *vdev) static int init_vq(struct virtio_blk *vblk) { - int err = 0; + int err; int i; vq_callback_t **callbacks; const char **names; @@ -390,13 +390,13 @@ static int init_vq(struct virtio_blk *vblk) if (err) num_vqs = 1; - vblk->vqs = kmalloc(sizeof(*vblk->vqs) * num_vqs, GFP_KERNEL); + vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL); if (!vblk->vqs) return -ENOMEM; - names = kmalloc(sizeof(*names) * num_vqs, GFP_KERNEL); - callbacks = kmalloc(sizeof(*callbacks) * num_vqs, GFP_KERNEL); - vqs = kmalloc(sizeof(*vqs) * num_vqs, GFP_KERNEL); + names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL); + callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL); + vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL); if (!names || !callbacks || !vqs) { err = -ENOMEM; goto out; diff --git a/drivers/bluetooth/btwilink.c b/drivers/bluetooth/btwilink.c index ef51c9c864c5..b6bb58c41df5 100644 --- a/drivers/bluetooth/btwilink.c +++ b/drivers/bluetooth/btwilink.c @@ -310,7 +310,7 @@ static int bt_ti_probe(struct platform_device *pdev) BT_DBG("HCI device registered (hdev %p)", hdev); dev_set_drvdata(&pdev->dev, hst); - return err; + return 0; } static int bt_ti_remove(struct platform_device *pdev) diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c index 5ccb90ef0146..8f6c23c20c52 100644 --- a/drivers/bluetooth/hci_bcm.c +++ b/drivers/bluetooth/hci_bcm.c @@ -643,6 +643,14 @@ static const struct dmi_system_id bcm_wrong_irq_dmi_table[] = { }, .driver_data = &acpi_active_low, }, + { /* Handle ThinkPad 8 tablets with BCM2E55 chipset ACPI ID */ + .ident = "Lenovo ThinkPad 8", + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_EXACT_MATCH(DMI_PRODUCT_VERSION, "ThinkPad 8"), + }, + .driver_data = &acpi_active_low, + }, { } }; diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig index 7010dcac9328..78751057164a 100644 --- a/drivers/bus/Kconfig +++ b/drivers/bus/Kconfig @@ -111,6 +111,7 @@ config OMAP_OCP2SCP config QCOM_EBI2 bool "Qualcomm External Bus Interface 2 (EBI2)" depends on HAS_IOMEM + depends on ARCH_QCOM || COMPILE_TEST help Say y here to enable support for the Qualcomm External Bus Interface 2, which can be used to connect things like NAND Flash, diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 482794526e8c..d2d2c89de5b4 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -84,14 +84,14 @@ static size_t rng_buffer_size(void) static void add_early_randomness(struct hwrng *rng) { - unsigned char bytes[16]; int bytes_read; + size_t size = min_t(size_t, 16, rng_buffer_size()); mutex_lock(&reading_mutex); - bytes_read = rng_get_data(rng, bytes, sizeof(bytes), 1); + bytes_read = rng_get_data(rng, rng_buffer, size, 1); mutex_unlock(&reading_mutex); if (bytes_read > 0) - add_device_randomness(bytes, bytes_read); + add_device_randomness(rng_buffer, bytes_read); } static inline void cleanup_rng(struct kref *kref) diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index 8de61876f633..3a9149cf0110 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -813,9 +813,6 @@ int tpm_do_selftest(struct tpm_chip *chip) continue; } - if (rc < TPM_HEADER_SIZE) - return -EFAULT; - if (rc == TPM_ERR_DISABLED || rc == TPM_ERR_DEACTIVATED) { dev_info(&chip->dev, "TPM is disabled/deactivated (0x%X)\n", rc); diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index d433b1db1fdd..5649234b7316 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1539,19 +1539,29 @@ static void remove_port_data(struct port *port) spin_lock_irq(&port->inbuf_lock); /* Remove unused data this port might have received. */ discard_port_data(port); + spin_unlock_irq(&port->inbuf_lock); /* Remove buffers we queued up for the Host to send us data in. */ - while ((buf = virtqueue_detach_unused_buf(port->in_vq))) - free_buf(buf, true); - spin_unlock_irq(&port->inbuf_lock); + do { + spin_lock_irq(&port->inbuf_lock); + buf = virtqueue_detach_unused_buf(port->in_vq); + spin_unlock_irq(&port->inbuf_lock); + if (buf) + free_buf(buf, true); + } while (buf); spin_lock_irq(&port->outvq_lock); reclaim_consumed_buffers(port); + spin_unlock_irq(&port->outvq_lock); /* Free pending buffers from the out-queue. */ - while ((buf = virtqueue_detach_unused_buf(port->out_vq))) - free_buf(buf, true); - spin_unlock_irq(&port->outvq_lock); + do { + spin_lock_irq(&port->outvq_lock); + buf = virtqueue_detach_unused_buf(port->out_vq); + spin_unlock_irq(&port->outvq_lock); + if (buf) + free_buf(buf, true); + } while (buf); } /* diff --git a/drivers/clk/at91/clk-programmable.c b/drivers/clk/at91/clk-programmable.c index 190122e64a3a..85a449cf61e3 100644 --- a/drivers/clk/at91/clk-programmable.c +++ b/drivers/clk/at91/clk-programmable.c @@ -203,7 +203,7 @@ at91_clk_register_programmable(struct regmap *regmap, ret = clk_hw_register(NULL, &prog->hw); if (ret) { kfree(prog); - hw = &prog->hw; + hw = ERR_PTR(ret); } return hw; diff --git a/drivers/clk/bcm/clk-bcm2835.c b/drivers/clk/bcm/clk-bcm2835.c index b68bf573dcfb..8c7763fd9efc 100644 --- a/drivers/clk/bcm/clk-bcm2835.c +++ b/drivers/clk/bcm/clk-bcm2835.c @@ -502,8 +502,12 @@ static long bcm2835_pll_rate_from_divisors(unsigned long parent_rate, static long bcm2835_pll_round_rate(struct clk_hw *hw, unsigned long rate, unsigned long *parent_rate) { + struct bcm2835_pll *pll = container_of(hw, struct bcm2835_pll, hw); + const struct bcm2835_pll_data *data = pll->data; u32 ndiv, fdiv; + rate = clamp(rate, data->min_rate, data->max_rate); + bcm2835_pll_choose_ndiv_and_fdiv(rate, *parent_rate, &ndiv, &fdiv); return bcm2835_pll_rate_from_divisors(*parent_rate, ndiv, fdiv, 1); @@ -608,13 +612,6 @@ static int bcm2835_pll_set_rate(struct clk_hw *hw, u32 ana[4]; int i; - if (rate < data->min_rate || rate > data->max_rate) { - dev_err(cprman->dev, "%s: rate out of spec: %lu vs (%lu, %lu)\n", - clk_hw_get_name(hw), rate, - data->min_rate, data->max_rate); - return -EINVAL; - } - if (rate > data->max_fb_rate) { use_fb_prediv = true; rate /= 2; diff --git a/drivers/clk/clk-max77686.c b/drivers/clk/clk-max77686.c index b637f5979023..eb953d3b0b69 100644 --- a/drivers/clk/clk-max77686.c +++ b/drivers/clk/clk-max77686.c @@ -216,6 +216,7 @@ static int max77686_clk_probe(struct platform_device *pdev) return -EINVAL; } + drv_data->num_clks = num_clks; drv_data->max_clk_data = devm_kcalloc(dev, num_clks, sizeof(*drv_data->max_clk_data), GFP_KERNEL); diff --git a/drivers/clk/hisilicon/clk-hi6220.c b/drivers/clk/hisilicon/clk-hi6220.c index fe364e63f8de..c0e8e1f196aa 100644 --- a/drivers/clk/hisilicon/clk-hi6220.c +++ b/drivers/clk/hisilicon/clk-hi6220.c @@ -195,7 +195,7 @@ static void __init hi6220_clk_sys_init(struct device_node *np) hi6220_clk_register_divider(hi6220_div_clks_sys, ARRAY_SIZE(hi6220_div_clks_sys), clk_data); } -CLK_OF_DECLARE(hi6220_clk_sys, "hisilicon,hi6220-sysctrl", hi6220_clk_sys_init); +CLK_OF_DECLARE_DRIVER(hi6220_clk_sys, "hisilicon,hi6220-sysctrl", hi6220_clk_sys_init); /* clocks in media controller */ @@ -252,7 +252,7 @@ static void __init hi6220_clk_media_init(struct device_node *np) hi6220_clk_register_divider(hi6220_div_clks_media, ARRAY_SIZE(hi6220_div_clks_media), clk_data); } -CLK_OF_DECLARE(hi6220_clk_media, "hisilicon,hi6220-mediactrl", hi6220_clk_media_init); +CLK_OF_DECLARE_DRIVER(hi6220_clk_media, "hisilicon,hi6220-mediactrl", hi6220_clk_media_init); /* clocks in pmctrl */ diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 380c372d528e..f042bd2a6a99 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -8,6 +8,7 @@ config COMMON_CLK_MEDIATEK config COMMON_CLK_MT8135 bool "Clock driver for Mediatek MT8135" + depends on ARCH_MEDIATEK || COMPILE_TEST select COMMON_CLK_MEDIATEK default ARCH_MEDIATEK ---help--- @@ -15,6 +16,7 @@ config COMMON_CLK_MT8135 config COMMON_CLK_MT8173 bool "Clock driver for Mediatek MT8173" + depends on ARCH_MEDIATEK || COMPILE_TEST select COMMON_CLK_MEDIATEK default ARCH_MEDIATEK ---help--- diff --git a/drivers/clk/mvebu/armada-37xx-periph.c b/drivers/clk/mvebu/armada-37xx-periph.c index 45905fc0d75b..cecb0fdfaef6 100644 --- a/drivers/clk/mvebu/armada-37xx-periph.c +++ b/drivers/clk/mvebu/armada-37xx-periph.c @@ -305,7 +305,7 @@ static const struct of_device_id armada_3700_periph_clock_of_match[] = { }; static int armada_3700_add_composite_clk(const struct clk_periph_data *data, void __iomem *reg, spinlock_t *lock, - struct device *dev, struct clk_hw *hw) + struct device *dev, struct clk_hw **hw) { const struct clk_ops *mux_ops = NULL, *gate_ops = NULL, *rate_ops = NULL; @@ -329,6 +329,7 @@ static int armada_3700_add_composite_clk(const struct clk_periph_data *data, gate->lock = lock; gate_ops = gate_hw->init->ops; gate->reg = reg + (u64)gate->reg; + gate->flags = CLK_GATE_SET_TO_DISABLE; } if (data->rate_hw) { @@ -353,13 +354,13 @@ static int armada_3700_add_composite_clk(const struct clk_periph_data *data, } } - hw = clk_hw_register_composite(dev, data->name, data->parent_names, + *hw = clk_hw_register_composite(dev, data->name, data->parent_names, data->num_parents, mux_hw, mux_ops, rate_hw, rate_ops, gate_hw, gate_ops, CLK_IGNORE_UNUSED); - if (IS_ERR(hw)) - return PTR_ERR(hw); + if (IS_ERR(*hw)) + return PTR_ERR(*hw); return 0; } @@ -400,7 +401,7 @@ static int armada_3700_periph_clock_probe(struct platform_device *pdev) spin_lock_init(&driver_data->lock); for (i = 0; i < num_periph; i++) { - struct clk_hw *hw = driver_data->hw_data->hws[i]; + struct clk_hw **hw = &driver_data->hw_data->hws[i]; if (armada_3700_add_composite_clk(&data[i], reg, &driver_data->lock, dev, hw)) diff --git a/drivers/clk/samsung/clk-exynos-audss.c b/drivers/clk/samsung/clk-exynos-audss.c index 51d152f735cc..17e68a724945 100644 --- a/drivers/clk/samsung/clk-exynos-audss.c +++ b/drivers/clk/samsung/clk-exynos-audss.c @@ -106,6 +106,7 @@ static const struct of_device_id exynos_audss_clk_of_match[] = { }, { }, }; +MODULE_DEVICE_TABLE(of, exynos_audss_clk_of_match); static void exynos_audss_clk_teardown(void) { diff --git a/drivers/clk/uniphier/clk-uniphier-core.c b/drivers/clk/uniphier/clk-uniphier-core.c index 5ffb898d0839..26c53f7963a4 100644 --- a/drivers/clk/uniphier/clk-uniphier-core.c +++ b/drivers/clk/uniphier/clk-uniphier-core.c @@ -79,7 +79,7 @@ static int uniphier_clk_probe(struct platform_device *pdev) hw_data->num = clk_num; /* avoid returning NULL for unused idx */ - for (; clk_num >= 0; clk_num--) + while (--clk_num >= 0) hw_data->hws[clk_num] = ERR_PTR(-EINVAL); for (p = data; p->name; p++) { @@ -111,6 +111,10 @@ static int uniphier_clk_remove(struct platform_device *pdev) static const struct of_device_id uniphier_clk_match[] = { /* System clock */ { + .compatible = "socionext,uniphier-sld3-clock", + .data = uniphier_sld3_sys_clk_data, + }, + { .compatible = "socionext,uniphier-ld4-clock", .data = uniphier_ld4_sys_clk_data, }, @@ -138,7 +142,7 @@ static const struct of_device_id uniphier_clk_match[] = { .compatible = "socionext,uniphier-ld20-clock", .data = uniphier_ld20_sys_clk_data, }, - /* Media I/O clock */ + /* Media I/O clock, SD clock */ { .compatible = "socionext,uniphier-sld3-mio-clock", .data = uniphier_sld3_mio_clk_data, @@ -156,20 +160,20 @@ static const struct of_device_id uniphier_clk_match[] = { .data = uniphier_sld3_mio_clk_data, }, { - .compatible = "socionext,uniphier-pro5-mio-clock", - .data = uniphier_pro5_mio_clk_data, + .compatible = "socionext,uniphier-pro5-sd-clock", + .data = uniphier_pro5_sd_clk_data, }, { - .compatible = "socionext,uniphier-pxs2-mio-clock", - .data = uniphier_pro5_mio_clk_data, + .compatible = "socionext,uniphier-pxs2-sd-clock", + .data = uniphier_pro5_sd_clk_data, }, { .compatible = "socionext,uniphier-ld11-mio-clock", .data = uniphier_sld3_mio_clk_data, }, { - .compatible = "socionext,uniphier-ld20-mio-clock", - .data = uniphier_pro5_mio_clk_data, + .compatible = "socionext,uniphier-ld20-sd-clock", + .data = uniphier_pro5_sd_clk_data, }, /* Peripheral clock */ { diff --git a/drivers/clk/uniphier/clk-uniphier-mio.c b/drivers/clk/uniphier/clk-uniphier-mio.c index 6aa7ec768d0b..218d20f099ce 100644 --- a/drivers/clk/uniphier/clk-uniphier-mio.c +++ b/drivers/clk/uniphier/clk-uniphier-mio.c @@ -93,7 +93,7 @@ const struct uniphier_clk_data uniphier_sld3_mio_clk_data[] = { { /* sentinel */ } }; -const struct uniphier_clk_data uniphier_pro5_mio_clk_data[] = { +const struct uniphier_clk_data uniphier_pro5_sd_clk_data[] = { UNIPHIER_MIO_CLK_SD_FIXED, UNIPHIER_MIO_CLK_SD(0, 0), UNIPHIER_MIO_CLK_SD(1, 1), diff --git a/drivers/clk/uniphier/clk-uniphier-mux.c b/drivers/clk/uniphier/clk-uniphier-mux.c index 15a2f2cbe0d9..2c243a894f3b 100644 --- a/drivers/clk/uniphier/clk-uniphier-mux.c +++ b/drivers/clk/uniphier/clk-uniphier-mux.c @@ -42,7 +42,7 @@ static u8 uniphier_clk_mux_get_parent(struct clk_hw *hw) struct uniphier_clk_mux *mux = to_uniphier_clk_mux(hw); int num_parents = clk_hw_get_num_parents(hw); int ret; - u32 val; + unsigned int val; u8 i; ret = regmap_read(mux->regmap, mux->reg, &val); diff --git a/drivers/clk/uniphier/clk-uniphier.h b/drivers/clk/uniphier/clk-uniphier.h index 3ae184062388..0244dba1f4cf 100644 --- a/drivers/clk/uniphier/clk-uniphier.h +++ b/drivers/clk/uniphier/clk-uniphier.h @@ -115,7 +115,7 @@ extern const struct uniphier_clk_data uniphier_pxs2_sys_clk_data[]; extern const struct uniphier_clk_data uniphier_ld11_sys_clk_data[]; extern const struct uniphier_clk_data uniphier_ld20_sys_clk_data[]; extern const struct uniphier_clk_data uniphier_sld3_mio_clk_data[]; -extern const struct uniphier_clk_data uniphier_pro5_mio_clk_data[]; +extern const struct uniphier_clk_data uniphier_pro5_sd_clk_data[]; extern const struct uniphier_clk_data uniphier_ld4_peri_clk_data[]; extern const struct uniphier_clk_data uniphier_pro4_peri_clk_data[]; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index f535f8123258..4737520ec823 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -179,6 +179,7 @@ struct _pid { /** * struct cpudata - Per CPU instance data storage * @cpu: CPU number for this instance data + * @policy: CPUFreq policy value * @update_util: CPUFreq utility callback information * @update_util_set: CPUFreq utility callback is set * @iowait_boost: iowait-related boost fraction @@ -201,6 +202,7 @@ struct _pid { struct cpudata { int cpu; + unsigned int policy; struct update_util_data update_util; bool update_util_set; @@ -1142,10 +1144,8 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); } -static void intel_pstate_set_min_pstate(struct cpudata *cpu) +static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) { - int pstate = cpu->pstate.min_pstate; - trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); cpu->pstate.current_pstate = pstate; /* @@ -1157,6 +1157,20 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu) pstate_funcs.get_val(cpu, pstate)); } +static void intel_pstate_set_min_pstate(struct cpudata *cpu) +{ + intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate); +} + +static void intel_pstate_max_within_limits(struct cpudata *cpu) +{ + int min_pstate, max_pstate; + + update_turbo_state(); + intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate); + intel_pstate_set_pstate(cpu, max_pstate); +} + static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) { cpu->pstate.min_pstate = pstate_funcs.get_min(); @@ -1325,7 +1339,8 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) from = cpu->pstate.current_pstate; - target_pstate = pstate_funcs.get_target_pstate(cpu); + target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ? + cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu); intel_pstate_update_pstate(cpu, target_pstate); @@ -1491,7 +1506,9 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) pr_debug("set_policy cpuinfo.max %u policy->max %u\n", policy->cpuinfo.max_freq, policy->max); - cpu = all_cpu_data[0]; + cpu = all_cpu_data[policy->cpu]; + cpu->policy = policy->policy; + if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate && policy->max < policy->cpuinfo.max_freq && policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) { @@ -1499,7 +1516,7 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) policy->max = policy->cpuinfo.max_freq; } - if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) { + if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { limits = &performance_limits; if (policy->max >= policy->cpuinfo.max_freq) { pr_debug("set performance\n"); @@ -1535,6 +1552,15 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) limits->max_perf = round_up(limits->max_perf, FRAC_BITS); out: + if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { + /* + * NOHZ_FULL CPUs need this as the governor callback may not + * be invoked on them. + */ + intel_pstate_clear_update_util_hook(policy->cpu); + intel_pstate_max_within_limits(cpu); + } + intel_pstate_set_update_util_hook(policy->cpu); intel_pstate_hwp_set_policy(policy); diff --git a/drivers/dax/Kconfig b/drivers/dax/Kconfig index daadd20aa936..3e2ab3b14eea 100644 --- a/drivers/dax/Kconfig +++ b/drivers/dax/Kconfig @@ -14,7 +14,7 @@ if DEV_DAX config DEV_DAX_PMEM tristate "PMEM DAX: direct access to persistent memory" - depends on NVDIMM_DAX + depends on LIBNVDIMM && NVDIMM_DAX default DEV_DAX help Support raw access to persistent memory. Note that this diff --git a/drivers/dax/pmem.c b/drivers/dax/pmem.c index 9630d8837ba9..4a15fa5df98b 100644 --- a/drivers/dax/pmem.c +++ b/drivers/dax/pmem.c @@ -44,7 +44,6 @@ static void dax_pmem_percpu_exit(void *data) dev_dbg(dax_pmem->dev, "%s\n", __func__); percpu_ref_exit(ref); - wait_for_completion(&dax_pmem->cmp); } static void dax_pmem_percpu_kill(void *data) @@ -54,6 +53,7 @@ static void dax_pmem_percpu_kill(void *data) dev_dbg(dax_pmem->dev, "%s\n", __func__); percpu_ref_kill(ref); + wait_for_completion(&dax_pmem->cmp); } static int dax_pmem_probe(struct device *dev) diff --git a/drivers/extcon/extcon-qcom-spmi-misc.c b/drivers/extcon/extcon-qcom-spmi-misc.c index ca957a5f4291..b8cde096a808 100644 --- a/drivers/extcon/extcon-qcom-spmi-misc.c +++ b/drivers/extcon/extcon-qcom-spmi-misc.c @@ -51,7 +51,7 @@ static void qcom_usb_extcon_detect_cable(struct work_struct *work) if (ret) return; - extcon_set_state(info->edev, EXTCON_USB_HOST, !id); + extcon_set_state_sync(info->edev, EXTCON_USB_HOST, !id); } static irqreturn_t qcom_usb_irq_handler(int irq, void *dev_id) diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c index 309311b1faae..15475892af0c 100644 --- a/drivers/firewire/net.c +++ b/drivers/firewire/net.c @@ -73,13 +73,13 @@ struct rfc2734_header { #define fwnet_get_hdr_lf(h) (((h)->w0 & 0xc0000000) >> 30) #define fwnet_get_hdr_ether_type(h) (((h)->w0 & 0x0000ffff)) -#define fwnet_get_hdr_dg_size(h) (((h)->w0 & 0x0fff0000) >> 16) +#define fwnet_get_hdr_dg_size(h) ((((h)->w0 & 0x0fff0000) >> 16) + 1) #define fwnet_get_hdr_fg_off(h) (((h)->w0 & 0x00000fff)) #define fwnet_get_hdr_dgl(h) (((h)->w1 & 0xffff0000) >> 16) -#define fwnet_set_hdr_lf(lf) ((lf) << 30) +#define fwnet_set_hdr_lf(lf) ((lf) << 30) #define fwnet_set_hdr_ether_type(et) (et) -#define fwnet_set_hdr_dg_size(dgs) ((dgs) << 16) +#define fwnet_set_hdr_dg_size(dgs) (((dgs) - 1) << 16) #define fwnet_set_hdr_fg_off(fgo) (fgo) #define fwnet_set_hdr_dgl(dgl) ((dgl) << 16) @@ -578,6 +578,9 @@ static int fwnet_incoming_packet(struct fwnet_device *dev, __be32 *buf, int len, int retval; u16 ether_type; + if (len <= RFC2374_UNFRAG_HDR_SIZE) + return 0; + hdr.w0 = be32_to_cpu(buf[0]); lf = fwnet_get_hdr_lf(&hdr); if (lf == RFC2374_HDR_UNFRAG) { @@ -602,7 +605,12 @@ static int fwnet_incoming_packet(struct fwnet_device *dev, __be32 *buf, int len, return fwnet_finish_incoming_packet(net, skb, source_node_id, is_broadcast, ether_type); } + /* A datagram fragment has been received, now the fun begins. */ + + if (len <= RFC2374_FRAG_HDR_SIZE) + return 0; + hdr.w1 = ntohl(buf[1]); buf += 2; len -= RFC2374_FRAG_HDR_SIZE; @@ -614,7 +622,10 @@ static int fwnet_incoming_packet(struct fwnet_device *dev, __be32 *buf, int len, fg_off = fwnet_get_hdr_fg_off(&hdr); } datagram_label = fwnet_get_hdr_dgl(&hdr); - dg_size = fwnet_get_hdr_dg_size(&hdr); /* ??? + 1 */ + dg_size = fwnet_get_hdr_dg_size(&hdr); + + if (fg_off + len > dg_size) + return 0; spin_lock_irqsave(&dev->lock, flags); @@ -722,6 +733,22 @@ static void fwnet_receive_packet(struct fw_card *card, struct fw_request *r, fw_send_response(card, r, rcode); } +static int gasp_source_id(__be32 *p) +{ + return be32_to_cpu(p[0]) >> 16; +} + +static u32 gasp_specifier_id(__be32 *p) +{ + return (be32_to_cpu(p[0]) & 0xffff) << 8 | + (be32_to_cpu(p[1]) & 0xff000000) >> 24; +} + +static u32 gasp_version(__be32 *p) +{ + return be32_to_cpu(p[1]) & 0xffffff; +} + static void fwnet_receive_broadcast(struct fw_iso_context *context, u32 cycle, size_t header_length, void *header, void *data) { @@ -731,9 +758,6 @@ static void fwnet_receive_broadcast(struct fw_iso_context *context, __be32 *buf_ptr; int retval; u32 length; - u16 source_node_id; - u32 specifier_id; - u32 ver; unsigned long offset; unsigned long flags; @@ -750,22 +774,17 @@ static void fwnet_receive_broadcast(struct fw_iso_context *context, spin_unlock_irqrestore(&dev->lock, flags); - specifier_id = (be32_to_cpu(buf_ptr[0]) & 0xffff) << 8 - | (be32_to_cpu(buf_ptr[1]) & 0xff000000) >> 24; - ver = be32_to_cpu(buf_ptr[1]) & 0xffffff; - source_node_id = be32_to_cpu(buf_ptr[0]) >> 16; - - if (specifier_id == IANA_SPECIFIER_ID && - (ver == RFC2734_SW_VERSION + if (length > IEEE1394_GASP_HDR_SIZE && + gasp_specifier_id(buf_ptr) == IANA_SPECIFIER_ID && + (gasp_version(buf_ptr) == RFC2734_SW_VERSION #if IS_ENABLED(CONFIG_IPV6) - || ver == RFC3146_SW_VERSION + || gasp_version(buf_ptr) == RFC3146_SW_VERSION #endif - )) { - buf_ptr += 2; - length -= IEEE1394_GASP_HDR_SIZE; - fwnet_incoming_packet(dev, buf_ptr, length, source_node_id, + )) + fwnet_incoming_packet(dev, buf_ptr + 2, + length - IEEE1394_GASP_HDR_SIZE, + gasp_source_id(buf_ptr), context->card->generation, true); - } packet.payload_length = dev->rcv_buffer_size; packet.interrupt = 1; diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 26ee00f6bd58..d011cb89d25e 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -284,7 +284,7 @@ config GPIO_MM_LANTIQ config GPIO_MOCKUP tristate "GPIO Testing Driver" - depends on GPIOLIB + depends on GPIOLIB && SYSFS select GPIO_SYSFS help This enables GPIO Testing driver, which provides a way to test GPIO diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c index 9457e2022bf6..dc37dbe4b46d 100644 --- a/drivers/gpio/gpio-ath79.c +++ b/drivers/gpio/gpio-ath79.c @@ -219,6 +219,7 @@ static const struct of_device_id ath79_gpio_of_match[] = { { .compatible = "qca,ar9340-gpio" }, {}, }; +MODULE_DEVICE_TABLE(of, ath79_gpio_of_match); static int ath79_gpio_probe(struct platform_device *pdev) { diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c index 425501c39527..793518a30afe 100644 --- a/drivers/gpio/gpio-mpc8xxx.c +++ b/drivers/gpio/gpio-mpc8xxx.c @@ -239,7 +239,7 @@ static int mpc8xxx_gpio_irq_map(struct irq_domain *h, unsigned int irq, irq_hw_number_t hwirq) { irq_set_chip_data(irq, h->host_data); - irq_set_chip_and_handler(irq, &mpc8xxx_irq_chip, handle_level_irq); + irq_set_chip_and_handler(irq, &mpc8xxx_irq_chip, handle_edge_irq); return 0; } diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index cd5dc27320a2..1ed6132b993c 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -293,10 +293,10 @@ static void mvebu_gpio_irq_ack(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; - u32 mask = ~(1 << (d->irq - gc->irq_base)); + u32 mask = d->mask; irq_gc_lock(gc); - writel_relaxed(mask, mvebu_gpioreg_edge_cause(mvchip)); + writel_relaxed(~mask, mvebu_gpioreg_edge_cause(mvchip)); irq_gc_unlock(gc); } @@ -305,7 +305,7 @@ static void mvebu_gpio_edge_irq_mask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); - u32 mask = 1 << (d->irq - gc->irq_base); + u32 mask = d->mask; irq_gc_lock(gc); ct->mask_cache_priv &= ~mask; @@ -319,8 +319,7 @@ static void mvebu_gpio_edge_irq_unmask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); - - u32 mask = 1 << (d->irq - gc->irq_base); + u32 mask = d->mask; irq_gc_lock(gc); ct->mask_cache_priv |= mask; @@ -333,8 +332,7 @@ static void mvebu_gpio_level_irq_mask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); - - u32 mask = 1 << (d->irq - gc->irq_base); + u32 mask = d->mask; irq_gc_lock(gc); ct->mask_cache_priv &= ~mask; @@ -347,8 +345,7 @@ static void mvebu_gpio_level_irq_unmask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); - - u32 mask = 1 << (d->irq - gc->irq_base); + u32 mask = d->mask; irq_gc_lock(gc); ct->mask_cache_priv |= mask; @@ -462,7 +459,7 @@ static void mvebu_gpio_irq_handler(struct irq_desc *desc) for (i = 0; i < mvchip->chip.ngpio; i++) { int irq; - irq = mvchip->irqbase + i; + irq = irq_find_mapping(mvchip->domain, i); if (!(cause & (1 << i))) continue; @@ -655,6 +652,7 @@ static int mvebu_gpio_probe(struct platform_device *pdev) struct irq_chip_type *ct; struct clk *clk; unsigned int ngpios; + bool have_irqs; int soc_variant; int i, cpu, id; int err; @@ -665,6 +663,9 @@ static int mvebu_gpio_probe(struct platform_device *pdev) else soc_variant = MVEBU_GPIO_SOC_VARIANT_ORION; + /* Some gpio controllers do not provide irq support */ + have_irqs = of_irq_count(np) != 0; + mvchip = devm_kzalloc(&pdev->dev, sizeof(struct mvebu_gpio_chip), GFP_KERNEL); if (!mvchip) @@ -697,7 +698,8 @@ static int mvebu_gpio_probe(struct platform_device *pdev) mvchip->chip.get = mvebu_gpio_get; mvchip->chip.direction_output = mvebu_gpio_direction_output; mvchip->chip.set = mvebu_gpio_set; - mvchip->chip.to_irq = mvebu_gpio_to_irq; + if (have_irqs) + mvchip->chip.to_irq = mvebu_gpio_to_irq; mvchip->chip.base = id * MVEBU_MAX_GPIO_PER_BANK; mvchip->chip.ngpio = ngpios; mvchip->chip.can_sleep = false; @@ -758,34 +760,30 @@ static int mvebu_gpio_probe(struct platform_device *pdev) devm_gpiochip_add_data(&pdev->dev, &mvchip->chip, mvchip); /* Some gpio controllers do not provide irq support */ - if (!of_irq_count(np)) + if (!have_irqs) return 0; - /* Setup the interrupt handlers. Each chip can have up to 4 - * interrupt handlers, with each handler dealing with 8 GPIO - * pins. */ - for (i = 0; i < 4; i++) { - int irq = platform_get_irq(pdev, i); - - if (irq < 0) - continue; - irq_set_chained_handler_and_data(irq, mvebu_gpio_irq_handler, - mvchip); - } - - mvchip->irqbase = irq_alloc_descs(-1, 0, ngpios, -1); - if (mvchip->irqbase < 0) { - dev_err(&pdev->dev, "no irqs\n"); - return mvchip->irqbase; + mvchip->domain = + irq_domain_add_linear(np, ngpios, &irq_generic_chip_ops, NULL); + if (!mvchip->domain) { + dev_err(&pdev->dev, "couldn't allocate irq domain %s (DT).\n", + mvchip->chip.label); + return -ENODEV; } - gc = irq_alloc_generic_chip("mvebu_gpio_irq", 2, mvchip->irqbase, - mvchip->membase, handle_level_irq); - if (!gc) { - dev_err(&pdev->dev, "Cannot allocate generic irq_chip\n"); - return -ENOMEM; + err = irq_alloc_domain_generic_chips( + mvchip->domain, ngpios, 2, np->name, handle_level_irq, + IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_LEVEL, 0, 0); + if (err) { + dev_err(&pdev->dev, "couldn't allocate irq chips %s (DT).\n", + mvchip->chip.label); + goto err_domain; } + /* NOTE: The common accessors cannot be used because of the percpu + * access to the mask registers + */ + gc = irq_get_domain_generic_chip(mvchip->domain, 0); gc->private = mvchip; ct = &gc->chip_types[0]; ct->type = IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW; @@ -803,27 +801,23 @@ static int mvebu_gpio_probe(struct platform_device *pdev) ct->handler = handle_edge_irq; ct->chip.name = mvchip->chip.label; - irq_setup_generic_chip(gc, IRQ_MSK(ngpios), 0, - IRQ_NOREQUEST, IRQ_LEVEL | IRQ_NOPROBE); + /* Setup the interrupt handlers. Each chip can have up to 4 + * interrupt handlers, with each handler dealing with 8 GPIO + * pins. + */ + for (i = 0; i < 4; i++) { + int irq = platform_get_irq(pdev, i); - /* Setup irq domain on top of the generic chip. */ - mvchip->domain = irq_domain_add_simple(np, mvchip->chip.ngpio, - mvchip->irqbase, - &irq_domain_simple_ops, - mvchip); - if (!mvchip->domain) { - dev_err(&pdev->dev, "couldn't allocate irq domain %s (DT).\n", - mvchip->chip.label); - err = -ENODEV; - goto err_generic_chip; + if (irq < 0) + continue; + irq_set_chained_handler_and_data(irq, mvebu_gpio_irq_handler, + mvchip); } return 0; -err_generic_chip: - irq_remove_generic_chip(gc, IRQ_MSK(ngpios), IRQ_NOREQUEST, - IRQ_LEVEL | IRQ_NOPROBE); - kfree(gc); +err_domain: + irq_domain_remove(mvchip->domain); return err; } diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c index b9daa0bf32a4..ee1724806f46 100644 --- a/drivers/gpio/gpio-mxs.c +++ b/drivers/gpio/gpio-mxs.c @@ -308,8 +308,10 @@ static int mxs_gpio_probe(struct platform_device *pdev) writel(~0U, port->base + PINCTRL_IRQSTAT(port) + MXS_CLR); irq_base = irq_alloc_descs(-1, 0, 32, numa_node_id()); - if (irq_base < 0) - return irq_base; + if (irq_base < 0) { + err = irq_base; + goto out_iounmap; + } port->domain = irq_domain_add_legacy(np, 32, irq_base, 0, &irq_domain_simple_ops, NULL); @@ -349,6 +351,8 @@ out_irqdomain_remove: irq_domain_remove(port->domain); out_irqdesc_free: irq_free_descs(irq_base, 32); +out_iounmap: + iounmap(port->base); return err; } diff --git a/drivers/gpio/gpio-stmpe.c b/drivers/gpio/gpio-stmpe.c index e7d422a6b90b..5b0042776ec7 100644 --- a/drivers/gpio/gpio-stmpe.c +++ b/drivers/gpio/gpio-stmpe.c @@ -409,7 +409,7 @@ static irqreturn_t stmpe_gpio_irq(int irq, void *dev) * 801/1801/1600, bits are cleared when read. * Edge detect register is not present on 801/1600/1801 */ - if (stmpe->partnum != STMPE801 || stmpe->partnum != STMPE1600 || + if (stmpe->partnum != STMPE801 && stmpe->partnum != STMPE1600 && stmpe->partnum != STMPE1801) { stmpe_reg_write(stmpe, statmsbreg + i, status[i]); stmpe_reg_write(stmpe, diff --git a/drivers/gpio/gpio-ts4800.c b/drivers/gpio/gpio-ts4800.c index 99256115bea5..c2a80b4cbf32 100644 --- a/drivers/gpio/gpio-ts4800.c +++ b/drivers/gpio/gpio-ts4800.c @@ -66,6 +66,7 @@ static const struct of_device_id ts4800_gpio_of_match[] = { { .compatible = "technologic,ts4800-gpio", }, {}, }; +MODULE_DEVICE_TABLE(of, ts4800_gpio_of_match); static struct platform_driver ts4800_gpio_driver = { .driver = { diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 58ece201b8e6..72a4b326fd0d 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -653,14 +653,17 @@ int acpi_dev_gpio_irq_get(struct acpi_device *adev, int index) { int idx, i; unsigned int irq_flags; + int ret = -ENOENT; for (i = 0, idx = 0; idx <= index; i++) { struct acpi_gpio_info info; struct gpio_desc *desc; desc = acpi_get_gpiod_by_index(adev, NULL, i, &info); - if (IS_ERR(desc)) + if (IS_ERR(desc)) { + ret = PTR_ERR(desc); break; + } if (info.gpioint && idx++ == index) { int irq = gpiod_to_irq(desc); @@ -679,7 +682,7 @@ int acpi_dev_gpio_irq_get(struct acpi_device *adev, int index) } } - return -ENOENT; + return ret; } EXPORT_SYMBOL_GPL(acpi_dev_gpio_irq_get); diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index ecad3f0e3b77..193f15d50bba 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -26,14 +26,18 @@ #include "gpiolib.h" -static int of_gpiochip_match_node(struct gpio_chip *chip, void *data) +static int of_gpiochip_match_node_and_xlate(struct gpio_chip *chip, void *data) { - return chip->gpiodev->dev.of_node == data; + struct of_phandle_args *gpiospec = data; + + return chip->gpiodev->dev.of_node == gpiospec->np && + chip->of_xlate(chip, gpiospec, NULL) >= 0; } -static struct gpio_chip *of_find_gpiochip_by_node(struct device_node *np) +static struct gpio_chip *of_find_gpiochip_by_xlate( + struct of_phandle_args *gpiospec) { - return gpiochip_find(np, of_gpiochip_match_node); + return gpiochip_find(gpiospec, of_gpiochip_match_node_and_xlate); } static struct gpio_desc *of_xlate_and_get_gpiod_flags(struct gpio_chip *chip, @@ -79,7 +83,7 @@ struct gpio_desc *of_get_named_gpiod_flags(struct device_node *np, return ERR_PTR(ret); } - chip = of_find_gpiochip_by_node(gpiospec.np); + chip = of_find_gpiochip_by_xlate(&gpiospec); if (!chip) { desc = ERR_PTR(-EPROBE_DEFER); goto out; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index f0fc3a0d37c8..93ed0e00c578 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -21,6 +21,7 @@ #include <linux/uaccess.h> #include <linux/compat.h> #include <linux/anon_inodes.h> +#include <linux/file.h> #include <linux/kfifo.h> #include <linux/poll.h> #include <linux/timekeeping.h> @@ -333,6 +334,13 @@ struct linehandle_state { u32 numdescs; }; +#define GPIOHANDLE_REQUEST_VALID_FLAGS \ + (GPIOHANDLE_REQUEST_INPUT | \ + GPIOHANDLE_REQUEST_OUTPUT | \ + GPIOHANDLE_REQUEST_ACTIVE_LOW | \ + GPIOHANDLE_REQUEST_OPEN_DRAIN | \ + GPIOHANDLE_REQUEST_OPEN_SOURCE) + static long linehandle_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -344,6 +352,8 @@ static long linehandle_ioctl(struct file *filep, unsigned int cmd, if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { int val; + memset(&ghd, 0, sizeof(ghd)); + /* TODO: check if descriptors are really input */ for (i = 0; i < lh->numdescs; i++) { val = gpiod_get_value_cansleep(lh->descs[i]); @@ -414,6 +424,7 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) { struct gpiohandle_request handlereq; struct linehandle_state *lh; + struct file *file; int fd, i, ret; if (copy_from_user(&handlereq, ip, sizeof(handlereq))) @@ -444,6 +455,17 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) u32 lflags = handlereq.flags; struct gpio_desc *desc; + if (offset >= gdev->ngpio) { + ret = -EINVAL; + goto out_free_descs; + } + + /* Return an error if a unknown flag is set */ + if (lflags & ~GPIOHANDLE_REQUEST_VALID_FLAGS) { + ret = -EINVAL; + goto out_free_descs; + } + desc = &gdev->descs[offset]; ret = gpiod_request(desc, lh->label); if (ret) @@ -479,26 +501,41 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) i--; lh->numdescs = handlereq.lines; - fd = anon_inode_getfd("gpio-linehandle", - &linehandle_fileops, - lh, - O_RDONLY | O_CLOEXEC); + fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); if (fd < 0) { ret = fd; goto out_free_descs; } + file = anon_inode_getfile("gpio-linehandle", + &linehandle_fileops, + lh, + O_RDONLY | O_CLOEXEC); + if (IS_ERR(file)) { + ret = PTR_ERR(file); + goto out_put_unused_fd; + } + handlereq.fd = fd; if (copy_to_user(ip, &handlereq, sizeof(handlereq))) { - ret = -EFAULT; - goto out_free_descs; + /* + * fput() will trigger the release() callback, so do not go onto + * the regular error cleanup path here. + */ + fput(file); + put_unused_fd(fd); + return -EFAULT; } + fd_install(fd, file); + dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", lh->numdescs); return 0; +out_put_unused_fd: + put_unused_fd(fd); out_free_descs: for (; i >= 0; i--) gpiod_free(lh->descs[i]); @@ -536,6 +573,10 @@ struct lineevent_state { struct mutex read_lock; }; +#define GPIOEVENT_REQUEST_VALID_FLAGS \ + (GPIOEVENT_REQUEST_RISING_EDGE | \ + GPIOEVENT_REQUEST_FALLING_EDGE) + static unsigned int lineevent_poll(struct file *filep, struct poll_table_struct *wait) { @@ -623,6 +664,8 @@ static long lineevent_ioctl(struct file *filep, unsigned int cmd, if (cmd == GPIOHANDLE_GET_LINE_VALUES_IOCTL) { int val; + memset(&ghd, 0, sizeof(ghd)); + val = gpiod_get_value_cansleep(le->desc); if (val < 0) return val; @@ -695,6 +738,7 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) struct gpioevent_request eventreq; struct lineevent_state *le; struct gpio_desc *desc; + struct file *file; u32 offset; u32 lflags; u32 eflags; @@ -726,6 +770,18 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) lflags = eventreq.handleflags; eflags = eventreq.eventflags; + if (offset >= gdev->ngpio) { + ret = -EINVAL; + goto out_free_label; + } + + /* Return an error if a unknown flag is set */ + if ((lflags & ~GPIOHANDLE_REQUEST_VALID_FLAGS) || + (eflags & ~GPIOEVENT_REQUEST_VALID_FLAGS)) { + ret = -EINVAL; + goto out_free_label; + } + /* This is just wrong: we don't look for events on output lines */ if (lflags & GPIOHANDLE_REQUEST_OUTPUT) { ret = -EINVAL; @@ -777,23 +833,38 @@ static int lineevent_create(struct gpio_device *gdev, void __user *ip) if (ret) goto out_free_desc; - fd = anon_inode_getfd("gpio-event", - &lineevent_fileops, - le, - O_RDONLY | O_CLOEXEC); + fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); if (fd < 0) { ret = fd; goto out_free_irq; } + file = anon_inode_getfile("gpio-event", + &lineevent_fileops, + le, + O_RDONLY | O_CLOEXEC); + if (IS_ERR(file)) { + ret = PTR_ERR(file); + goto out_put_unused_fd; + } + eventreq.fd = fd; if (copy_to_user(ip, &eventreq, sizeof(eventreq))) { - ret = -EFAULT; - goto out_free_irq; + /* + * fput() will trigger the release() callback, so do not go onto + * the regular error cleanup path here. + */ + fput(file); + put_unused_fd(fd); + return -EFAULT; } + fd_install(fd, file); + return 0; +out_put_unused_fd: + put_unused_fd(fd); out_free_irq: free_irq(le->irq, le); out_free_desc: @@ -823,6 +894,8 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (cmd == GPIO_GET_CHIPINFO_IOCTL) { struct gpiochip_info chipinfo; + memset(&chipinfo, 0, sizeof(chipinfo)); + strncpy(chipinfo.name, dev_name(&gdev->dev), sizeof(chipinfo.name)); chipinfo.name[sizeof(chipinfo.name)-1] = '\0'; @@ -839,7 +912,7 @@ static long gpio_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) if (copy_from_user(&lineinfo, ip, sizeof(lineinfo))) return -EFAULT; - if (lineinfo.line_offset > gdev->ngpio) + if (lineinfo.line_offset >= gdev->ngpio) return -EINVAL; desc = &gdev->descs[lineinfo.line_offset]; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index b4bc83aa5999..78da52f90099 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -530,7 +530,8 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates); if (unlikely(r != 0)) { - DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); + if (r != -ERESTARTSYS) + DRM_ERROR("ttm_eu_reserve_buffers failed.\n"); goto error_free_pages; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 6958d4af017f..9e16e975f31a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1981,6 +1981,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon) */ amdgpu_bo_evict_vram(adev); + amdgpu_atombios_scratch_regs_save(adev); pci_save_state(dev->pdev); if (suspend) { /* Shut down the device */ @@ -2032,6 +2033,7 @@ int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) return r; } } + amdgpu_atombios_scratch_regs_restore(adev); /* post card */ if (!amdgpu_card_posted(adev) || !resume) { @@ -2290,8 +2292,6 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev) } if (need_full_reset) { - /* save scratch */ - amdgpu_atombios_scratch_regs_save(adev); r = amdgpu_suspend(adev); retry: @@ -2301,8 +2301,9 @@ retry: amdgpu_display_stop_mc_access(adev, &save); amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC); } - + amdgpu_atombios_scratch_regs_save(adev); r = amdgpu_asic_reset(adev); + amdgpu_atombios_scratch_regs_restore(adev); /* post card */ amdgpu_atom_asic_init(adev->mode_info.atom_context); @@ -2310,8 +2311,6 @@ retry: dev_info(adev->dev, "GPU reset succeeded, trying to resume\n"); r = amdgpu_resume(adev); } - /* restore scratch */ - amdgpu_atombios_scratch_regs_restore(adev); } if (!r) { amdgpu_irq_gpu_reset_resume_helper(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 57552c79ec58..97928d7281f6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -68,6 +68,7 @@ int amdgpu_fence_slab_init(void) void amdgpu_fence_slab_fini(void) { + rcu_barrier(); kmem_cache_destroy(amdgpu_fence_slab); } /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 278708f5a744..9fa809876339 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -239,6 +239,7 @@ int amdgpu_irq_init(struct amdgpu_device *adev) if (r) { adev->irq.installed = false; flush_work(&adev->hotplug_work); + cancel_work_sync(&adev->reset_work); return r; } @@ -264,6 +265,7 @@ void amdgpu_irq_fini(struct amdgpu_device *adev) if (adev->irq.msi_enabled) pci_disable_msi(adev->pdev); flush_work(&adev->hotplug_work); + cancel_work_sync(&adev->reset_work); } for (i = 0; i < AMDGPU_MAX_IRQ_SRC_ID; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 06e145b5afd7..ad908612aff9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -489,10 +489,8 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file /* return all clocks in KHz */ dev_info.gpu_counter_freq = amdgpu_asic_get_xclk(adev) * 10; if (adev->pm.dpm_enabled) { - dev_info.max_engine_clock = - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk * 10; - dev_info.max_memory_clock = - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk * 10; + dev_info.max_engine_clock = amdgpu_dpm_get_sclk(adev, false) * 10; + dev_info.max_memory_clock = amdgpu_dpm_get_mclk(adev, false) * 10; } else { dev_info.max_engine_clock = adev->pm.default_sclk * 10; dev_info.max_memory_clock = adev->pm.default_mclk * 10; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index f0a0513ef4c2..1479d09bd4dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -766,6 +766,10 @@ static const char *amdgpu_vram_names[] = { int amdgpu_bo_init(struct amdgpu_device *adev) { + /* reserve PAT memory space to WC for VRAM */ + arch_io_reserve_memtype_wc(adev->mc.aper_base, + adev->mc.aper_size); + /* Add an MTRR for the VRAM */ adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base, adev->mc.aper_size); @@ -781,6 +785,7 @@ void amdgpu_bo_fini(struct amdgpu_device *adev) { amdgpu_ttm_fini(adev); arch_phys_wc_del(adev->mc.vram_mtrr); + arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size); } int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index e480263387e1..337c5b31d18d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1775,5 +1775,6 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) dma_fence_put(adev->vm_manager.ids[i].first); amdgpu_sync_free(&adev->vm_manager.ids[i].active); dma_fence_put(id->flushed_updates); + dma_fence_put(id->last_flush); } } diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c index e9b1964d4e61..1caff75ab9fc 100644 --- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c @@ -4083,7 +4083,7 @@ static int ci_enable_uvd_dpm(struct amdgpu_device *adev, bool enable) pi->dpm_level_enable_mask.mclk_dpm_enable_mask); } } else { - if (pi->last_mclk_dpm_enable_mask & 0x1) { + if (pi->uvd_enabled) { pi->uvd_enabled = false; pi->dpm_level_enable_mask.mclk_dpm_enable_mask |= 1; amdgpu_ci_send_msg_to_smc_with_parameter(adev, @@ -6308,6 +6308,8 @@ static int ci_dpm_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + flush_work(&adev->pm.dpm.thermal.work); + mutex_lock(&adev->pm.mutex); amdgpu_pm_sysfs_fini(adev); ci_dpm_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index 199d3f7235d6..679dd7320279 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -3068,10 +3068,6 @@ static int dce_v10_0_hw_fini(void *handle) static int dce_v10_0_suspend(void *handle) { - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_atombios_scratch_regs_save(adev); - return dce_v10_0_hw_fini(handle); } @@ -3082,8 +3078,6 @@ static int dce_v10_0_resume(void *handle) ret = dce_v10_0_hw_init(handle); - amdgpu_atombios_scratch_regs_restore(adev); - /* turn on the BL */ if (adev->mode_info.bl_encoder) { u8 bl_level = amdgpu_display_backlight_get_level(adev, diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index ecd000e35981..807dfedb3610 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -3132,10 +3132,6 @@ static int dce_v11_0_hw_fini(void *handle) static int dce_v11_0_suspend(void *handle) { - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_atombios_scratch_regs_save(adev); - return dce_v11_0_hw_fini(handle); } @@ -3146,8 +3142,6 @@ static int dce_v11_0_resume(void *handle) ret = dce_v11_0_hw_init(handle); - amdgpu_atombios_scratch_regs_restore(adev); - /* turn on the BL */ if (adev->mode_info.bl_encoder) { u8 bl_level = amdgpu_display_backlight_get_level(adev, diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c index 44547f951d92..57423332bf75 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v6_0.c @@ -2403,10 +2403,6 @@ static int dce_v6_0_hw_fini(void *handle) static int dce_v6_0_suspend(void *handle) { - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_atombios_scratch_regs_save(adev); - return dce_v6_0_hw_fini(handle); } @@ -2417,8 +2413,6 @@ static int dce_v6_0_resume(void *handle) ret = dce_v6_0_hw_init(handle); - amdgpu_atombios_scratch_regs_restore(adev); - /* turn on the BL */ if (adev->mode_info.bl_encoder) { u8 bl_level = amdgpu_display_backlight_get_level(adev, diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index 979aedf4b74d..6f7656d525e2 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -2913,10 +2913,6 @@ static int dce_v8_0_hw_fini(void *handle) static int dce_v8_0_suspend(void *handle) { - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - amdgpu_atombios_scratch_regs_save(adev); - return dce_v8_0_hw_fini(handle); } @@ -2927,8 +2923,6 @@ static int dce_v8_0_resume(void *handle) ret = dce_v8_0_hw_init(handle); - amdgpu_atombios_scratch_regs_restore(adev); - /* turn on the BL */ if (adev->mode_info.bl_encoder) { u8 bl_level = amdgpu_display_backlight_get_level(adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 86a7ca5d8511..23f1bc94ad3e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -640,7 +640,6 @@ static const u32 stoney_mgcg_cgcg_init[] = mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201, mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201, mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200, - mmATC_MISC_CG, 0xffffffff, 0x000c0200, }; static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index 74d7cc3f7e8c..f7372d32b8e7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -100,6 +100,7 @@ static const u32 cz_mgcg_cgcg_init[] = static const u32 stoney_mgcg_cgcg_init[] = { + mmATC_MISC_CG, 0xffffffff, 0x000c0200, mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 }; diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c index b6f2e50636a9..61172d4a0657 100644 --- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c @@ -3063,6 +3063,8 @@ static int kv_dpm_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + flush_work(&adev->pm.dpm.thermal.work); + mutex_lock(&adev->pm.mutex); amdgpu_pm_sysfs_fini(adev); kv_dpm_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/si_dpm.c b/drivers/gpu/drm/amd/amdgpu/si_dpm.c index 917213396787..f0f2f6c9718e 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dpm.c @@ -3479,6 +3479,49 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, int i; struct si_dpm_quirk *p = si_dpm_quirk_list; + /* limit all SI kickers */ + if (adev->asic_type == CHIP_PITCAIRN) { + if ((adev->pdev->revision == 0x81) || + (adev->pdev->device == 0x6810) || + (adev->pdev->device == 0x6811) || + (adev->pdev->device == 0x6816) || + (adev->pdev->device == 0x6817) || + (adev->pdev->device == 0x6806)) + max_mclk = 120000; + } else if (adev->asic_type == CHIP_VERDE) { + if ((adev->pdev->revision == 0x81) || + (adev->pdev->revision == 0x83) || + (adev->pdev->revision == 0x87) || + (adev->pdev->device == 0x6820) || + (adev->pdev->device == 0x6821) || + (adev->pdev->device == 0x6822) || + (adev->pdev->device == 0x6823) || + (adev->pdev->device == 0x682A) || + (adev->pdev->device == 0x682B)) { + max_sclk = 75000; + max_mclk = 80000; + } + } else if (adev->asic_type == CHIP_OLAND) { + if ((adev->pdev->revision == 0xC7) || + (adev->pdev->revision == 0x80) || + (adev->pdev->revision == 0x81) || + (adev->pdev->revision == 0x83) || + (adev->pdev->device == 0x6604) || + (adev->pdev->device == 0x6605)) { + max_sclk = 75000; + max_mclk = 80000; + } + } else if (adev->asic_type == CHIP_HAINAN) { + if ((adev->pdev->revision == 0x81) || + (adev->pdev->revision == 0x83) || + (adev->pdev->revision == 0xC3) || + (adev->pdev->device == 0x6664) || + (adev->pdev->device == 0x6665) || + (adev->pdev->device == 0x6667)) { + max_sclk = 75000; + max_mclk = 80000; + } + } /* Apply dpm quirks */ while (p && p->chip_device != 0) { if (adev->pdev->vendor == p->chip_vendor && @@ -3491,22 +3534,6 @@ static void si_apply_state_adjust_rules(struct amdgpu_device *adev, } ++p; } - /* limit mclk on all R7 370 parts for stability */ - if (adev->pdev->device == 0x6811 && - adev->pdev->revision == 0x81) - max_mclk = 120000; - /* limit sclk/mclk on Jet parts for stability */ - if (adev->pdev->device == 0x6665 && - adev->pdev->revision == 0xc3) { - max_sclk = 75000; - max_mclk = 80000; - } - /* Limit clocks for some HD8600 parts */ - if (adev->pdev->device == 0x6660 && - adev->pdev->revision == 0x83) { - max_sclk = 75000; - max_mclk = 80000; - } if (rps->vce_active) { rps->evclk = adev->pm.dpm.vce_states[adev->pm.dpm.vce_level].evclk; @@ -7779,6 +7806,8 @@ static int si_dpm_sw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + flush_work(&adev->pm.dpm.thermal.work); + mutex_lock(&adev->pm.mutex); amdgpu_pm_sysfs_fini(adev); si_dpm_fini(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 5ed2930a8568..39f03f137a56 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -52,6 +52,8 @@ #define VCE_V3_0_STACK_SIZE (64 * 1024) #define VCE_V3_0_DATA_SIZE ((16 * 1024 * AMDGPU_MAX_VCE_HANDLES) + (52 * 1024)) +#define FW_52_8_3 ((52 << 24) | (8 << 16) | (3 << 8)) + static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx); static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev); static void vce_v3_0_set_irq_funcs(struct amdgpu_device *adev); @@ -382,6 +384,10 @@ static int vce_v3_0_sw_init(void *handle) if (r) return r; + /* 52.8.3 required for 3 ring support */ + if (adev->vce.fw_version < FW_52_8_3) + adev->vce.num_rings = 2; + r = amdgpu_vce_resume(adev); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index 25c0a71b257d..52d0a83e6ad1 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -988,7 +988,7 @@ static int vi_common_early_init(void *handle) AMD_CG_SUPPORT_SDMA_MGCG | AMD_CG_SUPPORT_SDMA_LS | AMD_CG_SUPPORT_VCE_MGCG; - adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | + adev->pg_flags = AMD_PG_SUPPORT_GFX_PG | AMD_PG_SUPPORT_GFX_SMG | AMD_PG_SUPPORT_GFX_PIPELINE | AMD_PG_SUPPORT_UVD | diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c index 1167205057b3..2ba7937d2545 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c @@ -716,7 +716,7 @@ int phm_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type, *voltage = 1150; } else { ret = atomctrl_get_voltage_evv_on_sclk_ai(hwmgr, voltage_type, sclk, id, &vol); - *voltage = (uint16_t)vol/100; + *voltage = (uint16_t)(vol/100); } return ret; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c index 1126bd4f74dc..0894527d932f 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c @@ -1320,7 +1320,8 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_ if (0 != result) return result; - *voltage = le32_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel); + *voltage = le32_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *) + (&get_voltage_info_param_space))->ulVoltageLevel); return result; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c index baf0f3d4c2f0..c45bd2560468 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/process_pptables_v1_0.c @@ -1201,12 +1201,15 @@ static uint32_t make_classification_flags(struct pp_hwmgr *hwmgr, static int ppt_get_num_of_vce_state_table_entries_v1_0(struct pp_hwmgr *hwmgr) { const ATOM_Tonga_POWERPLAYTABLE *pp_table = get_powerplay_table(hwmgr); - const ATOM_Tonga_VCE_State_Table *vce_state_table = - (ATOM_Tonga_VCE_State_Table *)(((unsigned long)pp_table) + le16_to_cpu(pp_table->usVCEStateTableOffset)); + const ATOM_Tonga_VCE_State_Table *vce_state_table; - if (vce_state_table == NULL) + + if (pp_table == NULL) return 0; + vce_state_table = (void *)pp_table + + le16_to_cpu(pp_table->usVCEStateTableOffset); + return vce_state_table->ucNumEntries; } diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c index 073e0bfa22a0..9e49f2777143 100644 --- a/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/powerplay/hwmgr/smu7_hwmgr.c @@ -1168,8 +1168,8 @@ static int smu7_enable_dpm_tasks(struct pp_hwmgr *hwmgr) tmp_result = (!smum_is_dpm_running(hwmgr)) ? 0 : -1; PP_ASSERT_WITH_CODE(tmp_result == 0, - "DPM is already running right now, no need to enable DPM!", - return 0); + "DPM is already running", + ); if (smu7_voltage_control(hwmgr)) { tmp_result = smu7_enable_voltage_control(hwmgr); @@ -2141,15 +2141,18 @@ static int smu7_patch_acp_vddc(struct pp_hwmgr *hwmgr, } static int smu7_patch_limits_vddc(struct pp_hwmgr *hwmgr, - struct phm_clock_and_voltage_limits *tab) + struct phm_clock_and_voltage_limits *tab) { + uint32_t vddc, vddci; struct smu7_hwmgr *data = (struct smu7_hwmgr *)(hwmgr->backend); if (tab) { - smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, (uint32_t *)&tab->vddc, - &data->vddc_leakage); - smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, (uint32_t *)&tab->vddci, - &data->vddci_leakage); + smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &vddc, + &data->vddc_leakage); + tab->vddc = vddc; + smu7_patch_ppt_v0_with_vdd_leakage(hwmgr, &vddci, + &data->vddci_leakage); + tab->vddci = vddci; } return 0; diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c index 5364e6a7ec8f..09b2cf6ccfa4 100644 --- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -646,6 +646,7 @@ void amd_sched_fini(struct amd_gpu_scheduler *sched) { if (sched->thread) kthread_stop(sched->thread); + rcu_barrier(); if (atomic_dec_and_test(&sched_fence_slab_ref)) kmem_cache_destroy(sched_fence_slab); } diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c index c26fa298fe9e..91530e25aaff 100644 --- a/drivers/gpu/drm/amd/scheduler/sched_fence.c +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -107,7 +107,7 @@ static void amd_sched_fence_free(struct rcu_head *rcu) } /** - * amd_sched_fence_release - callback that fence can be freed + * amd_sched_fence_release_scheduled - callback that fence can be freed * * @fence: fence * @@ -122,7 +122,7 @@ static void amd_sched_fence_release_scheduled(struct dma_fence *f) } /** - * amd_sched_fence_release_scheduled - drop extra reference + * amd_sched_fence_release_finished - drop extra reference * * @f: fence * diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c index 7134fdf49210..2a1368fac1d1 100644 --- a/drivers/gpu/drm/ast/ast_ttm.c +++ b/drivers/gpu/drm/ast/ast_ttm.c @@ -268,6 +268,8 @@ int ast_mm_init(struct ast_private *ast) return ret; } + arch_io_reserve_memtype_wc(pci_resource_start(dev->pdev, 0), + pci_resource_len(dev->pdev, 0)); ast->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0), pci_resource_len(dev->pdev, 0)); @@ -276,11 +278,15 @@ int ast_mm_init(struct ast_private *ast) void ast_mm_fini(struct ast_private *ast) { + struct drm_device *dev = ast->dev; + ttm_bo_device_release(&ast->ttm.bdev); ast_ttm_global_release(ast); arch_phys_wc_del(ast->fb_mtrr); + arch_io_free_memtype_wc(pci_resource_start(dev->pdev, 0), + pci_resource_len(dev->pdev, 0)); } void ast_ttm_placement(struct ast_bo *bo, int domain) diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c index de52b20800e1..d6da848f7c6f 100644 --- a/drivers/gpu/drm/cirrus/cirrus_ttm.c +++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c @@ -268,6 +268,9 @@ int cirrus_mm_init(struct cirrus_device *cirrus) return ret; } + arch_io_reserve_memtype_wc(pci_resource_start(dev->pdev, 0), + pci_resource_len(dev->pdev, 0)); + cirrus->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0), pci_resource_len(dev->pdev, 0)); @@ -277,6 +280,8 @@ int cirrus_mm_init(struct cirrus_device *cirrus) void cirrus_mm_fini(struct cirrus_device *cirrus) { + struct drm_device *dev = cirrus->dev; + if (!cirrus->mm_inited) return; @@ -286,6 +291,8 @@ void cirrus_mm_fini(struct cirrus_device *cirrus) arch_phys_wc_del(cirrus->fb_mtrr); cirrus->fb_mtrr = 0; + arch_io_free_memtype_wc(pci_resource_start(dev->pdev, 0), + pci_resource_len(dev->pdev, 0)); } void cirrus_ttm_placement(struct cirrus_bo *bo, int domain) diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index b096c1673969..f5ea7db4a48a 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -417,18 +417,21 @@ drm_atomic_replace_property_blob_from_id(struct drm_crtc *crtc, ssize_t expected_size, bool *replaced) { - struct drm_device *dev = crtc->dev; struct drm_property_blob *new_blob = NULL; if (blob_id != 0) { - new_blob = drm_property_lookup_blob(dev, blob_id); + new_blob = drm_property_lookup_blob(crtc->dev, blob_id); if (new_blob == NULL) return -EINVAL; - if (expected_size > 0 && expected_size != new_blob->length) + + if (expected_size > 0 && expected_size != new_blob->length) { + drm_property_unreference_blob(new_blob); return -EINVAL; + } } drm_atomic_replace_property_blob(blob, new_blob, replaced); + drm_property_unreference_blob(new_blob); return 0; } diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 75ad01d595fd..50077961228a 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -595,10 +595,6 @@ drm_atomic_helper_check_planes(struct drm_device *dev, struct drm_plane_state *plane_state; int i, ret = 0; - ret = drm_atomic_normalize_zpos(dev, state); - if (ret) - return ret; - for_each_plane_in_state(state, plane, plane_state, i) { const struct drm_plane_helper_funcs *funcs; diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index 04e457117980..aa644487749c 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -914,6 +914,7 @@ static void drm_dp_destroy_port(struct kref *kref) /* no need to clean up vcpi * as if we have no connector we never setup a vcpi */ drm_dp_port_teardown_pdt(port, port->pdt); + port->pdt = DP_PEER_DEVICE_NONE; } kfree(port); } @@ -1159,7 +1160,9 @@ static void drm_dp_add_port(struct drm_dp_mst_branch *mstb, drm_dp_put_port(port); goto out; } - if (port->port_num >= DP_MST_LOGICAL_PORT_0) { + if ((port->pdt == DP_PEER_DEVICE_DP_LEGACY_CONV || + port->pdt == DP_PEER_DEVICE_SST_SINK) && + port->port_num >= DP_MST_LOGICAL_PORT_0) { port->cached_edid = drm_get_edid(port->connector, &port->aux.ddc); drm_mode_connector_set_tile_property(port->connector); } @@ -2919,6 +2922,7 @@ static void drm_dp_destroy_connector_work(struct work_struct *work) mgr->cbs->destroy_connector(mgr, port->connector); drm_dp_port_teardown_pdt(port, port->pdt); + port->pdt = DP_PEER_DEVICE_NONE; if (!port->input && port->vcpi.vcpi > 0) { drm_dp_mst_reset_vcpi_slots(mgr, port); diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 83dbae0fabcf..36797c465edc 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -131,7 +131,12 @@ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper) return 0; fail: for (i = 0; i < fb_helper->connector_count; i++) { - kfree(fb_helper->connector_info[i]); + struct drm_fb_helper_connector *fb_helper_connector = + fb_helper->connector_info[i]; + + drm_connector_unreference(fb_helper_connector->connector); + + kfree(fb_helper_connector); fb_helper->connector_info[i] = NULL; } fb_helper->connector_count = 0; @@ -600,6 +605,24 @@ int drm_fb_helper_blank(int blank, struct fb_info *info) } EXPORT_SYMBOL(drm_fb_helper_blank); +static void drm_fb_helper_modeset_release(struct drm_fb_helper *helper, + struct drm_mode_set *modeset) +{ + int i; + + for (i = 0; i < modeset->num_connectors; i++) { + drm_connector_unreference(modeset->connectors[i]); + modeset->connectors[i] = NULL; + } + modeset->num_connectors = 0; + + drm_mode_destroy(helper->dev, modeset->mode); + modeset->mode = NULL; + + /* FIXME should hold a ref? */ + modeset->fb = NULL; +} + static void drm_fb_helper_crtc_free(struct drm_fb_helper *helper) { int i; @@ -609,10 +632,12 @@ static void drm_fb_helper_crtc_free(struct drm_fb_helper *helper) kfree(helper->connector_info[i]); } kfree(helper->connector_info); + for (i = 0; i < helper->crtc_count; i++) { - kfree(helper->crtc_info[i].mode_set.connectors); - if (helper->crtc_info[i].mode_set.mode) - drm_mode_destroy(helper->dev, helper->crtc_info[i].mode_set.mode); + struct drm_mode_set *modeset = &helper->crtc_info[i].mode_set; + + drm_fb_helper_modeset_release(helper, modeset); + kfree(modeset->connectors); } kfree(helper->crtc_info); } @@ -641,7 +666,9 @@ static void drm_fb_helper_dirty_work(struct work_struct *work) clip->x2 = clip->y2 = 0; spin_unlock_irqrestore(&helper->dirty_lock, flags); - helper->fb->funcs->dirty(helper->fb, NULL, 0, 0, &clip_copy, 1); + /* call dirty callback only when it has been really touched */ + if (clip_copy.x1 < clip_copy.x2 && clip_copy.y1 < clip_copy.y2) + helper->fb->funcs->dirty(helper->fb, NULL, 0, 0, &clip_copy, 1); } /** @@ -2085,7 +2112,6 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper) struct drm_fb_helper_crtc **crtcs; struct drm_display_mode **modes; struct drm_fb_offset *offsets; - struct drm_mode_set *modeset; bool *enabled; int width, height; int i; @@ -2133,45 +2159,35 @@ static void drm_setup_crtcs(struct drm_fb_helper *fb_helper) /* need to set the modesets up here for use later */ /* fill out the connector<->crtc mappings into the modesets */ - for (i = 0; i < fb_helper->crtc_count; i++) { - modeset = &fb_helper->crtc_info[i].mode_set; - modeset->num_connectors = 0; - modeset->fb = NULL; - } + for (i = 0; i < fb_helper->crtc_count; i++) + drm_fb_helper_modeset_release(fb_helper, + &fb_helper->crtc_info[i].mode_set); for (i = 0; i < fb_helper->connector_count; i++) { struct drm_display_mode *mode = modes[i]; struct drm_fb_helper_crtc *fb_crtc = crtcs[i]; struct drm_fb_offset *offset = &offsets[i]; - modeset = &fb_crtc->mode_set; + struct drm_mode_set *modeset = &fb_crtc->mode_set; if (mode && fb_crtc) { + struct drm_connector *connector = + fb_helper->connector_info[i]->connector; + DRM_DEBUG_KMS("desired mode %s set on crtc %d (%d,%d)\n", mode->name, fb_crtc->mode_set.crtc->base.id, offset->x, offset->y); + fb_crtc->desired_mode = mode; fb_crtc->x = offset->x; fb_crtc->y = offset->y; - if (modeset->mode) - drm_mode_destroy(dev, modeset->mode); modeset->mode = drm_mode_duplicate(dev, fb_crtc->desired_mode); - modeset->connectors[modeset->num_connectors++] = fb_helper->connector_info[i]->connector; + drm_connector_reference(connector); + modeset->connectors[modeset->num_connectors++] = connector; modeset->fb = fb_helper->fb; modeset->x = offset->x; modeset->y = offset->y; } } - - /* Clear out any old modes if there are no more connected outputs. */ - for (i = 0; i < fb_helper->crtc_count; i++) { - modeset = &fb_helper->crtc_info[i].mode_set; - if (modeset->num_connectors == 0) { - BUG_ON(modeset->fb); - if (modeset->mode) - drm_mode_destroy(dev, modeset->mode); - modeset->mode = NULL; - } - } out: kfree(crtcs); kfree(modes); diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c index b24714976d24..739180ac3da5 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_drv.c +++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c @@ -263,6 +263,26 @@ int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state, return 0; } +int exynos_atomic_check(struct drm_device *dev, + struct drm_atomic_state *state) +{ + int ret; + + ret = drm_atomic_helper_check_modeset(dev, state); + if (ret) + return ret; + + ret = drm_atomic_normalize_zpos(dev, state); + if (ret) + return ret; + + ret = drm_atomic_helper_check_planes(dev, state); + if (ret) + return ret; + + return ret; +} + static int exynos_drm_open(struct drm_device *dev, struct drm_file *file) { struct drm_exynos_file_private *file_priv; diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h index d215149e737b..80c4d5b81689 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_drv.h +++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h @@ -301,6 +301,7 @@ static inline int exynos_dpi_bind(struct drm_device *dev, int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state, bool nonblock); +int exynos_atomic_check(struct drm_device *dev, struct drm_atomic_state *state); extern struct platform_driver fimd_driver; diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c index 40ce841eb952..23cce0a3f5fc 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fb.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c @@ -190,7 +190,7 @@ dma_addr_t exynos_drm_fb_dma_addr(struct drm_framebuffer *fb, int index) static const struct drm_mode_config_funcs exynos_drm_mode_config_funcs = { .fb_create = exynos_user_fb_create, .output_poll_changed = exynos_drm_output_poll_changed, - .atomic_check = drm_atomic_helper_check, + .atomic_check = exynos_atomic_check, .atomic_commit = exynos_atomic_commit, }; diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 1c1b19ccb92f..df96aed6975a 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -11,6 +11,7 @@ config DRM_I915 select DRM_KMS_HELPER select DRM_PANEL select DRM_MIPI_DSI + select RELAY # i915 depends on ACPI_VIDEO when ACPI is enabled # but for select to work, need to select ACPI_VIDEO's dependencies, ick select BACKLIGHT_LCD_SUPPORT if ACPI @@ -24,16 +25,17 @@ config DRM_I915 including 830M, 845G, 852GM, 855GM, 865G, 915G, 945G, 965G, G35, G41, G43, G45 chipsets and Celeron, Pentium, Core i3, Core i5, Core i7 as well as Atom CPUs with integrated graphics. - If M is selected, the module will be called i915. AGP support - is required for this driver to work. This driver is used by - the Intel driver in X.org 6.8 and XFree86 4.4 and above. It - replaces the older i830 module that supported a subset of the - hardware in older X.org releases. + + This driver is used by the Intel driver in X.org 6.8 and + XFree86 4.4 and above. It replaces the older i830 module that + supported a subset of the hardware in older X.org releases. Note that the older i810/i815 chipsets require the use of the i810 driver instead, and the Atom z5xx series has an entirely different implementation. + If "M" is selected, the module will be called i915. + config DRM_I915_PRELIMINARY_HW_SUPPORT bool "Enable preliminary support for prerelease Intel hardware by default" depends on DRM_I915 @@ -85,6 +87,7 @@ config DRM_I915_USERPTR config DRM_I915_GVT bool "Enable Intel GVT-g graphics virtualization host support" depends on DRM_I915 + depends on 64BIT default n help Choose this option if you want to enable Intel GVT-g graphics diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 612340097f4b..0857e5035f4d 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -35,16 +35,19 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \ i915_gem_shrinker.o \ i915_gem_stolen.o \ i915_gem_tiling.o \ + i915_gem_timeline.o \ i915_gem_userptr.o \ i915_trace_points.o \ intel_breadcrumbs.o \ intel_engine_cs.o \ + intel_hangcheck.o \ intel_lrc.o \ intel_mocs.o \ intel_ringbuffer.o \ diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index aafb57e26288..0084ece8d8ff 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -1145,7 +1145,7 @@ static int skl_decode_mi_display_flip(struct parser_exec_state *s, info->event = PRIMARY_B_FLIP_DONE; break; case MI_DISPLAY_FLIP_SKL_PLANE_1_C: - info->pipe = PIPE_B; + info->pipe = PIPE_C; info->event = PRIMARY_C_FLIP_DONE; break; default: @@ -1201,20 +1201,19 @@ static int gen8_update_plane_mmio_from_mi_display_flip( struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; struct intel_vgpu *vgpu = s->vgpu; -#define write_bits(reg, e, s, v) do { \ - vgpu_vreg(vgpu, reg) &= ~GENMASK(e, s); \ - vgpu_vreg(vgpu, reg) |= (v << s); \ -} while (0) - - write_bits(info->surf_reg, 31, 12, info->surf_val); - if (IS_SKYLAKE(dev_priv)) - write_bits(info->stride_reg, 9, 0, info->stride_val); - else - write_bits(info->stride_reg, 15, 6, info->stride_val); - write_bits(info->ctrl_reg, IS_SKYLAKE(dev_priv) ? 12 : 10, - 10, info->tile_val); - -#undef write_bits + set_mask_bits(&vgpu_vreg(vgpu, info->surf_reg), GENMASK(31, 12), + info->surf_val << 12); + if (IS_SKYLAKE(dev_priv)) { + set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(9, 0), + info->stride_val); + set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(12, 10), + info->tile_val << 10); + } else { + set_mask_bits(&vgpu_vreg(vgpu, info->stride_reg), GENMASK(15, 6), + info->stride_val << 6); + set_mask_bits(&vgpu_vreg(vgpu, info->ctrl_reg), GENMASK(10, 10), + info->tile_val << 10); + } vgpu_vreg(vgpu, PIPE_FRMCOUNT_G4X(info->pipe))++; intel_vgpu_trigger_virtual_event(vgpu, info->event); diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 2cc761328569..6554da9f9f5b 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -276,7 +276,7 @@ static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index) pte = readq(addr); #else pte = ioread32(addr); - pte |= ioread32(addr + 4) << 32; + pte |= (u64)ioread32(addr + 4) << 32; #endif return pte; } @@ -1944,7 +1944,7 @@ static int create_scratch_page(struct intel_vgpu *vgpu) mfn = intel_gvt_hypervisor_virt_to_mfn(vaddr); if (mfn == INTEL_GVT_INVALID_ADDR) { - gvt_err("fail to translate vaddr:0x%llx\n", (u64)vaddr); + gvt_err("fail to translate vaddr: 0x%p\n", vaddr); __free_page(gtt->scratch_page); gtt->scratch_page = NULL; return -ENXIO; diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index 31b59d40f3fb..385969a89216 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -65,6 +65,8 @@ struct intel_gvt_io_emulation_ops intel_gvt_io_emulation_ops = { */ int intel_gvt_init_host(void) { + int ret; + if (intel_gvt_host.initialized) return 0; @@ -90,7 +92,8 @@ int intel_gvt_init_host(void) return -EINVAL; /* Try to detect if we're running in host instead of VM. */ - if (!intel_gvt_hypervisor_detect_host()) + ret = intel_gvt_hypervisor_detect_host(); + if (ret) return -ENODEV; gvt_dbg_core("Running with hypervisor %s in host mode\n", @@ -103,19 +106,20 @@ int intel_gvt_init_host(void) static void init_device_info(struct intel_gvt *gvt) { struct intel_gvt_device_info *info = &gvt->device_info; + struct pci_dev *pdev = gvt->dev_priv->drm.pdev; if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)) { info->max_support_vgpus = 8; info->cfg_space_size = 256; info->mmio_size = 2 * 1024 * 1024; info->mmio_bar = 0; - info->msi_cap_offset = IS_SKYLAKE(gvt->dev_priv) ? 0xac : 0x90; info->gtt_start_offset = 8 * 1024 * 1024; info->gtt_entry_size = 8; info->gtt_entry_size_shift = 3; info->gmadr_bytes_in_cmd = 8; info->max_surface_size = 36 * 1024 * 1024; } + info->msi_cap_offset = pdev->msi_cap; } static int gvt_service_thread(void *data) diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 11df62b542b1..62fc9e3ac5c6 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -382,6 +382,8 @@ void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu); int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa); int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci); +int setup_vgpu_mmio(struct intel_vgpu *vgpu); +void populate_pvinfo_page(struct intel_vgpu *vgpu); #include "mpt.h" diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 3e74fb3d4aa9..9ab1f95dddc5 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -239,7 +239,11 @@ static int handle_device_reset(struct intel_vgpu *vgpu, unsigned int offset, vgpu->resetting = true; intel_vgpu_stop_schedule(vgpu); - if (scheduler->current_vgpu == vgpu) { + /* + * The current_vgpu will set to NULL after stopping the + * scheduler when the reset is triggered by current vgpu. + */ + if (scheduler->current_vgpu == NULL) { mutex_unlock(&vgpu->gvt->lock); intel_gvt_wait_vgpu_idle(vgpu); mutex_lock(&vgpu->gvt->lock); @@ -247,6 +251,16 @@ static int handle_device_reset(struct intel_vgpu *vgpu, unsigned int offset, intel_vgpu_reset_execlist(vgpu, bitmap); + /* full GPU reset */ + if (bitmap == 0xff) { + mutex_unlock(&vgpu->gvt->lock); + intel_vgpu_clean_gtt(vgpu); + mutex_lock(&vgpu->gvt->lock); + setup_vgpu_mmio(vgpu); + populate_pvinfo_page(vgpu); + intel_vgpu_init_gtt(vgpu); + } + vgpu->resetting = false; return 0; @@ -258,6 +272,7 @@ static int gdrst_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, u32 data; u64 bitmap = 0; + write_vreg(vgpu, offset, p_data, bytes); data = vgpu_vreg(vgpu, offset); if (data & GEN6_GRDOM_FULL) { @@ -1305,7 +1320,7 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset); struct intel_vgpu_execlist *execlist; u32 data = *(u32 *)p_data; - int ret; + int ret = 0; if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1)) return -EINVAL; @@ -1313,12 +1328,15 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, execlist = &vgpu->execlist[ring_id]; execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data; - if (execlist->elsp_dwords.index == 3) + if (execlist->elsp_dwords.index == 3) { ret = intel_vgpu_submit_execlist(vgpu, ring_id); + if(ret) + gvt_err("fail submit workload on ring %d\n", ring_id); + } ++execlist->elsp_dwords.index; execlist->elsp_dwords.index &= 0x3; - return 0; + return ret; } static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c index 973c8a9d0b15..95218913b0bc 100644 --- a/drivers/gpu/drm/i915/gvt/opregion.c +++ b/drivers/gpu/drm/i915/gvt/opregion.c @@ -163,7 +163,7 @@ int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa) */ void intel_gvt_clean_opregion(struct intel_gvt *gvt) { - iounmap(gvt->opregion.opregion_va); + memunmap(gvt->opregion.opregion_va); gvt->opregion.opregion_va = NULL; } @@ -181,8 +181,8 @@ int intel_gvt_init_opregion(struct intel_gvt *gvt) pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION, &gvt->opregion.opregion_pa); - gvt->opregion.opregion_va = acpi_os_ioremap(gvt->opregion.opregion_pa, - INTEL_GVT_OPREGION_SIZE); + gvt->opregion.opregion_va = memremap(gvt->opregion.opregion_pa, + INTEL_GVT_OPREGION_SIZE, MEMREMAP_WB); if (!gvt->opregion.opregion_va) { gvt_err("fail to map host opregion\n"); return -EFAULT; diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c index feebb65ba641..3af894b3d257 100644 --- a/drivers/gpu/drm/i915/gvt/render.c +++ b/drivers/gpu/drm/i915/gvt/render.c @@ -118,6 +118,7 @@ static u32 gen9_render_mocs_L3[32]; static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) { struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; + enum forcewake_domains fw; i915_reg_t reg; u32 regs[] = { [RCS] = 0x4260, @@ -135,11 +136,25 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id) reg = _MMIO(regs[ring_id]); - I915_WRITE(reg, 0x1); + /* WaForceWakeRenderDuringMmioTLBInvalidate:skl + * we need to put a forcewake when invalidating RCS TLB caches, + * otherwise device can go to RC6 state and interrupt invalidation + * process + */ + fw = intel_uncore_forcewake_for_reg(dev_priv, reg, + FW_REG_READ | FW_REG_WRITE); + if (ring_id == RCS && IS_SKYLAKE(dev_priv)) + fw |= FORCEWAKE_RENDER; - if (wait_for_atomic((I915_READ(reg) == 0), 50)) + intel_uncore_forcewake_get(dev_priv, fw); + + I915_WRITE_FW(reg, 0x1); + + if (wait_for_atomic((I915_READ_FW(reg) == 0), 50)) gvt_err("timeout in invalidate ring (%d) tlb\n", ring_id); + intel_uncore_forcewake_put(dev_priv, fw); + gvt_dbg_core("invalidate TLB for ring %d\n", ring_id); } @@ -162,6 +177,7 @@ static void load_mocs(struct intel_vgpu *vgpu, int ring_id) if (!IS_SKYLAKE(dev_priv)) return; + offset.reg = regs[ring_id]; for (i = 0; i < 64; i++) { gen9_render_mocs[ring_id][i] = I915_READ(offset); I915_WRITE(offset, vgpu_vreg(vgpu, offset)); @@ -199,6 +215,7 @@ static void restore_mocs(struct intel_vgpu *vgpu, int ring_id) if (!IS_SKYLAKE(dev_priv)) return; + offset.reg = regs[ring_id]; for (i = 0; i < 64; i++) { vgpu_vreg(vgpu, offset) = I915_READ(offset); I915_WRITE(offset, gen9_render_mocs[ring_id][i]); diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index e96eaeebeb0a..18acb45dd14d 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -400,21 +400,27 @@ static int workload_thread(void *priv) int ring_id = p->ring_id; struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; struct intel_vgpu_workload *workload = NULL; + long lret; int ret; bool need_force_wake = IS_SKYLAKE(gvt->dev_priv); + DEFINE_WAIT_FUNC(wait, woken_wake_function); kfree(p); gvt_dbg_core("workload thread for ring %d started\n", ring_id); while (!kthread_should_stop()) { - ret = wait_event_interruptible(scheduler->waitq[ring_id], - kthread_should_stop() || - (workload = pick_next_workload(gvt, ring_id))); - - WARN_ON_ONCE(ret); - - if (kthread_should_stop()) + add_wait_queue(&scheduler->waitq[ring_id], &wait); + do { + workload = pick_next_workload(gvt, ring_id); + if (workload) + break; + wait_woken(&wait, TASK_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT); + } while (!kthread_should_stop()); + remove_wait_queue(&scheduler->waitq[ring_id], &wait); + + if (!workload) break; mutex_lock(&scheduler_mutex); @@ -444,10 +450,12 @@ static int workload_thread(void *priv) gvt_dbg_sched("ring id %d wait workload %p\n", workload->ring_id, workload); - workload->status = i915_wait_request(workload->req, - 0, NULL, NULL); - if (workload->status != 0) + lret = i915_wait_request(workload->req, + 0, MAX_SCHEDULE_TIMEOUT); + if (lret < 0) { + workload->status = lret; gvt_err("fail to wait workload, skip\n"); + } complete: gvt_dbg_sched("will complete workload %p\n, status: %d\n", diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index 9401436d721f..4f54005b976d 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -41,7 +41,7 @@ static void clean_vgpu_mmio(struct intel_vgpu *vgpu) vgpu->mmio.vreg = vgpu->mmio.sreg = NULL; } -static int setup_vgpu_mmio(struct intel_vgpu *vgpu) +int setup_vgpu_mmio(struct intel_vgpu *vgpu) { struct intel_gvt *gvt = vgpu->gvt; const struct intel_gvt_device_info *info = &gvt->device_info; @@ -103,7 +103,7 @@ static void setup_vgpu_cfg_space(struct intel_vgpu *vgpu, } } -static void populate_pvinfo_page(struct intel_vgpu *vgpu) +void populate_pvinfo_page(struct intel_vgpu *vgpu) { /* setup the ballooning information */ vgpu_vreg64(vgpu, vgtif_reg(magic)) = VGT_MAGIC; diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c index f191d7b66b1d..f5039f4f988f 100644 --- a/drivers/gpu/drm/i915/i915_cmd_parser.c +++ b/drivers/gpu/drm/i915/i915_cmd_parser.c @@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, } if (ret == 0 && needs_clflush_after) - drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len); + drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len); i915_gem_object_unpin_map(shadow_batch_obj); return ret; diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 20638d22bbad..c9465fbff2df 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -107,12 +107,12 @@ static char get_tiling_flag(struct drm_i915_gem_object *obj) static char get_global_flag(struct drm_i915_gem_object *obj) { - return obj->fault_mappable ? 'g' : ' '; + return !list_empty(&obj->userfault_link) ? 'g' : ' '; } static char get_pin_mapped_flag(struct drm_i915_gem_object *obj) { - return obj->mapping ? 'M' : ' '; + return obj->mm.mapping ? 'M' : ' '; } static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj) @@ -136,11 +136,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) struct i915_vma *vma; unsigned int frontbuffer_bits; int pin_count = 0; - enum intel_engine_id id; lockdep_assert_held(&obj->base.dev->struct_mutex); - seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x [ ", + seq_printf(m, "%pK: %c%c%c%c%c %8zdKiB %02x %02x %s%s%s", &obj->base, get_active_flag(obj), get_pin_flag(obj), @@ -149,17 +148,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) get_pin_mapped_flag(obj), obj->base.size / 1024, obj->base.read_domains, - obj->base.write_domain); - for_each_engine(engine, dev_priv, id) - seq_printf(m, "%x ", - i915_gem_active_get_seqno(&obj->last_read[id], - &obj->base.dev->struct_mutex)); - seq_printf(m, "] %x %s%s%s", - i915_gem_active_get_seqno(&obj->last_write, - &obj->base.dev->struct_mutex), + obj->base.write_domain, i915_cache_level_str(dev_priv, obj->cache_level), - obj->dirty ? " dirty" : "", - obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); + obj->mm.dirty ? " dirty" : "", + obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) seq_printf(m, " (name: %d)", obj->base.name); list_for_each_entry(vma, &obj->vma_list, obj_link) { @@ -187,8 +179,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (obj->stolen) seq_printf(m, " (stolen: %08llx)", obj->stolen->start); - engine = i915_gem_active_get_engine(&obj->last_write, - &dev_priv->drm.struct_mutex); + engine = i915_gem_object_last_write_engine(obj); if (engine) seq_printf(m, " (%s)", engine->name); @@ -226,7 +217,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) return ret; total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (obj->stolen == NULL) continue; @@ -236,7 +227,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data) total_gtt_size += i915_gem_obj_total_ggtt_size(obj); count++; } - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { if (obj->stolen == NULL) continue; @@ -399,16 +390,16 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size = count = 0; mapped_size = mapped_count = 0; purgeable_size = purgeable_count = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { size += obj->base.size; ++count; - if (obj->madv == I915_MADV_DONTNEED) { + if (obj->mm.madv == I915_MADV_DONTNEED) { purgeable_size += obj->base.size; ++purgeable_count; } - if (obj->mapping) { + if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; } @@ -416,7 +407,7 @@ static int i915_gem_object_info(struct seq_file *m, void *data) seq_printf(m, "%u unbound objects, %llu bytes\n", count, size); size = count = dpy_size = dpy_count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { size += obj->base.size; ++count; @@ -425,12 +416,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) ++dpy_count; } - if (obj->madv == I915_MADV_DONTNEED) { + if (obj->mm.madv == I915_MADV_DONTNEED) { purgeable_size += obj->base.size; ++purgeable_count; } - if (obj->mapping) { + if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; } @@ -502,7 +493,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data) return ret; total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (show_pin_display_only && !obj->pin_display) continue; @@ -561,7 +552,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) seq_printf(m, "Flip queued on %s at seqno %x, next seqno %x [current breadcrumb %x], completed? %d\n", engine->name, i915_gem_request_get_seqno(work->flip_queued_req), - dev_priv->next_seqno, + atomic_read(&dev_priv->gt.global_timeline.next_seqno), intel_engine_get_seqno(engine), i915_gem_request_completed(work->flip_queued_req)); } else @@ -640,17 +631,10 @@ static void print_request(struct seq_file *m, struct drm_i915_gem_request *rq, const char *prefix) { - struct pid *pid = rq->ctx->pid; - struct task_struct *task; - - rcu_read_lock(); - task = pid ? pid_task(pid, PIDTYPE_PID) : NULL; - seq_printf(m, "%s%x [%x:%x] @ %d: %s [%d]\n", prefix, - rq->fence.seqno, rq->ctx->hw_id, rq->fence.seqno, + seq_printf(m, "%s%x [%x:%x] @ %d: %s\n", prefix, + rq->global_seqno, rq->ctx->hw_id, rq->fence.seqno, jiffies_to_msecs(jiffies - rq->emitted_jiffies), - task ? task->comm : "<unknown>", - task ? task->pid : -1); - rcu_read_unlock(); + rq->timeline->common->name); } static int i915_gem_request_info(struct seq_file *m, void *data) @@ -671,13 +655,13 @@ static int i915_gem_request_info(struct seq_file *m, void *data) int count; count = 0; - list_for_each_entry(req, &engine->request_list, link) + list_for_each_entry(req, &engine->timeline->requests, link) count++; if (count == 0) continue; seq_printf(m, "%s requests: %d\n", engine->name, count); - list_for_each_entry(req, &engine->request_list, link) + list_for_each_entry(req, &engine->timeline->requests, link) print_request(m, req, " "); any++; @@ -699,14 +683,14 @@ static void i915_ring_seqno_info(struct seq_file *m, seq_printf(m, "Current sequence (%s): %x\n", engine->name, intel_engine_get_seqno(engine)); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "Waiting (%s): %s [%d] on %x\n", engine->name, w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static int i915_gem_seqno_info(struct seq_file *m, void *data) @@ -743,17 +727,32 @@ static int i915_interrupt_info(struct seq_file *m, void *data) I915_READ(VLV_IIR_RW)); seq_printf(m, "Display IMR:\t%08x\n", I915_READ(VLV_IMR)); - for_each_pipe(dev_priv, pipe) + for_each_pipe(dev_priv, pipe) { + enum intel_display_power_domain power_domain; + + power_domain = POWER_DOMAIN_PIPE(pipe); + if (!intel_display_power_get_if_enabled(dev_priv, + power_domain)) { + seq_printf(m, "Pipe %c power disabled\n", + pipe_name(pipe)); + continue; + } + seq_printf(m, "Pipe %c stat:\t%08x\n", pipe_name(pipe), I915_READ(PIPESTAT(pipe))); + intel_display_power_put(dev_priv, power_domain); + } + + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); seq_printf(m, "Port hotplug:\t%08x\n", I915_READ(PORT_HOTPLUG_EN)); seq_printf(m, "DPFLIPSTAT:\t%08x\n", I915_READ(VLV_DPFLIPSTAT)); seq_printf(m, "DPINVGTT:\t%08x\n", I915_READ(DPINVGTT)); + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); for (i = 0; i < 4; i++) { seq_printf(m, "GT Interrupt IMR %d:\t%08x\n", @@ -1046,15 +1045,8 @@ static int i915_next_seqno_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - int ret; - - ret = mutex_lock_interruptible(&dev_priv->drm.struct_mutex); - if (ret) - return ret; - - *val = dev_priv->next_seqno; - mutex_unlock(&dev_priv->drm.struct_mutex); + *val = atomic_read(&dev_priv->gt.global_timeline.next_seqno); return 0; } @@ -1069,7 +1061,7 @@ i915_next_seqno_set(void *data, u64 val) if (ret) return ret; - ret = i915_gem_set_seqno(dev, val); + ret = i915_gem_set_global_seqno(dev, val); mutex_unlock(&dev->struct_mutex); return ret; @@ -1356,21 +1348,20 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) seq_printf(m, "%s:\n", engine->name); seq_printf(m, "\tseqno = %x [current %x, last %x]\n", - engine->hangcheck.seqno, - seqno[id], - engine->last_submitted_seqno); + engine->hangcheck.seqno, seqno[id], + intel_engine_last_submit(engine)); seq_printf(m, "\twaiters? %s, fake irq active? %s\n", yesno(intel_engine_has_waiter(engine)), yesno(test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings))); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "\t%s [%d] waiting for %x\n", w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n", (long long)engine->hangcheck.acthd, @@ -1396,14 +1387,9 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused) static int ironlake_drpc_info(struct seq_file *m) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; u32 rgvmodectl, rstdbyctl; u16 crstandvid; - int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); rgvmodectl = I915_READ(MEMMODECTL); @@ -1411,7 +1397,6 @@ static int ironlake_drpc_info(struct seq_file *m) crstandvid = I915_READ16(CRSTANDVID); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN)); seq_printf(m, "Boost freq: %d\n", @@ -1674,11 +1659,13 @@ static int i915_fbc_status(struct seq_file *m, void *unused) seq_printf(m, "FBC disabled: %s\n", dev_priv->fbc.no_fbc_reason); - if (intel_fbc_is_active(dev_priv) && - INTEL_GEN(dev_priv) >= 7) + if (intel_fbc_is_active(dev_priv) && INTEL_GEN(dev_priv) >= 7) { + uint32_t mask = INTEL_GEN(dev_priv) >= 8 ? + BDW_FBC_COMPRESSION_MASK : + IVB_FBC_COMPRESSION_MASK; seq_printf(m, "Compressing: %s\n", - yesno(I915_READ(FBC_STATUS2) & - FBC_COMPRESSION_MASK)); + yesno(I915_READ(FBC_STATUS2) & mask)); + } mutex_unlock(&dev_priv->fbc.lock); intel_runtime_pm_put(dev_priv); @@ -1757,6 +1744,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) bool sr_enabled = false; intel_runtime_pm_get(dev_priv); + intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); if (HAS_PCH_SPLIT(dev_priv)) sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; @@ -1770,6 +1758,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) sr_enabled = I915_READ(FW_BLC_SELF_VLV) & FW_CSPWRDWNEN; + intel_display_power_put(dev_priv, POWER_DOMAIN_INIT); intel_runtime_pm_put(dev_priv); seq_printf(m, "self-refresh: %s\n", @@ -2015,7 +2004,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, seq_printf(m, "\tBound in GGTT at 0x%08x\n", i915_ggtt_offset(vma)); - if (i915_gem_object_get_pages(vma->obj)) { + if (i915_gem_object_pin_pages(vma->obj)) { seq_puts(m, "\tFailed to get pages for context object\n\n"); return; } @@ -2034,6 +2023,7 @@ static void i915_dump_lrc_obj(struct seq_file *m, kunmap_atomic(reg_state); } + i915_gem_object_unpin_pages(vma->obj); seq_putc(m, '\n'); } @@ -2091,12 +2081,7 @@ static const char *swizzle_string(unsigned swizzle) static int i915_swizzle_info(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); - struct drm_device *dev = &dev_priv->drm; - int ret; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); seq_printf(m, "bit6 swizzle for X-tiling = %s\n", @@ -2136,7 +2121,6 @@ static int i915_swizzle_info(struct seq_file *m, void *data) seq_puts(m, "L-shaped memory detected\n"); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); return 0; } @@ -2292,8 +2276,8 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) struct drm_file *file; seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled); - seq_printf(m, "GPU busy? %s [%x]\n", - yesno(dev_priv->gt.awake), dev_priv->gt.active_engines); + seq_printf(m, "GPU busy? %s [%d requests]\n", + yesno(dev_priv->gt.awake), dev_priv->gt.active_requests); seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv)); seq_printf(m, "Frequency requested %d\n", intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq)); @@ -2328,7 +2312,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) if (INTEL_GEN(dev_priv) >= 6 && dev_priv->rps.enabled && - dev_priv->gt.active_engines) { + dev_priv->gt.active_requests) { u32 rpup, rpupei; u32 rpdown, rpdownei; @@ -2409,6 +2393,32 @@ static int i915_guc_load_status_info(struct seq_file *m, void *data) return 0; } +static void i915_guc_log_info(struct seq_file *m, + struct drm_i915_private *dev_priv) +{ + struct intel_guc *guc = &dev_priv->guc; + + seq_puts(m, "\nGuC logging stats:\n"); + + seq_printf(m, "\tISR: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_ISR_LOG_BUFFER], + guc->log.total_overflow_count[GUC_ISR_LOG_BUFFER]); + + seq_printf(m, "\tDPC: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_DPC_LOG_BUFFER], + guc->log.total_overflow_count[GUC_DPC_LOG_BUFFER]); + + seq_printf(m, "\tCRASH: flush count %10u, overflow count %10u\n", + guc->log.flush_count[GUC_CRASH_DUMP_LOG_BUFFER], + guc->log.total_overflow_count[GUC_CRASH_DUMP_LOG_BUFFER]); + + seq_printf(m, "\tTotal flush interrupt count: %u\n", + guc->log.flush_interrupt_count); + + seq_printf(m, "\tCapture miss count: %u\n", + guc->log.capture_miss_count); +} + static void i915_guc_client_info(struct seq_file *m, struct drm_i915_private *dev_priv, struct i915_guc_client *client) @@ -2482,6 +2492,8 @@ static int i915_guc_info(struct seq_file *m, void *data) seq_printf(m, "\nGuC execbuf client @ %p:\n", guc.execbuf_client); i915_guc_client_info(m, dev_priv, &client); + i915_guc_log_info(m, dev_priv); + /* Add more as required ... */ return 0; @@ -2493,10 +2505,10 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) struct drm_i915_gem_object *obj; int i = 0, pg; - if (!dev_priv->guc.log_vma) + if (!dev_priv->guc.log.vma) return 0; - obj = dev_priv->guc.log_vma->obj; + obj = dev_priv->guc.log.vma->obj; for (pg = 0; pg < obj->base.size / PAGE_SIZE; pg++) { u32 *log = kmap_atomic(i915_gem_object_get_page(obj, pg)); @@ -2513,6 +2525,44 @@ static int i915_guc_log_dump(struct seq_file *m, void *data) return 0; } +static int i915_guc_log_control_get(void *data, u64 *val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = to_i915(dev); + + if (!dev_priv->guc.log.vma) + return -EINVAL; + + *val = i915.guc_log_level; + + return 0; +} + +static int i915_guc_log_control_set(void *data, u64 val) +{ + struct drm_device *dev = data; + struct drm_i915_private *dev_priv = to_i915(dev); + int ret; + + if (!dev_priv->guc.log.vma) + return -EINVAL; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + intel_runtime_pm_get(dev_priv); + ret = i915_guc_log_control(dev_priv, val); + intel_runtime_pm_put(dev_priv); + + mutex_unlock(&dev->struct_mutex); + return ret; +} + +DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_control_fops, + i915_guc_log_control_get, i915_guc_log_control_set, + "%lld\n"); + static int i915_edp_psr_status(struct seq_file *m, void *data) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -2542,11 +2592,22 @@ static int i915_edp_psr_status(struct seq_file *m, void *data) enabled = I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE; else { for_each_pipe(dev_priv, pipe) { + enum transcoder cpu_transcoder = + intel_pipe_to_cpu_transcoder(dev_priv, pipe); + enum intel_display_power_domain power_domain; + + power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); + if (!intel_display_power_get_if_enabled(dev_priv, + power_domain)) + continue; + stat[pipe] = I915_READ(VLV_PSRSTAT(pipe)) & VLV_EDP_PSR_CURR_STATE_MASK; if ((stat[pipe] == VLV_EDP_PSR_ACTIVE_NORFB_UP) || (stat[pipe] == VLV_EDP_PSR_ACTIVE_SF_UPDATE)) enabled = true; + + intel_display_power_put(dev_priv, power_domain); } } @@ -3094,6 +3155,8 @@ static int i915_engine_info(struct seq_file *m, void *unused) struct intel_engine_cs *engine; enum intel_engine_id id; + intel_runtime_pm_get(dev_priv); + for_each_engine(engine, dev_priv, id) { struct intel_breadcrumbs *b = &engine->breadcrumbs; struct drm_i915_gem_request *rq; @@ -3103,7 +3166,7 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "%s\n", engine->name); seq_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [score %d]\n", intel_engine_get_seqno(engine), - engine->last_submitted_seqno, + intel_engine_last_submit(engine), engine->hangcheck.seqno, engine->hangcheck.score); @@ -3111,14 +3174,14 @@ static int i915_engine_info(struct seq_file *m, void *unused) seq_printf(m, "\tRequests:\n"); - rq = list_first_entry(&engine->request_list, - struct drm_i915_gem_request, link); - if (&rq->link != &engine->request_list) + rq = list_first_entry(&engine->timeline->requests, + struct drm_i915_gem_request, link); + if (&rq->link != &engine->timeline->requests) print_request(m, rq, "\t\tfirst "); - rq = list_last_entry(&engine->request_list, - struct drm_i915_gem_request, link); - if (&rq->link != &engine->request_list) + rq = list_last_entry(&engine->timeline->requests, + struct drm_i915_gem_request, link); + if (&rq->link != &engine->timeline->requests) print_request(m, rq, "\t\tlast "); rq = i915_gem_find_active_request(engine); @@ -3192,6 +3255,12 @@ static int i915_engine_info(struct seq_file *m, void *unused) else seq_printf(m, "\t\tELSP[1] idle\n"); rcu_read_unlock(); + + spin_lock_irq(&engine->execlist_lock); + list_for_each_entry(rq, &engine->execlist_queue, execlist_link) { + print_request(m, rq, "\t\tQ "); + } + spin_unlock_irq(&engine->execlist_lock); } else if (INTEL_GEN(dev_priv) > 6) { seq_printf(m, "\tPP_DIR_BASE: 0x%08x\n", I915_READ(RING_PP_DIR_BASE(engine))); @@ -3201,18 +3270,20 @@ static int i915_engine_info(struct seq_file *m, void *unused) I915_READ(RING_PP_DIR_DCLV(engine))); } - spin_lock(&b->lock); + spin_lock_irq(&b->lock); for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) { struct intel_wait *w = container_of(rb, typeof(*w), node); seq_printf(m, "\t%s [%d] waiting for %x\n", w->tsk->comm, w->tsk->pid, w->seqno); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); seq_puts(m, "\n"); } + intel_runtime_pm_put(dev_priv); + return 0; } @@ -3274,15 +3345,6 @@ static int i915_semaphore_status(struct seq_file *m, void *unused) seq_putc(m, '\n'); } - seq_puts(m, "\nSync seqno:\n"); - for_each_engine(engine, dev_priv, id) { - for (j = 0; j < num_rings; j++) - seq_printf(m, " 0x%08x ", - engine->semaphore.sync_seqno[j]); - seq_putc(m, '\n'); - } - seq_putc(m, '\n'); - intel_runtime_pm_put(dev_priv); mutex_unlock(&dev->struct_mutex); return 0; @@ -3375,7 +3437,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused) for_each_pipe(dev_priv, pipe) { seq_printf(m, "Pipe %c\n", pipe_name(pipe)); - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { entry = &ddb->plane[pipe][plane]; seq_printf(m, " Plane%-8d%8u%8u%8u\n", plane + 1, entry->start, entry->end, @@ -4009,8 +4071,7 @@ static void hsw_trans_edp_pipe_A_crc_wa(struct drm_i915_private *dev_priv, bool enable) { struct drm_device *dev = &dev_priv->drm; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A); struct intel_crtc_state *pipe_config; struct drm_atomic_state *state; int ret = 0; @@ -4076,10 +4137,8 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, enum pipe pipe, enum intel_pipe_crc_source source) { - struct drm_device *dev = &dev_priv->drm; struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; - struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); enum intel_display_power_domain power_domain; u32 val = 0; /* shut up gcc */ int ret; @@ -4150,15 +4209,15 @@ static int pipe_crc_set_source(struct drm_i915_private *dev_priv, /* real source -> none transition */ if (source == INTEL_PIPE_CRC_SOURCE_NONE) { struct intel_pipe_crc_entry *entries; - struct intel_crtc *crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n", pipe_name(pipe)); drm_modeset_lock(&crtc->base.mutex, NULL); if (crtc->base.state->active) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); drm_modeset_unlock(&crtc->base.mutex); spin_lock_irq(&pipe_crc->lock); @@ -4798,13 +4857,9 @@ i915_wedged_set(void *data, u64 val) if (i915_reset_in_progress(&dev_priv->gpu_error)) return -EAGAIN; - intel_runtime_pm_get(dev_priv); - i915_handle_error(dev_priv, val, "Manually setting wedged to %llu", val); - intel_runtime_pm_put(dev_priv); - return 0; } @@ -4872,10 +4927,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops, #define DROP_BOUND 0x2 #define DROP_RETIRE 0x4 #define DROP_ACTIVE 0x8 -#define DROP_ALL (DROP_UNBOUND | \ - DROP_BOUND | \ - DROP_RETIRE | \ - DROP_ACTIVE) +#define DROP_FREED 0x10 +#define DROP_ALL (DROP_UNBOUND | \ + DROP_BOUND | \ + DROP_RETIRE | \ + DROP_ACTIVE | \ + DROP_FREED) static int i915_drop_caches_get(void *data, u64 *val) { @@ -4919,6 +4976,11 @@ i915_drop_caches_set(void *data, u64 val) unlock: mutex_unlock(&dev->struct_mutex); + if (val & DROP_FREED) { + synchronize_rcu(); + flush_work(&dev_priv->mm.free_work); + } + return ret; } @@ -5039,22 +5101,16 @@ static int i915_cache_sharing_get(void *data, u64 *val) { struct drm_i915_private *dev_priv = data; - struct drm_device *dev = &dev_priv->drm; u32 snpcr; - int ret; if (!(IS_GEN6(dev_priv) || IS_GEN7(dev_priv))) return -ENODEV; - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) - return ret; intel_runtime_pm_get(dev_priv); snpcr = I915_READ(GEN6_MBCUNIT_SNPCR); intel_runtime_pm_put(dev_priv); - mutex_unlock(&dev->struct_mutex); *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT; @@ -5414,7 +5470,8 @@ static const struct i915_debugfs_files { {"i915_fbc_false_color", &i915_fbc_fc_fops}, {"i915_dp_test_data", &i915_displayport_test_data_fops}, {"i915_dp_test_type", &i915_displayport_test_type_fops}, - {"i915_dp_test_active", &i915_displayport_test_active_fops} + {"i915_dp_test_active", &i915_displayport_test_active_fops}, + {"i915_guc_log_control", &i915_guc_log_control_fops} }; void intel_display_crc_init(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index cf583d4e3a9f..0213a3090ab3 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -537,14 +537,17 @@ static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { .can_switch = i915_switcheroo_can_switch, }; -static void i915_gem_fini(struct drm_device *dev) +static void i915_gem_fini(struct drm_i915_private *dev_priv) { - mutex_lock(&dev->struct_mutex); - i915_gem_cleanup_engines(dev); - i915_gem_context_fini(dev); - mutex_unlock(&dev->struct_mutex); + mutex_lock(&dev_priv->drm.struct_mutex); + i915_gem_cleanup_engines(&dev_priv->drm); + i915_gem_context_fini(&dev_priv->drm); + mutex_unlock(&dev_priv->drm.struct_mutex); + + rcu_barrier(); + flush_work(&dev_priv->mm.free_work); - WARN_ON(!list_empty(&to_i915(dev)->context_list)); + WARN_ON(!list_empty(&dev_priv->context_list)); } static int i915_load_modeset_init(struct drm_device *dev) @@ -592,7 +595,9 @@ static int i915_load_modeset_init(struct drm_device *dev) /* Important: The output setup functions called by modeset_init need * working irqs for e.g. gmbus and dp aux transfers. */ - intel_modeset_init(dev); + ret = intel_modeset_init(dev); + if (ret) + goto cleanup_irq; intel_guc_init(dev); @@ -619,7 +624,7 @@ static int i915_load_modeset_init(struct drm_device *dev) cleanup_gem: if (i915_gem_suspend(dev)) DRM_ERROR("failed to idle hardware; continuing to unload!\n"); - i915_gem_fini(dev); + i915_gem_fini(dev_priv); cleanup_irq: intel_guc_fini(dev); drm_irq_uninstall(dev); @@ -825,10 +830,13 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv, intel_init_dpio(dev_priv); intel_power_domains_init(dev_priv); intel_irq_init(dev_priv); + intel_hangcheck_init(dev_priv); intel_init_display_hooks(dev_priv); intel_init_clock_gating_hooks(dev_priv); intel_init_audio_hooks(dev_priv); - i915_gem_load_init(&dev_priv->drm); + ret = i915_gem_load_init(&dev_priv->drm); + if (ret < 0) + goto err_gvt; intel_display_crc_init(dev_priv); @@ -838,6 +846,8 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv, return 0; +err_gvt: + intel_gvt_cleanup(dev_priv); err_workqueues: i915_workqueues_cleanup(dev_priv); return ret; @@ -972,7 +982,6 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv) static int i915_driver_init_hw(struct drm_i915_private *dev_priv) { struct pci_dev *pdev = dev_priv->drm.pdev; - struct drm_device *dev = &dev_priv->drm; int ret; if (i915_inject_load_failure()) @@ -1030,7 +1039,7 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv) * behaviour if any general state is accessed within a page above 4GB, * which also needs to be handled carefully. */ - if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) { + if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv)) { ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); if (ret) { @@ -1111,6 +1120,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) /* Reveal our presence to userspace */ if (drm_dev_register(dev, 0) == 0) { i915_debugfs_register(dev_priv); + i915_guc_register(dev_priv); i915_setup_sysfs(dev_priv); } else DRM_ERROR("Failed to register driver for userspace access!\n"); @@ -1149,6 +1159,7 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv) intel_opregion_unregister(dev_priv); i915_teardown_sysfs(dev_priv); + i915_guc_unregister(dev_priv); i915_debugfs_unregister(dev_priv); drm_dev_unregister(&dev_priv->drm); @@ -1303,7 +1314,7 @@ void i915_driver_unload(struct drm_device *dev) drain_workqueue(dev_priv->wq); intel_guc_fini(dev); - i915_gem_fini(dev); + i915_gem_fini(dev_priv); intel_fbc_cleanup_cfb(dev_priv); intel_power_domains_fini(dev_priv); @@ -1425,7 +1436,7 @@ static int i915_drm_suspend(struct drm_device *dev) intel_suspend_encoders(dev_priv); - intel_suspend_hw(dev); + intel_suspend_hw(dev_priv); i915_gem_suspend_gtt_mappings(dev); @@ -1589,6 +1600,8 @@ static int i915_drm_resume(struct drm_device *dev) intel_display_resume(dev); + drm_kms_helper_poll_enable(dev); + /* * ... but also need to make sure that hotplug processing * doesn't cause havoc. Like in the driver load code we don't @@ -1596,8 +1609,6 @@ static int i915_drm_resume(struct drm_device *dev) * notifications. * */ intel_hpd_init(dev_priv); - /* Config may have changed between suspend and resume */ - drm_helper_hpd_irq_event(dev); intel_opregion_register(dev_priv); @@ -1610,7 +1621,6 @@ static int i915_drm_resume(struct drm_device *dev) intel_opregion_notify_adapter(dev_priv, PCI_D0); intel_autoenable_gt_powersave(dev_priv); - drm_kms_helper_poll_enable(dev); enable_rpm_wakeref_asserts(dev_priv); @@ -2254,7 +2264,6 @@ err1: static int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) { - struct drm_device *dev = &dev_priv->drm; int err; int ret; @@ -2278,10 +2287,8 @@ static int vlv_resume_prepare(struct drm_i915_private *dev_priv, vlv_check_no_gt_access(dev_priv); - if (rpm_resume) { - intel_init_clock_gating(dev); - i915_gem_restore_fences(dev); - } + if (rpm_resume) + intel_init_clock_gating(dev_priv); return ret; } @@ -2301,32 +2308,13 @@ static int intel_runtime_suspend(struct device *kdev) DRM_DEBUG_KMS("Suspending device\n"); - /* - * We could deadlock here in case another thread holding struct_mutex - * calls RPM suspend concurrently, since the RPM suspend will wait - * first for this RPM suspend to finish. In this case the concurrent - * RPM resume will be followed by its RPM suspend counterpart. Still - * for consistency return -EAGAIN, which will reschedule this suspend. - */ - if (!mutex_trylock(&dev->struct_mutex)) { - DRM_DEBUG_KMS("device lock contention, deffering suspend\n"); - /* - * Bump the expiration timestamp, otherwise the suspend won't - * be rescheduled. - */ - pm_runtime_mark_last_busy(kdev); - - return -EAGAIN; - } - disable_rpm_wakeref_asserts(dev_priv); /* * We are safe here against re-faults, since the fault handler takes * an RPM reference. */ - i915_gem_release_all_mmaps(dev_priv); - mutex_unlock(&dev->struct_mutex); + i915_gem_runtime_suspend(dev_priv); intel_guc_suspend(dev); @@ -2589,7 +2577,7 @@ static struct drm_driver driver = { .set_busid = drm_pci_set_busid, .gem_close_object = i915_gem_close_object, - .gem_free_object = i915_gem_free_object, + .gem_free_object_unlocked = i915_gem_free_object, .gem_vm_ops = &i915_gem_vm_ops, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 74476f55d05e..6f4a6bcf6ed4 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -41,6 +41,7 @@ #include <linux/intel-iommu.h> #include <linux/kref.h> #include <linux/pm_qos.h> +#include <linux/reservation.h> #include <linux/shmem_fs.h> #include <drm/drmP.h> @@ -62,6 +63,7 @@ #include "i915_gem_gtt.h" #include "i915_gem_render_state.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" #include "intel_gvt.h" @@ -70,8 +72,8 @@ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20161024" -#define DRIVER_TIMESTAMP 1477290335 +#define DRIVER_DATE "20161108" +#define DRIVER_TIMESTAMP 1478587895 #undef WARN_ON /* Many gcc seem to no see through this and fall over :( */ @@ -183,7 +185,7 @@ enum plane { }; #define plane_name(p) ((p) + 'A') -#define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A') +#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') enum port { PORT_NONE = -1, @@ -312,7 +314,7 @@ struct i915_hotplug { #define for_each_pipe_masked(__dev_priv, __p, __mask) \ for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ for_each_if ((__mask) & (1 << (__p))) -#define for_each_plane(__dev_priv, __pipe, __p) \ +#define for_each_universal_plane(__dev_priv, __pipe, __p) \ for ((__p) = 0; \ (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ (__p)++) @@ -492,8 +494,8 @@ struct intel_limit; struct dpll; struct drm_i915_display_funcs { - int (*get_display_clock_speed)(struct drm_device *dev); - int (*get_fifo_size)(struct drm_device *dev, int plane); + int (*get_display_clock_speed)(struct drm_i915_private *dev_priv); + int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane); int (*compute_pipe_wm)(struct intel_crtc_state *cstate); int (*compute_intermediate_wm)(struct drm_device *dev, struct intel_crtc *intel_crtc, @@ -501,7 +503,7 @@ struct drm_i915_display_funcs { void (*initial_watermarks)(struct intel_crtc_state *cstate); void (*optimize_watermarks)(struct intel_crtc_state *cstate); int (*compute_global_watermarks)(struct drm_atomic_state *state); - void (*update_wm)(struct drm_crtc *crtc); + void (*update_wm)(struct intel_crtc *crtc); int (*modeset_calc_cdclk)(struct drm_atomic_state *state); void (*modeset_commit_cdclk)(struct drm_atomic_state *state); /* Returns the active state of the crtc, and if the crtc is active, @@ -523,7 +525,7 @@ struct drm_i915_display_funcs { const struct drm_display_mode *adjusted_mode); void (*audio_codec_disable)(struct intel_encoder *encoder); void (*fdi_link_train)(struct drm_crtc *crtc); - void (*init_clock_gating)(struct drm_device *dev); + void (*init_clock_gating)(struct drm_i915_private *dev_priv); int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, @@ -668,6 +670,7 @@ struct intel_csr { func(is_kabylake); \ func(is_preliminary); \ /* Keep has_* in alphabetical order */ \ + func(has_64bit_reloc); \ func(has_csr); \ func(has_ddi); \ func(has_dp_mst); \ @@ -746,6 +749,8 @@ struct intel_display_error_state; struct drm_i915_error_state { struct kref ref; struct timeval time; + struct timeval boottime; + struct timeval uptime; struct drm_i915_private *i915; @@ -778,6 +783,7 @@ struct drm_i915_error_state { struct intel_overlay_error_state *overlay; struct intel_display_error_state *display; struct drm_i915_error_object *semaphore; + struct drm_i915_error_object *guc_log; struct drm_i915_error_engine { int engine_id; @@ -797,7 +803,6 @@ struct drm_i915_error_state { u32 cpu_ring_tail; u32 last_seqno; - u32 semaphore_seqno[I915_NUM_ENGINES - 1]; /* Register state */ u32 start; @@ -933,6 +938,7 @@ struct i915_gem_context { struct drm_i915_file_private *file_priv; struct i915_hw_ppgtt *ppgtt; struct pid *pid; + const char *name; struct i915_ctx_hang_stats hang_stats; @@ -1319,6 +1325,12 @@ struct i915_power_well { /* cached hw enabled state */ bool hw_enabled; unsigned long domains; + /* unique identifier for this power well */ + unsigned long id; + /* + * Arbitraty data associated with this power well. Platform and power + * well specific. + */ unsigned long data; const struct i915_power_well_ops *ops; }; @@ -1356,11 +1368,22 @@ struct i915_gem_mm { struct list_head bound_list; /** * List of objects which are not bound to the GTT (thus - * are idle and not used by the GPU) but still have - * (presumably uncached) pages still attached. + * are idle and not used by the GPU). These objects may or may + * not actually have any pages attached. */ struct list_head unbound_list; + /** List of all objects in gtt_space, currently mmaped by userspace. + * All objects within this list must also be on bound_list. + */ + struct list_head userfault_list; + + /** + * List of objects which are pending destruction. + */ + struct llist_head free_list; + struct work_struct free_work; + /** Usable portion of the GTT for GEM */ unsigned long stolen_base; /* limited to low memory (32-bit) */ @@ -1409,6 +1432,9 @@ struct i915_error_state_file_priv { struct drm_i915_error_state *error; }; +#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ +#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ + struct i915_gpu_error { /* For hangcheck timer */ #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ @@ -1682,7 +1708,6 @@ struct skl_wm_level { */ struct i915_runtime_pm { atomic_t wakeref_count; - atomic_t atomic_seq; bool suspended; bool irqs_enabled; }; @@ -1807,7 +1832,6 @@ struct drm_i915_private { struct i915_gem_context *kernel_context; struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct i915_vma *semaphore; - u32 next_seqno; struct drm_dma_handle *status_page_dmah; struct resource mch_res; @@ -1832,8 +1856,10 @@ struct drm_i915_private { u32 de_irq_mask[I915_MAX_PIPES]; }; u32 gt_irq_mask; - u32 pm_irq_mask; + u32 pm_imr; + u32 pm_ier; u32 pm_rps_events; + u32 pm_guc_events; u32 pipestat_irq_mask[I915_MAX_PIPES]; struct i915_hotplug hotplug; @@ -1910,8 +1936,8 @@ struct drm_i915_private { /* Kernel Modesetting */ - struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; - struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; wait_queue_head_t pending_flip_queue; #ifdef CONFIG_DEBUG_FS @@ -2065,6 +2091,10 @@ struct drm_i915_private { void (*resume)(struct drm_i915_private *); void (*cleanup_engine)(struct intel_engine_cs *engine); + struct list_head timelines; + struct i915_gem_timeline global_timeline; + u32 active_requests; + /** * Is the GPU currently considered idle, or busy executing * userspace requests? Whilst idle, we allow runtime power @@ -2072,7 +2102,6 @@ struct drm_i915_private { * In order to reduce the effect on performance, there * is a slight delay before we do so. */ - unsigned int active_engines; bool awake; /** @@ -2092,6 +2121,8 @@ struct drm_i915_private { * off the idle_work. */ struct delayed_work idle_work; + + ktime_t last_init_time; } gt; /* perform PHY state sanity checks? */ @@ -2151,6 +2182,7 @@ enum hdmi_force_audio { struct drm_i915_gem_object_ops { unsigned int flags; #define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1 +#define I915_GEM_OBJECT_IS_SHRINKABLE 0x2 /* Interface between the GEM object and its backing storage. * get_pages() is called once prior to the use of the associated set @@ -2165,8 +2197,8 @@ struct drm_i915_gem_object_ops { * being released or under memory pressure (where we attempt to * reap pages for the shrinker). */ - int (*get_pages)(struct drm_i915_gem_object *); - void (*put_pages)(struct drm_i915_gem_object *); + struct sg_table *(*get_pages)(struct drm_i915_gem_object *); + void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *); int (*dmabuf_export)(struct drm_i915_gem_object *); void (*release)(struct drm_i915_gem_object *); @@ -2200,10 +2232,20 @@ struct drm_i915_gem_object { /** List of VMAs backed by this object */ struct list_head vma_list; + struct rb_root vma_tree; /** Stolen memory for this object, instead of being backed by shmem. */ struct drm_mm_node *stolen; - struct list_head global_list; + struct list_head global_link; + union { + struct rcu_head rcu; + struct llist_node freed; + }; + + /** + * Whether the object is currently in the GGTT mmap. + */ + struct list_head userfault_link; /** Used in execbuf to temporarily hold a ref */ struct list_head obj_exec_link; @@ -2211,33 +2253,12 @@ struct drm_i915_gem_object { struct list_head batch_pool_link; unsigned long flags; - /** - * This is set if the object is on the active lists (has pending - * rendering and so a non-zero seqno), and is not set if it i s on - * inactive (ready to be unbound) list. - */ -#define I915_BO_ACTIVE_SHIFT 0 -#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1) -#define __I915_BO_ACTIVE(bo) \ - ((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK) /** - * This is set if the object has been written to since last bound - * to the GTT + * Have we taken a reference for the object for incomplete GPU + * activity? */ - unsigned int dirty:1; - - /** - * Advice: are the backing pages purgeable? - */ - unsigned int madv:2; - - /** - * Whether the current gtt mapping needs to be mappable (and isn't just - * mappable by accident). Track pin and fault separate for a more - * accurate mappable working set. - */ - unsigned int fault_mappable:1; +#define I915_BO_ACTIVE_REF 0 /* * Is the object to be mapped as read-only to the GPU @@ -2258,15 +2279,41 @@ struct drm_i915_gem_object { /** Count of VMA actually bound by this object */ unsigned int bind_count; + unsigned int active_count; unsigned int pin_display; - struct sg_table *pages; - int pages_pin_count; - struct get_page { - struct scatterlist *sg; - int last; - } get_page; - void *mapping; + struct { + struct mutex lock; /* protects the pages and their use */ + atomic_t pages_pin_count; + + struct sg_table *pages; + void *mapping; + + struct i915_gem_object_page_iter { + struct scatterlist *sg_pos; + unsigned int sg_idx; /* in pages, but 32bit eek! */ + + struct radix_tree_root radix; + struct mutex lock; /* protects this cache */ + } get_page; + + /** + * Advice: are the backing pages purgeable? + */ + unsigned int madv:2; + + /** + * This is set if the object has been written to since the + * pages were last acquired. + */ + bool dirty:1; + + /** + * This is set if the object has been pinned due to unknown + * swizzling. + */ + bool quirked:1; + } mm; /** Breadcrumb of last rendering to the buffer. * There can only be one writer, but we allow for multiple readers. @@ -2278,8 +2325,7 @@ struct drm_i915_gem_object { * read request. This allows for the CPU to read from an active * buffer by only waiting for the write to complete. */ - struct i915_gem_active last_read[I915_NUM_ENGINES]; - struct i915_gem_active last_write; + struct reservation_object *resv; /** References from framebuffers, locks out tiling changes. */ unsigned long framebuffer_references; @@ -2290,8 +2336,6 @@ struct drm_i915_gem_object { struct i915_gem_userptr { uintptr_t ptr; unsigned read_only :1; - unsigned workers :4; -#define I915_GEM_USERPTR_MAX_WORKERS 15 struct i915_mm_struct *mm; struct i915_mmu_object *mmu_object; @@ -2300,6 +2344,8 @@ struct drm_i915_gem_object { /** for phys allocated objects */ struct drm_dma_handle *phys_handle; + + struct reservation_object __builtin_resv; }; static inline struct drm_i915_gem_object * @@ -2311,10 +2357,38 @@ to_intel_bo(struct drm_gem_object *gem) return container_of(gem, struct drm_i915_gem_object, base); } +/** + * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle + * @filp: DRM file private date + * @handle: userspace handle + * + * Returns: + * + * A pointer to the object named by the handle if such exists on @filp, NULL + * otherwise. This object is only valid whilst under the RCU read lock, and + * note carefully the object may be in the process of being destroyed. + */ +static inline struct drm_i915_gem_object * +i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle) +{ +#ifdef CONFIG_LOCKDEP + WARN_ON(debug_locks && !lock_is_held(&rcu_lock_map)); +#endif + return idr_find(&file->object_idr, handle); +} + static inline struct drm_i915_gem_object * i915_gem_object_lookup(struct drm_file *file, u32 handle) { - return to_intel_bo(drm_gem_object_lookup(file, handle)); + struct drm_i915_gem_object *obj; + + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, handle); + if (obj && !kref_get_unless_zero(&obj->base.refcount)) + obj = NULL; + rcu_read_unlock(); + + return obj; } __deprecated @@ -2336,59 +2410,61 @@ __attribute__((nonnull)) static inline void i915_gem_object_put(struct drm_i915_gem_object *obj) { - drm_gem_object_unreference(&obj->base); + __drm_gem_object_unreference(&obj->base); } __deprecated extern void drm_gem_object_unreference(struct drm_gem_object *); -__attribute__((nonnull)) -static inline void -i915_gem_object_put_unlocked(struct drm_i915_gem_object *obj) -{ - drm_gem_object_unreference_unlocked(&obj->base); -} - __deprecated extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *); static inline bool +i915_gem_object_is_dead(const struct drm_i915_gem_object *obj) +{ + return atomic_read(&obj->base.refcount.refcount) == 0; +} + +static inline bool i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) { return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; } -static inline unsigned long -i915_gem_object_get_active(const struct drm_i915_gem_object *obj) +static inline bool +i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj) { - return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK; + return obj->ops->flags & I915_GEM_OBJECT_IS_SHRINKABLE; } static inline bool i915_gem_object_is_active(const struct drm_i915_gem_object *obj) { - return i915_gem_object_get_active(obj); + return obj->active_count; } -static inline void -i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine) +static inline bool +i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj) { - obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT); + return test_bit(I915_BO_ACTIVE_REF, &obj->flags); } static inline void -i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine) +i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj) { - obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT); + lockdep_assert_held(&obj->base.dev->struct_mutex); + __set_bit(I915_BO_ACTIVE_REF, &obj->flags); } -static inline bool -i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj, - int engine) +static inline void +i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj) { - return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT); + lockdep_assert_held(&obj->base.dev->struct_mutex); + __clear_bit(I915_BO_ACTIVE_REF, &obj->flags); } +void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj); + static inline unsigned int i915_gem_object_get_tiling(struct drm_i915_gem_object *obj) { @@ -2407,6 +2483,23 @@ i915_gem_object_get_stride(struct drm_i915_gem_object *obj) return obj->tiling_and_stride & STRIDE_MASK; } +static inline struct intel_engine_cs * +i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj) +{ + struct intel_engine_cs *engine = NULL; + struct dma_fence *fence; + + rcu_read_lock(); + fence = reservation_object_get_excl_rcu(obj->resv); + rcu_read_unlock(); + + if (fence && dma_fence_is_i915(fence) && !dma_fence_is_signaled(fence)) + engine = to_request(fence)->engine; + dma_fence_put(fence); + + return engine; +} + static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) { i915_gem_object_get(vma->obj); @@ -2415,7 +2508,6 @@ static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) static inline void i915_vma_put(struct i915_vma *vma) { - lockdep_assert_held(&vma->vm->dev->struct_mutex); i915_gem_object_put(vma->obj); } @@ -2445,6 +2537,14 @@ static __always_inline struct sgt_iter { return s; } +static inline struct scatterlist *____sg_next(struct scatterlist *sg) +{ + ++sg; + if (unlikely(sg_is_chain(sg))) + sg = sg_chain_ptr(sg); + return sg; +} + /** * __sg_next - return the next scatterlist entry in a list * @sg: The current sg entry @@ -2459,9 +2559,7 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg) #ifdef CONFIG_DEBUG_SG BUG_ON(sg->sg_magic != SG_MAGIC); #endif - return sg_is_last(sg) ? NULL : - likely(!sg_is_chain(++sg)) ? sg : - sg_chain_ptr(sg); + return sg_is_last(sg) ? NULL : ____sg_next(sg); } /** @@ -2633,20 +2731,20 @@ struct drm_i915_cmd_table { #define IS_I830(dev_priv) (INTEL_DEVID(dev_priv) == 0x3577) #define IS_845G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2562) -#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) +#define IS_I85X(dev_priv) ((dev_priv)->info.is_i85x) #define IS_I865G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2572) -#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) +#define IS_I915G(dev_priv) ((dev_priv)->info.is_i915g) #define IS_I915GM(dev_priv) (INTEL_DEVID(dev_priv) == 0x2592) #define IS_I945G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2772) -#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) -#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) -#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) +#define IS_I945GM(dev_priv) ((dev_priv)->info.is_i945gm) +#define IS_BROADWATER(dev_priv) ((dev_priv)->info.is_broadwater) +#define IS_CRESTLINE(dev_priv) ((dev_priv)->info.is_crestline) #define IS_GM45(dev_priv) (INTEL_DEVID(dev_priv) == 0x2A42) #define IS_G4X(dev_priv) ((dev_priv)->info.is_g4x) #define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001) #define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011) -#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) -#define IS_G33(dev) (INTEL_INFO(dev)->is_g33) +#define IS_PINEVIEW(dev_priv) ((dev_priv)->info.is_pineview) +#define IS_G33(dev_priv) ((dev_priv)->info.is_g33) #define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046) #define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.is_ivybridge) #define IS_IVB_GT1(dev_priv) (INTEL_DEVID(dev_priv) == 0x0156 || \ @@ -2659,7 +2757,7 @@ struct drm_i915_cmd_table { #define IS_SKYLAKE(dev_priv) ((dev_priv)->info.is_skylake) #define IS_BROXTON(dev_priv) ((dev_priv)->info.is_broxton) #define IS_KABYLAKE(dev_priv) ((dev_priv)->info.is_kabylake) -#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) +#define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile) #define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) #define IS_BDW_ULT(dev_priv) (IS_BROADWELL(dev_priv) && \ @@ -2803,7 +2901,7 @@ struct drm_i915_cmd_table { #define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) #define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) -#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) +#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) #define HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) @@ -2820,6 +2918,8 @@ struct drm_i915_cmd_table { #define HAS_CSR(dev) (INTEL_INFO(dev)->has_csr) #define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) +#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) + /* * For now, anything with a GuC requires uCode loading, and then supports * command submission once loaded. But these are logically independent @@ -2914,6 +3014,7 @@ extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); extern void i915_reset(struct drm_i915_private *dev_priv); extern int intel_guc_reset(struct drm_i915_private *dev_priv); extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); +extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); @@ -3097,7 +3198,7 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); -void i915_gem_load_init(struct drm_device *dev); +int i915_gem_load_init(struct drm_device *dev); void i915_gem_load_cleanup(struct drm_device *dev); void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); int i915_gem_freeze(struct drm_i915_private *dev_priv); @@ -3129,70 +3230,85 @@ void i915_vma_close(struct i915_vma *vma); void i915_vma_destroy(struct i915_vma *vma); int i915_gem_object_unbind(struct drm_i915_gem_object *obj); -int i915_gem_object_put_pages(struct drm_i915_gem_object *obj); -void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); -int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj); +void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); -static inline int __sg_page_count(struct scatterlist *sg) +static inline int __sg_page_count(const struct scatterlist *sg) { return sg->length >> PAGE_SHIFT; } +struct scatterlist * +i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + unsigned int n, unsigned int *offset); + +struct page * +i915_gem_object_get_page(struct drm_i915_gem_object *obj, + unsigned int n); + struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n); +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n); -static inline dma_addr_t -i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n) +dma_addr_t +i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, + unsigned long n); + +void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); + +static inline int __must_check +i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - } + might_lock(&obj->mm.lock); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { - obj->get_page.last += __sg_page_count(obj->get_page.sg++); - if (unlikely(sg_is_chain(obj->get_page.sg))) - obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); - } + if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) + return 0; - return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT); + return __i915_gem_object_get_pages(obj); } -static inline struct page * -i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n) +static inline void +__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT)) - return NULL; - - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - } + GEM_BUG_ON(!obj->mm.pages); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { - obj->get_page.last += __sg_page_count(obj->get_page.sg++); - if (unlikely(sg_is_chain(obj->get_page.sg))) - obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); - } + atomic_inc(&obj->mm.pages_pin_count); +} - return nth_page(sg_page(obj->get_page.sg), n - obj->get_page.last); +static inline bool +i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) +{ + return atomic_read(&obj->mm.pages_pin_count); } -static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) +static inline void +__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - GEM_BUG_ON(obj->pages == NULL); - obj->pages_pin_count++; + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + GEM_BUG_ON(!obj->mm.pages); + + atomic_dec(&obj->mm.pages_pin_count); + GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); } -static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) +static inline void +i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - GEM_BUG_ON(obj->pages_pin_count == 0); - obj->pages_pin_count--; - GEM_BUG_ON(obj->pages_pin_count < obj->bind_count); + __i915_gem_object_unpin_pages(obj); } +enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */ + I915_MM_NORMAL = 0, + I915_MM_SHRINKER +}; + +void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass); +void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); + enum i915_map_type { I915_MAP_WB = 0, I915_MAP_WC, @@ -3208,8 +3324,8 @@ enum i915_map_type { * the kernel address space. Based on the @type of mapping, the PTE will be * set to either WriteBack or WriteCombine (via pgprot_t). * - * The caller must hold the struct_mutex, and is responsible for calling - * i915_gem_object_unpin_map() when the mapping is no longer required. + * The caller is responsible for calling i915_gem_object_unpin_map() when the + * mapping is no longer required. * * Returns the pointer through which to access the mapped object, or an * ERR_PTR() on error. @@ -3225,12 +3341,9 @@ void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, * with your access, call i915_gem_object_unpin_map() to release the pin * upon the mapping. Once the pin count reaches zero, that mapping may be * removed. - * - * The caller must hold the struct_mutex. */ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) { - lockdep_assert_held(&obj->base.dev->struct_mutex); i915_gem_object_unpin_pages(obj); } @@ -3263,7 +3376,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, struct drm_i915_gem_object *new, unsigned frontbuffer_bits); -int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); +int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *engine); @@ -3302,9 +3415,10 @@ int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, int __must_check i915_gem_suspend(struct drm_device *dev); void i915_gem_resume(struct drm_device *dev); int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); -int __must_check -i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, - bool readonly); +int i915_gem_object_wait(struct drm_i915_gem_object *obj, + unsigned int flags, + long timeout, + struct intel_rps_client *rps); int __must_check i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); @@ -3386,6 +3500,7 @@ int __must_check i915_vma_put_fence(struct i915_vma *vma); static inline bool i915_vma_pin_fence(struct i915_vma *vma) { + lockdep_assert_held(&vma->vm->dev->struct_mutex); if (vma->fence) { vma->fence->pin_count++; return true; @@ -3404,6 +3519,7 @@ i915_vma_pin_fence(struct i915_vma *vma) static inline void i915_vma_unpin_fence(struct i915_vma *vma) { + lockdep_assert_held(&vma->vm->dev->struct_mutex); if (vma->fence) { GEM_BUG_ON(vma->fence->pin_count <= 0); vma->fence->pin_count--; @@ -3413,8 +3529,10 @@ i915_vma_unpin_fence(struct i915_vma *vma) void i915_gem_restore_fences(struct drm_device *dev); void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); +void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages); +void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages); /* i915_gem_context.c */ int __must_check i915_gem_context_init(struct drm_device *dev); @@ -3424,6 +3542,9 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); int i915_switch_context(struct drm_i915_gem_request *req); int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv); +struct i915_vma * +i915_gem_context_pin_legacy(struct i915_gem_context *ctx, + unsigned int flags); void i915_gem_context_free(struct kref *ctx_ref); struct drm_i915_gem_object * i915_gem_alloc_context_obj(struct drm_device *dev, size_t size); @@ -3457,6 +3578,16 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx) kref_put(&ctx->ref, i915_gem_context_free); } +static inline struct intel_timeline * +i915_gem_context_lookup_timeline(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + struct i915_address_space *vm; + + vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base; + return &vm->timeline.engine[engine->id]; +} + static inline bool i915_gem_context_is_default(const struct i915_gem_context *c) { return c->user_handle == DEFAULT_CONTEXT_HANDLE; @@ -3510,6 +3641,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal(struct drm_i915_private *dev_priv, + unsigned int size); + /* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target, @@ -3692,7 +3828,7 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv); /* modesetting */ extern void intel_modeset_init_hw(struct drm_device *dev); -extern void intel_modeset_init(struct drm_device *dev); +extern int intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); extern int intel_connector_register(struct drm_connector *); @@ -3747,6 +3883,23 @@ u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); /* intel_dpio_phy.c */ +void bxt_port_to_phy_channel(enum port port, + enum dpio_phy *phy, enum dpio_channel *ch); +void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, + enum port port, u32 margin, u32 scale, + u32 enable, u32 deemphasis); +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, + uint8_t lane_count); +void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, + uint8_t lane_lat_optim_mask); +uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); + void chv_set_phy_signal_level(struct intel_encoder *encoder, u32 deemph_reg_value, u32 margin_reg_value, bool uniq_trans_scale); @@ -3836,11 +3989,30 @@ __raw_write(64, q) #undef __raw_write /* These are untraced mmio-accessors that are only valid to be used inside - * critical sections inside IRQ handlers where forcewake is explicitly + * critical sections, such as inside IRQ handlers, where forcewake is explicitly * controlled. + * * Think twice, and think again, before using these. - * Note: Should only be used between intel_uncore_forcewake_irqlock() and - * intel_uncore_forcewake_irqunlock(). + * + * As an example, these accessors can possibly be used between: + * + * spin_lock_irq(&dev_priv->uncore.lock); + * intel_uncore_forcewake_get__locked(); + * + * and + * + * intel_uncore_forcewake_put__locked(); + * spin_unlock_irq(&dev_priv->uncore.lock); + * + * + * Note: some registers may not need forcewake held, so + * intel_uncore_forcewake_{get,put} can be omitted, see + * intel_uncore_forcewake_for_reg(). + * + * Certain architectures will die if the same cacheline is concurrently accessed + * by different clients (e.g. on Ivybridge). Access to registers should + * therefore generally be serialised, by either the dev_priv->uncore.lock or + * a more localised lock guarding all access to that bank of registers. */ #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) @@ -3917,7 +4089,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) /* Before we do the heavier coherent read of the seqno, * check the value (hopefully) in the CPU cacheline. */ - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; /* Ensure our read of the seqno is coherent so that we @@ -3968,7 +4140,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) wake_up_process(tsk); rcu_read_unlock(); - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 8ed8e24025ac..41e697e5dbcd 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -29,7 +29,6 @@ #include <drm/drm_vma_manager.h> #include <drm/i915_drm.h> #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "i915_vgpu.h" #include "i915_trace.h" #include "intel_drv.h" @@ -42,6 +41,7 @@ #include <linux/pci.h> #include <linux/dma-buf.h> +static void i915_gem_flush_free_objects(struct drm_i915_private *i915); static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); @@ -63,13 +63,13 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) } static int -insert_mappable_node(struct drm_i915_private *i915, +insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size) { memset(node, 0, sizeof(*node)); - return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node, - size, 0, 0, 0, - i915->ggtt.mappable_end, + return drm_mm_insert_node_in_range_generic(&ggtt->base.mm, node, + size, 0, -1, + 0, ggtt->mappable_end, DRM_MM_SEARCH_DEFAULT, DRM_MM_CREATE_DEFAULT); } @@ -104,6 +104,8 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) { int ret; + might_sleep(); + if (!i915_reset_in_progress(error)) return 0; @@ -114,7 +116,7 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) */ ret = wait_event_interruptible_timeout(error->reset_queue, !i915_reset_in_progress(error), - 10*HZ); + I915_RESET_TIMEOUT); if (ret == 0) { DRM_ERROR("Timed out waiting for the gpu reset to complete\n"); return -EIO; @@ -167,7 +169,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, return 0; } -static int +static struct sg_table * i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) { struct address_space *mapping = obj->base.filp->f_mapping; @@ -177,7 +179,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) int i; if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) - return -EINVAL; + return ERR_PTR(-EINVAL); for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { struct page *page; @@ -185,7 +187,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) page = shmem_read_mapping_page(mapping, i); if (IS_ERR(page)) - return PTR_ERR(page); + return ERR_CAST(page); src = kmap_atomic(page); memcpy(vaddr, src, PAGE_SIZE); @@ -200,11 +202,11 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) st = kmalloc(sizeof(*st), GFP_KERNEL); if (st == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); if (sg_alloc_table(st, 1, GFP_KERNEL)) { kfree(st); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } sg = st->sgl; @@ -214,29 +216,31 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) sg_dma_address(sg) = obj->phys_handle->busaddr; sg_dma_len(sg) = obj->base.size; - obj->pages = st; - return 0; + return st; } static void -i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) +__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj) { - int ret; + GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED); - BUG_ON(obj->madv == __I915_MADV_PURGED); + if (obj->mm.madv == I915_MADV_DONTNEED) + obj->mm.dirty = false; - ret = i915_gem_object_set_to_cpu_domain(obj, true); - if (WARN_ON(ret)) { - /* In the event of a disaster, abandon all caches and - * hope for the best. - */ - obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; - } + if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) + i915_gem_clflush_object(obj, false); - if (obj->madv == I915_MADV_DONTNEED) - obj->dirty = 0; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; +} - if (obj->dirty) { +static void +i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, + struct sg_table *pages) +{ + __i915_gem_object_release_shmem(obj); + + if (obj->mm.dirty) { struct address_space *mapping = obj->base.filp->f_mapping; char *vaddr = obj->phys_handle->vaddr; int i; @@ -255,22 +259,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) kunmap_atomic(dst); set_page_dirty(page); - if (obj->madv == I915_MADV_WILLNEED) + if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); put_page(page); vaddr += PAGE_SIZE; } - obj->dirty = 0; + obj->mm.dirty = false; } - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } static void i915_gem_object_release_phys(struct drm_i915_gem_object *obj) { drm_pci_free(obj->base.dev, obj->phys_handle); + i915_gem_object_unpin_pages(obj); } static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { @@ -292,7 +297,12 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj) * must wait for all rendering to complete to the object (as unbinding * must anyway), and retire the requests. */ - ret = i915_gem_object_wait_rendering(obj, false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -311,88 +321,143 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj) return ret; } -/** - * Ensures that all rendering to the object has completed and the object is - * safe to unbind from the GTT or access from the CPU. - * @obj: i915 gem object - * @readonly: waiting for just read access or read-write access - */ -int -i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, - bool readonly) +static long +i915_gem_object_wait_fence(struct dma_fence *fence, + unsigned int flags, + long timeout, + struct intel_rps_client *rps) { - struct reservation_object *resv; - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct drm_i915_gem_request *rq; - lockdep_assert_held(&obj->base.dev->struct_mutex); + BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1); - if (!readonly) { - active = obj->last_read; - active_mask = i915_gem_object_get_active(obj); - } else { - active_mask = 1; - active = &obj->last_write; - } + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return timeout; - for_each_active(active_mask, idx) { - int ret; + if (!dma_fence_is_i915(fence)) + return dma_fence_wait_timeout(fence, + flags & I915_WAIT_INTERRUPTIBLE, + timeout); - ret = i915_gem_active_wait(&active[idx], - &obj->base.dev->struct_mutex); - if (ret) - return ret; + rq = to_request(fence); + if (i915_gem_request_completed(rq)) + goto out; + + /* This client is about to stall waiting for the GPU. In many cases + * this is undesirable and limits the throughput of the system, as + * many clients cannot continue processing user input/output whilst + * blocked. RPS autotuning may take tens of milliseconds to respond + * to the GPU load and thus incurs additional latency for the client. + * We can circumvent that by promoting the GPU frequency to maximum + * before we wait. This makes the GPU throttle up much more quickly + * (good for benchmarks and user experience, e.g. window animations), + * but at a cost of spending more power processing the workload + * (bad for battery). Not all clients even want their results + * immediately and for them we should just let the GPU select its own + * frequency to maximise efficiency. To prevent a single client from + * forcing the clocks too high for the whole system, we only allow + * each client to waitboost once in a busy period. + */ + if (rps) { + if (INTEL_GEN(rq->i915) >= 6) + gen6_rps_boost(rq->i915, rps, rq->emitted_jiffies); + else + rps = NULL; } - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - long err; + timeout = i915_wait_request(rq, flags, timeout); - err = reservation_object_wait_timeout_rcu(resv, !readonly, true, - MAX_SCHEDULE_TIMEOUT); - if (err < 0) - return err; +out: + if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq)) + i915_gem_request_retire_upto(rq); + + if (rps && rq->global_seqno == intel_engine_last_submit(rq->engine)) { + /* The GPU is now idle and this client has stalled. + * Since no other client has submitted a request in the + * meantime, assume that this client is the only one + * supplying work to the GPU but is unable to keep that + * work supplied because it is waiting. Since the GPU is + * then never kept fully busy, RPS autoclocking will + * keep the clocks relatively low, causing further delays. + * Compensate by giving the synchronous client credit for + * a waitboost next time. + */ + spin_lock(&rq->i915->rps.client_lock); + list_del_init(&rps->link); + spin_unlock(&rq->i915->rps.client_lock); } - return 0; + return timeout; } -/* A nonblocking variant of the above wait. Must be called prior to - * acquiring the mutex for the object, as the object state may change - * during this call. A reference must be held by the caller for the object. - */ -static __must_check int -__unsafe_wait_rendering(struct drm_i915_gem_object *obj, - struct intel_rps_client *rps, - bool readonly) +static long +i915_gem_object_wait_reservation(struct reservation_object *resv, + unsigned int flags, + long timeout, + struct intel_rps_client *rps) { - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct dma_fence *excl; - active_mask = __I915_BO_ACTIVE(obj); - if (!active_mask) - return 0; - - if (!readonly) { - active = obj->last_read; - } else { - active_mask = 1; - active = &obj->last_write; - } - - for_each_active(active_mask, idx) { + if (flags & I915_WAIT_ALL) { + struct dma_fence **shared; + unsigned int count, i; int ret; - ret = i915_gem_active_wait_unlocked(&active[idx], - I915_WAIT_INTERRUPTIBLE, - NULL, rps); + ret = reservation_object_get_fences_rcu(resv, + &excl, &count, &shared); if (ret) return ret; + + for (i = 0; i < count; i++) { + timeout = i915_gem_object_wait_fence(shared[i], + flags, timeout, + rps); + if (timeout <= 0) + break; + + dma_fence_put(shared[i]); + } + + for (; i < count; i++) + dma_fence_put(shared[i]); + kfree(shared); + } else { + excl = reservation_object_get_excl_rcu(resv); } - return 0; + if (excl && timeout > 0) + timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps); + + dma_fence_put(excl); + + return timeout; +} + +/** + * Waits for rendering to the object to be completed + * @obj: i915 gem object + * @flags: how to wait (under a lock, for all rendering or just for writes etc) + * @timeout: how long to wait + * @rps: client (user process) to charge for any waitboosting + */ +int +i915_gem_object_wait(struct drm_i915_gem_object *obj, + unsigned int flags, + long timeout, + struct intel_rps_client *rps) +{ + might_sleep(); +#if IS_ENABLED(CONFIG_LOCKDEP) + GEM_BUG_ON(debug_locks && + !!lockdep_is_held(&obj->base.dev->struct_mutex) != + !!(flags & I915_WAIT_LOCKED)); +#endif + GEM_BUG_ON(timeout < 0); + + timeout = i915_gem_object_wait_reservation(obj->resv, + flags, timeout, + rps); + return timeout < 0 ? timeout : 0; } static struct intel_rps_client *to_rps_client(struct drm_file *file) @@ -416,7 +481,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, return 0; } - if (obj->madv != I915_MADV_WILLNEED) + if (obj->mm.madv != I915_MADV_WILLNEED) return -EFAULT; if (obj->base.filp == NULL) @@ -426,9 +491,9 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, if (ret) return ret; - ret = i915_gem_object_put_pages(obj); - if (ret) - return ret; + __i915_gem_object_put_pages(obj, I915_MM_NORMAL); + if (obj->mm.pages) + return -EBUSY; /* create a new object */ phys = drm_pci_alloc(obj->base.dev, obj->base.size, align); @@ -438,23 +503,29 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, obj->phys_handle = phys; obj->ops = &i915_gem_phys_ops; - return i915_gem_object_get_pages(obj); + return i915_gem_object_pin_pages(obj); } static int i915_gem_phys_pwrite(struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_device *dev = obj->base.dev; void *vaddr = obj->phys_handle->vaddr + args->offset; char __user *user_data = u64_to_user_ptr(args->data_ptr); - int ret = 0; + int ret; /* We manually control the domain here and pretend that it * remains coherent i.e. in the GTT domain, like shmem_pwrite. */ - ret = i915_gem_object_wait_rendering(obj, false); + lockdep_assert_held(&obj->base.dev->struct_mutex); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); if (ret) return ret; @@ -516,7 +587,7 @@ i915_gem_create(struct drm_file *file, ret = drm_gem_handle_create(file, &obj->base, &handle); /* drop reference from allocate - handle holds it now */ - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); if (ret) return ret; @@ -548,6 +619,8 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_create *args = data; + i915_gem_flush_free_objects(to_i915(dev)); + return i915_gem_create(file, dev, args->size, &args->handle); } @@ -614,21 +687,24 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj, { int ret; - *needs_clflush = 0; + lockdep_assert_held(&obj->base.dev->struct_mutex); + *needs_clflush = 0; if (!i915_gem_object_has_struct_page(obj)) return -ENODEV; - ret = i915_gem_object_wait_rendering(obj, true); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - i915_gem_object_flush_gtt_write_domain(obj); /* If we're not in the cpu read domain, set ourself into the gtt @@ -661,20 +737,25 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, { int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + *needs_clflush = 0; if (!i915_gem_object_has_struct_page(obj)) return -ENODEV; - ret = i915_gem_object_wait_rendering(obj, false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - i915_gem_object_flush_gtt_write_domain(obj); /* If we're not in the cpu write domain, set ourself into the @@ -704,7 +785,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, obj->cache_dirty = true; intel_fb_obj_invalidate(obj, ORIGIN_CPU); - obj->dirty = 1; + obj->mm.dirty = true; /* return with the pages pinned */ return 0; @@ -713,32 +794,6 @@ err_unpin: return ret; } -/* Per-page copy function for the shmem pread fastpath. - * Flushes invalid cachelines before reading the target if - * needs_clflush is set. */ -static int -shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length, - char __user *user_data, - bool page_do_bit17_swizzling, bool needs_clflush) -{ - char *vaddr; - int ret; - - if (unlikely(page_do_bit17_swizzling)) - return -EINVAL; - - vaddr = kmap_atomic(page); - if (needs_clflush) - drm_clflush_virt_range(vaddr + shmem_page_offset, - page_length); - ret = __copy_to_user_inatomic(user_data, - vaddr + shmem_page_offset, - page_length); - kunmap_atomic(vaddr); - - return ret ? -EFAULT : 0; -} - static void shmem_clflush_swizzled_range(char *addr, unsigned long length, bool swizzled) @@ -764,7 +819,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length, /* Only difference to the fast-path function is that this can handle bit17 * and uses non-atomic copy and kmap functions. */ static int -shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, +shmem_pread_slow(struct page *page, int offset, int length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush) { @@ -773,60 +828,130 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, vaddr = kmap(page); if (needs_clflush) - shmem_clflush_swizzled_range(vaddr + shmem_page_offset, - page_length, + shmem_clflush_swizzled_range(vaddr + offset, length, page_do_bit17_swizzling); if (page_do_bit17_swizzling) - ret = __copy_to_user_swizzled(user_data, - vaddr, shmem_page_offset, - page_length); + ret = __copy_to_user_swizzled(user_data, vaddr, offset, length); else - ret = __copy_to_user(user_data, - vaddr + shmem_page_offset, - page_length); + ret = __copy_to_user(user_data, vaddr + offset, length); kunmap(page); return ret ? - EFAULT : 0; } -static inline unsigned long -slow_user_access(struct io_mapping *mapping, - uint64_t page_base, int page_offset, - char __user *user_data, - unsigned long length, bool pwrite) +static int +shmem_pread(struct page *page, int offset, int length, char __user *user_data, + bool page_do_bit17_swizzling, bool needs_clflush) +{ + int ret; + + ret = -ENODEV; + if (!page_do_bit17_swizzling) { + char *vaddr = kmap_atomic(page); + + if (needs_clflush) + drm_clflush_virt_range(vaddr + offset, length); + ret = __copy_to_user_inatomic(user_data, vaddr + offset, length); + kunmap_atomic(vaddr); + } + if (ret == 0) + return 0; + + return shmem_pread_slow(page, offset, length, user_data, + page_do_bit17_swizzling, needs_clflush); +} + +static int +i915_gem_shmem_pread(struct drm_i915_gem_object *obj, + struct drm_i915_gem_pread *args) +{ + char __user *user_data; + u64 remain; + unsigned int obj_do_bit17_swizzling; + unsigned int needs_clflush; + unsigned int idx, offset; + int ret; + + obj_do_bit17_swizzling = 0; + if (i915_gem_object_needs_bit17_swizzle(obj)) + obj_do_bit17_swizzling = BIT(17); + + ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex); + if (ret) + return ret; + + ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); + mutex_unlock(&obj->base.dev->struct_mutex); + if (ret) + return ret; + + remain = args->size; + user_data = u64_to_user_ptr(args->data_ptr); + offset = offset_in_page(args->offset); + for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { + struct page *page = i915_gem_object_get_page(obj, idx); + int length; + + length = remain; + if (offset + length > PAGE_SIZE) + length = PAGE_SIZE - offset; + + ret = shmem_pread(page, offset, length, user_data, + page_to_phys(page) & obj_do_bit17_swizzling, + needs_clflush); + if (ret) + break; + + remain -= length; + user_data += length; + offset = 0; + } + + i915_gem_obj_finish_shmem_access(obj); + return ret; +} + +static inline bool +gtt_user_read(struct io_mapping *mapping, + loff_t base, int offset, + char __user *user_data, int length) { - void __iomem *ioaddr; void *vaddr; - uint64_t unwritten; + unsigned long unwritten; - ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE); /* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force *)ioaddr + page_offset; - if (pwrite) - unwritten = __copy_from_user(vaddr, user_data, length); - else - unwritten = __copy_to_user(user_data, vaddr, length); - - io_mapping_unmap(ioaddr); + vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base); + unwritten = __copy_to_user_inatomic(user_data, vaddr + offset, length); + io_mapping_unmap_atomic(vaddr); + if (unwritten) { + vaddr = (void __force *) + io_mapping_map_wc(mapping, base, PAGE_SIZE); + unwritten = copy_to_user(user_data, vaddr + offset, length); + io_mapping_unmap(vaddr); + } return unwritten; } static int -i915_gem_gtt_pread(struct drm_device *dev, - struct drm_i915_gem_object *obj, uint64_t size, - uint64_t data_offset, uint64_t data_ptr) +i915_gem_gtt_pread(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pread *args) { - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; - struct i915_vma *vma; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_ggtt *ggtt = &i915->ggtt; struct drm_mm_node node; - char __user *user_data; - uint64_t remain; - uint64_t offset; + struct i915_vma *vma; + void __user *user_data; + u64 remain, offset; int ret; - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); + ret = mutex_lock_interruptible(&i915->drm.struct_mutex); + if (ret) + return ret; + + intel_runtime_pm_get(i915); + vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, + PIN_MAPPABLE | PIN_NONBLOCK); if (!IS_ERR(vma)) { node.start = i915_ggtt_offset(vma); node.allocated = false; @@ -837,35 +962,21 @@ i915_gem_gtt_pread(struct drm_device *dev, } } if (IS_ERR(vma)) { - ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE); + ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); if (ret) - goto out; - - ret = i915_gem_object_get_pages(obj); - if (ret) { - remove_mappable_node(&node); - goto out; - } - - i915_gem_object_pin_pages(obj); + goto out_unlock; + GEM_BUG_ON(!node.allocated); } ret = i915_gem_object_set_to_gtt_domain(obj, false); if (ret) goto out_unpin; - user_data = u64_to_user_ptr(data_ptr); - remain = size; - offset = data_offset; + mutex_unlock(&i915->drm.struct_mutex); - mutex_unlock(&dev->struct_mutex); - if (likely(!i915.prefault_disable)) { - ret = fault_in_pages_writeable(user_data, remain); - if (ret) { - mutex_lock(&dev->struct_mutex); - goto out_unpin; - } - } + user_data = u64_to_user_ptr(args->data_ptr); + remain = args->size; + offset = args->offset; while (remain > 0) { /* Operation in this page @@ -882,19 +993,14 @@ i915_gem_gtt_pread(struct drm_device *dev, wmb(); ggtt->base.insert_page(&ggtt->base, i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT), - node.start, - I915_CACHE_NONE, 0); + node.start, I915_CACHE_NONE, 0); wmb(); } else { page_base += offset & PAGE_MASK; } - /* This is a slow read/write as it tries to read from - * and write to user memory which may result into page - * faults, and so we cannot perform this under struct_mutex. - */ - if (slow_user_access(&ggtt->mappable, page_base, - page_offset, user_data, - page_length, false)) { + + if (gtt_user_read(&ggtt->mappable, page_base, page_offset, + user_data, page_length)) { ret = -EFAULT; break; } @@ -904,110 +1010,19 @@ i915_gem_gtt_pread(struct drm_device *dev, offset += page_length; } - mutex_lock(&dev->struct_mutex); - if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { - /* The user has modified the object whilst we tried - * reading from it, and we now have no idea what domain - * the pages should be in. As we have just been touching - * them directly, flush everything back to the GTT - * domain. - */ - ret = i915_gem_object_set_to_gtt_domain(obj, false); - } - + mutex_lock(&i915->drm.struct_mutex); out_unpin: if (node.allocated) { wmb(); ggtt->base.clear_range(&ggtt->base, node.start, node.size); - i915_gem_object_unpin_pages(obj); remove_mappable_node(&node); } else { i915_vma_unpin(vma); } -out: - return ret; -} - -static int -i915_gem_shmem_pread(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pread *args, - struct drm_file *file) -{ - char __user *user_data; - ssize_t remain; - loff_t offset; - int shmem_page_offset, page_length, ret = 0; - int obj_do_bit17_swizzling, page_do_bit17_swizzling; - int prefaulted = 0; - int needs_clflush = 0; - struct sg_page_iter sg_iter; - - ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); - if (ret) - return ret; - - obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - user_data = u64_to_user_ptr(args->data_ptr); - offset = args->offset; - remain = args->size; - - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, - offset >> PAGE_SHIFT) { - struct page *page = sg_page_iter_page(&sg_iter); - - if (remain <= 0) - break; - - /* Operation in this page - * - * shmem_page_offset = offset within page in shmem file - * page_length = bytes to copy for this page - */ - shmem_page_offset = offset_in_page(offset); - page_length = remain; - if ((shmem_page_offset + page_length) > PAGE_SIZE) - page_length = PAGE_SIZE - shmem_page_offset; - - page_do_bit17_swizzling = obj_do_bit17_swizzling && - (page_to_phys(page) & (1 << 17)) != 0; - - ret = shmem_pread_fast(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - needs_clflush); - if (ret == 0) - goto next_page; - - mutex_unlock(&dev->struct_mutex); - - if (likely(!i915.prefault_disable) && !prefaulted) { - ret = fault_in_pages_writeable(user_data, remain); - /* Userspace is tricking us, but we've already clobbered - * its pages with the prefault and promised to write the - * data up to the first fault. Hence ignore any errors - * and just continue. */ - (void)ret; - prefaulted = 1; - } - - ret = shmem_pread_slow(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - needs_clflush); - - mutex_lock(&dev->struct_mutex); - - if (ret) - goto out; - -next_page: - remain -= page_length; - user_data += page_length; - offset += page_length; - } - -out: - i915_gem_obj_finish_shmem_access(obj); +out_unlock: + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); return ret; } @@ -1026,7 +1041,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_pread *args = data; struct drm_i915_gem_object *obj; - int ret = 0; + int ret; if (args->size == 0) return 0; @@ -1044,36 +1059,29 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (args->offset > obj->base.size || args->size > obj->base.size - args->offset) { ret = -EINVAL; - goto err; + goto out; } trace_i915_gem_object_pread(obj, args->offset, args->size); - ret = __unsafe_wait_rendering(obj, to_rps_client(file), true); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); if (ret) - goto err; + goto out; - ret = i915_mutex_lock_interruptible(dev); + ret = i915_gem_object_pin_pages(obj); if (ret) - goto err; + goto out; - ret = i915_gem_shmem_pread(dev, obj, args, file); - - /* pread for non shmem backed objects */ - if (ret == -EFAULT || ret == -ENODEV) { - intel_runtime_pm_get(to_i915(dev)); - ret = i915_gem_gtt_pread(dev, obj, args->size, - args->offset, args->data_ptr); - intel_runtime_pm_put(to_i915(dev)); - } + ret = i915_gem_shmem_pread(obj, args); + if (ret == -EFAULT || ret == -ENODEV) + ret = i915_gem_gtt_pread(obj, args); + i915_gem_object_unpin_pages(obj); +out: i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); - - return ret; - -err: - i915_gem_object_put_unlocked(obj); return ret; } @@ -1081,51 +1089,52 @@ err: * page faults in the source data */ -static inline int -fast_user_write(struct io_mapping *mapping, - loff_t page_base, int page_offset, - char __user *user_data, - int length) +static inline bool +ggtt_write(struct io_mapping *mapping, + loff_t base, int offset, + char __user *user_data, int length) { - void __iomem *vaddr_atomic; void *vaddr; unsigned long unwritten; - vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base); /* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force*)vaddr_atomic + page_offset; - unwritten = __copy_from_user_inatomic_nocache(vaddr, + vaddr = (void __force *)io_mapping_map_atomic_wc(mapping, base); + unwritten = __copy_from_user_inatomic_nocache(vaddr + offset, user_data, length); - io_mapping_unmap_atomic(vaddr_atomic); + io_mapping_unmap_atomic(vaddr); + if (unwritten) { + vaddr = (void __force *) + io_mapping_map_wc(mapping, base, PAGE_SIZE); + unwritten = copy_from_user(vaddr + offset, user_data, length); + io_mapping_unmap(vaddr); + } + return unwritten; } /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. - * @i915: i915 device private data - * @obj: i915 gem object + * @obj: i915 GEM object * @args: pwrite arguments structure - * @file: drm file pointer */ static int -i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_ggtt *ggtt = &i915->ggtt; - struct drm_device *dev = obj->base.dev; - struct i915_vma *vma; struct drm_mm_node node; - uint64_t remain, offset; - char __user *user_data; + struct i915_vma *vma; + u64 remain, offset; + void __user *user_data; int ret; - bool hit_slow_path = false; - if (i915_gem_object_is_tiled(obj)) - return -EFAULT; + ret = mutex_lock_interruptible(&i915->drm.struct_mutex); + if (ret) + return ret; + intel_runtime_pm_get(i915); vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE | PIN_NONBLOCK); if (!IS_ERR(vma)) { @@ -1138,25 +1147,19 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, } } if (IS_ERR(vma)) { - ret = insert_mappable_node(i915, &node, PAGE_SIZE); + ret = insert_mappable_node(ggtt, &node, PAGE_SIZE); if (ret) - goto out; - - ret = i915_gem_object_get_pages(obj); - if (ret) { - remove_mappable_node(&node); - goto out; - } - - i915_gem_object_pin_pages(obj); + goto out_unlock; + GEM_BUG_ON(!node.allocated); } ret = i915_gem_object_set_to_gtt_domain(obj, true); if (ret) goto out_unpin; + mutex_unlock(&i915->drm.struct_mutex); + intel_fb_obj_invalidate(obj, ORIGIN_CPU); - obj->dirty = true; user_data = u64_to_user_ptr(args->data_ptr); offset = args->offset; @@ -1169,8 +1172,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, * page_length = bytes to copy for this page */ u32 page_base = node.start; - unsigned page_offset = offset_in_page(offset); - unsigned page_length = PAGE_SIZE - page_offset; + unsigned int page_offset = offset_in_page(offset); + unsigned int page_length = PAGE_SIZE - page_offset; page_length = remain < page_length ? remain : page_length; if (node.allocated) { wmb(); /* flush the write before we modify the GGTT */ @@ -1187,91 +1190,36 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, * If the object is non-shmem backed, we retry again with the * path that handles page fault. */ - if (fast_user_write(&ggtt->mappable, page_base, - page_offset, user_data, page_length)) { - hit_slow_path = true; - mutex_unlock(&dev->struct_mutex); - if (slow_user_access(&ggtt->mappable, - page_base, - page_offset, user_data, - page_length, true)) { - ret = -EFAULT; - mutex_lock(&dev->struct_mutex); - goto out_flush; - } - - mutex_lock(&dev->struct_mutex); + if (ggtt_write(&ggtt->mappable, page_base, page_offset, + user_data, page_length)) { + ret = -EFAULT; + break; } remain -= page_length; user_data += page_length; offset += page_length; } - -out_flush: - if (hit_slow_path) { - if (ret == 0 && - (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) { - /* The user has modified the object whilst we tried - * reading from it, and we now have no idea what domain - * the pages should be in. As we have just been touching - * them directly, flush everything back to the GTT - * domain. - */ - ret = i915_gem_object_set_to_gtt_domain(obj, false); - } - } - intel_fb_obj_flush(obj, false, ORIGIN_CPU); + + mutex_lock(&i915->drm.struct_mutex); out_unpin: if (node.allocated) { wmb(); ggtt->base.clear_range(&ggtt->base, node.start, node.size); - i915_gem_object_unpin_pages(obj); remove_mappable_node(&node); } else { i915_vma_unpin(vma); } -out: +out_unlock: + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); return ret; } -/* Per-page copy function for the shmem pwrite fastpath. - * Flushes invalid cachelines before writing to the target if - * needs_clflush_before is set and flushes out any written cachelines after - * writing if needs_clflush is set. */ -static int -shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length, - char __user *user_data, - bool page_do_bit17_swizzling, - bool needs_clflush_before, - bool needs_clflush_after) -{ - char *vaddr; - int ret; - - if (unlikely(page_do_bit17_swizzling)) - return -EINVAL; - - vaddr = kmap_atomic(page); - if (needs_clflush_before) - drm_clflush_virt_range(vaddr + shmem_page_offset, - page_length); - ret = __copy_from_user_inatomic(vaddr + shmem_page_offset, - user_data, page_length); - if (needs_clflush_after) - drm_clflush_virt_range(vaddr + shmem_page_offset, - page_length); - kunmap_atomic(vaddr); - - return ret ? -EFAULT : 0; -} - -/* Only difference to the fast-path function is that this can handle bit17 - * and uses non-atomic copy and kmap functions. */ static int -shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, +shmem_pwrite_slow(struct page *page, int offset, int length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush_before, @@ -1282,124 +1230,114 @@ shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, vaddr = kmap(page); if (unlikely(needs_clflush_before || page_do_bit17_swizzling)) - shmem_clflush_swizzled_range(vaddr + shmem_page_offset, - page_length, + shmem_clflush_swizzled_range(vaddr + offset, length, page_do_bit17_swizzling); if (page_do_bit17_swizzling) - ret = __copy_from_user_swizzled(vaddr, shmem_page_offset, - user_data, - page_length); + ret = __copy_from_user_swizzled(vaddr, offset, user_data, + length); else - ret = __copy_from_user(vaddr + shmem_page_offset, - user_data, - page_length); + ret = __copy_from_user(vaddr + offset, user_data, length); if (needs_clflush_after) - shmem_clflush_swizzled_range(vaddr + shmem_page_offset, - page_length, + shmem_clflush_swizzled_range(vaddr + offset, length, page_do_bit17_swizzling); kunmap(page); return ret ? -EFAULT : 0; } +/* Per-page copy function for the shmem pwrite fastpath. + * Flushes invalid cachelines before writing to the target if + * needs_clflush_before is set and flushes out any written cachelines after + * writing if needs_clflush is set. + */ static int -i915_gem_shmem_pwrite(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, + bool page_do_bit17_swizzling, + bool needs_clflush_before, + bool needs_clflush_after) { - ssize_t remain; - loff_t offset; - char __user *user_data; - int shmem_page_offset, page_length, ret = 0; - int obj_do_bit17_swizzling, page_do_bit17_swizzling; - int hit_slowpath = 0; - unsigned int needs_clflush; - struct sg_page_iter sg_iter; + int ret; - ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); - if (ret) - return ret; + ret = -ENODEV; + if (!page_do_bit17_swizzling) { + char *vaddr = kmap_atomic(page); - obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - user_data = u64_to_user_ptr(args->data_ptr); - offset = args->offset; - remain = args->size; + if (needs_clflush_before) + drm_clflush_virt_range(vaddr + offset, len); + ret = __copy_from_user_inatomic(vaddr + offset, user_data, len); + if (needs_clflush_after) + drm_clflush_virt_range(vaddr + offset, len); - for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, - offset >> PAGE_SHIFT) { - struct page *page = sg_page_iter_page(&sg_iter); - int partial_cacheline_write; + kunmap_atomic(vaddr); + } + if (ret == 0) + return ret; - if (remain <= 0) - break; + return shmem_pwrite_slow(page, offset, len, user_data, + page_do_bit17_swizzling, + needs_clflush_before, + needs_clflush_after); +} - /* Operation in this page - * - * shmem_page_offset = offset within page in shmem file - * page_length = bytes to copy for this page - */ - shmem_page_offset = offset_in_page(offset); - - page_length = remain; - if ((shmem_page_offset + page_length) > PAGE_SIZE) - page_length = PAGE_SIZE - shmem_page_offset; - - /* If we don't overwrite a cacheline completely we need to be - * careful to have up-to-date data by first clflushing. Don't - * overcomplicate things and flush the entire patch. */ - partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE && - ((shmem_page_offset | page_length) - & (boot_cpu_data.x86_clflush_size - 1)); - - page_do_bit17_swizzling = obj_do_bit17_swizzling && - (page_to_phys(page) & (1 << 17)) != 0; - - ret = shmem_pwrite_fast(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - partial_cacheline_write, - needs_clflush & CLFLUSH_AFTER); - if (ret == 0) - goto next_page; - - hit_slowpath = 1; - mutex_unlock(&dev->struct_mutex); - ret = shmem_pwrite_slow(page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - partial_cacheline_write, - needs_clflush & CLFLUSH_AFTER); +static int +i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + void __user *user_data; + u64 remain; + unsigned int obj_do_bit17_swizzling; + unsigned int partial_cacheline_write; + unsigned int needs_clflush; + unsigned int offset, idx; + int ret; - mutex_lock(&dev->struct_mutex); + ret = mutex_lock_interruptible(&i915->drm.struct_mutex); + if (ret) + return ret; - if (ret) - goto out; + ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); + mutex_unlock(&i915->drm.struct_mutex); + if (ret) + return ret; -next_page: - remain -= page_length; - user_data += page_length; - offset += page_length; - } + obj_do_bit17_swizzling = 0; + if (i915_gem_object_needs_bit17_swizzle(obj)) + obj_do_bit17_swizzling = BIT(17); -out: - i915_gem_obj_finish_shmem_access(obj); + /* If we don't overwrite a cacheline completely we need to be + * careful to have up-to-date data by first clflushing. Don't + * overcomplicate things and flush the entire patch. + */ + partial_cacheline_write = 0; + if (needs_clflush & CLFLUSH_BEFORE) + partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1; - if (hit_slowpath) { - /* - * Fixup: Flush cpu caches in case we didn't flush the dirty - * cachelines in-line while writing and the object moved - * out of the cpu write domain while we've dropped the lock. - */ - if (!(needs_clflush & CLFLUSH_AFTER) && - obj->base.write_domain != I915_GEM_DOMAIN_CPU) { - if (i915_gem_clflush_object(obj, obj->pin_display)) - needs_clflush |= CLFLUSH_AFTER; - } - } + user_data = u64_to_user_ptr(args->data_ptr); + remain = args->size; + offset = offset_in_page(args->offset); + for (idx = args->offset >> PAGE_SHIFT; remain; idx++) { + struct page *page = i915_gem_object_get_page(obj, idx); + int length; + + length = remain; + if (offset + length > PAGE_SIZE) + length = PAGE_SIZE - offset; + + ret = shmem_pwrite(page, offset, length, user_data, + page_to_phys(page) & obj_do_bit17_swizzling, + (offset | length) & partial_cacheline_write, + needs_clflush & CLFLUSH_AFTER); + if (ret) + break; - if (needs_clflush & CLFLUSH_AFTER) - i915_gem_chipset_flush(to_i915(dev)); + remain -= length; + user_data += length; + offset = 0; + } intel_fb_obj_flush(obj, false, ORIGIN_CPU); + i915_gem_obj_finish_shmem_access(obj); return ret; } @@ -1415,7 +1353,6 @@ int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_pwrite *args = data; struct drm_i915_gem_object *obj; int ret; @@ -1428,13 +1365,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, args->size)) return -EFAULT; - if (likely(!i915.prefault_disable)) { - ret = fault_in_pages_readable(u64_to_user_ptr(args->data_ptr), - args->size); - if (ret) - return -EFAULT; - } - obj = i915_gem_object_lookup(file, args->handle); if (!obj) return -ENOENT; @@ -1448,15 +1378,17 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, trace_i915_gem_object_pwrite(obj, args->offset, args->size); - ret = __unsafe_wait_rendering(obj, to_rps_client(file), false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); if (ret) goto err; - intel_runtime_pm_get(dev_priv); - - ret = i915_mutex_lock_interruptible(dev); + ret = i915_gem_object_pin_pages(obj); if (ret) - goto err_rpm; + goto err; ret = -EFAULT; /* We can only do the GTT pwrite on untiled buffers, as otherwise @@ -1466,30 +1398,23 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, * perspective, requiring manual detiling by the client. */ if (!i915_gem_object_has_struct_page(obj) || - cpu_write_needs_clflush(obj)) { - ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file); + cpu_write_needs_clflush(obj)) /* Note that the gtt paths might fail with non-page-backed user * pointers (e.g. gtt mappings when moving data between - * textures). Fallback to the shmem path in that case. */ - } + * textures). Fallback to the shmem path in that case. + */ + ret = i915_gem_gtt_pwrite_fast(obj, args); if (ret == -EFAULT || ret == -ENOSPC) { if (obj->phys_handle) ret = i915_gem_phys_pwrite(obj, args, file); else - ret = i915_gem_shmem_pwrite(dev, obj, args, file); + ret = i915_gem_shmem_pwrite(obj, args); } - i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); - intel_runtime_pm_put(dev_priv); - - return ret; - -err_rpm: - intel_runtime_pm_put(dev_priv); + i915_gem_object_unpin_pages(obj); err: - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return ret; } @@ -1500,6 +1425,30 @@ write_origin(struct drm_i915_gem_object *obj, unsigned domain) obj->frontbuffer_ggtt_origin : ORIGIN_CPU); } +static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *i915; + struct list_head *list; + struct i915_vma *vma; + + list_for_each_entry(vma, &obj->vma_list, obj_link) { + if (!i915_vma_is_ggtt(vma)) + continue; + + if (i915_vma_is_active(vma)) + continue; + + if (!drm_mm_node_allocated(&vma->node)) + continue; + + list_move_tail(&vma->vm_link, &vma->vm->inactive_list); + } + + i915 = to_i915(obj->base.dev); + list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; + list_move_tail(&obj->global_link, list); +} + /** * Called when user space prepares to use an object with the CPU, either * through the mmap ioctl's mapping or a GTT mapping. @@ -1515,7 +1464,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; uint32_t read_domains = args->read_domains; uint32_t write_domain = args->write_domain; - int ret; + int err; /* Only handle setting domains to types used by the CPU. */ if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS) @@ -1535,29 +1484,48 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, * We will repeat the flush holding the lock in the normal manner * to catch cases where we are gazumped. */ - ret = __unsafe_wait_rendering(obj, to_rps_client(file), !write_domain); - if (ret) - goto err; + err = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + (write_domain ? I915_WAIT_ALL : 0), + MAX_SCHEDULE_TIMEOUT, + to_rps_client(file)); + if (err) + goto out; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - goto err; + /* Flush and acquire obj->pages so that we are coherent through + * direct access in memory with previous cached writes through + * shmemfs and that our cache domain tracking remains valid. + * For example, if the obj->filp was moved to swap without us + * being notified and releasing the pages, we would mistakenly + * continue to assume that the obj remained out of the CPU cached + * domain. + */ + err = i915_gem_object_pin_pages(obj); + if (err) + goto out; + + err = i915_mutex_lock_interruptible(dev); + if (err) + goto out_unpin; if (read_domains & I915_GEM_DOMAIN_GTT) - ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); + err = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); else - ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); + err = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); + + /* And bump the LRU for this access */ + i915_gem_object_bump_inactive_ggtt(obj); + + mutex_unlock(&dev->struct_mutex); if (write_domain != 0) intel_fb_obj_invalidate(obj, write_origin(obj, write_domain)); +out_unpin: + i915_gem_object_unpin_pages(obj); +out: i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); - return ret; - -err: - i915_gem_object_put_unlocked(obj); - return ret; + return err; } /** @@ -1587,7 +1555,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, } } - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return err; } @@ -1633,7 +1601,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, * pages from. */ if (!obj->base.filp) { - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return -EINVAL; } @@ -1645,7 +1613,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct vm_area_struct *vma; if (down_write_killable(&mm->mmap_sem)) { - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return -EINTR; } vma = find_vma(mm, addr); @@ -1659,7 +1627,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, /* This may race, but that's ok, it only gets set */ WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU); } - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); if (IS_ERR((void *)addr)) return addr; @@ -1771,7 +1739,14 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) * repeat the flush holding the lock in the normal manner to catch cases * where we are gazumped. */ - ret = __unsafe_wait_rendering(obj, NULL, !write); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT, + NULL); + if (ret) + goto err; + + ret = i915_gem_object_pin_pages(obj); if (ret) goto err; @@ -1804,7 +1779,7 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) /* Use a partial view if it is bigger than available space */ chunk_size = MIN_CHUNK_PAGES; if (i915_gem_object_is_tiled(obj)) - chunk_size = max(chunk_size, tile_row_pages(obj)); + chunk_size = roundup(chunk_size, tile_row_pages(obj)); memset(&view, 0, sizeof(view)); view.type = I915_GGTT_VIEW_PARTIAL; @@ -1839,22 +1814,25 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf) if (ret) goto err_unpin; + /* Mark as being mmapped into userspace for later revocation */ + assert_rpm_wakelock_held(dev_priv); + if (list_empty(&obj->userfault_link)) + list_add(&obj->userfault_link, &dev_priv->mm.userfault_list); + /* Finally, remap it using the new GTT offset */ ret = remap_io_mapping(area, area->vm_start + (vma->ggtt_view.params.partial.offset << PAGE_SHIFT), (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT, min_t(u64, vma->size, area->vm_end - area->vm_start), &ggtt->mappable); - if (ret) - goto err_unpin; - obj->fault_mappable = true; err_unpin: __i915_vma_unpin(vma); err_unlock: mutex_unlock(&dev->struct_mutex); err_rpm: intel_runtime_pm_put(dev_priv); + i915_gem_object_unpin_pages(obj); err: switch (ret) { case -EIO: @@ -1916,15 +1894,23 @@ err: void i915_gem_release_mmap(struct drm_i915_gem_object *obj) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + /* Serialisation between user GTT access and our code depends upon * revoking the CPU's PTE whilst the mutex is held. The next user * pagefault then has to wait until we release the mutex. + * + * Note that RPM complicates somewhat by adding an additional + * requirement that operations to the GGTT be made holding the RPM + * wakeref. */ - lockdep_assert_held(&obj->base.dev->struct_mutex); + lockdep_assert_held(&i915->drm.struct_mutex); + intel_runtime_pm_get(i915); - if (!obj->fault_mappable) - return; + if (list_empty(&obj->userfault_link)) + goto out; + list_del_init(&obj->userfault_link); drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->anon_inode->i_mapping); @@ -1937,16 +1923,45 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj) */ wmb(); - obj->fault_mappable = false; +out: + intel_runtime_pm_put(i915); } -void -i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv) +void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv) { - struct drm_i915_gem_object *obj; + struct drm_i915_gem_object *obj, *on; + int i; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) - i915_gem_release_mmap(obj); + /* + * Only called during RPM suspend. All users of the userfault_list + * must be holding an RPM wakeref to ensure that this can not + * run concurrently with themselves (and use the struct_mutex for + * protection between themselves). + */ + + list_for_each_entry_safe(obj, on, + &dev_priv->mm.userfault_list, userfault_link) { + list_del_init(&obj->userfault_link); + drm_vma_node_unmap(&obj->base.vma_node, + obj->base.dev->anon_inode->i_mapping); + } + + /* The fence will be lost when the device powers down. If any were + * in use by hardware (i.e. they are pinned), we should not be powering + * down! All other fences will be reacquired by the user upon waking. + */ + for (i = 0; i < dev_priv->num_fence_regs; i++) { + struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; + + if (WARN_ON(reg->pin_count)) + continue; + + if (!reg->vma) + continue; + + GEM_BUG_ON(!list_empty(®->vma->obj->userfault_link)); + reg->dirty = true; + } } /** @@ -2060,7 +2075,7 @@ i915_gem_mmap_gtt(struct drm_file *file, if (ret == 0) *offset = drm_vma_node_offset_addr(&obj->base.vma_node); - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return ret; } @@ -2103,16 +2118,18 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj) * backing pages, *now*. */ shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1); - obj->madv = __I915_MADV_PURGED; + obj->mm.madv = __I915_MADV_PURGED; } /* Try to discard unwanted pages */ -static void -i915_gem_object_invalidate(struct drm_i915_gem_object *obj) +void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj) { struct address_space *mapping; - switch (obj->madv) { + lockdep_assert_held(&obj->mm.lock); + GEM_BUG_ON(obj->mm.pages); + + switch (obj->mm.madv) { case I915_MADV_DONTNEED: i915_gem_object_truncate(obj); case __I915_MADV_PURGED: @@ -2127,82 +2144,83 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) } static void -i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) +i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct sgt_iter sgt_iter; struct page *page; - int ret; - BUG_ON(obj->madv == __I915_MADV_PURGED); + __i915_gem_object_release_shmem(obj); - ret = i915_gem_object_set_to_cpu_domain(obj, true); - if (WARN_ON(ret)) { - /* In the event of a disaster, abandon all caches and - * hope for the best. - */ - i915_gem_clflush_object(obj, true); - obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; - } - - i915_gem_gtt_finish_object(obj); + i915_gem_gtt_finish_pages(obj, pages); if (i915_gem_object_needs_bit17_swizzle(obj)) - i915_gem_object_save_bit_17_swizzle(obj); - - if (obj->madv == I915_MADV_DONTNEED) - obj->dirty = 0; + i915_gem_object_save_bit_17_swizzle(obj, pages); - for_each_sgt_page(page, sgt_iter, obj->pages) { - if (obj->dirty) + for_each_sgt_page(page, sgt_iter, pages) { + if (obj->mm.dirty) set_page_dirty(page); - if (obj->madv == I915_MADV_WILLNEED) + if (obj->mm.madv == I915_MADV_WILLNEED) mark_page_accessed(page); put_page(page); } - obj->dirty = 0; + obj->mm.dirty = false; - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } -int -i915_gem_object_put_pages(struct drm_i915_gem_object *obj) +static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj) { - const struct drm_i915_gem_object_ops *ops = obj->ops; + struct radix_tree_iter iter; + void **slot; - if (obj->pages == NULL) - return 0; + radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0) + radix_tree_delete(&obj->mm.get_page.radix, iter.index); +} - if (obj->pages_pin_count) - return -EBUSY; +void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass) +{ + struct sg_table *pages; + + if (i915_gem_object_has_pinned_pages(obj)) + return; GEM_BUG_ON(obj->bind_count); + if (!READ_ONCE(obj->mm.pages)) + return; + + /* May be called by shrinker from within get_pages() (on another bo) */ + mutex_lock_nested(&obj->mm.lock, subclass); + if (unlikely(atomic_read(&obj->mm.pages_pin_count))) + goto unlock; /* ->put_pages might need to allocate memory for the bit17 swizzle * array, hence protect them from being reaped by removing them from gtt * lists early. */ - list_del(&obj->global_list); + pages = fetch_and_zero(&obj->mm.pages); + GEM_BUG_ON(!pages); - if (obj->mapping) { + if (obj->mm.mapping) { void *ptr; - ptr = ptr_mask_bits(obj->mapping); + ptr = ptr_mask_bits(obj->mm.mapping); if (is_vmalloc_addr(ptr)) vunmap(ptr); else kunmap(kmap_to_page(ptr)); - obj->mapping = NULL; + obj->mm.mapping = NULL; } - ops->put_pages(obj); - obj->pages = NULL; - - i915_gem_object_invalidate(obj); + __i915_gem_object_reset_page_iter(obj); - return 0; + obj->ops->put_pages(obj, pages); +unlock: + mutex_unlock(&obj->mm.lock); } static unsigned int swiotlb_max_size(void) @@ -2214,7 +2232,7 @@ static unsigned int swiotlb_max_size(void) #endif } -static int +static struct sg_table * i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); @@ -2233,8 +2251,8 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) * wasn't in the GTT, there shouldn't be any way it could have been in * a GPU cache */ - BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); - BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); + GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); + GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); max_segment = swiotlb_max_size(); if (!max_segment) @@ -2242,12 +2260,12 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) st = kmalloc(sizeof(*st), GFP_KERNEL); if (st == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); page_count = obj->base.size / PAGE_SIZE; if (sg_alloc_table(st, page_count, GFP_KERNEL)) { kfree(st); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } /* Get the list of pages out of our struct file. They'll be pinned @@ -2298,20 +2316,15 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) } if (sg) /* loop terminated early; short sg table */ sg_mark_end(sg); - obj->pages = st; - ret = i915_gem_gtt_prepare_object(obj); + ret = i915_gem_gtt_prepare_pages(obj, st); if (ret) goto err_pages; if (i915_gem_object_needs_bit17_swizzle(obj)) - i915_gem_object_do_bit_17_swizzle(obj); + i915_gem_object_do_bit_17_swizzle(obj, st); - if (i915_gem_object_is_tiled(obj) && - dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) - i915_gem_object_pin_pages(obj); - - return 0; + return st; err_pages: sg_mark_end(sg); @@ -2331,43 +2344,73 @@ err_pages: if (ret == -ENOSPC) ret = -ENOMEM; - return ret; + return ERR_PTR(ret); } -/* Ensure that the associated pages are gathered from the backing storage - * and pinned into our object. i915_gem_object_get_pages() may be called - * multiple times before they are released by a single call to - * i915_gem_object_put_pages() - once the pages are no longer referenced - * either as a result of memory pressure (reaping pages under the shrinker) - * or as the object is itself released. - */ -int -i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - struct drm_i915_private *dev_priv = to_i915(obj->base.dev); - const struct drm_i915_gem_object_ops *ops = obj->ops; - int ret; + lockdep_assert_held(&obj->mm.lock); - if (obj->pages) - return 0; + obj->mm.get_page.sg_pos = pages->sgl; + obj->mm.get_page.sg_idx = 0; + + obj->mm.pages = pages; + + if (i915_gem_object_is_tiled(obj) && + to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES) { + GEM_BUG_ON(obj->mm.quirked); + __i915_gem_object_pin_pages(obj); + obj->mm.quirked = true; + } +} + +static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +{ + struct sg_table *pages; + + GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj)); - if (obj->madv != I915_MADV_WILLNEED) { + if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { DRM_DEBUG("Attempting to obtain a purgeable object\n"); return -EFAULT; } - BUG_ON(obj->pages_pin_count); + pages = obj->ops->get_pages(obj); + if (unlikely(IS_ERR(pages))) + return PTR_ERR(pages); - ret = ops->get_pages(obj); - if (ret) - return ret; + __i915_gem_object_set_pages(obj, pages); + return 0; +} - list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list); +/* Ensure that the associated pages are gathered from the backing storage + * and pinned into our object. i915_gem_object_pin_pages() may be called + * multiple times before they are released by a single call to + * i915_gem_object_unpin_pages() - once the pages are no longer referenced + * either as a result of memory pressure (reaping pages under the shrinker) + * or as the object is itself released. + */ +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) +{ + int err; - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; + err = mutex_lock_interruptible(&obj->mm.lock); + if (err) + return err; - return 0; + if (unlikely(!obj->mm.pages)) { + err = ____i915_gem_object_get_pages(obj); + if (err) + goto unlock; + + smp_mb__before_atomic(); + } + atomic_inc(&obj->mm.pages_pin_count); + +unlock: + mutex_unlock(&obj->mm.lock); + return err; } /* The 'mapping' part of i915_gem_object_pin_map() below */ @@ -2375,7 +2418,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, enum i915_map_type type) { unsigned long n_pages = obj->base.size >> PAGE_SHIFT; - struct sg_table *sgt = obj->pages; + struct sg_table *sgt = obj->mm.pages; struct sgt_iter sgt_iter; struct page *page; struct page *stack_pages[32]; @@ -2426,21 +2469,31 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, void *ptr; int ret; - lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); - ret = i915_gem_object_get_pages(obj); + ret = mutex_lock_interruptible(&obj->mm.lock); if (ret) return ERR_PTR(ret); - i915_gem_object_pin_pages(obj); - pinned = obj->pages_pin_count > 1; + pinned = true; + if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) { + if (unlikely(!obj->mm.pages)) { + ret = ____i915_gem_object_get_pages(obj); + if (ret) + goto err_unlock; - ptr = ptr_unpack_bits(obj->mapping, has_type); + smp_mb__before_atomic(); + } + atomic_inc(&obj->mm.pages_pin_count); + pinned = false; + } + GEM_BUG_ON(!obj->mm.pages); + + ptr = ptr_unpack_bits(obj->mm.mapping, has_type); if (ptr && has_type != type) { if (pinned) { ret = -EBUSY; - goto err; + goto err_unpin; } if (is_vmalloc_addr(ptr)) @@ -2448,59 +2501,28 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, else kunmap(kmap_to_page(ptr)); - ptr = obj->mapping = NULL; + ptr = obj->mm.mapping = NULL; } if (!ptr) { ptr = i915_gem_object_map(obj, type); if (!ptr) { ret = -ENOMEM; - goto err; + goto err_unpin; } - obj->mapping = ptr_pack_bits(ptr, type); + obj->mm.mapping = ptr_pack_bits(ptr, type); } +out_unlock: + mutex_unlock(&obj->mm.lock); return ptr; -err: - i915_gem_object_unpin_pages(obj); - return ERR_PTR(ret); -} - -static void -i915_gem_object_retire__write(struct i915_gem_active *active, - struct drm_i915_gem_request *request) -{ - struct drm_i915_gem_object *obj = - container_of(active, struct drm_i915_gem_object, last_write); - - intel_fb_obj_flush(obj, true, ORIGIN_CS); -} - -static void -i915_gem_object_retire__read(struct i915_gem_active *active, - struct drm_i915_gem_request *request) -{ - int idx = request->engine->id; - struct drm_i915_gem_object *obj = - container_of(active, struct drm_i915_gem_object, last_read[idx]); - - GEM_BUG_ON(!i915_gem_object_has_active_engine(obj, idx)); - - i915_gem_object_clear_active(obj, idx); - if (i915_gem_object_is_active(obj)) - return; - - /* Bump our place on the bound list to keep it roughly in LRU order - * so that we don't steal from recently used but inactive objects - * (unless we are forced to ofc!) - */ - if (obj->bind_count) - list_move_tail(&obj->global_list, - &request->i915->mm.bound_list); - - i915_gem_object_put(obj); +err_unpin: + atomic_dec(&obj->mm.pages_pin_count); +err_unlock: + ptr = ERR_PTR(ret); + goto out_unlock; } static bool i915_context_is_banned(const struct i915_gem_context *ctx) @@ -2547,13 +2569,10 @@ i915_gem_find_active_request(struct intel_engine_cs *engine) * extra delay for a recent interrupt is pointless. Hence, we do * not need an engine->irq_seqno_barrier() before the seqno reads. */ - list_for_each_entry(request, &engine->request_list, link) { - if (i915_gem_request_completed(request)) + list_for_each_entry(request, &engine->timeline->requests, link) { + if (__i915_gem_request_completed(request)) continue; - if (!i915_sw_fence_done(&request->submit)) - break; - return request; } @@ -2581,6 +2600,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request; struct i915_gem_context *incomplete_ctx; + struct intel_timeline *timeline; bool ring_hung; if (engine->irq_seqno_barrier) @@ -2599,7 +2619,7 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) return; DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n", - engine->name, request->fence.seqno); + engine->name, request->global_seqno); /* Setup the CS to resume from the breadcrumb of the hung request */ engine->reset_hw(engine, request); @@ -2616,9 +2636,13 @@ static void i915_gem_reset_engine(struct intel_engine_cs *engine) if (i915_gem_context_is_default(incomplete_ctx)) return; - list_for_each_entry_continue(request, &engine->request_list, link) + list_for_each_entry_continue(request, &engine->timeline->requests, link) if (request->ctx == incomplete_ctx) reset_request(request); + + timeline = i915_gem_context_lookup_timeline(incomplete_ctx, engine); + list_for_each_entry(request, &timeline->requests, link) + reset_request(request); } void i915_gem_reset(struct drm_i915_private *dev_priv) @@ -2626,6 +2650,8 @@ void i915_gem_reset(struct drm_i915_private *dev_priv) struct intel_engine_cs *engine; enum intel_engine_id id; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + i915_gem_retire_requests(dev_priv); for_each_engine(engine, dev_priv, id) @@ -2653,7 +2679,8 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) * (lockless) lookup doesn't try and wait upon the request as we * reset it. */ - intel_engine_init_seqno(engine, engine->last_submitted_seqno); + intel_engine_init_global_seqno(engine, + intel_engine_last_submit(engine)); /* * Clear the execlists queue up before freeing the requests, as those @@ -2669,8 +2696,6 @@ static void i915_gem_cleanup_engine(struct intel_engine_cs *engine) memset(engine->execlist_port, 0, sizeof(engine->execlist_port)); spin_unlock(&engine->execlist_lock); } - - engine->i915->gt.active_engines &= ~intel_engine_flag(engine); } void i915_gem_set_wedged(struct drm_i915_private *dev_priv) @@ -2727,7 +2752,14 @@ i915_gem_idle_work_handler(struct work_struct *work) if (!READ_ONCE(dev_priv->gt.awake)) return; - if (READ_ONCE(dev_priv->gt.active_engines)) + /* + * Wait for last execlists context complete, but bail out in case a + * new request is submitted. + */ + wait_for(READ_ONCE(dev_priv->gt.active_requests) || + intel_execlists_idle(dev_priv), 10); + + if (READ_ONCE(dev_priv->gt.active_requests)) return; rearm_hangcheck = @@ -2741,9 +2773,19 @@ i915_gem_idle_work_handler(struct work_struct *work) goto out_rearm; } - if (dev_priv->gt.active_engines) + /* + * New request retired after this work handler started, extend active + * period until next instance of the work. + */ + if (work_pending(work)) + goto out_unlock; + + if (dev_priv->gt.active_requests) goto out_unlock; + if (wait_for(intel_execlists_idle(dev_priv), 10)) + DRM_ERROR("Timeout waiting for engines to idle\n"); + for_each_engine(engine, dev_priv, id) i915_gem_batch_pool_fini(&engine->batch_pool); @@ -2774,9 +2816,26 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link) if (vma->vm->file == fpriv) i915_vma_close(vma); + + if (i915_gem_object_is_active(obj) && + !i915_gem_object_has_active_reference(obj)) { + i915_gem_object_set_active_reference(obj); + i915_gem_object_get(obj); + } mutex_unlock(&obj->base.dev->struct_mutex); } +static unsigned long to_wait_timeout(s64 timeout_ns) +{ + if (timeout_ns < 0) + return MAX_SCHEDULE_TIMEOUT; + + if (timeout_ns == 0) + return 0; + + return nsecs_to_jiffies_timeout(timeout_ns); +} + /** * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT * @dev: drm device pointer @@ -2805,10 +2864,9 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_wait *args = data; - struct intel_rps_client *rps = to_rps_client(file); struct drm_i915_gem_object *obj; - unsigned long active; - int idx, ret = 0; + ktime_t start; + long ret; if (args->flags != 0) return -EINVAL; @@ -2817,17 +2875,20 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (!obj) return -ENOENT; - active = __I915_BO_ACTIVE(obj); - for_each_active(active, idx) { - s64 *timeout = args->timeout_ns >= 0 ? &args->timeout_ns : NULL; - ret = i915_gem_active_wait_unlocked(&obj->last_read[idx], - I915_WAIT_INTERRUPTIBLE, - timeout, rps); - if (ret) - break; + start = ktime_get(); + + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL, + to_wait_timeout(args->timeout_ns), + to_rps_client(file)); + + if (args->timeout_ns > 0) { + args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start)); + if (args->timeout_ns < 0) + args->timeout_ns = 0; } - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return ret; } @@ -2848,6 +2909,8 @@ int i915_vma_unbind(struct i915_vma *vma) unsigned long active; int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + /* First wait upon any activity as retiring the request may * have side-effects such as unpinning or even unbinding this vma. */ @@ -2859,6 +2922,13 @@ int i915_vma_unbind(struct i915_vma *vma) * In order to prevent it from being recursively closed, * take a pin on the vma so that the second unbind is * aborted. + * + * Even more scary is that the retire callback may free + * the object (last active vma). To prevent the explosion + * we defer the actual object free to a worker that can + * only proceed once it acquires the struct_mutex (which + * we currently hold, therefore it cannot free this object + * before we are finished). */ __i915_vma_pin(vma); @@ -2883,7 +2953,7 @@ int i915_vma_unbind(struct i915_vma *vma) goto destroy; GEM_BUG_ON(obj->bind_count == 0); - GEM_BUG_ON(!obj->pages); + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); if (i915_vma_is_map_and_fenceable(vma)) { /* release the fence reg _after_ flushing */ @@ -2907,7 +2977,7 @@ int i915_vma_unbind(struct i915_vma *vma) drm_mm_remove_node(&vma->node); list_move_tail(&vma->vm_link, &vma->vm->unbound_list); - if (vma->pages != obj->pages) { + if (vma->pages != obj->mm.pages) { GEM_BUG_ON(!vma->pages); sg_free_table(vma->pages); kfree(vma->pages); @@ -2917,7 +2987,7 @@ int i915_vma_unbind(struct i915_vma *vma) /* Since the unbound list is global, only move to that list if * no more VMAs exist. */ if (--obj->bind_count == 0) - list_move_tail(&obj->global_list, + list_move_tail(&obj->global_link, &to_i915(obj->base.dev)->mm.unbound_list); /* And finally now the object is completely decoupled from this vma, @@ -2933,18 +3003,26 @@ destroy: return 0; } -int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, - unsigned int flags) +static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags) { - struct intel_engine_cs *engine; - enum intel_engine_id id; - int ret; + int ret, i; - for_each_engine(engine, dev_priv, id) { - if (engine->last_context == NULL) - continue; + for (i = 0; i < ARRAY_SIZE(tl->engine); i++) { + ret = i915_gem_active_wait(&tl->engine[i].last_request, flags); + if (ret) + return ret; + } - ret = intel_engine_idle(engine, flags); + return 0; +} + +int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags) +{ + struct i915_gem_timeline *tl; + int ret; + + list_for_each_entry(tl, &i915->gt.timelines, link) { + ret = wait_for_timeline(tl, flags); if (ret) return ret; } @@ -3040,12 +3118,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) return -E2BIG; } - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; - i915_gem_object_pin_pages(obj); - if (flags & PIN_OFFSET_FIXED) { u64 offset = flags & PIN_OFFSET_MASK; if (offset & (alignment - 1) || offset > end - size) { @@ -3108,9 +3184,10 @@ search_free: } GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); - list_move_tail(&obj->global_list, &dev_priv->mm.bound_list); + list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); list_move_tail(&vma->vm_link, &vma->vm->inactive_list); obj->bind_count++; + GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); return 0; @@ -3127,7 +3204,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, * to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ - if (obj->pages == NULL) + if (!obj->mm.pages) return false; /* @@ -3151,7 +3228,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, } trace_i915_gem_object_clflush(obj); - drm_clflush_sg(obj->pages); + drm_clflush_sg(obj->mm.pages); obj->cache_dirty = false; return true; @@ -3211,24 +3288,6 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) I915_GEM_DOMAIN_CPU); } -static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) -{ - struct i915_vma *vma; - - list_for_each_entry(vma, &obj->vma_list, obj_link) { - if (!i915_vma_is_ggtt(vma)) - continue; - - if (i915_vma_is_active(vma)) - continue; - - if (!drm_mm_node_allocated(&vma->node)) - continue; - - list_move_tail(&vma->vm_link, &vma->vm->inactive_list); - } -} - /** * Moves a single object to the GTT read, and possibly write domain. * @obj: object to act on @@ -3243,7 +3302,14 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) uint32_t old_write_domain, old_read_domains; int ret; - ret = i915_gem_object_wait_rendering(obj, !write); + lockdep_assert_held(&obj->base.dev->struct_mutex); + + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + (write ? I915_WAIT_ALL : 0), + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -3258,7 +3324,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) * continue to assume that the obj remained out of the CPU cached * domain. */ - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) return ret; @@ -3277,21 +3343,19 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); + GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); obj->base.read_domains |= I915_GEM_DOMAIN_GTT; if (write) { obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->base.write_domain = I915_GEM_DOMAIN_GTT; - obj->dirty = 1; + obj->mm.dirty = true; } trace_i915_gem_object_change_domain(obj, old_read_domains, old_write_domain); - /* And bump the LRU for this access */ - i915_gem_object_bump_inactive_ggtt(obj); - + i915_gem_object_unpin_pages(obj); return 0; } @@ -3316,6 +3380,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, struct i915_vma *vma; int ret = 0; + lockdep_assert_held(&obj->base.dev->struct_mutex); + if (obj->cache_level == cache_level) goto out; @@ -3360,7 +3426,12 @@ restart: * If we wait upon the object, we know that all the bound * VMA are no longer active. */ - ret = i915_gem_object_wait_rendering(obj, false); + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + I915_WAIT_ALL, + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -3428,10 +3499,14 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; + int err = 0; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (!obj) { + err = -ENOENT; + goto out; + } switch (obj->cache_level) { case I915_CACHE_LLC: @@ -3447,15 +3522,15 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, args->caching = I915_CACHING_NONE; break; } - - i915_gem_object_put_unlocked(obj); - return 0; +out: + rcu_read_unlock(); + return err; } int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_caching *args = data; struct drm_i915_gem_object *obj; enum i915_cache_level level; @@ -3472,23 +3547,21 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, * cacheline, whereas normally such cachelines would get * invalidated. */ - if (!HAS_LLC(dev) && !HAS_SNOOP(dev)) + if (!HAS_LLC(i915) && !HAS_SNOOP(i915)) return -ENODEV; level = I915_CACHE_LLC; break; case I915_CACHING_DISPLAY: - level = HAS_WT(dev_priv) ? I915_CACHE_WT : I915_CACHE_NONE; + level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE; break; default: return -EINVAL; } - intel_runtime_pm_get(dev_priv); - ret = i915_mutex_lock_interruptible(dev); if (ret) - goto rpm_put; + return ret; obj = i915_gem_object_lookup(file, args->handle); if (!obj) { @@ -3497,13 +3570,9 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, } ret = i915_gem_object_set_cache_level(obj, level); - i915_gem_object_put(obj); unlock: mutex_unlock(&dev->struct_mutex); -rpm_put: - intel_runtime_pm_put(dev_priv); - return ret; } @@ -3521,6 +3590,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, u32 old_read_domains, old_write_domain; int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + /* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. */ @@ -3554,15 +3625,27 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, if (view->type == I915_GGTT_VIEW_NORMAL) vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, PIN_MAPPABLE | PIN_NONBLOCK); - if (IS_ERR(vma)) - vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, 0); + if (IS_ERR(vma)) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned int flags; + + /* Valleyview is definitely limited to scanning out the first + * 512MiB. Lets presume this behaviour was inherited from the + * g4x display engine and that all earlier gen are similarly + * limited. Testing suggests that it is a little more + * complicated than this. For example, Cherryview appears quite + * happy to scanout from anywhere within its global aperture. + */ + flags = 0; + if (HAS_GMCH_DISPLAY(i915)) + flags = PIN_MAPPABLE; + vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags); + } if (IS_ERR(vma)) goto err_unpin_display; vma->display_alignment = max_t(u64, vma->display_alignment, alignment); - WARN_ON(obj->pin_display > i915_vma_pin_count(vma)); - i915_gem_object_flush_cpu_write_domain(obj); old_write_domain = obj->base.write_domain; @@ -3588,6 +3671,8 @@ err_unpin_display: void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) { + lockdep_assert_held(&vma->vm->dev->struct_mutex); + if (WARN_ON(vma->obj->pin_display == 0)) return; @@ -3599,7 +3684,6 @@ i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) list_move_tail(&vma->vm_link, &vma->vm->inactive_list); i915_vma_unpin(vma); - WARN_ON(vma->obj->pin_display > i915_vma_pin_count(vma)); } /** @@ -3616,7 +3700,14 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) uint32_t old_write_domain, old_read_domains; int ret; - ret = i915_gem_object_wait_rendering(obj, !write); + lockdep_assert_held(&obj->base.dev->struct_mutex); + + ret = i915_gem_object_wait(obj, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED | + (write ? I915_WAIT_ALL : 0), + MAX_SCHEDULE_TIMEOUT, + NULL); if (ret) return ret; @@ -3638,7 +3729,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); + GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); /* If we're writing through the CPU, then the GPU read domains will * need to be invalidated at next use. @@ -3672,11 +3763,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) struct drm_i915_file_private *file_priv = file->driver_priv; unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES; struct drm_i915_gem_request *request, *target = NULL; - int ret; - - ret = i915_gem_wait_for_error(&dev_priv->gpu_error); - if (ret) - return ret; + long ret; /* ABI: return -EIO if already wedged */ if (i915_terminally_wedged(&dev_priv->gpu_error)) @@ -3703,10 +3790,12 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) if (target == NULL) return 0; - ret = i915_wait_request(target, I915_WAIT_INTERRUPTIBLE, NULL, NULL); + ret = i915_wait_request(target, + I915_WAIT_INTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT); i915_gem_request_put(target); - return ret; + return ret < 0 ? ret : 0; } static bool @@ -3756,7 +3845,12 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) mappable = (vma->node.start + fence_size <= dev_priv->ggtt.mappable_end); - if (mappable && fenceable) + /* + * Explicitly disable for rotated VMA since the display does not + * need the fence and the VMA is not accessible to other users. + */ + if (mappable && fenceable && + vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED) vma->flags |= I915_VMA_CAN_FENCE; else vma->flags &= ~I915_VMA_CAN_FENCE; @@ -3768,6 +3862,7 @@ int __i915_vma_do_pin(struct i915_vma *vma, unsigned int bound = vma->flags; int ret; + lockdep_assert_held(&vma->vm->dev->struct_mutex); GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); @@ -3809,6 +3904,8 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, struct i915_vma *vma; int ret; + lockdep_assert_held(&obj->base.dev->struct_mutex); + vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view); if (IS_ERR(vma)) return vma; @@ -3899,83 +3996,42 @@ static __always_inline unsigned int __busy_write_id(unsigned int id) } static __always_inline unsigned int -__busy_set_if_active(const struct i915_gem_active *active, +__busy_set_if_active(const struct dma_fence *fence, unsigned int (*flag)(unsigned int id)) { - struct drm_i915_gem_request *request; - - request = rcu_dereference(active->request); - if (!request || i915_gem_request_completed(request)) - return 0; + struct drm_i915_gem_request *rq; - /* This is racy. See __i915_gem_active_get_rcu() for an in detail - * discussion of how to handle the race correctly, but for reporting - * the busy state we err on the side of potentially reporting the - * wrong engine as being busy (but we guarantee that the result - * is at least self-consistent). + /* We have to check the current hw status of the fence as the uABI + * guarantees forward progress. We could rely on the idle worker + * to eventually flush us, but to minimise latency just ask the + * hardware. * - * As we use SLAB_DESTROY_BY_RCU, the request may be reallocated - * whilst we are inspecting it, even under the RCU read lock as we are. - * This means that there is a small window for the engine and/or the - * seqno to have been overwritten. The seqno will always be in the - * future compared to the intended, and so we know that if that - * seqno is idle (on whatever engine) our request is idle and the - * return 0 above is correct. - * - * The issue is that if the engine is switched, it is just as likely - * to report that it is busy (but since the switch happened, we know - * the request should be idle). So there is a small chance that a busy - * result is actually the wrong engine. - * - * So why don't we care? - * - * For starters, the busy ioctl is a heuristic that is by definition - * racy. Even with perfect serialisation in the driver, the hardware - * state is constantly advancing - the state we report to the user - * is stale. - * - * The critical information for the busy-ioctl is whether the object - * is idle as userspace relies on that to detect whether its next - * access will stall, or if it has missed submitting commands to - * the hardware allowing the GPU to stall. We never generate a - * false-positive for idleness, thus busy-ioctl is reliable at the - * most fundamental level, and we maintain the guarantee that a - * busy object left to itself will eventually become idle (and stay - * idle!). - * - * We allow ourselves the leeway of potentially misreporting the busy - * state because that is an optimisation heuristic that is constantly - * in flux. Being quickly able to detect the busy/idle state is much - * more important than accurate logging of exactly which engines were - * busy. - * - * For accuracy in reporting the engine, we could use - * - * result = 0; - * request = __i915_gem_active_get_rcu(active); - * if (request) { - * if (!i915_gem_request_completed(request)) - * result = flag(request->engine->exec_id); - * i915_gem_request_put(request); - * } - * - * but that still remains susceptible to both hardware and userspace - * races. So we accept making the result of that race slightly worse, - * given the rarity of the race and its low impact on the result. + * Note we only report on the status of native fences. */ - return flag(READ_ONCE(request->engine->exec_id)); + if (!dma_fence_is_i915(fence)) + return 0; + + /* opencode to_request() in order to avoid const warnings */ + rq = container_of(fence, struct drm_i915_gem_request, fence); + if (i915_gem_request_completed(rq)) + return 0; + + return flag(rq->engine->exec_id); } static __always_inline unsigned int -busy_check_reader(const struct i915_gem_active *active) +busy_check_reader(const struct dma_fence *fence) { - return __busy_set_if_active(active, __busy_read_flag); + return __busy_set_if_active(fence, __busy_read_flag); } static __always_inline unsigned int -busy_check_writer(const struct i915_gem_active *active) +busy_check_writer(const struct dma_fence *fence) { - return __busy_set_if_active(active, __busy_write_id); + if (!fence) + return 0; + + return __busy_set_if_active(fence, __busy_write_id); } int @@ -3984,64 +4040,58 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, { struct drm_i915_gem_busy *args = data; struct drm_i915_gem_object *obj; - unsigned long active; + struct reservation_object_list *list; + unsigned int seq; + int err; - obj = i915_gem_object_lookup(file, args->handle); + err = -ENOENT; + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); if (!obj) - return -ENOENT; + goto out; - args->busy = 0; - active = __I915_BO_ACTIVE(obj); - if (active) { - int idx; + /* A discrepancy here is that we do not report the status of + * non-i915 fences, i.e. even though we may report the object as idle, + * a call to set-domain may still stall waiting for foreign rendering. + * This also means that wait-ioctl may report an object as busy, + * where busy-ioctl considers it idle. + * + * We trade the ability to warn of foreign fences to report on which + * i915 engines are active for the object. + * + * Alternatively, we can trade that extra information on read/write + * activity with + * args->busy = + * !reservation_object_test_signaled_rcu(obj->resv, true); + * to report the overall busyness. This is what the wait-ioctl does. + * + */ +retry: + seq = raw_read_seqcount(&obj->resv->seq); - /* Yes, the lookups are intentionally racy. - * - * First, we cannot simply rely on __I915_BO_ACTIVE. We have - * to regard the value as stale and as our ABI guarantees - * forward progress, we confirm the status of each active - * request with the hardware. - * - * Even though we guard the pointer lookup by RCU, that only - * guarantees that the pointer and its contents remain - * dereferencable and does *not* mean that the request we - * have is the same as the one being tracked by the object. - * - * Consider that we lookup the request just as it is being - * retired and freed. We take a local copy of the pointer, - * but before we add its engine into the busy set, the other - * thread reallocates it and assigns it to a task on another - * engine with a fresh and incomplete seqno. Guarding against - * that requires careful serialisation and reference counting, - * i.e. using __i915_gem_active_get_request_rcu(). We don't, - * instead we expect that if the result is busy, which engines - * are busy is not completely reliable - we only guarantee - * that the object was busy. - */ - rcu_read_lock(); + /* Translate the exclusive fence to the READ *and* WRITE engine */ + args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl)); - for_each_active(active, idx) - args->busy |= busy_check_reader(&obj->last_read[idx]); + /* Translate shared fences to READ set of engines */ + list = rcu_dereference(obj->resv->fence); + if (list) { + unsigned int shared_count = list->shared_count, i; - /* For ABI sanity, we only care that the write engine is in - * the set of read engines. This should be ensured by the - * ordering of setting last_read/last_write in - * i915_vma_move_to_active(), and then in reverse in retire. - * However, for good measure, we always report the last_write - * request as a busy read as well as being a busy write. - * - * We don't care that the set of active read/write engines - * may change during construction of the result, as it is - * equally liable to change before userspace can inspect - * the result. - */ - args->busy |= busy_check_writer(&obj->last_write); + for (i = 0; i < shared_count; ++i) { + struct dma_fence *fence = + rcu_dereference(list->shared[i]); - rcu_read_unlock(); + args->busy |= busy_check_reader(fence); + } } - i915_gem_object_put_unlocked(obj); - return 0; + if (args->busy && read_seqcount_retry(&obj->resv->seq, seq)) + goto retry; + + err = 0; +out: + rcu_read_unlock(); + return err; } int @@ -4058,7 +4108,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_madvise *args = data; struct drm_i915_gem_object *obj; - int ret; + int err; switch (args->madv) { case I915_MADV_DONTNEED: @@ -4068,65 +4118,72 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ret; - obj = i915_gem_object_lookup(file_priv, args->handle); - if (!obj) { - ret = -ENOENT; - goto unlock; - } + if (!obj) + return -ENOENT; + + err = mutex_lock_interruptible(&obj->mm.lock); + if (err) + goto out; - if (obj->pages && + if (obj->mm.pages && i915_gem_object_is_tiled(obj) && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { - if (obj->madv == I915_MADV_WILLNEED) - i915_gem_object_unpin_pages(obj); - if (args->madv == I915_MADV_WILLNEED) - i915_gem_object_pin_pages(obj); + if (obj->mm.madv == I915_MADV_WILLNEED) { + GEM_BUG_ON(!obj->mm.quirked); + __i915_gem_object_unpin_pages(obj); + obj->mm.quirked = false; + } + if (args->madv == I915_MADV_WILLNEED) { + GEM_BUG_ON(obj->mm.quirked); + __i915_gem_object_pin_pages(obj); + obj->mm.quirked = true; + } } - if (obj->madv != __I915_MADV_PURGED) - obj->madv = args->madv; + if (obj->mm.madv != __I915_MADV_PURGED) + obj->mm.madv = args->madv; /* if the object is no longer attached, discard its backing storage */ - if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL) + if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages) i915_gem_object_truncate(obj); - args->retained = obj->madv != __I915_MADV_PURGED; + args->retained = obj->mm.madv != __I915_MADV_PURGED; + mutex_unlock(&obj->mm.lock); +out: i915_gem_object_put(obj); -unlock: - mutex_unlock(&dev->struct_mutex); - return ret; + return err; } void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops) { - int i; + mutex_init(&obj->mm.lock); - INIT_LIST_HEAD(&obj->global_list); - for (i = 0; i < I915_NUM_ENGINES; i++) - init_request_active(&obj->last_read[i], - i915_gem_object_retire__read); - init_request_active(&obj->last_write, - i915_gem_object_retire__write); + INIT_LIST_HEAD(&obj->global_link); + INIT_LIST_HEAD(&obj->userfault_link); INIT_LIST_HEAD(&obj->obj_exec_link); INIT_LIST_HEAD(&obj->vma_list); INIT_LIST_HEAD(&obj->batch_pool_link); obj->ops = ops; + reservation_object_init(&obj->__builtin_resv); + obj->resv = &obj->__builtin_resv; + obj->frontbuffer_ggtt_origin = ORIGIN_GTT; - obj->madv = I915_MADV_WILLNEED; + + obj->mm.madv = I915_MADV_WILLNEED; + INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN); + mutex_init(&obj->mm.get_page.lock); i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size); } static const struct drm_i915_gem_object_ops i915_gem_object_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = i915_gem_object_get_pages_gtt, .put_pages = i915_gem_object_put_pages_gtt, }; @@ -4138,6 +4195,7 @@ static const struct drm_i915_gem_object_ops i915_gem_object_ops = { struct drm_i915_gem_object * i915_gem_object_create(struct drm_device *dev, u64 size) { + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; struct address_space *mapping; gfp_t mask; @@ -4163,7 +4221,7 @@ i915_gem_object_create(struct drm_device *dev, u64 size) goto fail; mask = GFP_HIGHUSER | __GFP_RECLAIMABLE; - if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) { + if (IS_CRESTLINE(dev_priv) || IS_BROADWATER(dev_priv)) { /* 965gm cannot relocate objects above 4GiB. */ mask &= ~__GFP_HIGHMEM; mask |= __GFP_DMA32; @@ -4200,7 +4258,6 @@ i915_gem_object_create(struct drm_device *dev, u64 size) fail: i915_gem_object_free(obj); - return ERR_PTR(ret); } @@ -4212,7 +4269,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj) * back the contents from the GPU. */ - if (obj->madv != I915_MADV_WILLNEED) + if (obj->mm.madv != I915_MADV_WILLNEED) return false; if (obj->base.filp == NULL) @@ -4228,16 +4285,72 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj) return atomic_long_read(&obj->base.filp->f_count) == 1; } -void i915_gem_free_object(struct drm_gem_object *gem_obj) +static void __i915_gem_free_objects(struct drm_i915_private *i915, + struct llist_node *freed) { - struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - struct drm_device *dev = obj->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct i915_vma *vma, *next; + struct drm_i915_gem_object *obj, *on; - intel_runtime_pm_get(dev_priv); + mutex_lock(&i915->drm.struct_mutex); + intel_runtime_pm_get(i915); + llist_for_each_entry(obj, freed, freed) { + struct i915_vma *vma, *vn; + + trace_i915_gem_object_destroy(obj); + + GEM_BUG_ON(i915_gem_object_is_active(obj)); + list_for_each_entry_safe(vma, vn, + &obj->vma_list, obj_link) { + GEM_BUG_ON(!i915_vma_is_ggtt(vma)); + GEM_BUG_ON(i915_vma_is_active(vma)); + vma->flags &= ~I915_VMA_PIN_MASK; + i915_vma_close(vma); + } + GEM_BUG_ON(!list_empty(&obj->vma_list)); + GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree)); + + list_del(&obj->global_link); + } + intel_runtime_pm_put(i915); + mutex_unlock(&i915->drm.struct_mutex); - trace_i915_gem_object_destroy(obj); + llist_for_each_entry_safe(obj, on, freed, freed) { + GEM_BUG_ON(obj->bind_count); + GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits)); + + if (obj->ops->release) + obj->ops->release(obj); + + if (WARN_ON(i915_gem_object_has_pinned_pages(obj))) + atomic_set(&obj->mm.pages_pin_count, 0); + __i915_gem_object_put_pages(obj, I915_MM_NORMAL); + GEM_BUG_ON(obj->mm.pages); + + if (obj->base.import_attach) + drm_prime_gem_destroy(&obj->base, NULL); + + reservation_object_fini(&obj->__builtin_resv); + drm_gem_object_release(&obj->base); + i915_gem_info_remove_obj(i915, obj->base.size); + + kfree(obj->bit_17); + i915_gem_object_free(obj); + } +} + +static void i915_gem_flush_free_objects(struct drm_i915_private *i915) +{ + struct llist_node *freed; + + freed = llist_del_all(&i915->mm.free_list); + if (unlikely(freed)) + __i915_gem_free_objects(i915, freed); +} + +static void __i915_gem_free_work(struct work_struct *work) +{ + struct drm_i915_private *i915 = + container_of(work, struct drm_i915_private, mm.free_work); + struct llist_node *freed; /* All file-owned VMA should have been released by this point through * i915_gem_close_object(), or earlier by i915_gem_context_close(). @@ -4246,47 +4359,62 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) * the GTT either for the user or for scanout). Those VMA still need to * unbound now. */ - list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) { - GEM_BUG_ON(!i915_vma_is_ggtt(vma)); - GEM_BUG_ON(i915_vma_is_active(vma)); - vma->flags &= ~I915_VMA_PIN_MASK; - i915_vma_close(vma); - } - GEM_BUG_ON(obj->bind_count); - /* Stolen objects don't hold a ref, but do hold pin count. Fix that up - * before progressing. */ - if (obj->stolen) - i915_gem_object_unpin_pages(obj); + while ((freed = llist_del_all(&i915->mm.free_list))) + __i915_gem_free_objects(i915, freed); +} - WARN_ON(atomic_read(&obj->frontbuffer_bits)); +static void __i915_gem_free_object_rcu(struct rcu_head *head) +{ + struct drm_i915_gem_object *obj = + container_of(head, typeof(*obj), rcu); + struct drm_i915_private *i915 = to_i915(obj->base.dev); - if (obj->pages && obj->madv == I915_MADV_WILLNEED && - dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES && - i915_gem_object_is_tiled(obj)) - i915_gem_object_unpin_pages(obj); + /* We can't simply use call_rcu() from i915_gem_free_object() + * as we need to block whilst unbinding, and the call_rcu + * task may be called from softirq context. So we take a + * detour through a worker. + */ + if (llist_add(&obj->freed, &i915->mm.free_list)) + schedule_work(&i915->mm.free_work); +} - if (WARN_ON(obj->pages_pin_count)) - obj->pages_pin_count = 0; - if (discard_backing_storage(obj)) - obj->madv = I915_MADV_DONTNEED; - i915_gem_object_put_pages(obj); +void i915_gem_free_object(struct drm_gem_object *gem_obj) +{ + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); - BUG_ON(obj->pages); + if (obj->mm.quirked) + __i915_gem_object_unpin_pages(obj); - if (obj->base.import_attach) - drm_prime_gem_destroy(&obj->base, NULL); + if (discard_backing_storage(obj)) + obj->mm.madv = I915_MADV_DONTNEED; - if (obj->ops->release) - obj->ops->release(obj); + /* Before we free the object, make sure any pure RCU-only + * read-side critical sections are complete, e.g. + * i915_gem_busy_ioctl(). For the corresponding synchronized + * lookup see i915_gem_object_lookup_rcu(). + */ + call_rcu(&obj->rcu, __i915_gem_free_object_rcu); +} - drm_gem_object_release(&obj->base); - i915_gem_info_remove_obj(dev_priv, obj->base.size); +void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj) +{ + lockdep_assert_held(&obj->base.dev->struct_mutex); - kfree(obj->bit_17); - i915_gem_object_free(obj); + GEM_BUG_ON(i915_gem_object_has_active_reference(obj)); + if (i915_gem_object_is_active(obj)) + i915_gem_object_set_active_reference(obj); + else + i915_gem_object_put(obj); +} - intel_runtime_pm_put(dev_priv); +static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, dev_priv, id) + GEM_BUG_ON(engine->last_context != dev_priv->kernel_context); } int i915_gem_suspend(struct drm_device *dev) @@ -4317,18 +4445,22 @@ int i915_gem_suspend(struct drm_device *dev) goto err; i915_gem_retire_requests(dev_priv); + GEM_BUG_ON(dev_priv->gt.active_requests); + assert_kernel_context_is_current(dev_priv); i915_gem_context_lost(dev_priv); mutex_unlock(&dev->struct_mutex); cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work); cancel_delayed_work_sync(&dev_priv->gt.retire_work); flush_delayed_work(&dev_priv->gt.idle_work); + flush_work(&dev_priv->mm.free_work); /* Assert that we sucessfully flushed all the work and * reset the GPU back to its idle, low power state. */ WARN_ON(dev_priv->gt.awake); + WARN_ON(!intel_execlists_idle(dev_priv)); /* * Neither the BIOS, ourselves or any other kernel @@ -4365,6 +4497,8 @@ void i915_gem_resume(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + WARN_ON(dev_priv->gt.awake); + mutex_lock(&dev->struct_mutex); i915_gem_restore_gtt_mappings(dev); @@ -4435,6 +4569,8 @@ i915_gem_init_hw(struct drm_device *dev) enum intel_engine_id id; int ret; + dev_priv->gt.last_init_time = ktime_get(); + /* Double layer security blanket, see i915_gem_init() */ intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); @@ -4613,33 +4749,43 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv) i915_gem_detect_bit_6_swizzle(dev); } -void +int i915_gem_load_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + int err = -ENOMEM; + + dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN); + if (!dev_priv->objects) + goto err_out; + + dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); + if (!dev_priv->vmas) + goto err_objects; - dev_priv->objects = - kmem_cache_create("i915_gem_object", - sizeof(struct drm_i915_gem_object), 0, - SLAB_HWCACHE_ALIGN, - NULL); - dev_priv->vmas = - kmem_cache_create("i915_gem_vma", - sizeof(struct i915_vma), 0, - SLAB_HWCACHE_ALIGN, - NULL); - dev_priv->requests = - kmem_cache_create("i915_gem_request", - sizeof(struct drm_i915_gem_request), 0, - SLAB_HWCACHE_ALIGN | - SLAB_RECLAIM_ACCOUNT | - SLAB_DESTROY_BY_RCU, - NULL); + dev_priv->requests = KMEM_CACHE(drm_i915_gem_request, + SLAB_HWCACHE_ALIGN | + SLAB_RECLAIM_ACCOUNT | + SLAB_DESTROY_BY_RCU); + if (!dev_priv->requests) + goto err_vmas; + + mutex_lock(&dev_priv->drm.struct_mutex); + INIT_LIST_HEAD(&dev_priv->gt.timelines); + err = i915_gem_timeline_init(dev_priv, + &dev_priv->gt.global_timeline, + "[execution]"); + mutex_unlock(&dev_priv->drm.struct_mutex); + if (err) + goto err_requests; INIT_LIST_HEAD(&dev_priv->context_list); + INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work); + init_llist_head(&dev_priv->mm.free_list); INIT_LIST_HEAD(&dev_priv->mm.unbound_list); INIT_LIST_HEAD(&dev_priv->mm.bound_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); + INIT_LIST_HEAD(&dev_priv->mm.userfault_list); INIT_DELAYED_WORK(&dev_priv->gt.retire_work, i915_gem_retire_work_handler); INIT_DELAYED_WORK(&dev_priv->gt.idle_work, @@ -4656,12 +4802,25 @@ i915_gem_load_init(struct drm_device *dev) atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0); spin_lock_init(&dev_priv->fb_tracking.lock); + + return 0; + +err_requests: + kmem_cache_destroy(dev_priv->requests); +err_vmas: + kmem_cache_destroy(dev_priv->vmas); +err_objects: + kmem_cache_destroy(dev_priv->objects); +err_out: + return err; } void i915_gem_load_cleanup(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); + WARN_ON(!llist_empty(&dev_priv->mm.free_list)); + kmem_cache_destroy(dev_priv->requests); kmem_cache_destroy(dev_priv->vmas); kmem_cache_destroy(dev_priv->objects); @@ -4710,7 +4869,7 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv) i915_gem_shrink(dev_priv, -1UL, I915_SHRINK_UNBOUND); for (p = phases; *p; p++) { - list_for_each_entry(obj, *p, global_list) { + list_for_each_entry(obj, *p, global_link) { obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->base.write_domain = I915_GEM_DOMAIN_CPU; } @@ -4802,21 +4961,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, } } -/* Like i915_gem_object_get_page(), but mark the returned page dirty */ -struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n) -{ - struct page *page; - - /* Only default objects have per-page dirty tracking */ - if (WARN_ON(!i915_gem_object_has_struct_page(obj))) - return NULL; - - page = i915_gem_object_get_page(obj, n); - set_page_dirty(page); - return page; -} - /* Allocate a new GEM object and fill it with the supplied data */ struct drm_i915_gem_object * i915_gem_object_create_from_data(struct drm_device *dev, @@ -4835,14 +4979,13 @@ i915_gem_object_create_from_data(struct drm_device *dev, if (ret) goto fail; - ret = i915_gem_object_get_pages(obj); + ret = i915_gem_object_pin_pages(obj); if (ret) goto fail; - i915_gem_object_pin_pages(obj); - sg = obj->pages; + sg = obj->mm.pages; bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size); - obj->dirty = 1; /* Backing store is now out of date */ + obj->mm.dirty = true; /* Backing store is now out of date */ i915_gem_object_unpin_pages(obj); if (WARN_ON(bytes != size)) { @@ -4857,3 +5000,156 @@ fail: i915_gem_object_put(obj); return ERR_PTR(ret); } + +struct scatterlist * +i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + unsigned int n, + unsigned int *offset) +{ + struct i915_gem_object_page_iter *iter = &obj->mm.get_page; + struct scatterlist *sg; + unsigned int idx, count; + + might_sleep(); + GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT); + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + + /* As we iterate forward through the sg, we record each entry in a + * radixtree for quick repeated (backwards) lookups. If we have seen + * this index previously, we will have an entry for it. + * + * Initial lookup is O(N), but this is amortized to O(1) for + * sequential page access (where each new request is consecutive + * to the previous one). Repeated lookups are O(lg(obj->base.size)), + * i.e. O(1) with a large constant! + */ + if (n < READ_ONCE(iter->sg_idx)) + goto lookup; + + mutex_lock(&iter->lock); + + /* We prefer to reuse the last sg so that repeated lookup of this + * (or the subsequent) sg are fast - comparing against the last + * sg is faster than going through the radixtree. + */ + + sg = iter->sg_pos; + idx = iter->sg_idx; + count = __sg_page_count(sg); + + while (idx + count <= n) { + unsigned long exception, i; + int ret; + + /* If we cannot allocate and insert this entry, or the + * individual pages from this range, cancel updating the + * sg_idx so that on this lookup we are forced to linearly + * scan onwards, but on future lookups we will try the + * insertion again (in which case we need to be careful of + * the error return reporting that we have already inserted + * this index). + */ + ret = radix_tree_insert(&iter->radix, idx, sg); + if (ret && ret != -EEXIST) + goto scan; + + exception = + RADIX_TREE_EXCEPTIONAL_ENTRY | + idx << RADIX_TREE_EXCEPTIONAL_SHIFT; + for (i = 1; i < count; i++) { + ret = radix_tree_insert(&iter->radix, idx + i, + (void *)exception); + if (ret && ret != -EEXIST) + goto scan; + } + + idx += count; + sg = ____sg_next(sg); + count = __sg_page_count(sg); + } + +scan: + iter->sg_pos = sg; + iter->sg_idx = idx; + + mutex_unlock(&iter->lock); + + if (unlikely(n < idx)) /* insertion completed by another thread */ + goto lookup; + + /* In case we failed to insert the entry into the radixtree, we need + * to look beyond the current sg. + */ + while (idx + count <= n) { + idx += count; + sg = ____sg_next(sg); + count = __sg_page_count(sg); + } + + *offset = n - idx; + return sg; + +lookup: + rcu_read_lock(); + + sg = radix_tree_lookup(&iter->radix, n); + GEM_BUG_ON(!sg); + + /* If this index is in the middle of multi-page sg entry, + * the radixtree will contain an exceptional entry that points + * to the start of that range. We will return the pointer to + * the base page and the offset of this page within the + * sg entry's range. + */ + *offset = 0; + if (unlikely(radix_tree_exception(sg))) { + unsigned long base = + (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT; + + sg = radix_tree_lookup(&iter->radix, base); + GEM_BUG_ON(!sg); + + *offset = n - base; + } + + rcu_read_unlock(); + + return sg; +} + +struct page * +i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) +{ + struct scatterlist *sg; + unsigned int offset; + + GEM_BUG_ON(!i915_gem_object_has_struct_page(obj)); + + sg = i915_gem_object_get_sg(obj, n, &offset); + return nth_page(sg_page(sg), offset); +} + +/* Like i915_gem_object_get_page(), but mark the returned page dirty */ +struct page * +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n) +{ + struct page *page; + + page = i915_gem_object_get_page(obj, n); + if (!obj->mm.dirty) + set_page_dirty(page); + + return page; +} + +dma_addr_t +i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, + unsigned long n) +{ + struct scatterlist *sg; + unsigned int offset; + + sg = i915_gem_object_get_sg(obj, n, &offset); + return sg_dma_address(sg) + (offset << PAGE_SHIFT); +} diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 8292e797d9b5..735580d72eb1 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -31,4 +31,6 @@ #define GEM_BUG_ON(expr) #endif +#define I915_NUM_ENGINES 5 + #endif /* __I915_GEM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c index ed989596d9a3..b3bc119ec1bb 100644 --- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c +++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c @@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool) list_for_each_entry_safe(obj, next, &pool->cache_list[n], batch_pool_link) - i915_gem_object_put(obj); + __i915_gem_object_release_unless_active(obj); INIT_LIST_HEAD(&pool->cache_list[n]); } @@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) { struct drm_i915_gem_object *obj = NULL; - struct drm_i915_gem_object *tmp, *next; + struct drm_i915_gem_object *tmp; struct list_head *list; - int n; + int n, ret; lockdep_assert_held(&pool->engine->i915->drm.struct_mutex); @@ -112,40 +112,35 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n = ARRAY_SIZE(pool->cache_list) - 1; list = &pool->cache_list[n]; - list_for_each_entry_safe(tmp, next, list, batch_pool_link) { + list_for_each_entry(tmp, list, batch_pool_link) { /* The batches are strictly LRU ordered */ - if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id], - &tmp->base.dev->struct_mutex)) + if (i915_gem_object_is_active(tmp)) break; - /* While we're looping, do some clean up */ - if (tmp->madv == __I915_MADV_PURGED) { - list_del(&tmp->batch_pool_link); - i915_gem_object_put(tmp); - continue; - } + GEM_BUG_ON(!reservation_object_test_signaled_rcu(tmp->resv, + true)); if (tmp->base.size >= size) { + /* Clear the set of shared fences early */ + ww_mutex_lock(&tmp->resv->lock, NULL); + reservation_object_add_excl_fence(tmp->resv, NULL); + ww_mutex_unlock(&tmp->resv->lock); + obj = tmp; break; } } if (obj == NULL) { - int ret; - - obj = i915_gem_object_create(&pool->engine->i915->drm, size); + obj = i915_gem_object_create_internal(pool->engine->i915, size); if (IS_ERR(obj)) return obj; - - ret = i915_gem_object_get_pages(obj); - if (ret) - return ERR_PTR(ret); - - obj->madv = I915_MADV_DONTNEED; } + ret = i915_gem_object_pin_pages(obj); + if (ret) + return ERR_PTR(ret); + list_move_tail(&obj->batch_pool_link, list); - i915_gem_object_pin_pages(obj); return obj; } diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 5dca32ac1c67..6dd475735f0a 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -155,9 +155,10 @@ void i915_gem_context_free(struct kref *ctx_ref) if (ce->ring) intel_ring_free(ce->ring); - i915_vma_put(ce->state); + __i915_gem_object_release_unless_active(ce->state->obj); } + kfree(ctx->name); put_pid(ctx->pid); list_del(&ctx->link); @@ -303,19 +304,28 @@ __create_hw_context(struct drm_device *dev, } /* Default context will never have a file_priv */ - if (file_priv != NULL) { + ret = DEFAULT_CONTEXT_HANDLE; + if (file_priv) { ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_HANDLE, 0, GFP_KERNEL); if (ret < 0) goto err_out; - } else - ret = DEFAULT_CONTEXT_HANDLE; + } + ctx->user_handle = ret; ctx->file_priv = file_priv; - if (file_priv) + if (file_priv) { ctx->pid = get_task_pid(current, PIDTYPE_PID); + ctx->name = kasprintf(GFP_KERNEL, "%s[%d]/%x", + current->comm, + pid_nr(ctx->pid), + ctx->user_handle); + if (!ctx->name) { + ret = -ENOMEM; + goto err_pid; + } + } - ctx->user_handle = ret; /* NB: Mark all slices as needing a remap so that when the context first * loads it will restore whatever remap state already exists. If there * is no remap info, it will be a NOP. */ @@ -329,6 +339,9 @@ __create_hw_context(struct drm_device *dev, return ctx; +err_pid: + put_pid(ctx->pid); + idr_remove(&file_priv->context_idr, ctx->user_handle); err_out: context_close(ctx); return ERR_PTR(ret); @@ -352,9 +365,9 @@ i915_gem_create_context(struct drm_device *dev, return ctx; if (USES_FULL_PPGTT(dev)) { - struct i915_hw_ppgtt *ppgtt = - i915_ppgtt_create(to_i915(dev), file_priv); + struct i915_hw_ppgtt *ppgtt; + ppgtt = i915_ppgtt_create(to_i915(dev), file_priv, ctx->name); if (IS_ERR(ppgtt)) { DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", PTR_ERR(ppgtt)); @@ -751,12 +764,36 @@ needs_pd_load_post(struct i915_hw_ppgtt *ppgtt, return false; } +struct i915_vma * +i915_gem_context_pin_legacy(struct i915_gem_context *ctx, + unsigned int flags) +{ + struct i915_vma *vma = ctx->engine[RCS].state; + int ret; + + /* Clear this page out of any CPU caches for coherent swap-in/out. + * We only want to do this on the first bind so that we do not stall + * on an active context (which by nature is already on the GPU). + */ + if (!(vma->flags & I915_VMA_GLOBAL_BIND)) { + ret = i915_gem_object_set_to_gtt_domain(vma->obj, false); + if (ret) + return ERR_PTR(ret); + } + + ret = i915_vma_pin(vma, 0, ctx->ggtt_alignment, PIN_GLOBAL | flags); + if (ret) + return ERR_PTR(ret); + + return vma; +} + static int do_rcs_switch(struct drm_i915_gem_request *req) { struct i915_gem_context *to = req->ctx; struct intel_engine_cs *engine = req->engine; struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt; - struct i915_vma *vma = to->engine[RCS].state; + struct i915_vma *vma; struct i915_gem_context *from; u32 hw_flags; int ret, i; @@ -764,17 +801,10 @@ static int do_rcs_switch(struct drm_i915_gem_request *req) if (skip_rcs_switch(ppgtt, engine, to)) return 0; - /* Clear this page out of any CPU caches for coherent swap-in/out. */ - if (!(vma->flags & I915_VMA_GLOBAL_BIND)) { - ret = i915_gem_object_set_to_gtt_domain(vma->obj, false); - if (ret) - return ret; - } - /* Trying to pin first makes error handling easier. */ - ret = i915_vma_pin(vma, 0, to->ggtt_alignment, PIN_GLOBAL); - if (ret) - return ret; + vma = i915_gem_context_pin_legacy(to, 0); + if (IS_ERR(vma)) + return PTR_ERR(vma); /* * Pin can switch back to the default context if we end up calling into @@ -931,22 +961,33 @@ int i915_switch_context(struct drm_i915_gem_request *req) int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; + struct i915_gem_timeline *timeline; enum intel_engine_id id; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + for_each_engine(engine, dev_priv, id) { struct drm_i915_gem_request *req; int ret; - if (engine->last_context == NULL) - continue; - - if (engine->last_context == dev_priv->kernel_context) - continue; - req = i915_gem_request_alloc(engine, dev_priv->kernel_context); if (IS_ERR(req)) return PTR_ERR(req); + /* Queue this switch after all other activity */ + list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { + struct drm_i915_gem_request *prev; + struct intel_timeline *tl; + + tl = &timeline->engine[engine->id]; + prev = i915_gem_active_raw(&tl->last_request, + &dev_priv->drm.struct_mutex); + if (prev) + i915_sw_fence_await_sw_fence_gfp(&req->submit, + &prev->submit, + GFP_KERNEL); + } + ret = i915_switch_context(req); i915_add_request_no_flush(req); if (ret) diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/i915_gem_dmabuf.c index 97c9d68b45df..5e38299b5df6 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.c @@ -44,51 +44,42 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme struct scatterlist *src, *dst; int ret, i; - ret = i915_mutex_lock_interruptible(obj->base.dev); + ret = i915_gem_object_pin_pages(obj); if (ret) goto err; - ret = i915_gem_object_get_pages(obj); - if (ret) - goto err_unlock; - - i915_gem_object_pin_pages(obj); - /* Copy sg so that we make an independent mapping */ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (st == NULL) { ret = -ENOMEM; - goto err_unpin; + goto err_unpin_pages; } - ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL); + ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); if (ret) goto err_free; - src = obj->pages->sgl; + src = obj->mm.pages->sgl; dst = st->sgl; - for (i = 0; i < obj->pages->nents; i++) { + for (i = 0; i < obj->mm.pages->nents; i++) { sg_set_page(dst, sg_page(src), src->length, 0); dst = sg_next(dst); src = sg_next(src); } if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) { - ret =-ENOMEM; + ret = -ENOMEM; goto err_free_sg; } - mutex_unlock(&obj->base.dev->struct_mutex); return st; err_free_sg: sg_free_table(st); err_free: kfree(st); -err_unpin: +err_unpin_pages: i915_gem_object_unpin_pages(obj); -err_unlock: - mutex_unlock(&obj->base.dev->struct_mutex); err: return ERR_PTR(ret); } @@ -103,36 +94,21 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment, sg_free_table(sg); kfree(sg); - mutex_lock(&obj->base.dev->struct_mutex); i915_gem_object_unpin_pages(obj); - mutex_unlock(&obj->base.dev->struct_mutex); } static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); - struct drm_device *dev = obj->base.dev; - void *addr; - int ret; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ERR_PTR(ret); - - addr = i915_gem_object_pin_map(obj, I915_MAP_WB); - mutex_unlock(&dev->struct_mutex); - - return addr; + return i915_gem_object_pin_map(obj, I915_MAP_WB); } static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); - struct drm_device *dev = obj->base.dev; - mutex_lock(&dev->struct_mutex); i915_gem_object_unpin_map(obj); - mutex_unlock(&dev->struct_mutex); } static void *i915_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf, unsigned long page_num) @@ -179,32 +155,45 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; - int ret; bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE); + int err; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ret; + err = i915_gem_object_pin_pages(obj); + if (err) + return err; + + err = i915_mutex_lock_interruptible(dev); + if (err) + goto out; - ret = i915_gem_object_set_to_cpu_domain(obj, write); + err = i915_gem_object_set_to_cpu_domain(obj, write); mutex_unlock(&dev->struct_mutex); - return ret; + +out: + i915_gem_object_unpin_pages(obj); + return err; } static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; - int ret; + int err; - ret = i915_mutex_lock_interruptible(dev); - if (ret) - return ret; + err = i915_gem_object_pin_pages(obj); + if (err) + return err; - ret = i915_gem_object_set_to_gtt_domain(obj, false); + err = i915_mutex_lock_interruptible(dev); + if (err) + goto out; + + err = i915_gem_object_set_to_gtt_domain(obj, false); mutex_unlock(&dev->struct_mutex); - return ret; +out: + i915_gem_object_unpin_pages(obj); + return err; } static const struct dma_buf_ops i915_dmabuf_ops = { @@ -222,60 +211,17 @@ static const struct dma_buf_ops i915_dmabuf_ops = { .end_cpu_access = i915_gem_end_cpu_access, }; -static void export_fences(struct drm_i915_gem_object *obj, - struct dma_buf *dma_buf) -{ - struct reservation_object *resv = dma_buf->resv; - struct drm_i915_gem_request *req; - unsigned long active; - int idx; - - active = __I915_BO_ACTIVE(obj); - if (!active) - return; - - /* Serialise with execbuf to prevent concurrent fence-loops */ - mutex_lock(&obj->base.dev->struct_mutex); - - /* Mark the object for future fences before racily adding old fences */ - obj->base.dma_buf = dma_buf; - - ww_mutex_lock(&resv->lock, NULL); - - for_each_active(active, idx) { - req = i915_gem_active_get(&obj->last_read[idx], - &obj->base.dev->struct_mutex); - if (!req) - continue; - - if (reservation_object_reserve_shared(resv) == 0) - reservation_object_add_shared_fence(resv, &req->fence); - - i915_gem_request_put(req); - } - - req = i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); - if (req) { - reservation_object_add_excl_fence(resv, &req->fence); - i915_gem_request_put(req); - } - - ww_mutex_unlock(&resv->lock); - mutex_unlock(&obj->base.dev->struct_mutex); -} - struct dma_buf *i915_gem_prime_export(struct drm_device *dev, struct drm_gem_object *gem_obj, int flags) { struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - struct dma_buf *dma_buf; exp_info.ops = &i915_dmabuf_ops; exp_info.size = gem_obj->size; exp_info.flags = flags; exp_info.priv = gem_obj; + exp_info.resv = obj->resv; if (obj->ops->dmabuf_export) { int ret = obj->ops->dmabuf_export(obj); @@ -283,30 +229,21 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev, return ERR_PTR(ret); } - dma_buf = drm_gem_dmabuf_export(dev, &exp_info); - if (IS_ERR(dma_buf)) - return dma_buf; - - export_fences(obj, dma_buf); - return dma_buf; + return drm_gem_dmabuf_export(dev, &exp_info); } -static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) { - struct sg_table *sg; - - sg = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL); - if (IS_ERR(sg)) - return PTR_ERR(sg); - - obj->pages = sg; - return 0; + return dma_buf_map_attachment(obj->base.import_attach, + DMA_BIDIRECTIONAL); } -static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - dma_buf_unmap_attachment(obj->base.import_attach, - obj->pages, DMA_BIDIRECTIONAL); + dma_buf_unmap_attachment(obj->base.import_attach, pages, + DMA_BIDIRECTIONAL); } static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = { @@ -350,6 +287,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, drm_gem_private_object_init(dev, &obj->base, dma_buf->size); i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops); obj->base.import_attach = attach; + obj->resv = dma_buf->resv; /* We use GTT as shorthand for a coherent domain, one that is * neither in the GPU cache nor in the CPU cache, where all diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index b5e9e669f50f..bd08814b015c 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -33,14 +33,17 @@ #include "intel_drv.h" #include "i915_trace.h" -static bool -gpu_is_idle(struct drm_i915_private *dev_priv) +static bool ggtt_is_idle(struct drm_i915_private *dev_priv) { + struct i915_ggtt *ggtt = &dev_priv->ggtt; struct intel_engine_cs *engine; enum intel_engine_id id; for_each_engine(engine, dev_priv, id) { - if (intel_engine_is_active(engine)) + struct intel_timeline *tl; + + tl = &ggtt->base.timeline.engine[engine->id]; + if (i915_gem_active_isset(&tl->last_request)) return false; } @@ -56,7 +59,7 @@ mark_free(struct i915_vma *vma, unsigned int flags, struct list_head *unwind) if (WARN_ON(!list_empty(&vma->exec_list))) return false; - if (flags & PIN_NONFAULT && vma->obj->fault_mappable) + if (flags & PIN_NONFAULT && !list_empty(&vma->obj->userfault_link)) return false; list_add(&vma->exec_list, unwind); @@ -103,6 +106,7 @@ i915_gem_evict_something(struct i915_address_space *vm, struct i915_vma *vma, *next; int ret; + lockdep_assert_held(&vm->dev->struct_mutex); trace_i915_gem_evict(vm, min_size, alignment, flags); /* @@ -153,7 +157,7 @@ search_again: if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK) return -ENOSPC; - if (gpu_is_idle(dev_priv)) { + if (ggtt_is_idle(dev_priv)) { /* If we still have pending pageflip completions, drop * back to userspace to give our workqueues time to * acquire our locks and unpin the old scanouts. @@ -213,6 +217,8 @@ i915_gem_evict_for_vma(struct i915_vma *target) { struct drm_mm_node *node, *next; + lockdep_assert_held(&target->vm->dev->struct_mutex); + list_for_each_entry_safe(node, next, &target->vm->mm.head_node.node_list, node_list) { @@ -266,7 +272,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle) struct i915_vma *vma, *next; int ret; - WARN_ON(!mutex_is_locked(&vm->dev->struct_mutex)); + lockdep_assert_held(&vm->dev->struct_mutex); trace_i915_gem_evict_vm(vm); if (do_idle) { diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index e52affdcc125..fb5b44339f71 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -34,7 +34,6 @@ #include <drm/i915_drm.h> #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "i915_trace.h" #include "intel_drv.h" #include "intel_frontbuffer.h" @@ -332,7 +331,8 @@ static void reloc_cache_init(struct reloc_cache *cache, cache->page = -1; cache->vaddr = 0; cache->i915 = i915; - cache->use_64bit_reloc = INTEL_GEN(cache->i915) >= 8; + /* Must be a variable in the struct to allow GCC to unroll. */ + cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); cache->node.allocated = false; } @@ -418,15 +418,6 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, unsigned long offset; void *vaddr; - if (cache->node.allocated) { - wmb(); - ggtt->base.insert_page(&ggtt->base, - i915_gem_object_get_dma_address(obj, page), - cache->node.start, I915_CACHE_NONE, 0); - cache->page = page; - return unmask_page(cache->vaddr); - } - if (cache->vaddr) { io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); } else { @@ -466,6 +457,7 @@ static void *reloc_iomap(struct drm_i915_gem_object *obj, offset = cache->node.start; if (cache->node.allocated) { + wmb(); ggtt->base.insert_page(&ggtt->base, i915_gem_object_get_dma_address(obj, page), offset, I915_CACHE_NONE, 0); @@ -1109,44 +1101,20 @@ err: return ret; } -static unsigned int eb_other_engines(struct drm_i915_gem_request *req) -{ - unsigned int mask; - - mask = ~intel_engine_flag(req->engine) & I915_BO_ACTIVE_MASK; - mask <<= I915_BO_ACTIVE_SHIFT; - - return mask; -} - static int i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req, struct list_head *vmas) { - const unsigned int other_rings = eb_other_engines(req); struct i915_vma *vma; int ret; list_for_each_entry(vma, vmas, exec_list) { struct drm_i915_gem_object *obj = vma->obj; - struct reservation_object *resv; - if (obj->flags & other_rings) { - ret = i915_gem_request_await_object - (req, obj, obj->base.pending_write_domain); - if (ret) - return ret; - } - - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - ret = i915_sw_fence_await_reservation - (&req->submit, resv, &i915_fence_ops, - obj->base.pending_write_domain, 10*HZ, - GFP_KERNEL | __GFP_NOWARN); - if (ret < 0) - return ret; - } + ret = i915_gem_request_await_object + (req, obj, obj->base.pending_write_domain); + if (ret) + return ret; if (obj->base.write_domain & I915_GEM_DOMAIN_CPU) i915_gem_clflush_object(obj, false); @@ -1279,6 +1247,12 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file, return ctx; } +static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj) +{ + return !(obj->cache_level == I915_CACHE_NONE || + obj->cache_level == I915_CACHE_WT); +} + void i915_vma_move_to_active(struct i915_vma *vma, struct drm_i915_gem_request *req, unsigned int flags) @@ -1288,8 +1262,6 @@ void i915_vma_move_to_active(struct i915_vma *vma, GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); - obj->dirty = 1; /* be paranoid */ - /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is * vital here: mark_active adds to the start of the callback list, @@ -1297,37 +1269,32 @@ void i915_vma_move_to_active(struct i915_vma *vma, * add the active reference first and queue for it to be dropped * *last*. */ - if (!i915_gem_object_is_active(obj)) - i915_gem_object_get(obj); - i915_gem_object_set_active(obj, idx); - i915_gem_active_set(&obj->last_read[idx], req); + if (!i915_vma_is_active(vma)) + obj->active_count++; + i915_vma_set_active(vma, idx); + i915_gem_active_set(&vma->last_read[idx], req); + list_move_tail(&vma->vm_link, &vma->vm->active_list); if (flags & EXEC_OBJECT_WRITE) { - i915_gem_active_set(&obj->last_write, req); + i915_gem_active_set(&vma->last_write, req); intel_fb_obj_invalidate(obj, ORIGIN_CS); /* update for the implicit flush after a batch */ obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS; + if (!obj->cache_dirty && gpu_write_needs_clflush(obj)) + obj->cache_dirty = true; } if (flags & EXEC_OBJECT_NEEDS_FENCE) i915_gem_active_set(&vma->last_fence, req); - - i915_vma_set_active(vma, idx); - i915_gem_active_set(&vma->last_read[idx], req); - list_move_tail(&vma->vm_link, &vma->vm->active_list); } static void eb_export_fence(struct drm_i915_gem_object *obj, struct drm_i915_gem_request *req, unsigned int flags) { - struct reservation_object *resv; - - resv = i915_gem_object_get_dmabuf_resv(obj); - if (!resv) - return; + struct reservation_object *resv = obj->resv; /* Ignore errors from failing to allocate the new fence, we can't * handle an error right now. Worst case should be missed diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c index a6daf2deab74..cd59dbc6588c 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence.c +++ b/drivers/gpu/drm/i915/i915_gem_fence.c @@ -343,6 +343,9 @@ i915_vma_get_fence(struct i915_vma *vma) struct drm_i915_fence_reg *fence; struct i915_vma *set = i915_gem_object_is_tiled(vma->obj) ? vma : NULL; + /* Note that we revoke fences on runtime suspend. Therefore the user + * must keep the device awake whilst using the fence. + */ assert_rpm_wakelock_held(to_i915(vma->vm->dev)); /* Just update our place in the LRU if our fence is getting reused. */ @@ -368,19 +371,14 @@ i915_vma_get_fence(struct i915_vma *vma) * @dev: DRM device * * Restore the hw fence state to match the software tracking again, to be called - * after a gpu reset and on resume. + * after a gpu reset and on resume. Note that on runtime suspend we only cancel + * the fences, to be reacquired by the user later. */ void i915_gem_restore_fences(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); int i; - /* Note that this may be called outside of struct_mutex, by - * runtime suspend/resume. The barrier we require is enforced by - * rpm itself - all access to fences/GTT are only within an rpm - * wakeref, and to acquire that wakeref you must pass through here. - */ - for (i = 0; i < dev_priv->num_fence_regs; i++) { struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; struct i915_vma *vma = reg->vma; @@ -391,7 +389,7 @@ void i915_gem_restore_fences(struct drm_device *dev) */ if (vma && !i915_gem_object_is_tiled(vma->obj)) { GEM_BUG_ON(!reg->dirty); - GEM_BUG_ON(vma->obj->fault_mappable); + GEM_BUG_ON(!list_empty(&vma->obj->userfault_link)); list_move(®->link, &dev_priv->mm.fence_list); vma->fence = NULL; @@ -646,6 +644,7 @@ i915_gem_swizzle_page(struct page *page) /** * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling * @obj: i915 GEM buffer object + * @pages: the scattergather list of physical pages * * This function fixes up the swizzling in case any page frame number for this * object has changed in bit 17 since that state has been saved with @@ -656,7 +655,8 @@ i915_gem_swizzle_page(struct page *page) * by swapping them out and back in again). */ void -i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) +i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct sgt_iter sgt_iter; struct page *page; @@ -666,10 +666,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) return; i = 0; - for_each_sgt_page(page, sgt_iter, obj->pages) { + for_each_sgt_page(page, sgt_iter, pages) { char new_bit_17 = page_to_phys(page) >> 17; - if ((new_bit_17 & 0x1) != - (test_bit(i, obj->bit_17) != 0)) { + if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) { i915_gem_swizzle_page(page); set_page_dirty(page); } @@ -680,17 +679,19 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) /** * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling * @obj: i915 GEM buffer object + * @pages: the scattergather list of physical pages * * This function saves the bit 17 of each page frame number so that swizzling * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must * be called before the backing storage can be unpinned. */ void -i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) +i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages) { + const unsigned int page_count = obj->base.size >> PAGE_SHIFT; struct sgt_iter sgt_iter; struct page *page; - int page_count = obj->base.size >> PAGE_SHIFT; int i; if (obj->bit_17 == NULL) { @@ -705,7 +706,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i = 0; - for_each_sgt_page(page, sgt_iter, obj->pages) { + for_each_sgt_page(page, sgt_iter, pages) { if (page_to_phys(page) & (1 << 17)) __set_bit(i, obj->bit_17); else diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index 062fb0ad75da..a5fafa3d4fc8 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -31,6 +31,7 @@ #include "i915_vgpu.h" #include "i915_trace.h" #include "intel_drv.h" +#include "intel_frontbuffer.h" #define I915_GFP_DMA (GFP_KERNEL | __GFP_HIGHMEM) @@ -175,7 +176,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma, { u32 pte_flags = 0; - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages; /* Currently applicable only to VLV */ if (vma->obj->gt_ro) @@ -706,6 +707,16 @@ static int gen8_48b_mm_switch(struct i915_hw_ppgtt *ppgtt, return gen8_write_pdp(req, 0, px_dma(&ppgtt->pml4)); } +/* PDE TLBs are a pain to invalidate on GEN8+. When we modify + * the page table structures, we mark them dirty so that + * context switching/execlist queuing code takes extra steps + * to ensure that tlbs are flushed. + */ +static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) +{ + ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; +} + /* Removes entries from a single page table, releasing it if it's empty. * Caller can use the return value to update higher-level entries. */ @@ -715,9 +726,9 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, uint64_t length) { struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); - unsigned int pte_start = gen8_pte_index(start); unsigned int num_entries = gen8_pte_count(start, length); - uint64_t pte; + unsigned int pte = gen8_pte_index(start); + unsigned int pte_end = pte + num_entries; gen8_pte_t *pt_vaddr; gen8_pte_t scratch_pte = gen8_pte_encode(vm->scratch_page.daddr, I915_CACHE_LLC); @@ -725,7 +736,9 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, if (WARN_ON(!px_page(pt))) return false; - bitmap_clear(pt->used_ptes, pte_start, num_entries); + GEM_BUG_ON(pte_end > GEN8_PTES); + + bitmap_clear(pt->used_ptes, pte, num_entries); if (bitmap_empty(pt->used_ptes, GEN8_PTES)) { free_pt(vm->dev, pt); @@ -734,8 +747,8 @@ static bool gen8_ppgtt_clear_pt(struct i915_address_space *vm, pt_vaddr = kmap_px(pt); - for (pte = pte_start; pte < num_entries; pte++) - pt_vaddr[pte] = scratch_pte; + while (pte < pte_end) + pt_vaddr[pte++] = scratch_pte; kunmap_px(ppgtt, pt_vaddr); @@ -806,6 +819,8 @@ static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm, } } + mark_tlbs_dirty(ppgtt); + if (USES_FULL_48BIT_PPGTT(vm->dev) && bitmap_empty(pdp->used_pdpes, I915_PDPES_PER_PDP(vm->dev))) { free_pdp(vm->dev, pdp); @@ -1280,16 +1295,6 @@ err_out: return -ENOMEM; } -/* PDE TLBs are a pain to invalidate on GEN8+. When we modify - * the page table structures, we mark them dirty so that - * context switching/execlist queuing code takes extra steps - * to ensure that tlbs are flushed. - */ -static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) -{ - ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask; -} - static int gen8_alloc_va_range_3lvl(struct i915_address_space *vm, struct i915_page_directory_pointer *pdp, uint64_t start, @@ -2184,8 +2189,10 @@ static int __hw_ppgtt_init(struct i915_hw_ppgtt *ppgtt, } static void i915_address_space_init(struct i915_address_space *vm, - struct drm_i915_private *dev_priv) + struct drm_i915_private *dev_priv, + const char *name) { + i915_gem_timeline_init(dev_priv, &vm->timeline, name); drm_mm_init(&vm->mm, vm->start, vm->total); INIT_LIST_HEAD(&vm->active_list); INIT_LIST_HEAD(&vm->inactive_list); @@ -2214,14 +2221,15 @@ static void gtt_write_workarounds(struct drm_device *dev) static int i915_ppgtt_init(struct i915_hw_ppgtt *ppgtt, struct drm_i915_private *dev_priv, - struct drm_i915_file_private *file_priv) + struct drm_i915_file_private *file_priv, + const char *name) { int ret; ret = __hw_ppgtt_init(ppgtt, dev_priv); if (ret == 0) { kref_init(&ppgtt->ref); - i915_address_space_init(&ppgtt->base, dev_priv); + i915_address_space_init(&ppgtt->base, dev_priv, name); ppgtt->base.file = file_priv; } @@ -2257,7 +2265,8 @@ int i915_ppgtt_init_hw(struct drm_device *dev) struct i915_hw_ppgtt * i915_ppgtt_create(struct drm_i915_private *dev_priv, - struct drm_i915_file_private *fpriv) + struct drm_i915_file_private *fpriv, + const char *name) { struct i915_hw_ppgtt *ppgtt; int ret; @@ -2266,7 +2275,7 @@ i915_ppgtt_create(struct drm_i915_private *dev_priv, if (!ppgtt) return ERR_PTR(-ENOMEM); - ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv); + ret = i915_ppgtt_init(ppgtt, dev_priv, fpriv, name); if (ret) { kfree(ppgtt); return ERR_PTR(ret); @@ -2289,6 +2298,7 @@ void i915_ppgtt_release(struct kref *kref) WARN_ON(!list_empty(&ppgtt->base.inactive_list)); WARN_ON(!list_empty(&ppgtt->base.unbound_list)); + i915_gem_timeline_fini(&ppgtt->base.timeline); list_del(&ppgtt->base.global_link); drm_mm_takedown(&ppgtt->base.mm); @@ -2370,14 +2380,15 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev) i915_ggtt_flush(dev_priv); } -int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) +int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { - if (!dma_map_sg(&obj->base.dev->pdev->dev, - obj->pages->sgl, obj->pages->nents, - PCI_DMA_BIDIRECTIONAL)) - return -ENOSPC; + if (dma_map_sg(&obj->base.dev->pdev->dev, + pages->sgl, pages->nents, + PCI_DMA_BIDIRECTIONAL)) + return 0; - return 0; + return -ENOSPC; } static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) @@ -2395,16 +2406,11 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm, gen8_pte_t __iomem *pte = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + (offset >> PAGE_SHIFT); - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); gen8_set_pte(pte, gen8_pte_encode(addr, level)); I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void gen8_ggtt_insert_entries(struct i915_address_space *vm, @@ -2418,11 +2424,8 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, gen8_pte_t __iomem *gtt_entries; gen8_pte_t gtt_entry; dma_addr_t addr; - int rpm_atomic_seq; int i = 0; - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT); for_each_sgt_dma(addr, sgt_iter, st) { @@ -2446,8 +2449,6 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, */ I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } struct insert_entries { @@ -2486,16 +2487,11 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm, gen6_pte_t __iomem *pte = (gen6_pte_t __iomem *)dev_priv->ggtt.gsm + (offset >> PAGE_SHIFT); - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); iowrite32(vm->pte_encode(addr, level, flags), pte); I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } /* @@ -2515,11 +2511,8 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, gen6_pte_t __iomem *gtt_entries; gen6_pte_t gtt_entry; dma_addr_t addr; - int rpm_atomic_seq; int i = 0; - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT); for_each_sgt_dma(addr, sgt_iter, st) { @@ -2542,8 +2535,6 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, */ I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); POSTING_READ(GFX_FLSH_CNTL_GEN6); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void nop_clear_range(struct i915_address_space *vm, @@ -2554,7 +2545,6 @@ static void nop_clear_range(struct i915_address_space *vm, static void gen8_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; @@ -2562,9 +2552,6 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, (gen8_pte_t __iomem *)ggtt->gsm + first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry; int i; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); if (WARN(num_entries > max_entries, "First entry = %d; Num entries = %d (max=%d)\n", @@ -2576,15 +2563,12 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm, for (i = 0; i < num_entries; i++) gen8_set_pte(>t_base[i], scratch_pte); readl(gtt_base); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void gen6_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); unsigned first_entry = start >> PAGE_SHIFT; unsigned num_entries = length >> PAGE_SHIFT; @@ -2592,9 +2576,6 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, (gen6_pte_t __iomem *)ggtt->gsm + first_entry; const int max_entries = ggtt_total_entries(ggtt) - first_entry; int i; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); if (WARN(num_entries > max_entries, "First entry = %d; Num entries = %d (max=%d)\n", @@ -2607,8 +2588,6 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, for (i = 0; i < num_entries; i++) iowrite32(scratch_pte, >t_base[i]); readl(gtt_base); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void i915_ggtt_insert_page(struct i915_address_space *vm, @@ -2617,16 +2596,10 @@ static void i915_ggtt_insert_page(struct i915_address_space *vm, enum i915_cache_level cache_level, u32 unused) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); } static void i915_ggtt_insert_entries(struct i915_address_space *vm, @@ -2634,39 +2607,25 @@ static void i915_ggtt_insert_entries(struct i915_address_space *vm, uint64_t start, enum i915_cache_level cache_level, u32 unused) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); unsigned int flags = (cache_level == I915_CACHE_NONE) ? AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags); - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); - } static void i915_ggtt_clear_range(struct i915_address_space *vm, uint64_t start, uint64_t length) { - struct drm_i915_private *dev_priv = to_i915(vm->dev); - unsigned first_entry = start >> PAGE_SHIFT; - unsigned num_entries = length >> PAGE_SHIFT; - int rpm_atomic_seq; - - rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv); - - intel_gtt_clear_range(first_entry, num_entries); - - assert_rpm_atomic_end(dev_priv, rpm_atomic_seq); + intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); } static int ggtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_private *i915 = to_i915(vma->vm->dev); struct drm_i915_gem_object *obj = vma->obj; u32 pte_flags = 0; int ret; @@ -2679,8 +2638,10 @@ static int ggtt_bind_vma(struct i915_vma *vma, if (obj->gt_ro) pte_flags |= PTE_READ_ONLY; + intel_runtime_pm_get(i915); vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, cache_level, pte_flags); + intel_runtime_pm_put(i915); /* * Without aliasing PPGTT there's no difference between @@ -2696,6 +2657,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, enum i915_cache_level cache_level, u32 flags) { + struct drm_i915_private *i915 = to_i915(vma->vm->dev); u32 pte_flags; int ret; @@ -2710,14 +2672,15 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, if (flags & I915_VMA_GLOBAL_BIND) { + intel_runtime_pm_get(i915); vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start, cache_level, pte_flags); + intel_runtime_pm_put(i915); } if (flags & I915_VMA_LOCAL_BIND) { - struct i915_hw_ppgtt *appgtt = - to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; appgtt->base.insert_entries(&appgtt->base, vma->pages, vma->node.start, cache_level, pte_flags); @@ -2728,19 +2691,24 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma, static void ggtt_unbind_vma(struct i915_vma *vma) { - struct i915_hw_ppgtt *appgtt = to_i915(vma->vm->dev)->mm.aliasing_ppgtt; + struct drm_i915_private *i915 = to_i915(vma->vm->dev); + struct i915_hw_ppgtt *appgtt = i915->mm.aliasing_ppgtt; const u64 size = min(vma->size, vma->node.size); - if (vma->flags & I915_VMA_GLOBAL_BIND) + if (vma->flags & I915_VMA_GLOBAL_BIND) { + intel_runtime_pm_get(i915); vma->vm->clear_range(vma->vm, vma->node.start, size); + intel_runtime_pm_put(i915); + } if (vma->flags & I915_VMA_LOCAL_BIND && appgtt) appgtt->base.clear_range(&appgtt->base, vma->node.start, size); } -void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) +void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); struct device *kdev = &dev_priv->drm.pdev->dev; @@ -2754,8 +2722,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } - dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents, - PCI_DMA_BIDIRECTIONAL); + dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL); } static void i915_gtt_color_adjust(struct drm_mm_node *node, @@ -3274,11 +3241,13 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv) /* Subtract the guard page before address space initialization to * shrink the range used by drm_mm. */ + mutex_lock(&dev_priv->drm.struct_mutex); ggtt->base.total -= PAGE_SIZE; - i915_address_space_init(&ggtt->base, dev_priv); + i915_address_space_init(&ggtt->base, dev_priv, "[global]"); ggtt->base.total += PAGE_SIZE; if (!HAS_LLC(dev_priv)) ggtt->base.mm.color_adjust = i915_gtt_color_adjust; + mutex_unlock(&dev_priv->drm.struct_mutex); if (!io_mapping_init_wc(&dev_priv->ggtt.mappable, dev_priv->ggtt.mappable_base, @@ -3327,7 +3296,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev) /* clflush objects bound into the GGTT and rebind them. */ list_for_each_entry_safe(obj, on, - &dev_priv->mm.bound_list, global_list) { + &dev_priv->mm.bound_list, global_link) { bool ggtt_bound = false; struct i915_vma *vma; @@ -3386,6 +3355,7 @@ i915_vma_retire(struct i915_gem_active *active, const unsigned int idx = rq->engine->id; struct i915_vma *vma = container_of(active, struct i915_vma, last_read[idx]); + struct drm_i915_gem_object *obj = vma->obj; GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); @@ -3396,6 +3366,34 @@ i915_vma_retire(struct i915_gem_active *active, list_move_tail(&vma->vm_link, &vma->vm->inactive_list); if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) WARN_ON(i915_vma_unbind(vma)); + + GEM_BUG_ON(!i915_gem_object_is_active(obj)); + if (--obj->active_count) + return; + + /* Bump our place on the bound list to keep it roughly in LRU order + * so that we don't steal from recently used but inactive objects + * (unless we are forced to ofc!) + */ + if (obj->bind_count) + list_move_tail(&obj->global_link, &rq->i915->mm.bound_list); + + obj->mm.dirty = true; /* be paranoid */ + + if (i915_gem_object_has_active_reference(obj)) { + i915_gem_object_clear_active_reference(obj); + i915_gem_object_put(obj); + } +} + +static void +i915_ggtt_retire__write(struct i915_gem_active *active, + struct drm_i915_gem_request *request) +{ + struct i915_vma *vma = + container_of(active, struct i915_vma, last_write); + + intel_fb_obj_flush(vma->obj, true, ORIGIN_CS); } void i915_vma_destroy(struct i915_vma *vma) @@ -3417,17 +3415,40 @@ void i915_vma_close(struct i915_vma *vma) GEM_BUG_ON(i915_vma_is_closed(vma)); vma->flags |= I915_VMA_CLOSED; - list_del_init(&vma->obj_link); + list_del(&vma->obj_link); + rb_erase(&vma->obj_node, &vma->obj->vma_tree); + if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma)) WARN_ON(i915_vma_unbind(vma)); } +static inline long vma_compare(struct i915_vma *vma, + struct i915_address_space *vm, + const struct i915_ggtt_view *view) +{ + GEM_BUG_ON(view && !i915_is_ggtt(vm)); + + if (vma->vm != vm) + return vma->vm - vm; + + if (!view) + return vma->ggtt_view.type; + + if (vma->ggtt_view.type != view->type) + return vma->ggtt_view.type - view->type; + + return memcmp(&vma->ggtt_view.params, + &view->params, + sizeof(view->params)); +} + static struct i915_vma * __i915_vma_create(struct drm_i915_gem_object *obj, struct i915_address_space *vm, const struct i915_ggtt_view *view) { struct i915_vma *vma; + struct rb_node *rb, **p; int i; GEM_BUG_ON(vm->closed); @@ -3439,6 +3460,8 @@ __i915_vma_create(struct drm_i915_gem_object *obj, INIT_LIST_HEAD(&vma->exec_list); for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) init_request_active(&vma->last_read[i], i915_vma_retire); + init_request_active(&vma->last_write, + i915_is_ggtt(vm) ? i915_ggtt_retire__write : NULL); init_request_active(&vma->last_fence, NULL); list_add(&vma->vm_link, &vm->unbound_list); vma->vm = vm; @@ -3459,33 +3482,28 @@ __i915_vma_create(struct drm_i915_gem_object *obj, if (i915_is_ggtt(vm)) { vma->flags |= I915_VMA_GGTT; + list_add(&vma->obj_link, &obj->vma_list); } else { i915_ppgtt_get(i915_vm_to_ppgtt(vm)); + list_add_tail(&vma->obj_link, &obj->vma_list); } - list_add_tail(&vma->obj_link, &obj->vma_list); - return vma; -} - -static inline bool vma_matches(struct i915_vma *vma, - struct i915_address_space *vm, - const struct i915_ggtt_view *view) -{ - if (vma->vm != vm) - return false; + rb = NULL; + p = &obj->vma_tree.rb_node; + while (*p) { + struct i915_vma *pos; - if (!i915_vma_is_ggtt(vma)) - return true; - - if (!view) - return vma->ggtt_view.type == 0; - - if (vma->ggtt_view.type != view->type) - return false; + rb = *p; + pos = rb_entry(rb, struct i915_vma, obj_node); + if (vma_compare(pos, vm, view) < 0) + p = &rb->rb_right; + else + p = &rb->rb_left; + } + rb_link_node(&vma->obj_node, rb, p); + rb_insert_color(&vma->obj_node, &obj->vma_tree); - return memcmp(&vma->ggtt_view.params, - &view->params, - sizeof(view->params)) == 0; + return vma; } struct i915_vma * @@ -3493,6 +3511,7 @@ i915_vma_create(struct drm_i915_gem_object *obj, struct i915_address_space *vm, const struct i915_ggtt_view *view) { + lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(view && !i915_is_ggtt(vm)); GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view)); @@ -3504,12 +3523,23 @@ i915_gem_obj_to_vma(struct drm_i915_gem_object *obj, struct i915_address_space *vm, const struct i915_ggtt_view *view) { - struct i915_vma *vma; + struct rb_node *rb; + + rb = obj->vma_tree.rb_node; + while (rb) { + struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); + long cmp; - list_for_each_entry_reverse(vma, &obj->vma_list, obj_link) - if (vma_matches(vma, vm, view)) + cmp = vma_compare(vma, vm, view); + if (cmp == 0) return vma; + if (cmp < 0) + rb = rb->rb_right; + else + rb = rb->rb_left; + } + return NULL; } @@ -3520,11 +3550,14 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, { struct i915_vma *vma; + lockdep_assert_held(&obj->base.dev->struct_mutex); GEM_BUG_ON(view && !i915_is_ggtt(vm)); vma = i915_gem_obj_to_vma(obj, vm, view); - if (!vma) + if (!vma) { vma = __i915_vma_create(obj, vm, view); + GEM_BUG_ON(vma != i915_gem_obj_to_vma(obj, vm, view)); + } GEM_BUG_ON(i915_vma_is_closed(vma)); return vma; @@ -3590,7 +3623,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info, /* Populate source page list from the object. */ i = 0; - for_each_sgt_dma(dma_addr, sgt_iter, obj->pages) + for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages) page_addr_list[i++] = dma_addr; GEM_BUG_ON(i != n_pages); @@ -3626,35 +3659,47 @@ intel_partial_pages(const struct i915_ggtt_view *view, struct drm_i915_gem_object *obj) { struct sg_table *st; - struct scatterlist *sg; - struct sg_page_iter obj_sg_iter; + struct scatterlist *sg, *iter; + unsigned int count = view->params.partial.size; + unsigned int offset; int ret = -ENOMEM; st = kmalloc(sizeof(*st), GFP_KERNEL); if (!st) goto err_st_alloc; - ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL); + ret = sg_alloc_table(st, count, GFP_KERNEL); if (ret) goto err_sg_alloc; + iter = i915_gem_object_get_sg(obj, + view->params.partial.offset, + &offset); + GEM_BUG_ON(!iter); + sg = st->sgl; st->nents = 0; - for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents, - view->params.partial.offset) - { - if (st->nents >= view->params.partial.size) - break; + do { + unsigned int len; - sg_set_page(sg, NULL, PAGE_SIZE, 0); - sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter); - sg_dma_len(sg) = PAGE_SIZE; + len = min(iter->length - (offset << PAGE_SHIFT), + count << PAGE_SHIFT); + sg_set_page(sg, NULL, len, 0); + sg_dma_address(sg) = + sg_dma_address(iter) + (offset << PAGE_SHIFT); + sg_dma_len(sg) = len; - sg = sg_next(sg); st->nents++; - } + count -= len >> PAGE_SHIFT; + if (count == 0) { + sg_mark_end(sg); + return st; + } - return st; + sg = __sg_next(sg); + iter = __sg_next(iter); + offset = 0; + } while (1); err_sg_alloc: kfree(st); @@ -3667,11 +3712,18 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma) { int ret = 0; + /* The vma->pages are only valid within the lifespan of the borrowed + * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so + * must be the vma->pages. A simple rule is that vma->pages must only + * be accessed when the obj->mm.pages are pinned. + */ + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); + if (vma->pages) return 0; if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages; else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED) vma->pages = intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj); @@ -3778,11 +3830,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) void i915_vma_unpin_and_release(struct i915_vma **p_vma) { struct i915_vma *vma; + struct drm_i915_gem_object *obj; vma = fetch_and_zero(p_vma); if (!vma) return; + obj = vma->obj; + i915_vma_unpin(vma); - i915_vma_put(vma); + i915_vma_close(vma); + + __i915_gem_object_release_unless_active(obj); } diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h index c241d8143255..c23ef9db1f53 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.h +++ b/drivers/gpu/drm/i915/i915_gem_gtt.h @@ -211,6 +211,7 @@ struct i915_vma { unsigned int active; struct i915_gem_active last_read[I915_NUM_ENGINES]; + struct i915_gem_active last_write; struct i915_gem_active last_fence; /** @@ -226,6 +227,7 @@ struct i915_vma { struct list_head vm_link; struct list_head obj_link; /* Link in the object's VMA list */ + struct rb_node obj_node; /** This vma's place in the batchbuffer or on the eviction list */ struct list_head exec_list; @@ -341,6 +343,7 @@ struct i915_pml4 { struct i915_address_space { struct drm_mm mm; + struct i915_gem_timeline timeline; struct drm_device *dev; /* Every address space belongs to a struct file - except for the global * GTT that is owned by the driver (and so @file is set to NULL). In @@ -612,7 +615,8 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv); int i915_ppgtt_init_hw(struct drm_device *dev); void i915_ppgtt_release(struct kref *kref); struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv, - struct drm_i915_file_private *fpriv); + struct drm_i915_file_private *fpriv, + const char *name); static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt) { if (ppgtt) @@ -628,8 +632,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv); void i915_gem_suspend_gtt_mappings(struct drm_device *dev); void i915_gem_restore_gtt_mappings(struct drm_device *dev); -int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj); -void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj); +int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); +void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); /* Flags used by pin/bind&friends. */ #define PIN_NONBLOCK BIT(0) diff --git a/drivers/gpu/drm/i915/i915_gem_internal.c b/drivers/gpu/drm/i915/i915_gem_internal.c new file mode 100644 index 000000000000..4b3ff3e5b911 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_internal.c @@ -0,0 +1,170 @@ +/* + * Copyright © 2014-2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include <drm/drmP.h> +#include <drm/i915_drm.h> +#include "i915_drv.h" + +#define QUIET (__GFP_NORETRY | __GFP_NOWARN) + +/* convert swiotlb segment size into sensible units (pages)! */ +#define IO_TLB_SEGPAGES (IO_TLB_SEGSIZE << IO_TLB_SHIFT >> PAGE_SHIFT) + +static void internal_free_pages(struct sg_table *st) +{ + struct scatterlist *sg; + + for (sg = st->sgl; sg; sg = __sg_next(sg)) + __free_pages(sg_page(sg), get_order(sg->length)); + + sg_free_table(st); + kfree(st); +} + +static struct sg_table * +i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) +{ + struct drm_i915_private *i915 = to_i915(obj->base.dev); + unsigned int npages = obj->base.size / PAGE_SIZE; + struct sg_table *st; + struct scatterlist *sg; + int max_order; + gfp_t gfp; + + st = kmalloc(sizeof(*st), GFP_KERNEL); + if (!st) + return ERR_PTR(-ENOMEM); + + if (sg_alloc_table(st, npages, GFP_KERNEL)) { + kfree(st); + return ERR_PTR(-ENOMEM); + } + + sg = st->sgl; + st->nents = 0; + + max_order = MAX_ORDER; +#ifdef CONFIG_SWIOTLB + if (swiotlb_nr_tbl()) /* minimum max swiotlb size is IO_TLB_SEGSIZE */ + max_order = min(max_order, ilog2(IO_TLB_SEGPAGES)); +#endif + + gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; + if (IS_CRESTLINE(i915) || IS_BROADWATER(i915)) { + /* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + + do { + int order = min(fls(npages) - 1, max_order); + struct page *page; + + do { + page = alloc_pages(gfp | (order ? QUIET : 0), order); + if (page) + break; + if (!order--) + goto err; + + /* Limit subsequent allocations as well */ + max_order = order; + } while (1); + + sg_set_page(sg, page, PAGE_SIZE << order, 0); + st->nents++; + + npages -= 1 << order; + if (!npages) { + sg_mark_end(sg); + break; + } + + sg = __sg_next(sg); + } while (1); + + if (i915_gem_gtt_prepare_pages(obj, st)) + goto err; + + /* Mark the pages as dontneed whilst they are still pinned. As soon + * as they are unpinned they are allowed to be reaped by the shrinker, + * and the caller is expected to repopulate - the contents of this + * object are only valid whilst active and pinned. + */ + obj->mm.madv = I915_MADV_DONTNEED; + return st; + +err: + sg_mark_end(sg); + internal_free_pages(st); + return ERR_PTR(-ENOMEM); +} + +static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, + struct sg_table *pages) +{ + i915_gem_gtt_finish_pages(obj, pages); + internal_free_pages(pages); + + obj->mm.dirty = false; + obj->mm.madv = I915_MADV_WILLNEED; +} + +static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_IS_SHRINKABLE, + .get_pages = i915_gem_object_get_pages_internal, + .put_pages = i915_gem_object_put_pages_internal, +}; + +/** + * Creates a new object that wraps some internal memory for private use. + * This object is not backed by swappable storage, and as such its contents + * are volatile and only valid whilst pinned. If the object is reaped by the + * shrinker, its pages and data will be discarded. Equally, it is not a full + * GEM object and so not valid for access from userspace. This makes it useful + * for hardware interfaces like ringbuffers (which are pinned from the time + * the request is written to the time the hardware stops accessing it), but + * not for contexts (which need to be preserved when not active for later + * reuse). Note that it is not cleared upon allocation. + */ +struct drm_i915_gem_object * +i915_gem_object_create_internal(struct drm_i915_private *i915, + unsigned int size) +{ + struct drm_i915_gem_object *obj; + + obj = i915_gem_object_alloc(&i915->drm); + if (!obj) + return ERR_PTR(-ENOMEM); + + drm_gem_private_object_init(&i915->drm, &obj->base, size); + i915_gem_object_init(obj, &i915_gem_object_internal_ops); + + obj->base.write_domain = I915_GEM_DOMAIN_CPU; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; + + return obj; +} diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c index a98c0f42badd..5af19b0bf713 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.c +++ b/drivers/gpu/drm/i915/i915_gem_render_state.c @@ -28,17 +28,19 @@ #include "i915_drv.h" #include "intel_renderstate.h" -struct render_state { +struct intel_render_state { const struct intel_renderstate_rodata *rodata; struct i915_vma *vma; - u32 aux_batch_size; - u32 aux_batch_offset; + u32 batch_offset; + u32 batch_size; + u32 aux_offset; + u32 aux_size; }; static const struct intel_renderstate_rodata * -render_state_get_rodata(const struct drm_i915_gem_request *req) +render_state_get_rodata(const struct intel_engine_cs *engine) { - switch (INTEL_GEN(req->i915)) { + switch (INTEL_GEN(engine->i915)) { case 6: return &gen6_null_state; case 7: @@ -63,29 +65,26 @@ render_state_get_rodata(const struct drm_i915_gem_request *req) */ #define OUT_BATCH(batch, i, val) \ do { \ - if (WARN_ON((i) >= PAGE_SIZE / sizeof(u32))) { \ - ret = -ENOSPC; \ - goto err_out; \ - } \ + if ((i) >= PAGE_SIZE / sizeof(u32)) \ + goto err; \ (batch)[(i)++] = (val); \ } while(0) -static int render_state_setup(struct render_state *so) +static int render_state_setup(struct intel_render_state *so, + struct drm_i915_private *i915) { - struct drm_i915_private *dev_priv = to_i915(so->vma->vm->dev); const struct intel_renderstate_rodata *rodata = so->rodata; - const bool has_64bit_reloc = INTEL_GEN(dev_priv) >= 8; + struct drm_i915_gem_object *obj = so->vma->obj; unsigned int i = 0, reloc_index = 0; - struct page *page; + unsigned int needs_clflush; u32 *d; int ret; - ret = i915_gem_object_set_to_cpu_domain(so->vma->obj, true); + ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); if (ret) return ret; - page = i915_gem_object_get_dirty_page(so->vma->obj, 0); - d = kmap(page); + d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0)); while (i < rodata->batch_items) { u32 s = rodata->batch[i]; @@ -93,12 +92,10 @@ static int render_state_setup(struct render_state *so) if (i * 4 == rodata->reloc[reloc_index]) { u64 r = s + so->vma->node.start; s = lower_32_bits(r); - if (has_64bit_reloc) { + if (HAS_64BIT_RELOC(i915)) { if (i + 1 >= rodata->batch_items || - rodata->batch[i + 1] != 0) { - ret = -EINVAL; - goto err_out; - } + rodata->batch[i + 1] != 0) + goto err; d[i++] = s; s = upper_32_bits(r); @@ -110,12 +107,20 @@ static int render_state_setup(struct render_state *so) d[i++] = s; } + if (rodata->reloc[reloc_index] != -1) { + DRM_ERROR("only %d relocs resolved\n", reloc_index); + goto err; + } + + so->batch_offset = so->vma->node.start; + so->batch_size = rodata->batch_items * sizeof(u32); + while (i % CACHELINE_DWORDS) OUT_BATCH(d, i, MI_NOOP); - so->aux_batch_offset = i * sizeof(u32); + so->aux_offset = i * sizeof(u32); - if (HAS_POOLED_EU(dev_priv)) { + if (HAS_POOLED_EU(i915)) { /* * We always program 3x6 pool config but depending upon which * subslice is disabled HW drops down to appropriate config @@ -143,88 +148,133 @@ static int render_state_setup(struct render_state *so) } OUT_BATCH(d, i, MI_BATCH_BUFFER_END); - so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset; - + so->aux_size = i * sizeof(u32) - so->aux_offset; + so->aux_offset += so->batch_offset; /* * Since we are sending length, we need to strictly conform to * all requirements. For Gen2 this must be a multiple of 8. */ - so->aux_batch_size = ALIGN(so->aux_batch_size, 8); - - kunmap(page); - - ret = i915_gem_object_set_to_gtt_domain(so->vma->obj, false); - if (ret) - return ret; - - if (rodata->reloc[reloc_index] != -1) { - DRM_ERROR("only %d relocs resolved\n", reloc_index); - return -EINVAL; - } + so->aux_size = ALIGN(so->aux_size, 8); - return 0; + if (needs_clflush) + drm_clflush_virt_range(d, i * sizeof(u32)); + kunmap_atomic(d); -err_out: - kunmap(page); + ret = i915_gem_object_set_to_gtt_domain(obj, false); +out: + i915_gem_obj_finish_shmem_access(obj); return ret; + +err: + kunmap_atomic(d); + ret = -EINVAL; + goto out; } #undef OUT_BATCH -int i915_gem_render_state_init(struct drm_i915_gem_request *req) +int i915_gem_render_state_init(struct intel_engine_cs *engine) { - struct render_state so; + struct intel_render_state *so; + const struct intel_renderstate_rodata *rodata; struct drm_i915_gem_object *obj; int ret; - if (WARN_ON(req->engine->id != RCS)) - return -ENOENT; + if (engine->id != RCS) + return 0; - so.rodata = render_state_get_rodata(req); - if (!so.rodata) + rodata = render_state_get_rodata(engine); + if (!rodata) return 0; - if (so.rodata->batch_items * 4 > 4096) + if (rodata->batch_items * 4 > 4096) return -EINVAL; - obj = i915_gem_object_create(&req->i915->drm, 4096); - if (IS_ERR(obj)) - return PTR_ERR(obj); + so = kmalloc(sizeof(*so), GFP_KERNEL); + if (!so) + return -ENOMEM; - so.vma = i915_vma_create(obj, &req->i915->ggtt.base, NULL); - if (IS_ERR(so.vma)) { - ret = PTR_ERR(so.vma); - goto err_obj; + obj = i915_gem_object_create_internal(engine->i915, 4096); + if (IS_ERR(obj)) { + ret = PTR_ERR(obj); + goto err_free; } - ret = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL); - if (ret) + so->vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL); + if (IS_ERR(so->vma)) { + ret = PTR_ERR(so->vma); goto err_obj; + } + + so->rodata = rodata; + engine->render_state = so; + return 0; + +err_obj: + i915_gem_object_put(obj); +err_free: + kfree(so); + return ret; +} + +int i915_gem_render_state_emit(struct drm_i915_gem_request *req) +{ + struct intel_render_state *so; + int ret; + + lockdep_assert_held(&req->i915->drm.struct_mutex); - ret = render_state_setup(&so); + so = req->engine->render_state; + if (!so) + return 0; + + /* Recreate the page after shrinking */ + if (!so->vma->obj->mm.pages) + so->batch_offset = -1; + + ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH); if (ret) - goto err_unpin; + return ret; + + if (so->vma->node.start != so->batch_offset) { + ret = render_state_setup(so, req->i915); + if (ret) + goto err_unpin; + } - ret = req->engine->emit_bb_start(req, so.vma->node.start, - so.rodata->batch_items * 4, + ret = req->engine->emit_bb_start(req, + so->batch_offset, so->batch_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; - if (so.aux_batch_size > 8) { + if (so->aux_size > 8) { ret = req->engine->emit_bb_start(req, - (so.vma->node.start + - so.aux_batch_offset), - so.aux_batch_size, + so->aux_offset, so->aux_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; } - i915_vma_move_to_active(so.vma, req, 0); + i915_vma_move_to_active(so->vma, req, 0); err_unpin: - i915_vma_unpin(so.vma); -err_obj: - i915_gem_object_put(obj); + i915_vma_unpin(so->vma); return ret; } + +void i915_gem_render_state_fini(struct intel_engine_cs *engine) +{ + struct intel_render_state *so; + struct drm_i915_gem_object *obj; + + so = fetch_and_zero(&engine->render_state); + if (!so) + return; + + obj = so->vma->obj; + + i915_vma_close(so->vma); + __i915_gem_object_release_unless_active(obj); + + kfree(so); +} diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h index 18cce3f06e9c..87481845799d 100644 --- a/drivers/gpu/drm/i915/i915_gem_render_state.h +++ b/drivers/gpu/drm/i915/i915_gem_render_state.h @@ -26,6 +26,8 @@ struct drm_i915_gem_request; -int i915_gem_render_state_init(struct drm_i915_gem_request *req); +int i915_gem_render_state_init(struct intel_engine_cs *engine); +int i915_gem_render_state_emit(struct drm_i915_gem_request *req); +void i915_gem_render_state_fini(struct intel_engine_cs *engine); #endif /* _I915_GEM_RENDER_STATE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c index f9af2a00625e..0b3b051a5683 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.c +++ b/drivers/gpu/drm/i915/i915_gem_request.c @@ -23,6 +23,7 @@ */ #include <linux/prefetch.h> +#include <linux/dma-fence-array.h> #include "i915_drv.h" @@ -33,13 +34,7 @@ static const char *i915_fence_get_driver_name(struct dma_fence *fence) static const char *i915_fence_get_timeline_name(struct dma_fence *fence) { - /* Timelines are bound by eviction to a VM. However, since - * we only have a global seqno at the moment, we only have - * a single timeline. Note that each timeline will have - * multiple execution contexts (fence contexts) as we allow - * engines within a single timeline to execute in parallel. - */ - return "global"; + return to_request(fence)->timeline->common->name; } static bool i915_fence_signaled(struct dma_fence *fence) @@ -58,43 +53,9 @@ static bool i915_fence_enable_signaling(struct dma_fence *fence) static signed long i915_fence_wait(struct dma_fence *fence, bool interruptible, - signed long timeout_jiffies) + signed long timeout) { - s64 timeout_ns, *timeout; - int ret; - - if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) { - timeout_ns = jiffies_to_nsecs(timeout_jiffies); - timeout = &timeout_ns; - } else { - timeout = NULL; - } - - ret = i915_wait_request(to_request(fence), - interruptible, timeout, - NO_WAITBOOST); - if (ret == -ETIME) - return 0; - - if (ret < 0) - return ret; - - if (timeout_jiffies != MAX_SCHEDULE_TIMEOUT) - timeout_jiffies = nsecs_to_jiffies(timeout_ns); - - return timeout_jiffies; -} - -static void i915_fence_value_str(struct dma_fence *fence, char *str, int size) -{ - snprintf(str, size, "%u", fence->seqno); -} - -static void i915_fence_timeline_value_str(struct dma_fence *fence, char *str, - int size) -{ - snprintf(str, size, "%u", - intel_engine_get_seqno(to_request(fence)->engine)); + return i915_wait_request(to_request(fence), interruptible, timeout); } static void i915_fence_release(struct dma_fence *fence) @@ -111,8 +72,6 @@ const struct dma_fence_ops i915_fence_ops = { .signaled = i915_fence_signaled, .wait = i915_fence_wait, .release = i915_fence_release, - .fence_value_str = i915_fence_value_str, - .timeline_value_str = i915_fence_timeline_value_str, }; int i915_gem_request_add_to_client(struct drm_i915_gem_request *req, @@ -164,8 +123,14 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) { struct i915_gem_active *active, *next; + lockdep_assert_held(&request->i915->drm.struct_mutex); + GEM_BUG_ON(!i915_gem_request_completed(request)); + trace_i915_gem_request_retire(request); - list_del(&request->link); + + spin_lock_irq(&request->engine->timeline->lock); + list_del_init(&request->link); + spin_unlock_irq(&request->engine->timeline->lock); /* We know the GPU must have read the request to have * sent us the seqno + interrupt, so use the position @@ -177,6 +142,7 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) */ list_del(&request->ring_link); request->ring->last_retired_head = request->postfix; + request->i915->gt.active_requests--; /* Walk through the active list, calling retire on each. This allows * objects to track their GPU activity and mark themselves as idle @@ -214,6 +180,8 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) } i915_gem_context_put(request->ctx); + + dma_fence_signal(&request->fence); i915_gem_request_put(request); } @@ -223,10 +191,11 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req) struct drm_i915_gem_request *tmp; lockdep_assert_held(&req->i915->drm.struct_mutex); - GEM_BUG_ON(list_empty(&req->link)); + if (list_empty(&req->link)) + return; do { - tmp = list_first_entry(&engine->request_list, + tmp = list_first_entry(&engine->timeline->requests, typeof(*tmp), link); i915_gem_request_retire(tmp); @@ -253,40 +222,51 @@ static int i915_gem_check_wedge(struct drm_i915_private *dev_priv) return 0; } -static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno) +static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno) { + struct i915_gem_timeline *timeline = &i915->gt.global_timeline; struct intel_engine_cs *engine; enum intel_engine_id id; int ret; /* Carefully retire all requests without writing to the rings */ - for_each_engine(engine, dev_priv, id) { - ret = intel_engine_idle(engine, - I915_WAIT_INTERRUPTIBLE | - I915_WAIT_LOCKED); - if (ret) - return ret; - } - i915_gem_retire_requests(dev_priv); + ret = i915_gem_wait_for_idle(i915, + I915_WAIT_INTERRUPTIBLE | + I915_WAIT_LOCKED); + if (ret) + return ret; + + i915_gem_retire_requests(i915); + GEM_BUG_ON(i915->gt.active_requests > 1); /* If the seqno wraps around, we need to clear the breadcrumb rbtree */ - if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) { - while (intel_kick_waiters(dev_priv) || - intel_kick_signalers(dev_priv)) + if (!i915_seqno_passed(seqno, atomic_read(&timeline->next_seqno))) { + while (intel_kick_waiters(i915) || intel_kick_signalers(i915)) yield(); + yield(); } + atomic_set(&timeline->next_seqno, seqno); /* Finally reset hw state */ - for_each_engine(engine, dev_priv, id) - intel_engine_init_seqno(engine, seqno); + for_each_engine(engine, i915, id) + intel_engine_init_global_seqno(engine, seqno); + + list_for_each_entry(timeline, &i915->gt.timelines, link) { + for_each_engine(engine, i915, id) { + struct intel_timeline *tl = &timeline->engine[id]; + + memset(tl->sync_seqno, 0, sizeof(tl->sync_seqno)); + } + } return 0; } -int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) +int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno) { struct drm_i915_private *dev_priv = to_i915(dev); - int ret; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); if (seqno == 0) return -EINVAL; @@ -294,48 +274,84 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno) /* HWS page needs to be set less than what we * will inject to ring */ - ret = i915_gem_init_seqno(dev_priv, seqno - 1); - if (ret) - return ret; - - dev_priv->next_seqno = seqno; - return 0; + return i915_gem_init_global_seqno(dev_priv, seqno - 1); } -static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno) +static int reserve_global_seqno(struct drm_i915_private *i915) { - /* reserve 0 for non-seqno */ - if (unlikely(dev_priv->next_seqno == 0)) { - int ret; + u32 active_requests = ++i915->gt.active_requests; + u32 next_seqno = atomic_read(&i915->gt.global_timeline.next_seqno); + int ret; - ret = i915_gem_init_seqno(dev_priv, 0); - if (ret) - return ret; + /* Reservation is fine until we need to wrap around */ + if (likely(next_seqno + active_requests > next_seqno)) + return 0; - dev_priv->next_seqno = 1; + ret = i915_gem_init_global_seqno(i915, 0); + if (ret) { + i915->gt.active_requests--; + return ret; } - *seqno = dev_priv->next_seqno++; return 0; } +static u32 __timeline_get_seqno(struct i915_gem_timeline *tl) +{ + /* next_seqno only incremented under a mutex */ + return ++tl->next_seqno.counter; +} + +static u32 timeline_get_seqno(struct i915_gem_timeline *tl) +{ + return atomic_inc_return(&tl->next_seqno); +} + static int __i915_sw_fence_call submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { struct drm_i915_gem_request *request = container_of(fence, typeof(*request), submit); + struct intel_engine_cs *engine = request->engine; + struct intel_timeline *timeline; + unsigned long flags; + u32 seqno; + + if (state != FENCE_COMPLETE) + return NOTIFY_DONE; + + /* Transfer from per-context onto the global per-engine timeline */ + timeline = engine->timeline; + GEM_BUG_ON(timeline == request->timeline); /* Will be called from irq-context when using foreign DMA fences */ + spin_lock_irqsave(&timeline->lock, flags); - switch (state) { - case FENCE_COMPLETE: - request->engine->last_submitted_seqno = request->fence.seqno; - request->engine->submit_request(request); - break; + seqno = timeline_get_seqno(timeline->common); + GEM_BUG_ON(!seqno); + GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno)); - case FENCE_FREE: - break; - } + GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, seqno)); + request->previous_seqno = timeline->last_submitted_seqno; + timeline->last_submitted_seqno = seqno; + + /* We may be recursing from the signal callback of another i915 fence */ + spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING); + request->global_seqno = seqno; + if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags)) + intel_engine_enable_signaling(request); + spin_unlock(&request->lock); + + GEM_BUG_ON(!request->global_seqno); + engine->emit_breadcrumb(request, + request->ring->vaddr + request->postfix); + engine->submit_request(request); + + spin_lock_nested(&request->timeline->lock, SINGLE_DEPTH_NESTING); + list_move_tail(&request->link, &timeline->requests); + spin_unlock(&request->timeline->lock); + + spin_unlock_irqrestore(&timeline->lock, flags); return NOTIFY_DONE; } @@ -358,9 +374,10 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, { struct drm_i915_private *dev_priv = engine->i915; struct drm_i915_gem_request *req; - u32 seqno; int ret; + lockdep_assert_held(&dev_priv->drm.struct_mutex); + /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex * and restart. @@ -369,10 +386,14 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, if (ret) return ERR_PTR(ret); + ret = reserve_global_seqno(dev_priv); + if (ret) + return ERR_PTR(ret); + /* Move the oldest request to the slab-cache (if not in use!) */ - req = list_first_entry_or_null(&engine->request_list, + req = list_first_entry_or_null(&engine->timeline->requests, typeof(*req), link); - if (req && i915_gem_request_completed(req)) + if (req && __i915_gem_request_completed(req)) i915_gem_request_retire(req); /* Beware: Dragons be flying overhead. @@ -383,7 +404,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * of being read by __i915_gem_active_get_rcu(). As such, * we have to be very careful when overwriting the contents. During * the RCU lookup, we change chase the request->engine pointer, - * read the request->fence.seqno and increment the reference count. + * read the request->global_seqno and increment the reference count. * * The reference count is incremented atomically. If it is zero, * the lookup knows the request is unallocated and complete. Otherwise, @@ -404,19 +425,20 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * Do not use kmem_cache_zalloc() here! */ req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL); - if (!req) - return ERR_PTR(-ENOMEM); + if (!req) { + ret = -ENOMEM; + goto err_unreserve; + } - ret = i915_gem_get_seqno(dev_priv, &seqno); - if (ret) - goto err; + req->timeline = i915_gem_context_lookup_timeline(ctx, engine); + GEM_BUG_ON(req->timeline == engine->timeline); spin_lock_init(&req->lock); dma_fence_init(&req->fence, &i915_fence_ops, &req->lock, - engine->fence_context, - seqno); + req->timeline->fence_context, + __timeline_get_seqno(req->timeline->common)); i915_sw_fence_init(&req->submit, submit_notify); @@ -426,6 +448,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, req->ctx = i915_gem_context_get(ctx); /* No zalloc, must clear what we need by hand */ + req->global_seqno = 0; req->previous_context = NULL; req->file_priv = NULL; req->batch = NULL; @@ -438,6 +461,7 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, * away, e.g. because a GPU scheduler has deferred it. */ req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST; + GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz); if (i915.enable_execlists) ret = intel_logical_ring_alloc_request_extras(req); @@ -457,8 +481,9 @@ i915_gem_request_alloc(struct intel_engine_cs *engine, err_ctx: i915_gem_context_put(ctx); -err: kmem_cache_free(dev_priv->requests, req); +err_unreserve: + dev_priv->gt.active_requests--; return ERR_PTR(ret); } @@ -466,15 +491,28 @@ static int i915_gem_request_await_request(struct drm_i915_gem_request *to, struct drm_i915_gem_request *from) { - int idx, ret; + int ret; GEM_BUG_ON(to == from); - if (to->engine == from->engine) + if (to->timeline == from->timeline) return 0; - idx = intel_engine_sync_index(from->engine, to->engine); - if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx]) + if (to->engine == from->engine) { + ret = i915_sw_fence_await_sw_fence_gfp(&to->submit, + &from->submit, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + + if (!from->global_seqno) { + ret = i915_sw_fence_await_dma_fence(&to->submit, + &from->fence, 0, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + + if (from->global_seqno <= to->timeline->sync_seqno[from->engine->id]) return 0; trace_i915_gem_ring_sync_to(to, from); @@ -492,7 +530,54 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to, return ret; } - from->engine->semaphore.sync_seqno[idx] = from->fence.seqno; + to->timeline->sync_seqno[from->engine->id] = from->global_seqno; + return 0; +} + +int +i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, + struct dma_fence *fence) +{ + struct dma_fence_array *array; + int ret; + int i; + + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return 0; + + if (dma_fence_is_i915(fence)) + return i915_gem_request_await_request(req, to_request(fence)); + + if (!dma_fence_is_array(fence)) { + ret = i915_sw_fence_await_dma_fence(&req->submit, + fence, I915_FENCE_TIMEOUT, + GFP_KERNEL); + return ret < 0 ? ret : 0; + } + + /* Note that if the fence-array was created in signal-on-any mode, + * we should *not* decompose it into its individual fences. However, + * we don't currently store which mode the fence-array is operating + * in. Fortunately, the only user of signal-on-any is private to + * amdgpu and we should not see any incoming fence-array from + * sync-file being in signal-on-any mode. + */ + + array = to_dma_fence_array(fence); + for (i = 0; i < array->num_fences; i++) { + struct dma_fence *child = array->fences[i]; + + if (dma_fence_is_i915(child)) + ret = i915_gem_request_await_request(req, + to_request(child)); + else + ret = i915_sw_fence_await_dma_fence(&req->submit, + child, I915_FENCE_TIMEOUT, + GFP_KERNEL); + if (ret < 0) + return ret; + } + return 0; } @@ -521,40 +606,47 @@ i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write) { - struct i915_gem_active *active; - unsigned long active_mask; - int idx; + struct dma_fence *excl; + int ret = 0; if (write) { - active_mask = i915_gem_object_get_active(obj); - active = obj->last_read; + struct dma_fence **shared; + unsigned int count, i; + + ret = reservation_object_get_fences_rcu(obj->resv, + &excl, &count, &shared); + if (ret) + return ret; + + for (i = 0; i < count; i++) { + ret = i915_gem_request_await_dma_fence(to, shared[i]); + if (ret) + break; + + dma_fence_put(shared[i]); + } + + for (; i < count; i++) + dma_fence_put(shared[i]); + kfree(shared); } else { - active_mask = 1; - active = &obj->last_write; + excl = reservation_object_get_excl_rcu(obj->resv); } - for_each_active(active_mask, idx) { - struct drm_i915_gem_request *request; - int ret; - - request = i915_gem_active_peek(&active[idx], - &obj->base.dev->struct_mutex); - if (!request) - continue; + if (excl) { + if (ret == 0) + ret = i915_gem_request_await_dma_fence(to, excl); - ret = i915_gem_request_await_request(to, request); - if (ret) - return ret; + dma_fence_put(excl); } - return 0; + return ret; } static void i915_gem_mark_busy(const struct intel_engine_cs *engine) { struct drm_i915_private *dev_priv = engine->i915; - dev_priv->gt.active_engines |= intel_engine_flag(engine); if (dev_priv->gt.awake) return; @@ -580,11 +672,11 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) { struct intel_engine_cs *engine = request->engine; struct intel_ring *ring = request->ring; + struct intel_timeline *timeline = request->timeline; struct drm_i915_gem_request *prev; - u32 request_start; - u32 reserved_tail; - int ret; + int err; + lockdep_assert_held(&request->i915->drm.struct_mutex); trace_i915_gem_request_add(request); /* @@ -592,8 +684,6 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * should already have been reserved in the ring buffer. Let the ring * know that it is time to use that space up. */ - request_start = ring->tail; - reserved_tail = request->reserved_space; request->reserved_space = 0; /* @@ -604,10 +694,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * what. */ if (flush_caches) { - ret = engine->emit_flush(request, EMIT_FLUSH); + err = engine->emit_flush(request, EMIT_FLUSH); /* Not allowed to fail! */ - WARN(ret, "engine->emit_flush() failed: %d!\n", ret); + WARN(err, "engine->emit_flush() failed: %d!\n", err); } /* Record the position of the start of the breadcrumb so that @@ -615,20 +705,10 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * GPU processing the request, we never over-estimate the * position of the ring's HEAD. */ + err = intel_ring_begin(request, engine->emit_breadcrumb_sz); + GEM_BUG_ON(err); request->postfix = ring->tail; - - /* Not allowed to fail! */ - ret = engine->emit_request(request); - WARN(ret, "(%s)->emit_request failed: %d!\n", engine->name, ret); - - /* Sanity check that the reserved size was large enough. */ - ret = ring->tail - request_start; - if (ret < 0) - ret += ring->size; - WARN_ONCE(ret > reserved_tail, - "Not enough space reserved (%d bytes) " - "for adding the request (%d bytes)\n", - reserved_tail, ret); + ring->tail += engine->emit_breadcrumb_sz * sizeof(u32); /* Seal the request and mark it as pending execution. Note that * we may inspect this state, without holding any locks, during @@ -636,18 +716,24 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches) * see a more recent value in the hws than we are tracking. */ - prev = i915_gem_active_raw(&engine->last_request, + prev = i915_gem_active_raw(&timeline->last_request, &request->i915->drm.struct_mutex); if (prev) i915_sw_fence_await_sw_fence(&request->submit, &prev->submit, &request->submitq); - request->emitted_jiffies = jiffies; - request->previous_seqno = engine->last_pending_seqno; - engine->last_pending_seqno = request->fence.seqno; - i915_gem_active_set(&engine->last_request, request); - list_add_tail(&request->link, &engine->request_list); + spin_lock_irq(&timeline->lock); + list_add_tail(&request->link, &timeline->requests); + spin_unlock_irq(&timeline->lock); + + GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, + request->fence.seqno)); + + timeline->last_submitted_seqno = request->fence.seqno; + i915_gem_active_set(&timeline->last_request, request); + list_add_tail(&request->ring_link, &ring->request_list); + request->emitted_jiffies = jiffies; i915_gem_mark_busy(engine); @@ -715,7 +801,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req, timeout_us += local_clock_us(&cpu); do { - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; if (signal_pending_state(state, current)) @@ -730,76 +816,101 @@ bool __i915_spin_request(const struct drm_i915_gem_request *req, return false; } +static long +__i915_request_wait_for_submit(struct drm_i915_gem_request *request, + unsigned int flags, + long timeout) +{ + const int state = flags & I915_WAIT_INTERRUPTIBLE ? + TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; + wait_queue_head_t *q = &request->i915->gpu_error.wait_queue; + DEFINE_WAIT(reset); + DEFINE_WAIT(wait); + + if (flags & I915_WAIT_LOCKED) + add_wait_queue(q, &reset); + + do { + prepare_to_wait(&request->submit.wait, &wait, state); + + if (i915_sw_fence_done(&request->submit)) + break; + + if (flags & I915_WAIT_LOCKED && + i915_reset_in_progress(&request->i915->gpu_error)) { + __set_current_state(TASK_RUNNING); + i915_reset(request->i915); + reset_wait_queue(q, &reset); + continue; + } + + if (signal_pending_state(state, current)) { + timeout = -ERESTARTSYS; + break; + } + + timeout = io_schedule_timeout(timeout); + } while (timeout); + finish_wait(&request->submit.wait, &wait); + + if (flags & I915_WAIT_LOCKED) + remove_wait_queue(q, &reset); + + return timeout; +} + /** * i915_wait_request - wait until execution of request has finished - * @req: duh! + * @req: the request to wait upon * @flags: how to wait - * @timeout: in - how long to wait (NULL forever); out - how much time remaining - * @rps: client to charge for RPS boosting + * @timeout: how long to wait in jiffies + * + * i915_wait_request() waits for the request to be completed, for a + * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an + * unbounded wait). * - * Note: It is of utmost importance that the passed in seqno and reset_counter - * values have been read by the caller in an smp safe manner. Where read-side - * locks are involved, it is sufficient to read the reset_counter before - * unlocking the lock that protects the seqno. For lockless tricks, the - * reset_counter _must_ be read before, and an appropriate smp_rmb must be - * inserted. + * If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED + * in via the flags, and vice versa if the struct_mutex is not held, the caller + * must not specify that the wait is locked. * - * Returns 0 if the request was found within the alloted time. Else returns the - * errno with remaining time filled in timeout argument. + * Returns the remaining time (in jiffies) if the request completed, which may + * be zero or -ETIME if the request is unfinished after the timeout expires. + * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is + * pending before the request completes. */ -int i915_wait_request(struct drm_i915_gem_request *req, - unsigned int flags, - s64 *timeout, - struct intel_rps_client *rps) +long i915_wait_request(struct drm_i915_gem_request *req, + unsigned int flags, + long timeout) { const int state = flags & I915_WAIT_INTERRUPTIBLE ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE; DEFINE_WAIT(reset); struct intel_wait wait; - unsigned long timeout_remain; - int ret = 0; might_sleep(); #if IS_ENABLED(CONFIG_LOCKDEP) - GEM_BUG_ON(!!lockdep_is_held(&req->i915->drm.struct_mutex) != + GEM_BUG_ON(debug_locks && + !!lockdep_is_held(&req->i915->drm.struct_mutex) != !!(flags & I915_WAIT_LOCKED)); #endif + GEM_BUG_ON(timeout < 0); if (i915_gem_request_completed(req)) - return 0; + return timeout; - timeout_remain = MAX_SCHEDULE_TIMEOUT; - if (timeout) { - if (WARN_ON(*timeout < 0)) - return -EINVAL; - - if (*timeout == 0) - return -ETIME; - - /* Record current time in case interrupted, or wedged */ - timeout_remain = nsecs_to_jiffies_timeout(*timeout); - *timeout += ktime_get_raw_ns(); - } + if (!timeout) + return -ETIME; trace_i915_gem_request_wait_begin(req); - /* This client is about to stall waiting for the GPU. In many cases - * this is undesirable and limits the throughput of the system, as - * many clients cannot continue processing user input/output whilst - * blocked. RPS autotuning may take tens of milliseconds to respond - * to the GPU load and thus incurs additional latency for the client. - * We can circumvent that by promoting the GPU frequency to maximum - * before we wait. This makes the GPU throttle up much more quickly - * (good for benchmarks and user experience, e.g. window animations), - * but at a cost of spending more power processing the workload - * (bad for battery). Not all clients even want their results - * immediately and for them we should just let the GPU select its own - * frequency to maximise efficiency. To prevent a single client from - * forcing the clocks too high for the whole system, we only allow - * each client to waitboost once in a busy period. - */ - if (IS_RPS_CLIENT(rps) && INTEL_GEN(req->i915) >= 6) - gen6_rps_boost(req->i915, rps, req->emitted_jiffies); + if (!i915_sw_fence_done(&req->submit)) { + timeout = __i915_request_wait_for_submit(req, flags, timeout); + if (timeout < 0) + goto complete; + + GEM_BUG_ON(!i915_sw_fence_done(&req->submit)); + } + GEM_BUG_ON(!req->global_seqno); /* Optimistic short spin before touching IRQs */ if (i915_spin_request(req, state, 5)) @@ -809,7 +920,7 @@ int i915_wait_request(struct drm_i915_gem_request *req, if (flags & I915_WAIT_LOCKED) add_wait_queue(&req->i915->gpu_error.wait_queue, &reset); - intel_wait_init(&wait, req->fence.seqno); + intel_wait_init(&wait, req->global_seqno); if (intel_engine_add_wait(req->engine, &wait)) /* In order to check that we haven't missed the interrupt * as we enabled it, we need to kick ourselves to do a @@ -819,16 +930,17 @@ int i915_wait_request(struct drm_i915_gem_request *req, for (;;) { if (signal_pending_state(state, current)) { - ret = -ERESTARTSYS; + timeout = -ERESTARTSYS; break; } - timeout_remain = io_schedule_timeout(timeout_remain); - if (timeout_remain == 0) { - ret = -ETIME; + if (!timeout) { + timeout = -ETIME; break; } + timeout = io_schedule_timeout(timeout); + if (intel_wait_complete(&wait)) break; @@ -875,74 +987,39 @@ wakeup: complete: trace_i915_gem_request_wait_end(req); - if (timeout) { - *timeout -= ktime_get_raw_ns(); - if (*timeout < 0) - *timeout = 0; - - /* - * Apparently ktime isn't accurate enough and occasionally has a - * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch - * things up to make the test happy. We allow up to 1 jiffy. - * - * This is a regrssion from the timespec->ktime conversion. - */ - if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000) - *timeout = 0; - } - - if (IS_RPS_USER(rps) && - req->fence.seqno == req->engine->last_submitted_seqno) { - /* The GPU is now idle and this client has stalled. - * Since no other client has submitted a request in the - * meantime, assume that this client is the only one - * supplying work to the GPU but is unable to keep that - * work supplied because it is waiting. Since the GPU is - * then never kept fully busy, RPS autoclocking will - * keep the clocks relatively low, causing further delays. - * Compensate by giving the synchronous client credit for - * a waitboost next time. - */ - spin_lock(&req->i915->rps.client_lock); - list_del_init(&rps->link); - spin_unlock(&req->i915->rps.client_lock); - } - - return ret; + return timeout; } -static bool engine_retire_requests(struct intel_engine_cs *engine) +static void engine_retire_requests(struct intel_engine_cs *engine) { struct drm_i915_gem_request *request, *next; - list_for_each_entry_safe(request, next, &engine->request_list, link) { - if (!i915_gem_request_completed(request)) - return false; + list_for_each_entry_safe(request, next, + &engine->timeline->requests, link) { + if (!__i915_gem_request_completed(request)) + return; i915_gem_request_retire(request); } - - return true; } void i915_gem_retire_requests(struct drm_i915_private *dev_priv) { struct intel_engine_cs *engine; - unsigned int tmp; + enum intel_engine_id id; lockdep_assert_held(&dev_priv->drm.struct_mutex); - if (dev_priv->gt.active_engines == 0) + if (!dev_priv->gt.active_requests) return; GEM_BUG_ON(!dev_priv->gt.awake); - for_each_engine_masked(engine, dev_priv, dev_priv->gt.active_engines, tmp) - if (engine_retire_requests(engine)) - dev_priv->gt.active_engines &= ~intel_engine_flag(engine); + for_each_engine(engine, dev_priv, id) + engine_retire_requests(engine); - if (dev_priv->gt.active_engines == 0) - queue_delayed_work(dev_priv->wq, - &dev_priv->gt.idle_work, - msecs_to_jiffies(100)); + if (!dev_priv->gt.active_requests) + mod_delayed_work(dev_priv->wq, + &dev_priv->gt.idle_work, + msecs_to_jiffies(100)); } diff --git a/drivers/gpu/drm/i915/i915_gem_request.h b/drivers/gpu/drm/i915/i915_gem_request.h index bceeaa3a5193..75f8360b3421 100644 --- a/drivers/gpu/drm/i915/i915_gem_request.h +++ b/drivers/gpu/drm/i915/i915_gem_request.h @@ -81,11 +81,14 @@ struct drm_i915_gem_request { struct i915_gem_context *ctx; struct intel_engine_cs *engine; struct intel_ring *ring; + struct intel_timeline *timeline; struct intel_signal_node signaling; struct i915_sw_fence submit; wait_queue_t submitq; + u32 global_seqno; + /** GEM sequence number associated with the previous request, * when the HWS breadcrumb is equal to this the GPU is processing * this request. @@ -147,7 +150,7 @@ struct drm_i915_gem_request { extern const struct dma_fence_ops i915_fence_ops; -static inline bool fence_is_i915(struct dma_fence *fence) +static inline bool dma_fence_is_i915(const struct dma_fence *fence) { return fence->ops == &i915_fence_ops; } @@ -162,7 +165,7 @@ void i915_gem_request_retire_upto(struct drm_i915_gem_request *req); static inline u32 i915_gem_request_get_seqno(struct drm_i915_gem_request *req) { - return req ? req->fence.seqno : 0; + return req ? req->global_seqno : 0; } static inline struct intel_engine_cs * @@ -176,7 +179,7 @@ to_request(struct dma_fence *fence) { /* We assume that NULL fence/request are interoperable */ BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0); - GEM_BUG_ON(fence && !fence_is_i915(fence)); + GEM_BUG_ON(fence && !dma_fence_is_i915(fence)); return container_of(fence, struct drm_i915_gem_request, fence); } @@ -214,6 +217,8 @@ int i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write); +int i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req, + struct dma_fence *fence); void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches); #define i915_add_request(req) \ @@ -226,13 +231,13 @@ struct intel_rps_client; #define IS_RPS_CLIENT(p) (!IS_ERR(p)) #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p)) -int i915_wait_request(struct drm_i915_gem_request *req, - unsigned int flags, - s64 *timeout, - struct intel_rps_client *rps) +long i915_wait_request(struct drm_i915_gem_request *req, + unsigned int flags, + long timeout) __attribute__((nonnull(1))); #define I915_WAIT_INTERRUPTIBLE BIT(0) #define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */ +#define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine); @@ -245,17 +250,37 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2) } static inline bool -i915_gem_request_started(const struct drm_i915_gem_request *req) +__i915_gem_request_started(const struct drm_i915_gem_request *req) { + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), req->previous_seqno); } static inline bool -i915_gem_request_completed(const struct drm_i915_gem_request *req) +i915_gem_request_started(const struct drm_i915_gem_request *req) +{ + if (!req->global_seqno) + return false; + + return __i915_gem_request_started(req); +} + +static inline bool +__i915_gem_request_completed(const struct drm_i915_gem_request *req) { + GEM_BUG_ON(!req->global_seqno); return i915_seqno_passed(intel_engine_get_seqno(req->engine), - req->fence.seqno); + req->global_seqno); +} + +static inline bool +i915_gem_request_completed(const struct drm_i915_gem_request *req) +{ + if (!req->global_seqno) + return false; + + return __i915_gem_request_completed(req); } bool __i915_spin_request(const struct drm_i915_gem_request *request, @@ -263,7 +288,7 @@ bool __i915_spin_request(const struct drm_i915_gem_request *request, static inline bool i915_spin_request(const struct drm_i915_gem_request *request, int state, unsigned long timeout_us) { - return (i915_gem_request_started(request) && + return (__i915_gem_request_started(request) && __i915_spin_request(request, state, timeout_us)); } @@ -552,53 +577,13 @@ i915_gem_active_isset(const struct i915_gem_active *active) } /** - * i915_gem_active_is_idle - report whether the active tracker is idle - * @active - the active tracker - * - * i915_gem_active_is_idle() returns true if the active tracker is currently - * unassigned or if the request is complete (but not yet retired). Requires - * the caller to hold struct_mutex (but that can be relaxed if desired). - */ -static inline bool -i915_gem_active_is_idle(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return !i915_gem_active_peek(active, mutex); -} - -/** * i915_gem_active_wait - waits until the request is completed * @active - the active request on which to wait - * - * i915_gem_active_wait() waits until the request is completed before - * returning. Note that it does not guarantee that the request is - * retired first, see i915_gem_active_retire(). - * - * i915_gem_active_wait() returns immediately if the active - * request is already complete. - */ -static inline int __must_check -i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex) -{ - struct drm_i915_gem_request *request; - - request = i915_gem_active_peek(active, mutex); - if (!request) - return 0; - - return i915_wait_request(request, - I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - NULL, NULL); -} - -/** - * i915_gem_active_wait_unlocked - waits until the request is completed - * @active - the active request on which to wait * @flags - how to wait * @timeout - how long to wait at most * @rps - userspace client to charge for a waitboost * - * i915_gem_active_wait_unlocked() waits until the request is completed before + * i915_gem_active_wait() waits until the request is completed before * returning, without requiring any locks to be held. Note that it does not * retire any requests before returning. * @@ -614,21 +599,18 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex) * Returns 0 if successful, or a negative error code. */ static inline int -i915_gem_active_wait_unlocked(const struct i915_gem_active *active, - unsigned int flags, - s64 *timeout, - struct intel_rps_client *rps) +i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags) { struct drm_i915_gem_request *request; - int ret = 0; + long ret = 0; request = i915_gem_active_get_unlocked(active); if (request) { - ret = i915_wait_request(request, flags, timeout, rps); + ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT); i915_gem_request_put(request); } - return ret; + return ret < 0 ? ret : 0; } /** @@ -645,7 +627,7 @@ i915_gem_active_retire(struct i915_gem_active *active, struct mutex *mutex) { struct drm_i915_gem_request *request; - int ret; + long ret; request = i915_gem_active_raw(active, mutex); if (!request) @@ -653,8 +635,8 @@ i915_gem_active_retire(struct i915_gem_active *active, ret = i915_wait_request(request, I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - NULL, NULL); - if (ret) + MAX_SCHEDULE_TIMEOUT); + if (ret < 0) return ret; list_del_init(&active->link); @@ -665,24 +647,6 @@ i915_gem_active_retire(struct i915_gem_active *active, return 0; } -/* Convenience functions for peeking at state inside active's request whilst - * guarded by the struct_mutex. - */ - -static inline uint32_t -i915_gem_active_get_seqno(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return i915_gem_request_get_seqno(i915_gem_active_peek(active, mutex)); -} - -static inline struct intel_engine_cs * -i915_gem_active_get_engine(const struct i915_gem_active *active, - struct mutex *mutex) -{ - return i915_gem_request_get_engine(i915_gem_active_peek(active, mutex)); -} - #define for_each_active(mask, idx) \ for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx)) diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index de25b6e0a101..a6fc1bdc48af 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -48,6 +48,20 @@ static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task) #endif } +static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) +{ + if (!mutex_trylock(&dev->struct_mutex)) { + if (!mutex_is_locked_by(&dev->struct_mutex, current)) + return false; + + *unlock = false; + } else { + *unlock = true; + } + + return true; +} + static bool any_vma_pinned(struct drm_i915_gem_object *obj) { struct i915_vma *vma; @@ -66,8 +80,11 @@ static bool swap_available(void) static bool can_release_pages(struct drm_i915_gem_object *obj) { - /* Only shmemfs objects are backed by swap */ - if (!obj->base.filp) + if (!obj->mm.pages) + return false; + + /* Consider only shrinkable ojects. */ + if (!i915_gem_object_is_shrinkable(obj)) return false; /* Only report true if by unbinding the object and putting its pages @@ -78,7 +95,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) * to the GPU, simply unbinding from the GPU is not going to succeed * in releasing our pin count on the pages themselves. */ - if (obj->pages_pin_count > obj->bind_count) + if (atomic_read(&obj->mm.pages_pin_count) > obj->bind_count) return false; if (any_vma_pinned(obj)) @@ -88,7 +105,14 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) * discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. */ - return swap_available() || obj->madv == I915_MADV_DONTNEED; + return swap_available() || obj->mm.madv == I915_MADV_DONTNEED; +} + +static bool unsafe_drop_pages(struct drm_i915_gem_object *obj) +{ + if (i915_gem_object_unbind(obj) == 0) + __i915_gem_object_put_pages(obj, I915_MM_SHRINKER); + return !READ_ONCE(obj->mm.pages); } /** @@ -128,6 +152,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, { NULL, 0 }, }, *phase; unsigned long count = 0; + bool unlock; + + if (!i915_gem_shrinker_lock(&dev_priv->drm, &unlock)) + return 0; trace_i915_gem_shrink(dev_priv, target, flags); i915_gem_retire_requests(dev_priv); @@ -171,15 +199,19 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, while (count < target && (obj = list_first_entry_or_null(phase->list, typeof(*obj), - global_list))) { - list_move_tail(&obj->global_list, &still_in_list); + global_link))) { + list_move_tail(&obj->global_link, &still_in_list); + if (!obj->mm.pages) { + list_del_init(&obj->global_link); + continue; + } if (flags & I915_SHRINK_PURGEABLE && - obj->madv != I915_MADV_DONTNEED) + obj->mm.madv != I915_MADV_DONTNEED) continue; if (flags & I915_SHRINK_VMAPS && - !is_vmalloc_addr(obj->mapping)) + !is_vmalloc_addr(obj->mm.mapping)) continue; if (!(flags & I915_SHRINK_ACTIVE) && @@ -190,22 +222,28 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, if (!can_release_pages(obj)) continue; - i915_gem_object_get(obj); - - /* For the unbound phase, this should be a no-op! */ - i915_gem_object_unbind(obj); - if (i915_gem_object_put_pages(obj) == 0) - count += obj->base.size >> PAGE_SHIFT; - - i915_gem_object_put(obj); + if (unsafe_drop_pages(obj)) { + /* May arrive from get_pages on another bo */ + mutex_lock_nested(&obj->mm.lock, + I915_MM_SHRINKER); + if (!obj->mm.pages) { + __i915_gem_object_invalidate(obj); + list_del_init(&obj->global_link); + count += obj->base.size >> PAGE_SHIFT; + } + mutex_unlock(&obj->mm.lock); + } } - list_splice(&still_in_list, phase->list); + list_splice_tail(&still_in_list, phase->list); } if (flags & I915_SHRINK_BOUND) intel_runtime_pm_put(dev_priv); i915_gem_retire_requests(dev_priv); + if (unlock) + mutex_unlock(&dev_priv->drm.struct_mutex); + /* expedite the RCU grace period to free some request slabs */ synchronize_rcu_expedited(); @@ -239,19 +277,6 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) return freed; } -static bool i915_gem_shrinker_lock(struct drm_device *dev, bool *unlock) -{ - if (!mutex_trylock(&dev->struct_mutex)) { - if (!mutex_is_locked_by(&dev->struct_mutex, current)) - return false; - - *unlock = false; - } else - *unlock = true; - - return true; -} - static unsigned long i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { @@ -268,11 +293,11 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) i915_gem_retire_requests(dev_priv); count = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) if (can_release_pages(obj)) count += obj->base.size >> PAGE_SHIFT; - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { if (!i915_gem_object_is_active(obj) && can_release_pages(obj)) count += obj->base.size >> PAGE_SHIFT; } @@ -373,13 +398,19 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) * being pointed to by hardware. */ unbound = bound = unevictable = 0; - list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_link) { + if (!obj->mm.pages) + continue; + if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; else unbound += obj->base.size >> PAGE_SHIFT; } - list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) { + list_for_each_entry(obj, &dev_priv->mm.bound_list, global_link) { + if (!obj->mm.pages) + continue; + if (!can_release_pages(obj)) unevictable += obj->base.size >> PAGE_SHIFT; else diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c index f4f6d3a48b05..b1d367dba347 100644 --- a/drivers/gpu/drm/i915/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/i915_gem_stolen.c @@ -109,7 +109,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) * */ base = 0; - if (INTEL_INFO(dev)->gen >= 3) { + if (INTEL_GEN(dev_priv) >= 3) { u32 bsm; pci_read_config_dword(pdev, INTEL_BSM, &bsm); @@ -138,7 +138,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev) I865_TOUD, &toud); base = (toud << 16) + tseg_size; - } else if (IS_I85X(dev)) { + } else if (IS_I85X(dev_priv)) { u32 tseg_size = 0; u32 tom; u8 tmp; @@ -546,25 +546,29 @@ i915_pages_create_for_stolen(struct drm_device *dev, return st; } -static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) { - BUG(); - return -EINVAL; + return i915_pages_create_for_stolen(obj->base.dev, + obj->stolen->start, + obj->stolen->size); } -static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, + struct sg_table *pages) { /* Should only be called during free */ - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } - static void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj) { struct drm_i915_private *dev_priv = to_i915(obj->base.dev); + __i915_gem_object_unpin_pages(obj); + if (obj->stolen) { i915_gem_stolen_remove_node(dev_priv, obj->stolen); kfree(obj->stolen); @@ -590,20 +594,13 @@ _i915_gem_object_create_stolen(struct drm_device *dev, drm_gem_private_object_init(dev, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); - obj->pages = i915_pages_create_for_stolen(dev, - stolen->start, stolen->size); - if (obj->pages == NULL) - goto cleanup; - - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - - i915_gem_object_pin_pages(obj); obj->stolen = stolen; - obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE; + if (i915_gem_object_pin_pages(obj)) + goto cleanup; + return obj; cleanup: @@ -698,10 +695,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, if (gtt_offset == I915_GTT_OFFSET_NONE) return obj; + ret = i915_gem_object_pin_pages(obj); + if (ret) + goto err; + vma = i915_gem_obj_lookup_or_create_vma(obj, &ggtt->base, NULL); if (IS_ERR(vma)) { ret = PTR_ERR(vma); - goto err; + goto err_pages; } /* To simplify the initialisation sequence between KMS and GTT, @@ -715,20 +716,20 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node); if (ret) { DRM_DEBUG_KMS("failed to allocate stolen GTT space\n"); - goto err; + goto err_pages; } - vma->pages = obj->pages; + vma->pages = obj->mm.pages; vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); list_move_tail(&vma->vm_link, &ggtt->base.inactive_list); + list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); obj->bind_count++; - list_add_tail(&obj->global_list, &dev_priv->mm.bound_list); - i915_gem_object_pin_pages(obj); - return obj; +err_pages: + i915_gem_object_unpin_pages(obj); err: i915_gem_object_put(obj); return NULL; diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index c21bc0068d20..251d51b01174 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -201,12 +201,10 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, if (!i915_tiling_ok(dev, args->stride, obj->base.size, args->tiling_mode)) { - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return -EINVAL; } - intel_runtime_pm_get(dev_priv); - mutex_lock(&dev->struct_mutex); if (obj->pin_display || obj->framebuffer_references) { err = -EBUSY; @@ -261,14 +259,22 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, if (!err) { struct i915_vma *vma; - if (obj->pages && - obj->madv == I915_MADV_WILLNEED && + mutex_lock(&obj->mm.lock); + if (obj->mm.pages && + obj->mm.madv == I915_MADV_WILLNEED && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { - if (args->tiling_mode == I915_TILING_NONE) - i915_gem_object_unpin_pages(obj); - if (!i915_gem_object_is_tiled(obj)) - i915_gem_object_pin_pages(obj); + if (args->tiling_mode == I915_TILING_NONE) { + GEM_BUG_ON(!obj->mm.quirked); + __i915_gem_object_unpin_pages(obj); + obj->mm.quirked = false; + } + if (!i915_gem_object_is_tiled(obj)) { + GEM_BUG_ON(!obj->mm.quirked); + __i915_gem_object_pin_pages(obj); + obj->mm.quirked = true; + } } + mutex_unlock(&obj->mm.lock); list_for_each_entry(vma, &obj->vma_list, obj_link) { if (!vma->fence) @@ -302,8 +308,6 @@ err: i915_gem_object_put(obj); mutex_unlock(&dev->struct_mutex); - intel_runtime_pm_put(dev_priv); - return err; } @@ -327,12 +331,19 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, struct drm_i915_gem_get_tiling *args = data; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; + int err = -ENOENT; + + rcu_read_lock(); + obj = i915_gem_object_lookup_rcu(file, args->handle); + if (obj) { + args->tiling_mode = + READ_ONCE(obj->tiling_and_stride) & TILING_MASK; + err = 0; + } + rcu_read_unlock(); + if (unlikely(err)) + return err; - obj = i915_gem_object_lookup(file, args->handle); - if (!obj) - return -ENOENT; - - args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK; switch (args->tiling_mode) { case I915_TILING_X: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; @@ -340,11 +351,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, case I915_TILING_Y: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; break; + default: case I915_TILING_NONE: args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; break; - default: - DRM_ERROR("unknown tiling mode\n"); } /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ @@ -357,6 +367,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; - i915_gem_object_put_unlocked(obj); return 0; } diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.c b/drivers/gpu/drm/i915/i915_gem_timeline.c new file mode 100644 index 000000000000..fc8f13a79f8f --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_timeline.c @@ -0,0 +1,65 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_drv.h" + +int i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *timeline, + const char *name) +{ + unsigned int i; + u64 fences; + + lockdep_assert_held(&i915->drm.struct_mutex); + + timeline->i915 = i915; + timeline->name = kstrdup(name ?: "[kernel]", GFP_KERNEL); + if (!timeline->name) + return -ENOMEM; + + list_add(&timeline->link, &i915->gt.timelines); + + /* Called during early_init before we know how many engines there are */ + fences = dma_fence_context_alloc(ARRAY_SIZE(timeline->engine)); + for (i = 0; i < ARRAY_SIZE(timeline->engine); i++) { + struct intel_timeline *tl = &timeline->engine[i]; + + tl->fence_context = fences++; + tl->common = timeline; + + spin_lock_init(&tl->lock); + init_request_active(&tl->last_request, NULL); + INIT_LIST_HEAD(&tl->requests); + } + + return 0; +} + +void i915_gem_timeline_fini(struct i915_gem_timeline *tl) +{ + lockdep_assert_held(&tl->i915->drm.struct_mutex); + + list_del(&tl->link); + kfree(tl->name); +} diff --git a/drivers/gpu/drm/i915/i915_gem_timeline.h b/drivers/gpu/drm/i915/i915_gem_timeline.h new file mode 100644 index 000000000000..f2bf7b1d49a1 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_timeline.h @@ -0,0 +1,72 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#ifndef I915_GEM_TIMELINE_H +#define I915_GEM_TIMELINE_H + +#include <linux/list.h> + +#include "i915_gem_request.h" + +struct i915_gem_timeline; + +struct intel_timeline { + u64 fence_context; + u32 last_submitted_seqno; + + spinlock_t lock; + + /** + * List of breadcrumbs associated with GPU requests currently + * outstanding. + */ + struct list_head requests; + + /* Contains an RCU guarded pointer to the last request. No reference is + * held to the request, users must carefully acquire a reference to + * the request using i915_gem_active_get_request_rcu(), or hold the + * struct_mutex. + */ + struct i915_gem_active last_request; + u32 sync_seqno[I915_NUM_ENGINES]; + + struct i915_gem_timeline *common; +}; + +struct i915_gem_timeline { + struct list_head link; + atomic_t next_seqno; + + struct drm_i915_private *i915; + const char *name; + + struct intel_timeline engine[I915_NUM_ENGINES]; +}; + +int i915_gem_timeline_init(struct drm_i915_private *i915, + struct i915_gem_timeline *tl, + const char *name); +void i915_gem_timeline_fini(struct i915_gem_timeline *tl); + +#endif diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c index c6f780f5abc9..64261639f547 100644 --- a/drivers/gpu/drm/i915/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c @@ -61,33 +61,26 @@ struct i915_mmu_object { bool attached; }; -static void wait_rendering(struct drm_i915_gem_object *obj) -{ - unsigned long active = __I915_BO_ACTIVE(obj); - int idx; - - for_each_active(active, idx) - i915_gem_active_wait_unlocked(&obj->last_read[idx], - 0, NULL, NULL); -} - static void cancel_userptr(struct work_struct *work) { struct i915_mmu_object *mo = container_of(work, typeof(*mo), work); struct drm_i915_gem_object *obj = mo->obj; struct drm_device *dev = obj->base.dev; - wait_rendering(obj); + i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL); mutex_lock(&dev->struct_mutex); /* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work = NULL; - if (obj->pages != NULL) { - /* We are inside a kthread context and can't be interrupted */ - WARN_ON(i915_gem_object_unbind(obj)); - WARN_ON(i915_gem_object_put_pages(obj)); - } + /* We are inside a kthread context and can't be interrupted */ + if (i915_gem_object_unbind(obj) == 0) + __i915_gem_object_put_pages(obj, I915_MM_NORMAL); + WARN_ONCE(obj->mm.pages, + "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n", + obj->bind_count, + atomic_read(&obj->mm.pages_pin_count), + obj->pin_display); i915_gem_object_put(obj); mutex_unlock(&dev->struct_mutex); @@ -436,24 +429,25 @@ err: return ret; } -static int +static struct sg_table * __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, struct page **pvec, int num_pages) { + struct sg_table *pages; int ret; - ret = st_set_pages(&obj->pages, pvec, num_pages); + ret = st_set_pages(&pages, pvec, num_pages); if (ret) - return ret; + return ERR_PTR(ret); - ret = i915_gem_gtt_prepare_object(obj); + ret = i915_gem_gtt_prepare_pages(obj, pages); if (ret) { - sg_free_table(obj->pages); - kfree(obj->pages); - obj->pages = NULL; + sg_free_table(pages); + kfree(pages); + return ERR_PTR(ret); } - return ret; + return pages; } static int @@ -497,7 +491,6 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) { struct get_pages_work *work = container_of(_work, typeof(*work), work); struct drm_i915_gem_object *obj = work->obj; - struct drm_device *dev = obj->base.dev; const int npages = obj->base.size >> PAGE_SHIFT; struct page **pvec; int pinned, ret; @@ -533,33 +526,32 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) } } - mutex_lock(&dev->struct_mutex); + mutex_lock(&obj->mm.lock); if (obj->userptr.work == &work->work) { + struct sg_table *pages = ERR_PTR(ret); + if (pinned == npages) { - ret = __i915_gem_userptr_set_pages(obj, pvec, npages); - if (ret == 0) { - list_add_tail(&obj->global_list, - &to_i915(dev)->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; + pages = __i915_gem_userptr_set_pages(obj, pvec, npages); + if (!IS_ERR(pages)) { + __i915_gem_object_set_pages(obj, pages); pinned = 0; + pages = NULL; } } - obj->userptr.work = ERR_PTR(ret); - } - obj->userptr.workers--; - i915_gem_object_put(obj); - mutex_unlock(&dev->struct_mutex); + obj->userptr.work = ERR_CAST(pages); + } + mutex_unlock(&obj->mm.lock); release_pages(pvec, pinned, 0); drm_free_large(pvec); + i915_gem_object_put(obj); put_task_struct(work->task); kfree(work); } -static int +static struct sg_table * __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, bool *active) { @@ -584,15 +576,11 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, * that error back to this function through * obj->userptr.work = ERR_PTR. */ - if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS) - return -EAGAIN; - work = kmalloc(sizeof(*work), GFP_KERNEL); if (work == NULL) - return -ENOMEM; + return ERR_PTR(-ENOMEM); obj->userptr.work = &work->work; - obj->userptr.workers++; work->obj = i915_gem_object_get(obj); @@ -603,14 +591,15 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj, schedule_work(&work->work); *active = true; - return -EAGAIN; + return ERR_PTR(-EAGAIN); } -static int +static struct sg_table * i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) { const int num_pages = obj->base.size >> PAGE_SHIFT; struct page **pvec; + struct sg_table *pages; int pinned, ret; bool active; @@ -634,15 +623,15 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) if (obj->userptr.work) { /* active flag should still be held for the pending work */ if (IS_ERR(obj->userptr.work)) - return PTR_ERR(obj->userptr.work); + return ERR_CAST(obj->userptr.work); else - return -EAGAIN; + return ERR_PTR(-EAGAIN); } /* Let the mmu-notifier know that we have begun and need cancellation */ ret = __i915_gem_userptr_set_active(obj, true); if (ret) - return ret; + return ERR_PTR(ret); pvec = NULL; pinned = 0; @@ -651,7 +640,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) GFP_TEMPORARY); if (pvec == NULL) { __i915_gem_userptr_set_active(obj, false); - return -ENOMEM; + return ERR_PTR(-ENOMEM); } pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, @@ -660,21 +649,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) active = false; if (pinned < 0) - ret = pinned, pinned = 0; + pages = ERR_PTR(pinned), pinned = 0; else if (pinned < num_pages) - ret = __i915_gem_userptr_get_pages_schedule(obj, &active); + pages = __i915_gem_userptr_get_pages_schedule(obj, &active); else - ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages); - if (ret) { + pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages); + if (IS_ERR(pages)) { __i915_gem_userptr_set_active(obj, active); release_pages(pvec, pinned, 0); } drm_free_large(pvec); - return ret; + return pages; } static void -i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) +i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages) { struct sgt_iter sgt_iter; struct page *page; @@ -682,22 +672,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) BUG_ON(obj->userptr.work != NULL); __i915_gem_userptr_set_active(obj, false); - if (obj->madv != I915_MADV_WILLNEED) - obj->dirty = 0; + if (obj->mm.madv != I915_MADV_WILLNEED) + obj->mm.dirty = false; - i915_gem_gtt_finish_object(obj); + i915_gem_gtt_finish_pages(obj, pages); - for_each_sgt_page(page, sgt_iter, obj->pages) { - if (obj->dirty) + for_each_sgt_page(page, sgt_iter, pages) { + if (obj->mm.dirty) set_page_dirty(page); mark_page_accessed(page); put_page(page); } - obj->dirty = 0; + obj->mm.dirty = false; - sg_free_table(obj->pages); - kfree(obj->pages); + sg_free_table(pages); + kfree(pages); } static void @@ -717,7 +707,8 @@ i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj) } static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | + I915_GEM_OBJECT_IS_SHRINKABLE, .get_pages = i915_gem_userptr_get_pages, .put_pages = i915_gem_userptr_put_pages, .dmabuf_export = i915_gem_userptr_dmabuf_export, @@ -816,7 +807,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file ret = drm_gem_handle_create(file, &obj->base, &handle); /* drop reference from allocate - handle holds it now */ - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 242b9a927899..204093f3eaa5 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -415,17 +415,13 @@ static void error_print_engine(struct drm_i915_error_state_buf *m, if (INTEL_GEN(m->i915) >= 6) { err_printf(m, " RC PSMI: 0x%08x\n", ee->rc_psmi); err_printf(m, " FAULT_REG: 0x%08x\n", ee->fault_reg); - err_printf(m, " SYNC_0: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[0], - ee->semaphore_seqno[0]); - err_printf(m, " SYNC_1: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[1], - ee->semaphore_seqno[1]); - if (HAS_VEBOX(m->i915)) { - err_printf(m, " SYNC_2: 0x%08x [last synced 0x%08x]\n", - ee->semaphore_mboxes[2], - ee->semaphore_seqno[2]); - } + err_printf(m, " SYNC_0: 0x%08x\n", + ee->semaphore_mboxes[0]); + err_printf(m, " SYNC_1: 0x%08x\n", + ee->semaphore_mboxes[1]); + if (HAS_VEBOX(m->i915)) + err_printf(m, " SYNC_2: 0x%08x\n", + ee->semaphore_mboxes[2]); } if (USES_PPGTT(m->i915)) { err_printf(m, " GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode); @@ -546,9 +542,13 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, } err_printf(m, "%s\n", error->error_msg); - err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec, - error->time.tv_usec); err_printf(m, "Kernel: " UTS_RELEASE "\n"); + err_printf(m, "Time: %ld s %ld us\n", + error->time.tv_sec, error->time.tv_usec); + err_printf(m, "Boottime: %ld s %ld us\n", + error->boottime.tv_sec, error->boottime.tv_usec); + err_printf(m, "Uptime: %ld s %ld us\n", + error->uptime.tv_sec, error->uptime.tv_usec); err_print_capabilities(m, &error->device_info); max_hangcheck_score = 0; for (i = 0; i < ARRAY_SIZE(error->engine); i++) { @@ -702,6 +702,8 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, print_error_obj(m, NULL, "Semaphores", error->semaphore); + print_error_obj(m, NULL, "GuC log buffer", error->guc_log); + if (error->overlay) intel_overlay_print_error_state(m, error->overlay); @@ -782,6 +784,7 @@ static void i915_error_state_free(struct kref *error_ref) } i915_error_object_free(error->semaphore); + i915_error_object_free(error->guc_log); for (i = 0; i < ARRAY_SIZE(error->active_bo); i++) kfree(error->active_bo[i]); @@ -880,17 +883,17 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->name = obj->base.name; for (i = 0; i < I915_NUM_ENGINES; i++) - err->rseqno[i] = __active_get_seqno(&obj->last_read[i]); - err->wseqno = __active_get_seqno(&obj->last_write); - err->engine = __active_get_engine_id(&obj->last_write); + err->rseqno[i] = __active_get_seqno(&vma->last_read[i]); + err->wseqno = __active_get_seqno(&vma->last_write); + err->engine = __active_get_engine_id(&vma->last_write); err->gtt_offset = vma->node.start; err->read_domains = obj->base.read_domains; err->write_domain = obj->base.write_domain; err->fence_reg = vma->fence ? vma->fence->id : -1; err->tiling = i915_gem_object_get_tiling(obj); - err->dirty = obj->dirty; - err->purgeable = obj->madv != I915_MADV_WILLNEED; + err->dirty = obj->mm.dirty; + err->purgeable = obj->mm.madv != I915_MADV_WILLNEED; err->userptr = obj->userptr.mm != NULL; err->cache_level = obj->cache_level; } @@ -965,6 +968,26 @@ static void i915_gem_record_fences(struct drm_i915_private *dev_priv, } } +static inline u32 +gen8_engine_sync_index(struct intel_engine_cs *engine, + struct intel_engine_cs *other) +{ + int idx; + + /* + * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; + * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; + * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; + * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; + * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; + */ + + idx = (other - engine) - 1; + if (idx < 0) + idx += I915_NUM_ENGINES; + + return idx; +} static void gen8_record_semaphore_state(struct drm_i915_error_state *error, struct intel_engine_cs *engine, @@ -988,10 +1011,9 @@ static void gen8_record_semaphore_state(struct drm_i915_error_state *error, signal_offset = (GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4; tmp = error->semaphore->pages[0]; - idx = intel_engine_sync_index(engine, to); + idx = gen8_engine_sync_index(engine, to); ee->semaphore_mboxes[idx] = tmp[signal_offset]; - ee->semaphore_seqno[idx] = engine->semaphore.sync_seqno[idx]; } } @@ -1002,14 +1024,9 @@ static void gen6_record_semaphore_state(struct intel_engine_cs *engine, ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base)); ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base)); - ee->semaphore_seqno[0] = engine->semaphore.sync_seqno[0]; - ee->semaphore_seqno[1] = engine->semaphore.sync_seqno[1]; - - if (HAS_VEBOX(dev_priv)) { + if (HAS_VEBOX(dev_priv)) ee->semaphore_mboxes[2] = I915_READ(RING_SYNC_2(engine->mmio_base)); - ee->semaphore_seqno[2] = engine->semaphore.sync_seqno[2]; - } } static void error_record_engine_waiters(struct intel_engine_cs *engine, @@ -1026,7 +1043,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (RB_EMPTY_ROOT(&b->waiters)) return; - if (!spin_trylock(&b->lock)) { + if (!spin_trylock_irq(&b->lock)) { ee->waiters = ERR_PTR(-EDEADLK); return; } @@ -1034,7 +1051,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, count = 0; for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb)) count++; - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); waiter = NULL; if (count) @@ -1044,7 +1061,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (!waiter) return; - if (!spin_trylock(&b->lock)) { + if (!spin_trylock_irq(&b->lock)) { kfree(waiter); ee->waiters = ERR_PTR(-EDEADLK); return; @@ -1062,7 +1079,7 @@ static void error_record_engine_waiters(struct intel_engine_cs *engine, if (++ee->num_waiters == count) break; } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static void error_record_engine_registers(struct drm_i915_error_state *error, @@ -1103,7 +1120,7 @@ static void error_record_engine_registers(struct drm_i915_error_state *error, ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base)); ee->acthd = intel_engine_get_active_head(engine); ee->seqno = intel_engine_get_seqno(engine); - ee->last_seqno = engine->last_submitted_seqno; + ee->last_seqno = intel_engine_last_submit(engine); ee->start = I915_READ_START(engine); ee->head = I915_READ_HEAD(engine); ee->tail = I915_READ_TAIL(engine); @@ -1169,7 +1186,7 @@ static void record_request(struct drm_i915_gem_request *request, struct drm_i915_error_request *erq) { erq->context = request->ctx->hw_id; - erq->seqno = request->fence.seqno; + erq->seqno = request->global_seqno; erq->jiffies = request->emitted_jiffies; erq->head = request->head; erq->tail = request->tail; @@ -1188,7 +1205,7 @@ static void engine_record_requests(struct intel_engine_cs *engine, count = 0; request = first; - list_for_each_entry_from(request, &engine->request_list, link) + list_for_each_entry_from(request, &engine->timeline->requests, link) count++; if (!count) return; @@ -1201,7 +1218,7 @@ static void engine_record_requests(struct intel_engine_cs *engine, count = 0; request = first; - list_for_each_entry_from(request, &engine->request_list, link) { + list_for_each_entry_from(request, &engine->timeline->requests, link) { if (count >= ee->num_requests) { /* * If the ring request list was changed in @@ -1408,6 +1425,17 @@ static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv, error->pinned_bo = bo; } +static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv, + struct drm_i915_error_state *error) +{ + /* Capturing log buf contents won't be useful if logging was disabled */ + if (!dev_priv->guc.log.vma || (i915.guc_log_level < 0)) + return; + + error->guc_log = i915_error_object_create(dev_priv, + dev_priv->guc.log.vma); +} + /* Capture all registers which don't fit into another category. */ static void i915_capture_reg_state(struct drm_i915_private *dev_priv, struct drm_i915_error_state *error) @@ -1532,8 +1560,13 @@ static int capture(void *data) i915_gem_record_rings(error->i915, error); i915_capture_active_buffers(error->i915, error); i915_capture_pinned_buffers(error->i915, error); + i915_gem_capture_guc_log_buffer(error->i915, error); do_gettimeofday(&error->time); + error->boottime = ktime_to_timeval(ktime_get_boottime()); + error->uptime = + ktime_to_timeval(ktime_sub(ktime_get(), + error->i915->gt.last_init_time)); error->overlay = intel_overlay_capture_error_state(error->i915); error->display = intel_display_capture_error_state(error->i915); diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c index a1f76c8f8cde..666dab7a675a 100644 --- a/drivers/gpu/drm/i915/i915_guc_submission.c +++ b/drivers/gpu/drm/i915/i915_guc_submission.c @@ -23,6 +23,8 @@ */ #include <linux/firmware.h> #include <linux/circ_buf.h> +#include <linux/debugfs.h> +#include <linux/relay.h> #include "i915_drv.h" #include "intel_guc.h" @@ -85,6 +87,7 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) if (WARN_ON(len < 1 || len > 15)) return -EINVAL; + mutex_lock(&guc->action_lock); intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); dev_priv->guc.action_count += 1; @@ -123,6 +126,7 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len) dev_priv->guc.action_status = status; intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); + mutex_unlock(&guc->action_lock); return ret; } @@ -170,6 +174,35 @@ static int host2guc_sample_forcewake(struct intel_guc *guc, return host2guc_action(guc, data, ARRAY_SIZE(data)); } +static int host2guc_logbuffer_flush_complete(struct intel_guc *guc) +{ + u32 data[1]; + + data[0] = HOST2GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE; + + return host2guc_action(guc, data, 1); +} + +static int host2guc_force_logbuffer_flush(struct intel_guc *guc) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_FORCE_LOG_BUFFER_FLUSH; + data[1] = 0; + + return host2guc_action(guc, data, 2); +} + +static int host2guc_logging_control(struct intel_guc *guc, u32 control_val) +{ + u32 data[2]; + + data[0] = HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING; + data[1] = control_val; + + return host2guc_action(guc, data, 2); +} + /* * Initialise, update, or clear doorbell data shared with the GuC * @@ -187,7 +220,7 @@ static int guc_update_doorbell_id(struct intel_guc *guc, struct guc_context_desc desc; size_t len; - doorbell = client->client_base + client->doorbell_offset; + doorbell = client->vaddr + client->doorbell_offset; if (client->doorbell_id != GUC_INVALID_DOORBELL_ID && test_bit(client->doorbell_id, doorbell_bitmap)) { @@ -293,7 +326,7 @@ static void guc_proc_desc_init(struct intel_guc *guc, { struct guc_process_desc *desc; - desc = client->client_base + client->proc_desc_offset; + desc = client->vaddr + client->proc_desc_offset; memset(desc, 0, sizeof(*desc)); @@ -380,8 +413,8 @@ static void guc_ctx_desc_init(struct intel_guc *guc, gfx_addr = i915_ggtt_offset(client->vma); desc.db_trigger_phy = sg_dma_address(client->vma->pages->sgl) + client->doorbell_offset; - desc.db_trigger_cpu = (uintptr_t)client->client_base + - client->doorbell_offset; + desc.db_trigger_cpu = + (uintptr_t)client->vaddr + client->doorbell_offset; desc.db_trigger_uk = gfx_addr + client->doorbell_offset; desc.process_desc = gfx_addr + client->proc_desc_offset; desc.wq_addr = gfx_addr + client->wq_offset; @@ -432,7 +465,7 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *request) { const size_t wqi_size = sizeof(struct guc_wq_item); struct i915_guc_client *gc = request->i915->guc.execbuf_client; - struct guc_process_desc *desc = gc->client_base + gc->proc_desc_offset; + struct guc_process_desc *desc = gc->vaddr + gc->proc_desc_offset; u32 freespace; int ret; @@ -473,10 +506,9 @@ static void guc_wq_item_append(struct i915_guc_client *gc, struct intel_engine_cs *engine = rq->engine; struct guc_process_desc *desc; struct guc_wq_item *wqi; - void *base; - u32 freespace, tail, wq_off, wq_page; + u32 freespace, tail, wq_off; - desc = gc->client_base + gc->proc_desc_offset; + desc = gc->vaddr + gc->proc_desc_offset; /* Free space is guaranteed, see i915_guc_wq_reserve() above */ freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size); @@ -506,10 +538,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc, gc->wq_rsvd -= wqi_size; /* WQ starts from the page after doorbell / process_desc */ - wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT; - wq_off &= PAGE_SIZE - 1; - base = kmap_atomic(i915_gem_object_get_page(gc->vma->obj, wq_page)); - wqi = (struct guc_wq_item *)((char *)base + wq_off); + wqi = gc->vaddr + wq_off + GUC_DB_SIZE; /* Now fill in the 4-word work queue item */ wqi->header = WQ_TYPE_INORDER | @@ -521,9 +550,7 @@ static void guc_wq_item_append(struct i915_guc_client *gc, wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine); wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT; - wqi->fence_id = rq->fence.seqno; - - kunmap_atomic(base); + wqi->fence_id = rq->global_seqno; } static int guc_ring_doorbell(struct i915_guc_client *gc) @@ -533,7 +560,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc) union guc_doorbell_qw *db; int attempt = 2, ret = -EAGAIN; - desc = gc->client_base + gc->proc_desc_offset; + desc = gc->vaddr + gc->proc_desc_offset; /* Update the tail so it is visible to GuC */ desc->tail = gc->wq_tail; @@ -549,7 +576,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc) db_exc.cookie = 1; /* pointer of current doorbell cacheline */ - db = gc->client_base + gc->doorbell_offset; + db = gc->vaddr + gc->doorbell_offset; while (attempt--) { /* lets ring the doorbell */ @@ -601,6 +628,7 @@ static int guc_ring_doorbell(struct i915_guc_client *gc) */ static void i915_guc_submit(struct drm_i915_gem_request *rq) { + struct drm_i915_private *dev_priv = rq->i915; unsigned int engine_id = rq->engine->id; struct intel_guc *guc = &rq->i915->guc; struct i915_guc_client *client = guc->execbuf_client; @@ -608,6 +636,11 @@ static void i915_guc_submit(struct drm_i915_gem_request *rq) spin_lock(&client->wq_lock); guc_wq_item_append(client, rq); + + /* WA to flush out the pending GMADR writes to ring buffer. */ + if (i915_vma_is_map_and_fenceable(rq->ring->vma)) + POSTING_READ_FW(GUC_STATUS); + b_ret = guc_ring_doorbell(client); client->submissions[engine_id] += 1; @@ -616,7 +649,7 @@ static void i915_guc_submit(struct drm_i915_gem_request *rq) client->b_fail += 1; guc->submissions[engine_id] += 1; - guc->last_seqno[engine_id] = rq->fence.seqno; + guc->last_seqno[engine_id] = rq->global_seqno; spin_unlock(&client->wq_lock); } @@ -685,14 +718,14 @@ guc_client_free(struct drm_i915_private *dev_priv, * Be sure to drop any locks */ - if (client->client_base) { + if (client->vaddr) { /* * If we got as far as setting up a doorbell, make sure we * shut it down before unmapping & deallocating the memory. */ guc_disable_doorbell(guc, client); - kunmap(kmap_to_page(client->client_base)); + i915_gem_object_unpin_map(client->vma->obj); } i915_vma_unpin_and_release(&client->vma); @@ -781,6 +814,7 @@ guc_client_alloc(struct drm_i915_private *dev_priv, struct i915_guc_client *client; struct intel_guc *guc = &dev_priv->guc; struct i915_vma *vma; + void *vaddr; uint16_t db_id; client = kzalloc(sizeof(*client), GFP_KERNEL); @@ -807,7 +841,12 @@ guc_client_alloc(struct drm_i915_private *dev_priv, /* We'll keep just the first (doorbell/proc) page permanently kmap'd. */ client->vma = vma; - client->client_base = kmap(i915_vma_first_page(vma)); + + vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB); + if (IS_ERR(vaddr)) + goto err; + + client->vaddr = vaddr; spin_lock_init(&client->wq_lock); client->wq_offset = GUC_DB_SIZE; @@ -847,15 +886,411 @@ err: return NULL; } +/* + * Sub buffer switch callback. Called whenever relay has to switch to a new + * sub buffer, relay stays on the same sub buffer if 0 is returned. + */ +static int subbuf_start_callback(struct rchan_buf *buf, + void *subbuf, + void *prev_subbuf, + size_t prev_padding) +{ + /* Use no-overwrite mode by default, where relay will stop accepting + * new data if there are no empty sub buffers left. + * There is no strict synchronization enforced by relay between Consumer + * and Producer. In overwrite mode, there is a possibility of getting + * inconsistent/garbled data, the producer could be writing on to the + * same sub buffer from which Consumer is reading. This can't be avoided + * unless Consumer is fast enough and can always run in tandem with + * Producer. + */ + if (relay_buf_full(buf)) + return 0; + + return 1; +} + +/* + * file_create() callback. Creates relay file in debugfs. + */ +static struct dentry *create_buf_file_callback(const char *filename, + struct dentry *parent, + umode_t mode, + struct rchan_buf *buf, + int *is_global) +{ + struct dentry *buf_file; + + /* This to enable the use of a single buffer for the relay channel and + * correspondingly have a single file exposed to User, through which + * it can collect the logs in order without any post-processing. + * Need to set 'is_global' even if parent is NULL for early logging. + */ + *is_global = 1; + + if (!parent) + return NULL; + + /* Not using the channel filename passed as an argument, since for each + * channel relay appends the corresponding CPU number to the filename + * passed in relay_open(). This should be fine as relay just needs a + * dentry of the file associated with the channel buffer and that file's + * name need not be same as the filename passed as an argument. + */ + buf_file = debugfs_create_file("guc_log", mode, + parent, buf, &relay_file_operations); + return buf_file; +} + +/* + * file_remove() default callback. Removes relay file in debugfs. + */ +static int remove_buf_file_callback(struct dentry *dentry) +{ + debugfs_remove(dentry); + return 0; +} + +/* relay channel callbacks */ +static struct rchan_callbacks relay_callbacks = { + .subbuf_start = subbuf_start_callback, + .create_buf_file = create_buf_file_callback, + .remove_buf_file = remove_buf_file_callback, +}; + +static void guc_log_remove_relay_file(struct intel_guc *guc) +{ + relay_close(guc->log.relay_chan); +} + +static int guc_log_create_relay_channel(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct rchan *guc_log_relay_chan; + size_t n_subbufs, subbuf_size; + + /* Keep the size of sub buffers same as shared log buffer */ + subbuf_size = guc->log.vma->obj->base.size; + + /* Store up to 8 snapshots, which is large enough to buffer sufficient + * boot time logs and provides enough leeway to User, in terms of + * latency, for consuming the logs from relay. Also doesn't take + * up too much memory. + */ + n_subbufs = 8; + + guc_log_relay_chan = relay_open(NULL, NULL, subbuf_size, + n_subbufs, &relay_callbacks, dev_priv); + if (!guc_log_relay_chan) { + DRM_ERROR("Couldn't create relay chan for GuC logging\n"); + return -ENOMEM; + } + + GEM_BUG_ON(guc_log_relay_chan->subbuf_size < subbuf_size); + guc->log.relay_chan = guc_log_relay_chan; + return 0; +} + +static int guc_log_create_relay_file(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + struct dentry *log_dir; + int ret; + + /* For now create the log file in /sys/kernel/debug/dri/0 dir */ + log_dir = dev_priv->drm.primary->debugfs_root; + + /* If /sys/kernel/debug/dri/0 location do not exist, then debugfs is + * not mounted and so can't create the relay file. + * The relay API seems to fit well with debugfs only, for availing relay + * there are 3 requirements which can be met for debugfs file only in a + * straightforward/clean manner :- + * i) Need the associated dentry pointer of the file, while opening the + * relay channel. + * ii) Should be able to use 'relay_file_operations' fops for the file. + * iii) Set the 'i_private' field of file's inode to the pointer of + * relay channel buffer. + */ + if (!log_dir) { + DRM_ERROR("Debugfs dir not available yet for GuC log file\n"); + return -ENODEV; + } + + ret = relay_late_setup_files(guc->log.relay_chan, "guc_log", log_dir); + if (ret) { + DRM_ERROR("Couldn't associate relay chan with file %d\n", ret); + return ret; + } + + return 0; +} + +static void guc_move_to_next_buf(struct intel_guc *guc) +{ + /* Make sure the updates made in the sub buffer are visible when + * Consumer sees the following update to offset inside the sub buffer. + */ + smp_wmb(); + + /* All data has been written, so now move the offset of sub buffer. */ + relay_reserve(guc->log.relay_chan, guc->log.vma->obj->base.size); + + /* Switch to the next sub buffer */ + relay_flush(guc->log.relay_chan); +} + +static void *guc_get_write_buffer(struct intel_guc *guc) +{ + if (!guc->log.relay_chan) + return NULL; + + /* Just get the base address of a new sub buffer and copy data into it + * ourselves. NULL will be returned in no-overwrite mode, if all sub + * buffers are full. Could have used the relay_write() to indirectly + * copy the data, but that would have been bit convoluted, as we need to + * write to only certain locations inside a sub buffer which cannot be + * done without using relay_reserve() along with relay_write(). So its + * better to use relay_reserve() alone. + */ + return relay_reserve(guc->log.relay_chan, 0); +} + +static bool +guc_check_log_buf_overflow(struct intel_guc *guc, + enum guc_log_buffer_type type, unsigned int full_cnt) +{ + unsigned int prev_full_cnt = guc->log.prev_overflow_count[type]; + bool overflow = false; + + if (full_cnt != prev_full_cnt) { + overflow = true; + + guc->log.prev_overflow_count[type] = full_cnt; + guc->log.total_overflow_count[type] += full_cnt - prev_full_cnt; + + if (full_cnt < prev_full_cnt) { + /* buffer_full_cnt is a 4 bit counter */ + guc->log.total_overflow_count[type] += 16; + } + DRM_ERROR_RATELIMITED("GuC log buffer overflow\n"); + } + + return overflow; +} + +static unsigned int guc_get_log_buffer_size(enum guc_log_buffer_type type) +{ + switch (type) { + case GUC_ISR_LOG_BUFFER: + return (GUC_LOG_ISR_PAGES + 1) * PAGE_SIZE; + case GUC_DPC_LOG_BUFFER: + return (GUC_LOG_DPC_PAGES + 1) * PAGE_SIZE; + case GUC_CRASH_DUMP_LOG_BUFFER: + return (GUC_LOG_CRASH_PAGES + 1) * PAGE_SIZE; + default: + MISSING_CASE(type); + } + + return 0; +} + +static void guc_read_update_log_buffer(struct intel_guc *guc) +{ + unsigned int buffer_size, read_offset, write_offset, bytes_to_copy, full_cnt; + struct guc_log_buffer_state *log_buf_state, *log_buf_snapshot_state; + struct guc_log_buffer_state log_buf_state_local; + enum guc_log_buffer_type type; + void *src_data, *dst_data; + bool new_overflow; + + if (WARN_ON(!guc->log.buf_addr)) + return; + + /* Get the pointer to shared GuC log buffer */ + log_buf_state = src_data = guc->log.buf_addr; + + /* Get the pointer to local buffer to store the logs */ + log_buf_snapshot_state = dst_data = guc_get_write_buffer(guc); + + /* Actual logs are present from the 2nd page */ + src_data += PAGE_SIZE; + dst_data += PAGE_SIZE; + + for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { + /* Make a copy of the state structure, inside GuC log buffer + * (which is uncached mapped), on the stack to avoid reading + * from it multiple times. + */ + memcpy(&log_buf_state_local, log_buf_state, + sizeof(struct guc_log_buffer_state)); + buffer_size = guc_get_log_buffer_size(type); + read_offset = log_buf_state_local.read_ptr; + write_offset = log_buf_state_local.sampled_write_ptr; + full_cnt = log_buf_state_local.buffer_full_cnt; + + /* Bookkeeping stuff */ + guc->log.flush_count[type] += log_buf_state_local.flush_to_file; + new_overflow = guc_check_log_buf_overflow(guc, type, full_cnt); + + /* Update the state of shared log buffer */ + log_buf_state->read_ptr = write_offset; + log_buf_state->flush_to_file = 0; + log_buf_state++; + + if (unlikely(!log_buf_snapshot_state)) + continue; + + /* First copy the state structure in snapshot buffer */ + memcpy(log_buf_snapshot_state, &log_buf_state_local, + sizeof(struct guc_log_buffer_state)); + + /* The write pointer could have been updated by GuC firmware, + * after sending the flush interrupt to Host, for consistency + * set write pointer value to same value of sampled_write_ptr + * in the snapshot buffer. + */ + log_buf_snapshot_state->write_ptr = write_offset; + log_buf_snapshot_state++; + + /* Now copy the actual logs. */ + if (unlikely(new_overflow)) { + /* copy the whole buffer in case of overflow */ + read_offset = 0; + write_offset = buffer_size; + } else if (unlikely((read_offset > buffer_size) || + (write_offset > buffer_size))) { + DRM_ERROR("invalid log buffer state\n"); + /* copy whole buffer as offsets are unreliable */ + read_offset = 0; + write_offset = buffer_size; + } + + /* Just copy the newly written data */ + if (read_offset > write_offset) { + i915_memcpy_from_wc(dst_data, src_data, write_offset); + bytes_to_copy = buffer_size - read_offset; + } else { + bytes_to_copy = write_offset - read_offset; + } + i915_memcpy_from_wc(dst_data + read_offset, + src_data + read_offset, bytes_to_copy); + + src_data += buffer_size; + dst_data += buffer_size; + } + + if (log_buf_snapshot_state) + guc_move_to_next_buf(guc); + else { + /* Used rate limited to avoid deluge of messages, logs might be + * getting consumed by User at a slow rate. + */ + DRM_ERROR_RATELIMITED("no sub-buffer to capture logs\n"); + guc->log.capture_miss_count++; + } +} + +static void guc_capture_logs_work(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, struct drm_i915_private, guc.log.flush_work); + + i915_guc_capture_logs(dev_priv); +} + +static void guc_log_cleanup(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + /* First disable the flush interrupt */ + gen9_disable_guc_interrupts(dev_priv); + + if (guc->log.flush_wq) + destroy_workqueue(guc->log.flush_wq); + + guc->log.flush_wq = NULL; + + if (guc->log.relay_chan) + guc_log_remove_relay_file(guc); + + guc->log.relay_chan = NULL; + + if (guc->log.buf_addr) + i915_gem_object_unpin_map(guc->log.vma->obj); + + guc->log.buf_addr = NULL; +} + +static int guc_log_create_extras(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + void *vaddr; + int ret; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + /* Nothing to do */ + if (i915.guc_log_level < 0) + return 0; + + if (!guc->log.buf_addr) { + /* Create a WC (Uncached for read) vmalloc mapping of log + * buffer pages, so that we can directly get the data + * (up-to-date) from memory. + */ + vaddr = i915_gem_object_pin_map(guc->log.vma->obj, I915_MAP_WC); + if (IS_ERR(vaddr)) { + ret = PTR_ERR(vaddr); + DRM_ERROR("Couldn't map log buffer pages %d\n", ret); + return ret; + } + + guc->log.buf_addr = vaddr; + } + + if (!guc->log.relay_chan) { + /* Create a relay channel, so that we have buffers for storing + * the GuC firmware logs, the channel will be linked with a file + * later on when debugfs is registered. + */ + ret = guc_log_create_relay_channel(guc); + if (ret) + return ret; + } + + if (!guc->log.flush_wq) { + INIT_WORK(&guc->log.flush_work, guc_capture_logs_work); + + /* + * GuC log buffer flush work item has to do register access to + * send the ack to GuC and this work item, if not synced before + * suspend, can potentially get executed after the GFX device is + * suspended. + * By marking the WQ as freezable, we don't have to bother about + * flushing of this work item from the suspend hooks, the pending + * work item if any will be either executed before the suspend + * or scheduled later on resume. This way the handling of work + * item can be kept same between system suspend & rpm suspend. + */ + guc->log.flush_wq = alloc_ordered_workqueue("i915-guc_log", + WQ_HIGHPRI | WQ_FREEZABLE); + if (guc->log.flush_wq == NULL) { + DRM_ERROR("Couldn't allocate the wq for GuC logging\n"); + return -ENOMEM; + } + } + + return 0; +} + static void guc_log_create(struct intel_guc *guc) { struct i915_vma *vma; unsigned long offset; uint32_t size, flags; - if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN) - return; - if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX) i915.guc_log_level = GUC_LOG_VERBOSITY_MAX; @@ -865,8 +1300,18 @@ static void guc_log_create(struct intel_guc *guc) GUC_LOG_ISR_PAGES + 1 + GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT; - vma = guc->log_vma; + vma = guc->log.vma; if (!vma) { + /* We require SSE 4.1 for fast reads from the GuC log buffer and + * it should be present on the chipsets supporting GuC based + * submisssions. + */ + if (WARN_ON(!i915_memcpy_from_wc(NULL, NULL, 0))) { + /* logging will not be enabled */ + i915.guc_log_level = -1; + return; + } + vma = guc_allocate_vma(guc, size); if (IS_ERR(vma)) { /* logging will be off */ @@ -874,7 +1319,14 @@ static void guc_log_create(struct intel_guc *guc) return; } - guc->log_vma = vma; + guc->log.vma = vma; + + if (guc_log_create_extras(guc)) { + guc_log_cleanup(guc); + i915_vma_unpin_and_release(&guc->log.vma); + i915.guc_log_level = -1; + return; + } } /* each allocated unit is a page */ @@ -884,7 +1336,37 @@ static void guc_log_create(struct intel_guc *guc) (GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT); offset = i915_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */ - guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; + guc->log.flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags; +} + +static int guc_log_late_setup(struct intel_guc *guc) +{ + struct drm_i915_private *dev_priv = guc_to_i915(guc); + int ret; + + lockdep_assert_held(&dev_priv->drm.struct_mutex); + + if (i915.guc_log_level < 0) + return -EINVAL; + + /* If log_level was set as -1 at boot time, then setup needed to + * handle log buffer flush interrupts would not have been done yet, + * so do that now. + */ + ret = guc_log_create_extras(guc); + if (ret) + goto err; + + ret = guc_log_create_relay_file(guc); + if (ret) + goto err; + + return 0; +err: + guc_log_cleanup(guc); + /* logging will remain off */ + i915.guc_log_level = -1; + return ret; } static void guc_policies_init(struct guc_policies *policies) @@ -1006,6 +1488,7 @@ int i915_guc_submission_init(struct drm_i915_private *dev_priv) guc->ctx_pool_vma = vma; ida_init(&guc->ctx_ids); + mutex_init(&guc->action_lock); guc_log_create(guc); guc_addon_create(guc); @@ -1039,7 +1522,8 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv) engine->submit_request = i915_guc_submit; /* Replay the current set of previously submitted requests */ - list_for_each_entry(request, &engine->request_list, link) { + list_for_each_entry(request, + &engine->timeline->requests, link) { client->wq_rsvd += sizeof(struct guc_wq_item); if (i915_sw_fence_done(&request->submit)) i915_guc_submit(request); @@ -1068,7 +1552,7 @@ void i915_guc_submission_fini(struct drm_i915_private *dev_priv) struct intel_guc *guc = &dev_priv->guc; i915_vma_unpin_and_release(&guc->ads_vma); - i915_vma_unpin_and_release(&guc->log_vma); + i915_vma_unpin_and_release(&guc->log.vma); if (guc->ctx_pool_vma) ida_destroy(&guc->ctx_ids); @@ -1089,6 +1573,8 @@ int intel_guc_suspend(struct drm_device *dev) if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS) return 0; + gen9_disable_guc_interrupts(dev_priv); + ctx = dev_priv->kernel_context; data[0] = HOST2GUC_ACTION_ENTER_S_STATE; @@ -1115,6 +1601,9 @@ int intel_guc_resume(struct drm_device *dev) if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS) return 0; + if (i915.guc_log_level >= 0) + gen9_enable_guc_interrupts(dev_priv); + ctx = dev_priv->kernel_context; data[0] = HOST2GUC_ACTION_EXIT_S_STATE; @@ -1124,3 +1613,104 @@ int intel_guc_resume(struct drm_device *dev) return host2guc_action(guc, data, ARRAY_SIZE(data)); } + +void i915_guc_capture_logs(struct drm_i915_private *dev_priv) +{ + guc_read_update_log_buffer(&dev_priv->guc); + + /* Generally device is expected to be active only at this + * time, so get/put should be really quick. + */ + intel_runtime_pm_get(dev_priv); + host2guc_logbuffer_flush_complete(&dev_priv->guc); + intel_runtime_pm_put(dev_priv); +} + +void i915_guc_flush_logs(struct drm_i915_private *dev_priv) +{ + if (!i915.enable_guc_submission || (i915.guc_log_level < 0)) + return; + + /* First disable the interrupts, will be renabled afterwards */ + gen9_disable_guc_interrupts(dev_priv); + + /* Before initiating the forceful flush, wait for any pending/ongoing + * flush to complete otherwise forceful flush may not actually happen. + */ + flush_work(&dev_priv->guc.log.flush_work); + + /* Ask GuC to update the log buffer state */ + host2guc_force_logbuffer_flush(&dev_priv->guc); + + /* GuC would have updated log buffer by now, so capture it */ + i915_guc_capture_logs(dev_priv); +} + +void i915_guc_unregister(struct drm_i915_private *dev_priv) +{ + if (!i915.enable_guc_submission) + return; + + mutex_lock(&dev_priv->drm.struct_mutex); + guc_log_cleanup(&dev_priv->guc); + mutex_unlock(&dev_priv->drm.struct_mutex); +} + +void i915_guc_register(struct drm_i915_private *dev_priv) +{ + if (!i915.enable_guc_submission) + return; + + mutex_lock(&dev_priv->drm.struct_mutex); + guc_log_late_setup(&dev_priv->guc); + mutex_unlock(&dev_priv->drm.struct_mutex); +} + +int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val) +{ + union guc_log_control log_param; + int ret; + + log_param.value = control_val; + + if (log_param.verbosity < GUC_LOG_VERBOSITY_MIN || + log_param.verbosity > GUC_LOG_VERBOSITY_MAX) + return -EINVAL; + + /* This combination doesn't make sense & won't have any effect */ + if (!log_param.logging_enabled && (i915.guc_log_level < 0)) + return 0; + + ret = host2guc_logging_control(&dev_priv->guc, log_param.value); + if (ret < 0) { + DRM_DEBUG_DRIVER("host2guc action failed %d\n", ret); + return ret; + } + + i915.guc_log_level = log_param.verbosity; + + /* If log_level was set as -1 at boot time, then the relay channel file + * wouldn't have been created by now and interrupts also would not have + * been enabled. + */ + if (!dev_priv->guc.log.relay_chan) { + ret = guc_log_late_setup(&dev_priv->guc); + if (!ret) + gen9_enable_guc_interrupts(dev_priv); + } else if (!log_param.logging_enabled) { + /* Once logging is disabled, GuC won't generate logs & send an + * interrupt. But there could be some data in the log buffer + * which is yet to be captured. So request GuC to update the log + * buffer state and then collect the left over logs. + */ + i915_guc_flush_logs(dev_priv); + + /* As logging is disabled, update log level to reflect that */ + i915.guc_log_level = -1; + } else { + /* In case interrupts were disabled, enable them now */ + gen9_enable_guc_interrupts(dev_priv); + } + + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 23315e5461bf..6d7505b5c5e7 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -170,6 +170,7 @@ static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv, } while (0) static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); +static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir); /* For display hotplug interrupt */ static inline void @@ -303,18 +304,18 @@ static void snb_update_pm_irq(struct drm_i915_private *dev_priv, assert_spin_locked(&dev_priv->irq_lock); - new_val = dev_priv->pm_irq_mask; + new_val = dev_priv->pm_imr; new_val &= ~interrupt_mask; new_val |= (~enabled_irq_mask & interrupt_mask); - if (new_val != dev_priv->pm_irq_mask) { - dev_priv->pm_irq_mask = new_val; - I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask); + if (new_val != dev_priv->pm_imr) { + dev_priv->pm_imr = new_val; + I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr); POSTING_READ(gen6_pm_imr(dev_priv)); } } -void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) +void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) { if (WARN_ON(!intel_irqs_enabled(dev_priv))) return; @@ -322,28 +323,54 @@ void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) snb_update_pm_irq(dev_priv, mask, mask); } -static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv, - uint32_t mask) +static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) { snb_update_pm_irq(dev_priv, mask, 0); } -void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask) +void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask) { if (WARN_ON(!intel_irqs_enabled(dev_priv))) return; - __gen6_disable_pm_irq(dev_priv, mask); + __gen6_mask_pm_irq(dev_priv, mask); } -void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) +void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask) { i915_reg_t reg = gen6_pm_iir(dev_priv); - spin_lock_irq(&dev_priv->irq_lock); - I915_WRITE(reg, dev_priv->pm_rps_events); - I915_WRITE(reg, dev_priv->pm_rps_events); + assert_spin_locked(&dev_priv->irq_lock); + + I915_WRITE(reg, reset_mask); + I915_WRITE(reg, reset_mask); POSTING_READ(reg); +} + +void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask) +{ + assert_spin_locked(&dev_priv->irq_lock); + + dev_priv->pm_ier |= enable_mask; + I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier); + gen6_unmask_pm_irq(dev_priv, enable_mask); + /* unmask_pm_irq provides an implicit barrier (POSTING_READ) */ +} + +void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask) +{ + assert_spin_locked(&dev_priv->irq_lock); + + dev_priv->pm_ier &= ~disable_mask; + __gen6_mask_pm_irq(dev_priv, disable_mask); + I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier); + /* though a barrier is missing here, but don't really need a one */ +} + +void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events); dev_priv->rps.pm_iir = 0; spin_unlock_irq(&dev_priv->irq_lock); } @@ -357,8 +384,6 @@ void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv) WARN_ON_ONCE(dev_priv->rps.pm_iir); WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events); dev_priv->rps.interrupts_enabled = true; - I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) | - dev_priv->pm_rps_events); gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); @@ -379,9 +404,7 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u)); - __gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events); - I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) & - ~dev_priv->pm_rps_events); + gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events); spin_unlock_irq(&dev_priv->irq_lock); synchronize_irq(dev_priv->drm.irq); @@ -395,6 +418,38 @@ void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv) gen6_reset_rps_interrupts(dev_priv); } +void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events); + spin_unlock_irq(&dev_priv->irq_lock); +} + +void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + if (!dev_priv->guc.interrupts_enabled) { + WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & + dev_priv->pm_guc_events); + dev_priv->guc.interrupts_enabled = true; + gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events); + } + spin_unlock_irq(&dev_priv->irq_lock); +} + +void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv) +{ + spin_lock_irq(&dev_priv->irq_lock); + dev_priv->guc.interrupts_enabled = false; + + gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events); + + spin_unlock_irq(&dev_priv->irq_lock); + synchronize_irq(dev_priv->drm.irq); + + gen9_reset_guc_interrupts(dev_priv); +} + /** * bdw_update_port_irq - update DE port interrupt * @dev_priv: driver private @@ -670,8 +725,8 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe) struct drm_i915_private *dev_priv = to_i915(dev); i915_reg_t high_frame, low_frame; u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal; - struct intel_crtc *intel_crtc = - to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); const struct drm_display_mode *mode = &intel_crtc->base.hwmode; htotal = mode->crtc_htotal; @@ -776,8 +831,8 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, const struct drm_display_mode *mode) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; bool in_vbl = true; @@ -912,21 +967,22 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, struct timeval *vblank_time, unsigned flags) { - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc; - if (pipe >= INTEL_INFO(dev)->num_pipes) { + if (pipe >= INTEL_INFO(dev_priv)->num_pipes) { DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } /* Get drm_crtc to timestamp: */ - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc == NULL) { DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } - if (!crtc->hwmode.crtc_clock) { + if (!crtc->base.hwmode.crtc_clock) { DRM_DEBUG_KMS("crtc %u is disabled\n", pipe); return -EBUSY; } @@ -934,7 +990,7 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe, /* Helper routine in DRM core does all the work: */ return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, vblank_time, flags, - &crtc->hwmode); + &crtc->base.hwmode); } static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv) @@ -1085,7 +1141,7 @@ static void gen6_pm_rps_work(struct work_struct *work) pm_iir = dev_priv->rps.pm_iir; dev_priv->rps.pm_iir = 0; /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */ - gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events); + gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events); client_boost = dev_priv->rps.client_boost; dev_priv->rps.client_boost = false; spin_unlock_irq(&dev_priv->irq_lock); @@ -1324,11 +1380,13 @@ static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv, DRM_ERROR("The master control interrupt lied (GT3)!\n"); } - if (master_ctl & GEN8_GT_PM_IRQ) { + if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) { gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2)); - if (gt_iir[2] & dev_priv->pm_rps_events) { + if (gt_iir[2] & (dev_priv->pm_rps_events | + dev_priv->pm_guc_events)) { I915_WRITE_FW(GEN8_GT_IIR(2), - gt_iir[2] & dev_priv->pm_rps_events); + gt_iir[2] & (dev_priv->pm_rps_events | + dev_priv->pm_guc_events)); ret = IRQ_HANDLED; } else DRM_ERROR("The master control interrupt lied (PM)!\n"); @@ -1360,6 +1418,9 @@ static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv, if (gt_iir[2] & dev_priv->pm_rps_events) gen6_rps_irq_handler(dev_priv, gt_iir[2]); + + if (gt_iir[2] & dev_priv->pm_guc_events) + gen9_guc_irq_handler(dev_priv, gt_iir[2]); } static bool bxt_port_hotplug_long_detect(enum port port, u32 val) @@ -1586,7 +1647,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) { if (pm_iir & dev_priv->pm_rps_events) { spin_lock(&dev_priv->irq_lock); - gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); + gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events); if (dev_priv->rps.interrupts_enabled) { dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events; schedule_work(&dev_priv->rps.work); @@ -1606,6 +1667,41 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir) } } +static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir) +{ + if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) { + /* Sample the log buffer flush related bits & clear them out now + * itself from the message identity register to minimize the + * probability of losing a flush interrupt, when there are back + * to back flush interrupts. + * There can be a new flush interrupt, for different log buffer + * type (like for ISR), whilst Host is handling one (for DPC). + * Since same bit is used in message register for ISR & DPC, it + * could happen that GuC sets the bit for 2nd interrupt but Host + * clears out the bit on handling the 1st interrupt. + */ + u32 msg, flush; + + msg = I915_READ(SOFT_SCRATCH(15)); + flush = msg & (GUC2HOST_MSG_CRASH_DUMP_POSTED | + GUC2HOST_MSG_FLUSH_LOG_BUFFER); + if (flush) { + /* Clear the message bits that are handled */ + I915_WRITE(SOFT_SCRATCH(15), msg & ~flush); + + /* Handle flush interrupt in bottom half */ + queue_work(dev_priv->guc.log.flush_wq, + &dev_priv->guc.log.flush_work); + + dev_priv->guc.log.flush_interrupt_count++; + } else { + /* Not clearing of unhandled event bits won't result in + * re-triggering of the interrupt. + */ + } + } +} + static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) { @@ -2408,7 +2504,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS; if (fault_errors) - DRM_ERROR("Fault errors on pipe %c\n: 0x%08x", + DRM_ERROR("Fault errors on pipe %c: 0x%08x\n", pipe_name(pipe), fault_errors); } @@ -2752,420 +2848,6 @@ static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe) spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } -static bool -ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr) -{ - if (INTEL_GEN(engine->i915) >= 8) { - return (ipehr >> 23) == 0x1c; - } else { - ipehr &= ~MI_SEMAPHORE_SYNC_MASK; - return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | - MI_SEMAPHORE_REGISTER); - } -} - -static struct intel_engine_cs * -semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, - u64 offset) -{ - struct drm_i915_private *dev_priv = engine->i915; - struct intel_engine_cs *signaller; - enum intel_engine_id id; - - if (INTEL_GEN(dev_priv) >= 8) { - for_each_engine(signaller, dev_priv, id) { - if (engine == signaller) - continue; - - if (offset == signaller->semaphore.signal_ggtt[engine->hw_id]) - return signaller; - } - } else { - u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; - - for_each_engine(signaller, dev_priv, id) { - if(engine == signaller) - continue; - - if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id]) - return signaller; - } - } - - DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n", - engine->name, ipehr, offset); - - return ERR_PTR(-ENODEV); -} - -static struct intel_engine_cs * -semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) -{ - struct drm_i915_private *dev_priv = engine->i915; - void __iomem *vaddr; - u32 cmd, ipehr, head; - u64 offset = 0; - int i, backwards; - - /* - * This function does not support execlist mode - any attempt to - * proceed further into this function will result in a kernel panic - * when dereferencing ring->buffer, which is not set up in execlist - * mode. - * - * The correct way of doing it would be to derive the currently - * executing ring buffer from the current context, which is derived - * from the currently running request. Unfortunately, to get the - * current request we would have to grab the struct_mutex before doing - * anything else, which would be ill-advised since some other thread - * might have grabbed it already and managed to hang itself, causing - * the hang checker to deadlock. - * - * Therefore, this function does not support execlist mode in its - * current form. Just return NULL and move on. - */ - if (engine->buffer == NULL) - return NULL; - - ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); - if (!ipehr_is_semaphore_wait(engine, ipehr)) - return NULL; - - /* - * HEAD is likely pointing to the dword after the actual command, - * so scan backwards until we find the MBOX. But limit it to just 3 - * or 4 dwords depending on the semaphore wait command size. - * Note that we don't care about ACTHD here since that might - * point at at batch, and semaphores are always emitted into the - * ringbuffer itself. - */ - head = I915_READ_HEAD(engine) & HEAD_ADDR; - backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4; - vaddr = (void __iomem *)engine->buffer->vaddr; - - for (i = backwards; i; --i) { - /* - * Be paranoid and presume the hw has gone off into the wild - - * our ring is smaller than what the hardware (and hence - * HEAD_ADDR) allows. Also handles wrap-around. - */ - head &= engine->buffer->size - 1; - - /* This here seems to blow up */ - cmd = ioread32(vaddr + head); - if (cmd == ipehr) - break; - - head -= 4; - } - - if (!i) - return NULL; - - *seqno = ioread32(vaddr + head + 4) + 1; - if (INTEL_GEN(dev_priv) >= 8) { - offset = ioread32(vaddr + head + 12); - offset <<= 32; - offset |= ioread32(vaddr + head + 8); - } - return semaphore_wait_to_signaller_ring(engine, ipehr, offset); -} - -static int semaphore_passed(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - struct intel_engine_cs *signaller; - u32 seqno; - - engine->hangcheck.deadlock++; - - signaller = semaphore_waits_for(engine, &seqno); - if (signaller == NULL) - return -1; - - if (IS_ERR(signaller)) - return 0; - - /* Prevent pathological recursion due to driver bugs */ - if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) - return -1; - - if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno)) - return 1; - - /* cursory check for an unkickable deadlock */ - if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && - semaphore_passed(signaller) < 0) - return -1; - - return 0; -} - -static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, dev_priv, id) - engine->hangcheck.deadlock = 0; -} - -static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone) -{ - u32 tmp = current_instdone | *old_instdone; - bool unchanged; - - unchanged = tmp == *old_instdone; - *old_instdone |= tmp; - - return unchanged; -} - -static bool subunits_stuck(struct intel_engine_cs *engine) -{ - struct drm_i915_private *dev_priv = engine->i915; - struct intel_instdone instdone; - struct intel_instdone *accu_instdone = &engine->hangcheck.instdone; - bool stuck; - int slice; - int subslice; - - if (engine->id != RCS) - return true; - - intel_engine_get_instdone(engine, &instdone); - - /* There might be unstable subunit states even when - * actual head is not moving. Filter out the unstable ones by - * accumulating the undone -> done transitions and only - * consider those as progress. - */ - stuck = instdone_unchanged(instdone.instdone, - &accu_instdone->instdone); - stuck &= instdone_unchanged(instdone.slice_common, - &accu_instdone->slice_common); - - for_each_instdone_slice_subslice(dev_priv, slice, subslice) { - stuck &= instdone_unchanged(instdone.sampler[slice][subslice], - &accu_instdone->sampler[slice][subslice]); - stuck &= instdone_unchanged(instdone.row[slice][subslice], - &accu_instdone->row[slice][subslice]); - } - - return stuck; -} - -static enum intel_engine_hangcheck_action -head_stuck(struct intel_engine_cs *engine, u64 acthd) -{ - if (acthd != engine->hangcheck.acthd) { - - /* Clear subunit states on head movement */ - memset(&engine->hangcheck.instdone, 0, - sizeof(engine->hangcheck.instdone)); - - return HANGCHECK_ACTIVE; - } - - if (!subunits_stuck(engine)) - return HANGCHECK_ACTIVE; - - return HANGCHECK_HUNG; -} - -static enum intel_engine_hangcheck_action -engine_stuck(struct intel_engine_cs *engine, u64 acthd) -{ - struct drm_i915_private *dev_priv = engine->i915; - enum intel_engine_hangcheck_action ha; - u32 tmp; - - ha = head_stuck(engine, acthd); - if (ha != HANGCHECK_HUNG) - return ha; - - if (IS_GEN2(dev_priv)) - return HANGCHECK_HUNG; - - /* Is the chip hanging on a WAIT_FOR_EVENT? - * If so we can simply poke the RB_WAIT bit - * and break the hang. This should work on - * all but the second generation chipsets. - */ - tmp = I915_READ_CTL(engine); - if (tmp & RING_WAIT) { - i915_handle_error(dev_priv, 0, - "Kicking stuck wait on %s", - engine->name); - I915_WRITE_CTL(engine, tmp); - return HANGCHECK_KICK; - } - - if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) { - switch (semaphore_passed(engine)) { - default: - return HANGCHECK_HUNG; - case 1: - i915_handle_error(dev_priv, 0, - "Kicking stuck semaphore on %s", - engine->name); - I915_WRITE_CTL(engine, tmp); - return HANGCHECK_KICK; - case 0: - return HANGCHECK_WAIT; - } - } - - return HANGCHECK_HUNG; -} - -/* - * This is called when the chip hasn't reported back with completed - * batchbuffers in a long time. We keep track per ring seqno progress and - * if there are no progress, hangcheck score for that ring is increased. - * Further, acthd is inspected to see if the ring is stuck. On stuck case - * we kick the ring. If we see no progress on three subsequent calls - * we assume chip is wedged and try to fix it by resetting the chip. - */ -static void i915_hangcheck_elapsed(struct work_struct *work) -{ - struct drm_i915_private *dev_priv = - container_of(work, typeof(*dev_priv), - gpu_error.hangcheck_work.work); - struct intel_engine_cs *engine; - enum intel_engine_id id; - unsigned int hung = 0, stuck = 0; - int busy_count = 0; -#define BUSY 1 -#define KICK 5 -#define HUNG 20 -#define ACTIVE_DECAY 15 - - if (!i915.enable_hangcheck) - return; - - if (!READ_ONCE(dev_priv->gt.awake)) - return; - - /* As enabling the GPU requires fairly extensive mmio access, - * periodically arm the mmio checker to see if we are triggering - * any invalid access. - */ - intel_uncore_arm_unclaimed_mmio_detection(dev_priv); - - for_each_engine(engine, dev_priv, id) { - bool busy = intel_engine_has_waiter(engine); - u64 acthd; - u32 seqno; - u32 submit; - - semaphore_clear_deadlocks(dev_priv); - - /* We don't strictly need an irq-barrier here, as we are not - * serving an interrupt request, be paranoid in case the - * barrier has side-effects (such as preventing a broken - * cacheline snoop) and so be sure that we can see the seqno - * advance. If the seqno should stick, due to a stale - * cacheline, we would erroneously declare the GPU hung. - */ - if (engine->irq_seqno_barrier) - engine->irq_seqno_barrier(engine); - - acthd = intel_engine_get_active_head(engine); - seqno = intel_engine_get_seqno(engine); - submit = READ_ONCE(engine->last_submitted_seqno); - - if (engine->hangcheck.seqno == seqno) { - if (i915_seqno_passed(seqno, submit)) { - engine->hangcheck.action = HANGCHECK_IDLE; - } else { - /* We always increment the hangcheck score - * if the engine is busy and still processing - * the same request, so that no single request - * can run indefinitely (such as a chain of - * batches). The only time we do not increment - * the hangcheck score on this ring, if this - * engine is in a legitimate wait for another - * engine. In that case the waiting engine is a - * victim and we want to be sure we catch the - * right culprit. Then every time we do kick - * the ring, add a small increment to the - * score so that we can catch a batch that is - * being repeatedly kicked and so responsible - * for stalling the machine. - */ - engine->hangcheck.action = - engine_stuck(engine, acthd); - - switch (engine->hangcheck.action) { - case HANGCHECK_IDLE: - case HANGCHECK_WAIT: - break; - case HANGCHECK_ACTIVE: - engine->hangcheck.score += BUSY; - break; - case HANGCHECK_KICK: - engine->hangcheck.score += KICK; - break; - case HANGCHECK_HUNG: - engine->hangcheck.score += HUNG; - break; - } - } - - if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { - hung |= intel_engine_flag(engine); - if (engine->hangcheck.action != HANGCHECK_HUNG) - stuck |= intel_engine_flag(engine); - } - } else { - engine->hangcheck.action = HANGCHECK_ACTIVE; - - /* Gradually reduce the count so that we catch DoS - * attempts across multiple batches. - */ - if (engine->hangcheck.score > 0) - engine->hangcheck.score -= ACTIVE_DECAY; - if (engine->hangcheck.score < 0) - engine->hangcheck.score = 0; - - /* Clear head and subunit states on seqno movement */ - acthd = 0; - - memset(&engine->hangcheck.instdone, 0, - sizeof(engine->hangcheck.instdone)); - } - - engine->hangcheck.seqno = seqno; - engine->hangcheck.acthd = acthd; - busy_count += busy; - } - - if (hung) { - char msg[80]; - unsigned int tmp; - int len; - - /* If some rings hung but others were still busy, only - * blame the hanging rings in the synopsis. - */ - if (stuck != hung) - hung &= ~stuck; - len = scnprintf(msg, sizeof(msg), - "%s on ", stuck == hung ? "No progress" : "Hang"); - for_each_engine_masked(engine, dev_priv, hung, tmp) - len += scnprintf(msg + len, sizeof(msg) - len, - "%s, ", engine->name); - msg[len-2] = '\0'; - - return i915_handle_error(dev_priv, hung, msg); - } - - /* Reset timer in case GPU hangs without another request being added */ - if (busy_count) - i915_queue_hangcheck(dev_priv); -} - static void ibx_irq_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); @@ -3545,11 +3227,13 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev) * RPS interrupts will get enabled/disabled on demand when RPS * itself is enabled/disabled. */ - if (HAS_VEBOX(dev)) + if (HAS_VEBOX(dev_priv)) { pm_irqs |= PM_VEBOX_USER_INTERRUPT; + dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT; + } - dev_priv->pm_irq_mask = 0xffffffff; - GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs); + dev_priv->pm_imr = 0xffffffff; + GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs); } } @@ -3669,14 +3353,15 @@ static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv) if (HAS_L3_DPF(dev_priv)) gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT; - dev_priv->pm_irq_mask = 0xffffffff; + dev_priv->pm_ier = 0x0; + dev_priv->pm_imr = ~dev_priv->pm_ier; GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]); GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]); /* * RPS interrupts will get enabled/disabled on demand when RPS itself - * is enabled/disabled. + * is enabled/disabled. Same wil be the case for GuC interrupts. */ - GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0); + GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier); GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]); } @@ -4460,6 +4145,9 @@ void intel_irq_init(struct drm_i915_private *dev_priv) INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work); INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work); + if (HAS_GUC_SCHED(dev)) + dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT; + /* Let's track the enabled rps events */ if (IS_VALLEYVIEW(dev_priv)) /* WaGsvRC0ResidencyMethod:vlv */ @@ -4481,9 +4169,6 @@ void intel_irq_init(struct drm_i915_private *dev_priv) if (INTEL_INFO(dev_priv)->gen >= 8) dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC; - INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work, - i915_hangcheck_elapsed); - if (IS_GEN2(dev_priv)) { /* Gen2 doesn't have a hardware frame counter */ dev->max_vblank_count = 0; diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 31e6edd08dd0..2a419500b81a 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -288,7 +288,8 @@ static const struct intel_device_info intel_haswell_info = { #define BDW_FEATURES \ HSW_FEATURES, \ BDW_COLORS, \ - .has_logical_ring_contexts = 1 + .has_logical_ring_contexts = 1, \ + .has_64bit_reloc = 1 static const struct intel_device_info intel_broadwell_info = { BDW_FEATURES, @@ -308,6 +309,7 @@ static const struct intel_device_info intel_cherryview_info = { .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .is_cherryview = 1, + .has_64bit_reloc = 1, .has_psr = 1, .has_runtime_pm = 1, .has_resource_streamer = 1, @@ -347,6 +349,7 @@ static const struct intel_device_info intel_broxton_info = { .has_hotplug = 1, .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, .num_pipes = 3, + .has_64bit_reloc = 1, .has_ddi = 1, .has_fpga_dbg = 1, .has_fbc = 1, diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 00efaa13974d..3361d7ffc63e 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -830,96 +830,7 @@ enum skl_disp_power_wells { #define CCK_FREQUENCY_STATUS_SHIFT 8 #define CCK_FREQUENCY_VALUES (0x1f << 0) -/** - * DOC: DPIO - * - * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI - * ports. DPIO is the name given to such a display PHY. These PHYs - * don't follow the standard programming model using direct MMIO - * registers, and instead their registers must be accessed trough IOSF - * sideband. VLV has one such PHY for driving ports B and C, and CHV - * adds another PHY for driving port D. Each PHY responds to specific - * IOSF-SB port. - * - * Each display PHY is made up of one or two channels. Each channel - * houses a common lane part which contains the PLL and other common - * logic. CH0 common lane also contains the IOSF-SB logic for the - * Common Register Interface (CRI) ie. the DPIO registers. CRI clock - * must be running when any DPIO registers are accessed. - * - * In addition to having their own registers, the PHYs are also - * controlled through some dedicated signals from the display - * controller. These include PLL reference clock enable, PLL enable, - * and CRI clock selection, for example. - * - * Eeach channel also has two splines (also called data lanes), and - * each spline is made up of one Physical Access Coding Sub-Layer - * (PCS) block and two TX lanes. So each channel has two PCS blocks - * and four TX lanes. The TX lanes are used as DP lanes or TMDS - * data/clock pairs depending on the output type. - * - * Additionally the PHY also contains an AUX lane with AUX blocks - * for each channel. This is used for DP AUX communication, but - * this fact isn't really relevant for the driver since AUX is - * controlled from the display controller side. No DPIO registers - * need to be accessed during AUX communication, - * - * Generally on VLV/CHV the common lane corresponds to the pipe and - * the spline (PCS/TX) corresponds to the port. - * - * For dual channel PHY (VLV/CHV): - * - * pipe A == CMN/PLL/REF CH0 - * - * pipe B == CMN/PLL/REF CH1 - * - * port B == PCS/TX CH0 - * - * port C == PCS/TX CH1 - * - * This is especially important when we cross the streams - * ie. drive port B with pipe B, or port C with pipe A. - * - * For single channel PHY (CHV): - * - * pipe C == CMN/PLL/REF CH0 - * - * port D == PCS/TX CH0 - * - * On BXT the entire PHY channel corresponds to the port. That means - * the PLL is also now associated with the port rather than the pipe, - * and so the clock needs to be routed to the appropriate transcoder. - * Port A PLL is directly connected to transcoder EDP and port B/C - * PLLs can be routed to any transcoder A/B/C. - * - * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is - * digital port D (CHV) or port A (BXT). :: - * - * - * Dual channel PHY (VLV/CHV/BXT) - * --------------------------------- - * | CH0 | CH1 | - * | CMN/PLL/REF | CMN/PLL/REF | - * |---------------|---------------| Display PHY - * | PCS01 | PCS23 | PCS01 | PCS23 | - * |-------|-------|-------|-------| - * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| - * --------------------------------- - * | DDI0 | DDI1 | DP/HDMI ports - * --------------------------------- - * - * Single channel PHY (CHV/BXT) - * ----------------- - * | CH0 | - * | CMN/PLL/REF | - * |---------------| Display PHY - * | PCS01 | PCS23 | - * |-------|-------| - * |TX0|TX1|TX2|TX3| - * ----------------- - * | DDI2 | DP/HDMI port - * ----------------- - */ +/* DPIO registers */ #define DPIO_DEVFN 0 #define DPIO_CTL _MMIO(VLV_DISPLAY_BASE + 0x2110) @@ -1275,7 +1186,19 @@ enum skl_disp_power_wells { #define DPIO_UPAR_SHIFT 30 /* BXT PHY registers */ -#define _BXT_PHY(phy, a, b) _MMIO_PIPE((phy), (a), (b)) +#define _BXT_PHY0_BASE 0x6C000 +#define _BXT_PHY1_BASE 0x162000 +#define BXT_PHY_BASE(phy) _PIPE((phy), _BXT_PHY0_BASE, \ + _BXT_PHY1_BASE) + +#define _BXT_PHY(phy, reg) \ + _MMIO(BXT_PHY_BASE(phy) - _BXT_PHY0_BASE + (reg)) + +#define _BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \ + (BXT_PHY_BASE(phy) + _PIPE((ch), (reg_ch0) - _BXT_PHY0_BASE, \ + (reg_ch1) - _BXT_PHY0_BASE)) +#define _MMIO_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1) \ + _MMIO(_BXT_PHY_CH(phy, ch, reg_ch0, reg_ch1)) #define BXT_P_CR_GT_DISP_PWRON _MMIO(0x138090) #define GT_DISPLAY_POWER_ON(phy) (1 << (phy)) @@ -1292,8 +1215,8 @@ enum skl_disp_power_wells { #define _PHY_CTL_FAMILY_EDP 0x64C80 #define _PHY_CTL_FAMILY_DDI 0x64C90 #define COMMON_RESET_DIS (1 << 31) -#define BXT_PHY_CTL_FAMILY(phy) _BXT_PHY((phy), _PHY_CTL_FAMILY_DDI, \ - _PHY_CTL_FAMILY_EDP) +#define BXT_PHY_CTL_FAMILY(phy) _MMIO_PIPE((phy), _PHY_CTL_FAMILY_DDI, \ + _PHY_CTL_FAMILY_EDP) /* BXT PHY PLL registers */ #define _PORT_PLL_A 0x46074 @@ -1313,18 +1236,18 @@ enum skl_disp_power_wells { #define PORT_PLL_P2_SHIFT 8 #define PORT_PLL_P2_MASK (0x1f << PORT_PLL_P2_SHIFT) #define PORT_PLL_P2(x) ((x) << PORT_PLL_P2_SHIFT) -#define BXT_PORT_PLL_EBB_0(port) _MMIO_PORT3(port, _PORT_PLL_EBB_0_A, \ - _PORT_PLL_EBB_0_B, \ - _PORT_PLL_EBB_0_C) +#define BXT_PORT_PLL_EBB_0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PLL_EBB_0_B, \ + _PORT_PLL_EBB_0_C) #define _PORT_PLL_EBB_4_A 0x162038 #define _PORT_PLL_EBB_4_B 0x6C038 #define _PORT_PLL_EBB_4_C 0x6C344 #define PORT_PLL_10BIT_CLK_ENABLE (1 << 13) #define PORT_PLL_RECALIBRATE (1 << 14) -#define BXT_PORT_PLL_EBB_4(port) _MMIO_PORT3(port, _PORT_PLL_EBB_4_A, \ - _PORT_PLL_EBB_4_B, \ - _PORT_PLL_EBB_4_C) +#define BXT_PORT_PLL_EBB_4(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PLL_EBB_4_B, \ + _PORT_PLL_EBB_4_C) #define _PORT_PLL_0_A 0x162100 #define _PORT_PLL_0_B 0x6C100 @@ -1355,57 +1278,56 @@ enum skl_disp_power_wells { #define PORT_PLL_DCO_AMP_DEFAULT 15 #define PORT_PLL_DCO_AMP_MASK 0x3c00 #define PORT_PLL_DCO_AMP(x) ((x)<<10) -#define _PORT_PLL_BASE(port) _PORT3(port, _PORT_PLL_0_A, \ - _PORT_PLL_0_B, \ - _PORT_PLL_0_C) -#define BXT_PORT_PLL(port, idx) _MMIO(_PORT_PLL_BASE(port) + (idx) * 4) +#define _PORT_PLL_BASE(phy, ch) _BXT_PHY_CH(phy, ch, \ + _PORT_PLL_0_B, \ + _PORT_PLL_0_C) +#define BXT_PORT_PLL(phy, ch, idx) _MMIO(_PORT_PLL_BASE(phy, ch) + \ + (idx) * 4) /* BXT PHY common lane registers */ #define _PORT_CL1CM_DW0_A 0x162000 #define _PORT_CL1CM_DW0_BC 0x6C000 #define PHY_POWER_GOOD (1 << 16) #define PHY_RESERVED (1 << 7) -#define BXT_PORT_CL1CM_DW0(phy) _BXT_PHY((phy), _PORT_CL1CM_DW0_BC, \ - _PORT_CL1CM_DW0_A) +#define BXT_PORT_CL1CM_DW0(phy) _BXT_PHY((phy), _PORT_CL1CM_DW0_BC) #define _PORT_CL1CM_DW9_A 0x162024 #define _PORT_CL1CM_DW9_BC 0x6C024 #define IREF0RC_OFFSET_SHIFT 8 #define IREF0RC_OFFSET_MASK (0xFF << IREF0RC_OFFSET_SHIFT) -#define BXT_PORT_CL1CM_DW9(phy) _BXT_PHY((phy), _PORT_CL1CM_DW9_BC, \ - _PORT_CL1CM_DW9_A) +#define BXT_PORT_CL1CM_DW9(phy) _BXT_PHY((phy), _PORT_CL1CM_DW9_BC) #define _PORT_CL1CM_DW10_A 0x162028 #define _PORT_CL1CM_DW10_BC 0x6C028 #define IREF1RC_OFFSET_SHIFT 8 #define IREF1RC_OFFSET_MASK (0xFF << IREF1RC_OFFSET_SHIFT) -#define BXT_PORT_CL1CM_DW10(phy) _BXT_PHY((phy), _PORT_CL1CM_DW10_BC, \ - _PORT_CL1CM_DW10_A) +#define BXT_PORT_CL1CM_DW10(phy) _BXT_PHY((phy), _PORT_CL1CM_DW10_BC) #define _PORT_CL1CM_DW28_A 0x162070 #define _PORT_CL1CM_DW28_BC 0x6C070 #define OCL1_POWER_DOWN_EN (1 << 23) #define DW28_OLDO_DYN_PWR_DOWN_EN (1 << 22) #define SUS_CLK_CONFIG 0x3 -#define BXT_PORT_CL1CM_DW28(phy) _BXT_PHY((phy), _PORT_CL1CM_DW28_BC, \ - _PORT_CL1CM_DW28_A) +#define BXT_PORT_CL1CM_DW28(phy) _BXT_PHY((phy), _PORT_CL1CM_DW28_BC) #define _PORT_CL1CM_DW30_A 0x162078 #define _PORT_CL1CM_DW30_BC 0x6C078 #define OCL2_LDOFUSE_PWR_DIS (1 << 6) -#define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC, \ - _PORT_CL1CM_DW30_A) +#define BXT_PORT_CL1CM_DW30(phy) _BXT_PHY((phy), _PORT_CL1CM_DW30_BC) -/* Defined for PHY0 only */ -#define BXT_PORT_CL2CM_DW6_BC _MMIO(0x6C358) +/* The spec defines this only for BXT PHY0, but lets assume that this + * would exist for PHY1 too if it had a second channel. + */ +#define _PORT_CL2CM_DW6_A 0x162358 +#define _PORT_CL2CM_DW6_BC 0x6C358 +#define BXT_PORT_CL2CM_DW6(phy) _BXT_PHY((phy), _PORT_CL2CM_DW6_BC) #define DW6_OLDO_DYN_PWR_DOWN_EN (1 << 28) /* BXT PHY Ref registers */ #define _PORT_REF_DW3_A 0x16218C #define _PORT_REF_DW3_BC 0x6C18C #define GRC_DONE (1 << 22) -#define BXT_PORT_REF_DW3(phy) _BXT_PHY((phy), _PORT_REF_DW3_BC, \ - _PORT_REF_DW3_A) +#define BXT_PORT_REF_DW3(phy) _BXT_PHY((phy), _PORT_REF_DW3_BC) #define _PORT_REF_DW6_A 0x162198 #define _PORT_REF_DW6_BC 0x6C198 @@ -1416,15 +1338,13 @@ enum skl_disp_power_wells { #define GRC_CODE_SLOW_SHIFT 8 #define GRC_CODE_SLOW_MASK (0xFF << GRC_CODE_SLOW_SHIFT) #define GRC_CODE_NOM_MASK 0xFF -#define BXT_PORT_REF_DW6(phy) _BXT_PHY((phy), _PORT_REF_DW6_BC, \ - _PORT_REF_DW6_A) +#define BXT_PORT_REF_DW6(phy) _BXT_PHY((phy), _PORT_REF_DW6_BC) #define _PORT_REF_DW8_A 0x1621A0 #define _PORT_REF_DW8_BC 0x6C1A0 #define GRC_DIS (1 << 15) #define GRC_RDY_OVRD (1 << 1) -#define BXT_PORT_REF_DW8(phy) _BXT_PHY((phy), _PORT_REF_DW8_BC, \ - _PORT_REF_DW8_A) +#define BXT_PORT_REF_DW8(phy) _BXT_PHY((phy), _PORT_REF_DW8_BC) /* BXT PHY PCS registers */ #define _PORT_PCS_DW10_LN01_A 0x162428 @@ -1433,12 +1353,13 @@ enum skl_disp_power_wells { #define _PORT_PCS_DW10_GRP_A 0x162C28 #define _PORT_PCS_DW10_GRP_B 0x6CC28 #define _PORT_PCS_DW10_GRP_C 0x6CE28 -#define BXT_PORT_PCS_DW10_LN01(port) _MMIO_PORT3(port, _PORT_PCS_DW10_LN01_A, \ - _PORT_PCS_DW10_LN01_B, \ - _PORT_PCS_DW10_LN01_C) -#define BXT_PORT_PCS_DW10_GRP(port) _MMIO_PORT3(port, _PORT_PCS_DW10_GRP_A, \ - _PORT_PCS_DW10_GRP_B, \ - _PORT_PCS_DW10_GRP_C) +#define BXT_PORT_PCS_DW10_LN01(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW10_LN01_B, \ + _PORT_PCS_DW10_LN01_C) +#define BXT_PORT_PCS_DW10_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW10_GRP_B, \ + _PORT_PCS_DW10_GRP_C) + #define TX2_SWING_CALC_INIT (1 << 31) #define TX1_SWING_CALC_INIT (1 << 30) @@ -1453,15 +1374,15 @@ enum skl_disp_power_wells { #define _PORT_PCS_DW12_GRP_C 0x6CE30 #define LANESTAGGER_STRAP_OVRD (1 << 6) #define LANE_STAGGER_MASK 0x1F -#define BXT_PORT_PCS_DW12_LN01(port) _MMIO_PORT3(port, _PORT_PCS_DW12_LN01_A, \ - _PORT_PCS_DW12_LN01_B, \ - _PORT_PCS_DW12_LN01_C) -#define BXT_PORT_PCS_DW12_LN23(port) _MMIO_PORT3(port, _PORT_PCS_DW12_LN23_A, \ - _PORT_PCS_DW12_LN23_B, \ - _PORT_PCS_DW12_LN23_C) -#define BXT_PORT_PCS_DW12_GRP(port) _MMIO_PORT3(port, _PORT_PCS_DW12_GRP_A, \ - _PORT_PCS_DW12_GRP_B, \ - _PORT_PCS_DW12_GRP_C) +#define BXT_PORT_PCS_DW12_LN01(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW12_LN01_B, \ + _PORT_PCS_DW12_LN01_C) +#define BXT_PORT_PCS_DW12_LN23(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW12_LN23_B, \ + _PORT_PCS_DW12_LN23_C) +#define BXT_PORT_PCS_DW12_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_PCS_DW12_GRP_B, \ + _PORT_PCS_DW12_GRP_C) /* BXT PHY TX registers */ #define _BXT_LANE_OFFSET(lane) (((lane) >> 1) * 0x200 + \ @@ -1473,12 +1394,12 @@ enum skl_disp_power_wells { #define _PORT_TX_DW2_GRP_A 0x162D08 #define _PORT_TX_DW2_GRP_B 0x6CD08 #define _PORT_TX_DW2_GRP_C 0x6CF08 -#define BXT_PORT_TX_DW2_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW2_GRP_A, \ - _PORT_TX_DW2_GRP_B, \ - _PORT_TX_DW2_GRP_C) -#define BXT_PORT_TX_DW2_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW2_LN0_A, \ - _PORT_TX_DW2_LN0_B, \ - _PORT_TX_DW2_LN0_C) +#define BXT_PORT_TX_DW2_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW2_LN0_B, \ + _PORT_TX_DW2_LN0_C) +#define BXT_PORT_TX_DW2_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW2_GRP_B, \ + _PORT_TX_DW2_GRP_C) #define MARGIN_000_SHIFT 16 #define MARGIN_000 (0xFF << MARGIN_000_SHIFT) #define UNIQ_TRANS_SCALE_SHIFT 8 @@ -1490,12 +1411,12 @@ enum skl_disp_power_wells { #define _PORT_TX_DW3_GRP_A 0x162D0C #define _PORT_TX_DW3_GRP_B 0x6CD0C #define _PORT_TX_DW3_GRP_C 0x6CF0C -#define BXT_PORT_TX_DW3_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW3_GRP_A, \ - _PORT_TX_DW3_GRP_B, \ - _PORT_TX_DW3_GRP_C) -#define BXT_PORT_TX_DW3_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW3_LN0_A, \ - _PORT_TX_DW3_LN0_B, \ - _PORT_TX_DW3_LN0_C) +#define BXT_PORT_TX_DW3_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW3_LN0_B, \ + _PORT_TX_DW3_LN0_C) +#define BXT_PORT_TX_DW3_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW3_GRP_B, \ + _PORT_TX_DW3_GRP_C) #define SCALE_DCOMP_METHOD (1 << 26) #define UNIQUE_TRANGE_EN_METHOD (1 << 27) @@ -1505,12 +1426,12 @@ enum skl_disp_power_wells { #define _PORT_TX_DW4_GRP_A 0x162D10 #define _PORT_TX_DW4_GRP_B 0x6CD10 #define _PORT_TX_DW4_GRP_C 0x6CF10 -#define BXT_PORT_TX_DW4_LN0(port) _MMIO_PORT3(port, _PORT_TX_DW4_LN0_A, \ - _PORT_TX_DW4_LN0_B, \ - _PORT_TX_DW4_LN0_C) -#define BXT_PORT_TX_DW4_GRP(port) _MMIO_PORT3(port, _PORT_TX_DW4_GRP_A, \ - _PORT_TX_DW4_GRP_B, \ - _PORT_TX_DW4_GRP_C) +#define BXT_PORT_TX_DW4_LN0(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW4_LN0_B, \ + _PORT_TX_DW4_LN0_C) +#define BXT_PORT_TX_DW4_GRP(phy, ch) _MMIO_BXT_PHY_CH(phy, ch, \ + _PORT_TX_DW4_GRP_B, \ + _PORT_TX_DW4_GRP_C) #define DEEMPH_SHIFT 24 #define DE_EMPHASIS (0xFF << DEEMPH_SHIFT) @@ -1519,10 +1440,10 @@ enum skl_disp_power_wells { #define _PORT_TX_DW14_LN0_C 0x6C938 #define LATENCY_OPTIM_SHIFT 30 #define LATENCY_OPTIM (1 << LATENCY_OPTIM_SHIFT) -#define BXT_PORT_TX_DW14_LN(port, lane) _MMIO(_PORT3((port), _PORT_TX_DW14_LN0_A, \ - _PORT_TX_DW14_LN0_B, \ - _PORT_TX_DW14_LN0_C) + \ - _BXT_LANE_OFFSET(lane)) +#define BXT_PORT_TX_DW14_LN(phy, ch, lane) \ + _MMIO(_BXT_PHY_CH(phy, ch, _PORT_TX_DW14_LN0_B, \ + _PORT_TX_DW14_LN0_C) + \ + _BXT_LANE_OFFSET(lane)) /* UAIMI scratch pad register 1 */ #define UAIMI_SPR1 _MMIO(0x4F074) @@ -2188,8 +2109,9 @@ enum skl_disp_power_wells { #define FBC_FENCE_OFF _MMIO(0x3218) /* BSpec typo has 321Bh */ #define FBC_TAG(i) _MMIO(0x3300 + (i) * 4) -#define FBC_STATUS2 _MMIO(0x43214) -#define FBC_COMPRESSION_MASK 0x7ff +#define FBC_STATUS2 _MMIO(0x43214) +#define IVB_FBC_COMPRESSION_MASK 0x7ff +#define BDW_FBC_COMPRESSION_MASK 0xfff #define FBC_LL_SIZE (1536) @@ -6015,6 +5937,7 @@ enum { #define GEN8_DE_PIPE_A_IRQ (1<<16) #define GEN8_DE_PIPE_IRQ(pipe) (1<<(16+(pipe))) #define GEN8_GT_VECS_IRQ (1<<6) +#define GEN8_GT_GUC_IRQ (1<<5) #define GEN8_GT_PM_IRQ (1<<4) #define GEN8_GT_VCS2_IRQ (1<<3) #define GEN8_GT_VCS1_IRQ (1<<2) @@ -6026,6 +5949,16 @@ enum { #define GEN8_GT_IIR(which) _MMIO(0x44308 + (0x10 * (which))) #define GEN8_GT_IER(which) _MMIO(0x4430c + (0x10 * (which))) +#define GEN9_GUC_TO_HOST_INT_EVENT (1<<31) +#define GEN9_GUC_EXEC_ERROR_EVENT (1<<30) +#define GEN9_GUC_DISPLAY_EVENT (1<<29) +#define GEN9_GUC_SEMA_SIGNAL_EVENT (1<<28) +#define GEN9_GUC_IOMMU_MSG_EVENT (1<<27) +#define GEN9_GUC_DB_RING_EVENT (1<<26) +#define GEN9_GUC_DMA_DONE_EVENT (1<<25) +#define GEN9_GUC_FATAL_ERROR_EVENT (1<<24) +#define GEN9_GUC_NOTIFICATION_EVENT (1<<23) + #define GEN8_RCS_IRQ_SHIFT 0 #define GEN8_BCS_IRQ_SHIFT 16 #define GEN8_VCS1_IRQ_SHIFT 0 @@ -7358,6 +7291,13 @@ enum { #define _HSW_AUD_MISC_CTRL_B 0x65110 #define HSW_AUD_MISC_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_MISC_CTRL_A, _HSW_AUD_MISC_CTRL_B) +#define _HSW_AUD_M_CTS_ENABLE_A 0x65028 +#define _HSW_AUD_M_CTS_ENABLE_B 0x65128 +#define HSW_AUD_M_CTS_ENABLE(pipe) _MMIO_PIPE(pipe, _HSW_AUD_M_CTS_ENABLE_A, _HSW_AUD_M_CTS_ENABLE_B) +#define AUD_M_CTS_M_VALUE_INDEX (1 << 21) +#define AUD_M_CTS_M_PROG_ENABLE (1 << 20) +#define AUD_CONFIG_M_MASK 0xfffff + #define _HSW_AUD_DIP_ELD_CTRL_ST_A 0x650b4 #define _HSW_AUD_DIP_ELD_CTRL_ST_B 0x651b4 #define HSW_AUD_DIP_ELD_CTRL(pipe) _MMIO_PIPE(pipe, _HSW_AUD_DIP_ELD_CTRL_ST_A, _HSW_AUD_DIP_ELD_CTRL_ST_B) diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c index 8185002d7ec8..95f2f12e0917 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/i915_sw_fence.c @@ -13,6 +13,8 @@ #include "i915_sw_fence.h" +#define I915_SW_FENCE_FLAG_ALLOC BIT(3) /* after WQ_FLAG_* for safety */ + static DEFINE_SPINLOCK(i915_sw_fence_lock); static int __i915_sw_fence_notify(struct i915_sw_fence *fence, @@ -135,6 +137,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void * list_del(&wq->task_list); __i915_sw_fence_complete(wq->private, key); i915_sw_fence_put(wq->private); + if (wq->flags & I915_SW_FENCE_FLAG_ALLOC) + kfree(wq); return 0; } @@ -192,9 +196,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence, return err; } -int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, - struct i915_sw_fence *signaler, - wait_queue_t *wq) +static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, + struct i915_sw_fence *signaler, + wait_queue_t *wq, gfp_t gfp) { unsigned long flags; int pending; @@ -206,8 +210,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, if (unlikely(i915_sw_fence_check_if_after(fence, signaler))) return -EINVAL; + pending = 0; + if (!wq) { + wq = kmalloc(sizeof(*wq), gfp); + if (!wq) { + if (!gfpflags_allow_blocking(gfp)) + return -ENOMEM; + + i915_sw_fence_wait(signaler); + return 0; + } + + pending |= I915_SW_FENCE_FLAG_ALLOC; + } + INIT_LIST_HEAD(&wq->task_list); - wq->flags = 0; + wq->flags = pending; wq->func = i915_sw_fence_wake; wq->private = i915_sw_fence_get(fence); @@ -226,6 +244,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, return pending; } +int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, + struct i915_sw_fence *signaler, + wait_queue_t *wq) +{ + return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0); +} + +int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, + struct i915_sw_fence *signaler, + gfp_t gfp) +{ + return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp); +} + struct i915_sw_dma_fence_cb { struct dma_fence_cb base; struct i915_sw_fence *fence; diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h index cd239e92f67f..707dfc4f0da5 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.h +++ b/drivers/gpu/drm/i915/i915_sw_fence.h @@ -46,6 +46,9 @@ void i915_sw_fence_commit(struct i915_sw_fence *fence); int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, struct i915_sw_fence *after, wait_queue_t *wq); +int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence, + struct i915_sw_fence *after, + gfp_t gfp); int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, struct dma_fence *dma, unsigned long timeout, @@ -62,4 +65,9 @@ static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) return atomic_read(&fence->pending) < 0; } +static inline void i915_sw_fence_wait(struct i915_sw_fence *fence) +{ + wait_event(fence->wait, i915_sw_fence_done(fence)); +} + #endif /* _I915_SW_FENCE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index 5c912c25f7d3..c5d210ebaa9a 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -466,7 +466,7 @@ TRACE_EVENT(i915_gem_ring_sync_to, __entry->dev = from->i915->drm.primary->index; __entry->sync_from = from->engine->id; __entry->sync_to = to->engine->id; - __entry->seqno = from->fence.seqno; + __entry->seqno = from->global_seqno; ), TP_printk("dev=%u, sync-from=%u, sync-to=%u, seqno=%u", @@ -489,7 +489,7 @@ TRACE_EVENT(i915_gem_ring_dispatch, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->flags = flags; dma_fence_enable_sw_signaling(&req->fence); ), @@ -534,7 +534,7 @@ DECLARE_EVENT_CLASS(i915_gem_request, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; ), TP_printk("dev=%u, ring=%u, seqno=%u", @@ -596,7 +596,7 @@ TRACE_EVENT(i915_gem_request_wait_begin, TP_fast_assign( __entry->dev = req->i915->drm.primary->index; __entry->ring = req->engine->id; - __entry->seqno = req->fence.seqno; + __entry->seqno = req->global_seqno; __entry->blocking = mutex_is_locked(&req->i915->drm.struct_mutex); ), diff --git a/drivers/gpu/drm/i915/intel_atomic_plane.c b/drivers/gpu/drm/i915/intel_atomic_plane.c index c762ae549a1c..cb5594411bb6 100644 --- a/drivers/gpu/drm/i915/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/intel_atomic_plane.c @@ -84,7 +84,6 @@ intel_plane_duplicate_state(struct drm_plane *plane) state = &intel_state->base; __drm_atomic_helper_plane_duplicate_state(plane, state); - intel_state->wait_req = NULL; return state; } @@ -101,7 +100,6 @@ void intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state) { - WARN_ON(state && to_intel_plane_state(state)->wait_req); drm_atomic_helper_plane_destroy_state(plane, state); } diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c index 7093cfbb62b1..813fd74d9c8d 100644 --- a/drivers/gpu/drm/i915/intel_audio.c +++ b/drivers/gpu/drm/i915/intel_audio.c @@ -57,6 +57,63 @@ * struct &i915_audio_component_audio_ops @audio_ops is called from i915 driver. */ +/* DP N/M table */ +#define LC_540M 540000 +#define LC_270M 270000 +#define LC_162M 162000 + +struct dp_aud_n_m { + int sample_rate; + int clock; + u16 m; + u16 n; +}; + +/* Values according to DP 1.4 Table 2-104 */ +static const struct dp_aud_n_m dp_aud_n_m[] = { + { 32000, LC_162M, 1024, 10125 }, + { 44100, LC_162M, 784, 5625 }, + { 48000, LC_162M, 512, 3375 }, + { 64000, LC_162M, 2048, 10125 }, + { 88200, LC_162M, 1568, 5625 }, + { 96000, LC_162M, 1024, 3375 }, + { 128000, LC_162M, 4096, 10125 }, + { 176400, LC_162M, 3136, 5625 }, + { 192000, LC_162M, 2048, 3375 }, + { 32000, LC_270M, 1024, 16875 }, + { 44100, LC_270M, 784, 9375 }, + { 48000, LC_270M, 512, 5625 }, + { 64000, LC_270M, 2048, 16875 }, + { 88200, LC_270M, 1568, 9375 }, + { 96000, LC_270M, 1024, 5625 }, + { 128000, LC_270M, 4096, 16875 }, + { 176400, LC_270M, 3136, 9375 }, + { 192000, LC_270M, 2048, 5625 }, + { 32000, LC_540M, 1024, 33750 }, + { 44100, LC_540M, 784, 18750 }, + { 48000, LC_540M, 512, 11250 }, + { 64000, LC_540M, 2048, 33750 }, + { 88200, LC_540M, 1568, 18750 }, + { 96000, LC_540M, 1024, 11250 }, + { 128000, LC_540M, 4096, 33750 }, + { 176400, LC_540M, 3136, 18750 }, + { 192000, LC_540M, 2048, 11250 }, +}; + +static const struct dp_aud_n_m * +audio_config_dp_get_n_m(struct intel_crtc *intel_crtc, int rate) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) { + if (rate == dp_aud_n_m[i].sample_rate && + intel_crtc->config->port_clock == dp_aud_n_m[i].clock) + return &dp_aud_n_m[i]; + } + + return NULL; +} + static const struct { int clock; u32 config; @@ -225,16 +282,43 @@ hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port, const struct drm_display_mode *adjusted_mode) { struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); + struct i915_audio_component *acomp = dev_priv->audio_component; + int rate = acomp ? acomp->aud_sample_rate[port] : 0; + const struct dp_aud_n_m *nm = audio_config_dp_get_n_m(intel_crtc, rate); enum pipe pipe = intel_crtc->pipe; u32 tmp; + if (nm) + DRM_DEBUG_KMS("using Maud %u, Naud %u\n", nm->m, nm->n); + else + DRM_DEBUG_KMS("using automatic Maud, Naud\n"); + tmp = I915_READ(HSW_AUD_CFG(pipe)); tmp &= ~AUD_CONFIG_N_VALUE_INDEX; tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK; tmp &= ~AUD_CONFIG_N_PROG_ENABLE; tmp |= AUD_CONFIG_N_VALUE_INDEX; + if (nm) { + tmp &= ~AUD_CONFIG_N_MASK; + tmp |= AUD_CONFIG_N(nm->n); + tmp |= AUD_CONFIG_N_PROG_ENABLE; + } + I915_WRITE(HSW_AUD_CFG(pipe), tmp); + + tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe)); + tmp &= ~AUD_CONFIG_M_MASK; + tmp &= ~AUD_M_CTS_M_VALUE_INDEX; + tmp &= ~AUD_M_CTS_M_PROG_ENABLE; + + if (nm) { + tmp |= nm->m; + tmp |= AUD_M_CTS_M_VALUE_INDEX; + tmp |= AUD_M_CTS_M_PROG_ENABLE; + } + + I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp); } static void @@ -254,19 +338,24 @@ hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port, tmp &= ~AUD_CONFIG_N_PROG_ENABLE; tmp |= audio_config_hdmi_pixel_clock(adjusted_mode); - if (adjusted_mode->crtc_clock == TMDS_296M || - adjusted_mode->crtc_clock == TMDS_297M) { - n = audio_config_hdmi_get_n(adjusted_mode, rate); - if (n != 0) { - tmp &= ~AUD_CONFIG_N_MASK; - tmp |= AUD_CONFIG_N(n); - tmp |= AUD_CONFIG_N_PROG_ENABLE; - } else { - DRM_DEBUG_KMS("no suitable N value is found\n"); - } + n = audio_config_hdmi_get_n(adjusted_mode, rate); + if (n != 0) { + DRM_DEBUG_KMS("using N %d\n", n); + + tmp &= ~AUD_CONFIG_N_MASK; + tmp |= AUD_CONFIG_N(n); + tmp |= AUD_CONFIG_N_PROG_ENABLE; + } else { + DRM_DEBUG_KMS("using automatic N\n"); } I915_WRITE(HSW_AUD_CFG(pipe), tmp); + + tmp = I915_READ(HSW_AUD_M_CTS_ENABLE(pipe)); + tmp &= ~AUD_CONFIG_M_MASK; + tmp &= ~AUD_M_CTS_M_VALUE_INDEX; + tmp |= AUD_M_CTS_M_PROG_ENABLE; + I915_WRITE(HSW_AUD_M_CTS_ENABLE(pipe), tmp); } static void @@ -687,7 +776,8 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port, /* 1. get the pipe */ intel_encoder = get_saved_enc(dev_priv, port, pipe); if (!intel_encoder || !intel_encoder->base.crtc || - intel_encoder->type != INTEL_OUTPUT_HDMI) { + (intel_encoder->type != INTEL_OUTPUT_HDMI && + intel_encoder->type != INTEL_OUTPUT_DP)) { DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port)); err = -ENODEV; goto unlock; diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c index 56efcc507ea2..c410d3d6465f 100644 --- a/drivers/gpu/drm/i915/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c @@ -83,16 +83,18 @@ static void irq_enable(struct intel_engine_cs *engine) */ engine->breadcrumbs.irq_posted = true; - spin_lock_irq(&engine->i915->irq_lock); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); engine->irq_enable(engine); - spin_unlock_irq(&engine->i915->irq_lock); + spin_unlock(&engine->i915->irq_lock); } static void irq_disable(struct intel_engine_cs *engine) { - spin_lock_irq(&engine->i915->irq_lock); + /* Caller disables interrupts */ + spin_lock(&engine->i915->irq_lock); engine->irq_disable(engine); - spin_unlock_irq(&engine->i915->irq_lock); + spin_unlock(&engine->i915->irq_lock); engine->breadcrumbs.irq_posted = false; } @@ -293,9 +295,9 @@ bool intel_engine_add_wait(struct intel_engine_cs *engine, struct intel_breadcrumbs *b = &engine->breadcrumbs; bool first; - spin_lock(&b->lock); + spin_lock_irq(&b->lock); first = __intel_engine_add_wait(engine, wait); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); return first; } @@ -326,7 +328,7 @@ void intel_engine_remove_wait(struct intel_engine_cs *engine, if (RB_EMPTY_NODE(&wait->node)) return; - spin_lock(&b->lock); + spin_lock_irq(&b->lock); if (RB_EMPTY_NODE(&wait->node)) goto out_unlock; @@ -400,7 +402,7 @@ out_unlock: GEM_BUG_ON(rb_first(&b->waiters) != (b->first_wait ? &b->first_wait->node : NULL)); GEM_BUG_ON(!rcu_access_pointer(b->irq_seqno_bh) ^ RB_EMPTY_ROOT(&b->waiters)); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } static bool signal_complete(struct drm_i915_gem_request *request) @@ -473,14 +475,14 @@ static int intel_breadcrumbs_signaler(void *arg) * we just completed - so double check we are still * the oldest before picking the next one. */ - spin_lock(&b->lock); + spin_lock_irq(&b->lock); if (request == b->first_signal) { struct rb_node *rb = rb_next(&request->signaling.node); b->first_signal = rb ? to_signaler(rb) : NULL; } rb_erase(&request->signaling.node, &b->signals); - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); i915_gem_request_put(request); } else { @@ -502,11 +504,20 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) struct rb_node *parent, **p; bool first, wakeup; - /* locked by dma_fence_enable_sw_signaling() */ + /* Note that we may be called from an interrupt handler on another + * device (e.g. nouveau signaling a fence completion causing us + * to submit a request, and so enable signaling). As such, + * we need to make sure that all other users of b->lock protect + * against interrupts, i.e. use spin_lock_irqsave. + */ + + /* locked by dma_fence_enable_sw_signaling() (irqsafe fence->lock) */ assert_spin_locked(&request->lock); + if (!request->global_seqno) + return; request->signaling.wait.tsk = b->signaler; - request->signaling.wait.seqno = request->fence.seqno; + request->signaling.wait.seqno = request->global_seqno; i915_gem_request_get(request); spin_lock(&b->lock); @@ -530,8 +541,8 @@ void intel_engine_enable_signaling(struct drm_i915_gem_request *request) p = &b->signals.rb_node; while (*p) { parent = *p; - if (i915_seqno_passed(request->fence.seqno, - to_signaler(parent)->fence.seqno)) { + if (i915_seqno_passed(request->global_seqno, + to_signaler(parent)->global_seqno)) { p = &parent->rb_right; first = false; } else { @@ -592,7 +603,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) struct intel_breadcrumbs *b = &engine->breadcrumbs; cancel_fake_irq(engine); - spin_lock(&b->lock); + spin_lock_irq(&b->lock); __intel_breadcrumbs_disable_irq(b); if (intel_engine_has_waiter(engine)) { @@ -605,7 +616,7 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) irq_disable(engine); } - spin_unlock(&b->lock); + spin_unlock_irq(&b->lock); } void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index a97151fcb9f4..30eb95b54dcf 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -573,7 +573,7 @@ intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe) POSTING_READ(pipeconf_reg); /* Wait for next Vblank to substitue * border color for Color info */ - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); st00 = I915_READ8(_VGA_MSR_WRITE); status = ((st00 & (1 << 4)) != 0) ? connector_status_connected : diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c index 1ea0e1f43397..d7a04bca8c28 100644 --- a/drivers/gpu/drm/i915/intel_csr.c +++ b/drivers/gpu/drm/i915/intel_csr.c @@ -168,12 +168,6 @@ struct stepping_info { char substepping; }; -static const struct stepping_info kbl_stepping_info[] = { - {'A', '0'}, {'B', '0'}, {'C', '0'}, - {'D', '0'}, {'E', '0'}, {'F', '0'}, - {'G', '0'}, {'H', '0'}, {'I', '0'}, -}; - static const struct stepping_info skl_stepping_info[] = { {'A', '0'}, {'B', '0'}, {'C', '0'}, {'D', '0'}, {'E', '0'}, {'F', '0'}, @@ -194,10 +188,7 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv) const struct stepping_info *si; unsigned int size; - if (IS_KABYLAKE(dev_priv)) { - size = ARRAY_SIZE(kbl_stepping_info); - si = kbl_stepping_info; - } else if (IS_SKYLAKE(dev_priv)) { + if (IS_SKYLAKE(dev_priv)) { size = ARRAY_SIZE(skl_stepping_info); si = skl_stepping_info; } else if (IS_BROXTON(dev_priv)) { diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index fb18d699ce10..938ac4dbcb45 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -1547,7 +1547,6 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv, { const struct bxt_ddi_buf_trans *ddi_translations; u32 n_entries, i; - uint32_t val; if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) { n_entries = ARRAY_SIZE(bxt_ddi_translations_edp); @@ -1576,38 +1575,11 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv, } } - /* - * While we write to the group register to program all lanes at once we - * can read only lane registers and we pick lanes 0/1 for that. - */ - val = I915_READ(BXT_PORT_PCS_DW10_LN01(port)); - val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT); - I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val); - - val = I915_READ(BXT_PORT_TX_DW2_LN0(port)); - val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); - val |= ddi_translations[level].margin << MARGIN_000_SHIFT | - ddi_translations[level].scale << UNIQ_TRANS_SCALE_SHIFT; - I915_WRITE(BXT_PORT_TX_DW2_GRP(port), val); - - val = I915_READ(BXT_PORT_TX_DW3_LN0(port)); - val &= ~SCALE_DCOMP_METHOD; - if (ddi_translations[level].enable) - val |= SCALE_DCOMP_METHOD; - - if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) - DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); - - I915_WRITE(BXT_PORT_TX_DW3_GRP(port), val); - - val = I915_READ(BXT_PORT_TX_DW4_LN0(port)); - val &= ~DE_EMPHASIS; - val |= ddi_translations[level].deemphasis << DEEMPH_SHIFT; - I915_WRITE(BXT_PORT_TX_DW4_GRP(port), val); - - val = I915_READ(BXT_PORT_PCS_DW10_LN01(port)); - val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT; - I915_WRITE(BXT_PORT_PCS_DW10_GRP(port), val); + bxt_ddi_phy_set_signal_level(dev_priv, port, + ddi_translations[level].margin, + ddi_translations[level].scale, + ddi_translations[level].enable, + ddi_translations[level].deemphasis); } static uint32_t translate_signal_level(int signal_levels) @@ -1923,332 +1895,14 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder, } } -bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, - enum dpio_phy phy) -{ - enum port port; - - if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy))) - return false; - - if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & - (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) { - DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n", - phy); - - return false; - } - - if (phy == DPIO_PHY1 && - !(I915_READ(BXT_PORT_REF_DW3(DPIO_PHY1)) & GRC_DONE)) { - DRM_DEBUG_DRIVER("DDI PHY 1 powered, but GRC isn't done\n"); - - return false; - } - - if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) { - DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n", - phy); - - return false; - } - - for_each_port_masked(port, - phy == DPIO_PHY0 ? BIT(PORT_B) | BIT(PORT_C) : - BIT(PORT_A)) { - u32 tmp = I915_READ(BXT_PHY_CTL(port)); - - if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) { - DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane " - "for port %c powered down " - "(PHY_CTL %08x)\n", - phy, port_name(port), tmp); - - return false; - } - } - - return true; -} - -static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) -{ - u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); - - return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; -} - -static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, - enum dpio_phy phy) -{ - if (intel_wait_for_register(dev_priv, - BXT_PORT_REF_DW3(phy), - GRC_DONE, GRC_DONE, - 10)) - DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); -} - -void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) -{ - u32 val; - - if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { - /* Still read out the GRC value for state verification */ - if (phy == DPIO_PHY0) - dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); - - if (bxt_ddi_phy_verify_state(dev_priv, phy)) { - DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " - "won't reprogram it\n", phy); - - return; - } - - DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " - "force reprogramming it\n", phy); - } - - val = I915_READ(BXT_P_CR_GT_DISP_PWRON); - val |= GT_DISPLAY_POWER_ON(phy); - I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); - - /* - * The PHY registers start out inaccessible and respond to reads with - * all 1s. Eventually they become accessible as they power up, then - * the reserved bit will give the default 0. Poll on the reserved bit - * becoming 0 to find when the PHY is accessible. - * HW team confirmed that the time to reach phypowergood status is - * anywhere between 50 us and 100us. - */ - if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & - (PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) { - DRM_ERROR("timeout during PHY%d power on\n", phy); - } - - /* Program PLL Rcomp code offset */ - val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); - val &= ~IREF0RC_OFFSET_MASK; - val |= 0xE4 << IREF0RC_OFFSET_SHIFT; - I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val); - - val = I915_READ(BXT_PORT_CL1CM_DW10(phy)); - val &= ~IREF1RC_OFFSET_MASK; - val |= 0xE4 << IREF1RC_OFFSET_SHIFT; - I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val); - - /* Program power gating */ - val = I915_READ(BXT_PORT_CL1CM_DW28(phy)); - val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | - SUS_CLK_CONFIG; - I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val); - - if (phy == DPIO_PHY0) { - val = I915_READ(BXT_PORT_CL2CM_DW6_BC); - val |= DW6_OLDO_DYN_PWR_DOWN_EN; - I915_WRITE(BXT_PORT_CL2CM_DW6_BC, val); - } - - val = I915_READ(BXT_PORT_CL1CM_DW30(phy)); - val &= ~OCL2_LDOFUSE_PWR_DIS; - /* - * On PHY1 disable power on the second channel, since no port is - * connected there. On PHY0 both channels have a port, so leave it - * enabled. - * TODO: port C is only connected on BXT-P, so on BXT0/1 we should - * power down the second channel on PHY0 as well. - * - * FIXME: Clarify programming of the following, the register is - * read-only with bit 6 fixed at 0 at least in stepping A. - */ - if (phy == DPIO_PHY1) - val |= OCL2_LDOFUSE_PWR_DIS; - I915_WRITE(BXT_PORT_CL1CM_DW30(phy), val); - - if (phy == DPIO_PHY0) { - uint32_t grc_code; - /* - * PHY0 isn't connected to an RCOMP resistor so copy over - * the corresponding calibrated value from PHY1, and disable - * the automatic calibration on PHY0. - */ - val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, DPIO_PHY1); - grc_code = val << GRC_CODE_FAST_SHIFT | - val << GRC_CODE_SLOW_SHIFT | - val; - I915_WRITE(BXT_PORT_REF_DW6(DPIO_PHY0), grc_code); - - val = I915_READ(BXT_PORT_REF_DW8(DPIO_PHY0)); - val |= GRC_DIS | GRC_RDY_OVRD; - I915_WRITE(BXT_PORT_REF_DW8(DPIO_PHY0), val); - } - - val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); - val |= COMMON_RESET_DIS; - I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); - - if (phy == DPIO_PHY1) - bxt_phy_wait_grc_done(dev_priv, DPIO_PHY1); -} - -void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) -{ - uint32_t val; - - val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); - val &= ~COMMON_RESET_DIS; - I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); - - val = I915_READ(BXT_P_CR_GT_DISP_PWRON); - val &= ~GT_DISPLAY_POWER_ON(phy); - I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); -} - -static bool __printf(6, 7) -__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, - i915_reg_t reg, u32 mask, u32 expected, - const char *reg_fmt, ...) -{ - struct va_format vaf; - va_list args; - u32 val; - - val = I915_READ(reg); - if ((val & mask) == expected) - return true; - - va_start(args, reg_fmt); - vaf.fmt = reg_fmt; - vaf.va = &args; - - DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: " - "current %08x, expected %08x (mask %08x)\n", - phy, &vaf, reg.reg, val, (val & ~mask) | expected, - mask); - - va_end(args); - - return false; -} - -bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, - enum dpio_phy phy) -{ - uint32_t mask; - bool ok; - -#define _CHK(reg, mask, exp, fmt, ...) \ - __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ - ## __VA_ARGS__) - - if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) - return false; - - ok = true; - - /* PLL Rcomp code offset */ - ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), - IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, - "BXT_PORT_CL1CM_DW9(%d)", phy); - ok &= _CHK(BXT_PORT_CL1CM_DW10(phy), - IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT, - "BXT_PORT_CL1CM_DW10(%d)", phy); - - /* Power gating */ - mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG; - ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask, - "BXT_PORT_CL1CM_DW28(%d)", phy); - - if (phy == DPIO_PHY0) - ok &= _CHK(BXT_PORT_CL2CM_DW6_BC, - DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN, - "BXT_PORT_CL2CM_DW6_BC"); - - /* - * TODO: Verify BXT_PORT_CL1CM_DW30 bit OCL2_LDOFUSE_PWR_DIS, - * at least on stepping A this bit is read-only and fixed at 0. - */ - - if (phy == DPIO_PHY0) { - u32 grc_code = dev_priv->bxt_phy_grc; - - grc_code = grc_code << GRC_CODE_FAST_SHIFT | - grc_code << GRC_CODE_SLOW_SHIFT | - grc_code; - mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK | - GRC_CODE_NOM_MASK; - ok &= _CHK(BXT_PORT_REF_DW6(DPIO_PHY0), mask, grc_code, - "BXT_PORT_REF_DW6(%d)", DPIO_PHY0); - - mask = GRC_DIS | GRC_RDY_OVRD; - ok &= _CHK(BXT_PORT_REF_DW8(DPIO_PHY0), mask, mask, - "BXT_PORT_REF_DW8(%d)", DPIO_PHY0); - } - - return ok; -#undef _CHK -} - -static uint8_t -bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - switch (pipe_config->lane_count) { - case 1: - return 0; - case 2: - return BIT(2) | BIT(0); - case 4: - return BIT(3) | BIT(2) | BIT(0); - default: - MISSING_CASE(pipe_config->lane_count); - - return 0; - } -} - static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); - enum port port = dport->port; struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc); - int lane; - - for (lane = 0; lane < 4; lane++) { - u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); - - /* - * Note that on CHV this flag is called UPAR, but has - * the same function. - */ - val &= ~LATENCY_OPTIM; - if (intel_crtc->config->lane_lat_optim_mask & BIT(lane)) - val |= LATENCY_OPTIM; - - I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val); - } -} - -static uint8_t -bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) -{ - struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); - struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); - enum port port = dport->port; - int lane; - uint8_t mask; - - mask = 0; - for (lane = 0; lane < 4; lane++) { - u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane)); - - if (val & LATENCY_OPTIM) - mask |= BIT(lane); - } + uint8_t mask = intel_crtc->config->lane_lat_optim_mask; - return mask; + bxt_ddi_phy_set_lane_optim_mask(encoder, mask); } void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) @@ -2417,7 +2071,7 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder, if (IS_BROXTON(dev_priv) && ret) pipe_config->lane_lat_optim_mask = bxt_ddi_phy_calc_lane_lat_optim_mask(encoder, - pipe_config); + pipe_config->lane_count); return ret; diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index d6a8f11813d5..185e3bbc9ec9 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -282,12 +282,13 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) info->num_sprites[PIPE_A] = 2; info->num_sprites[PIPE_B] = 2; info->num_sprites[PIPE_C] = 1; - } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 2; - else + } else if (INTEL_GEN(dev_priv) >= 5) { for_each_pipe(dev_priv, pipe) info->num_sprites[pipe] = 1; + } if (i915.disable_display) { DRM_INFO("Display disabled (module parameter)\n"); diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index 63423d2f934a..f4e321853fa4 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -37,7 +37,6 @@ #include "intel_frontbuffer.h" #include <drm/i915_drm.h> #include "i915_drv.h" -#include "i915_gem_dmabuf.h" #include "intel_dsi.h" #include "i915_trace.h" #include <drm/drm_atomic.h> @@ -116,8 +115,9 @@ static void chv_prepare_pll(struct intel_crtc *crtc, const struct intel_crtc_state *pipe_config); static void intel_begin_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); static void intel_finish_crtc_commit(struct drm_crtc *, struct drm_crtc_state *); -static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc, - struct intel_crtc_state *crtc_state); +static void skl_init_scalers(struct drm_i915_private *dev_priv, + struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state); static void skylake_pfit_enable(struct intel_crtc *crtc); static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force); static void ironlake_pfit_enable(struct intel_crtc *crtc); @@ -1008,10 +1008,8 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, target_clock, refclk, NULL, best_clock); } -bool intel_crtc_active(struct drm_crtc *crtc) +bool intel_crtc_active(struct intel_crtc *crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - /* Be paranoid as we can arrive here with only partial * state retrieved from the hardware during setup. * @@ -1025,17 +1023,16 @@ bool intel_crtc_active(struct drm_crtc *crtc) * crtc->state->active once we have proper CRTC states wired up * for atomic. */ - return intel_crtc->active && crtc->primary->state->fb && - intel_crtc->config->base.adjusted_mode.crtc_clock; + return crtc->active && crtc->base.primary->state->fb && + crtc->config->base.adjusted_mode.crtc_clock; } enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); - return intel_crtc->config->cpu_transcoder; + return crtc->config->cpu_transcoder; } static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe) @@ -1788,8 +1785,8 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); i915_reg_t reg; uint32_t val, pipeconf_val; @@ -2979,7 +2976,8 @@ int skl_check_plane_surface(struct intel_plane_state *plane_state) /* Rotate src coordinates to match rotated GTT view */ if (drm_rotation_90_or_270(rotation)) drm_rect_rotate(&plane_state->base.src, - fb->width, fb->height, DRM_ROTATE_270); + fb->width << 16, fb->height << 16, + DRM_ROTATE_270); /* * Handle the AUX surface first since @@ -4246,6 +4244,7 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc) bool intel_has_pending_fb_unpin(struct drm_device *dev) { + struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *crtc; /* Note that we don't need to be called with mode_config.lock here @@ -4260,7 +4259,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev) continue; if (crtc->flip_work) - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); return true; } @@ -4937,7 +4936,7 @@ void hsw_disable_ips(struct intel_crtc *crtc) } /* We need to wait for a vblank before we can disable the plane. */ - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc) @@ -5049,7 +5048,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc) if (HAS_GMCH_DISPLAY(dev_priv)) { intel_set_memory_cxsr(dev_priv, false); dev_priv->wm.vlv.cxsr = false; - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); } } @@ -5068,7 +5067,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state) crtc->wm.cxsr_allowed = true; if (pipe_config->update_wm_post && pipe_config->base.active) - intel_update_watermarks(&crtc->base); + intel_update_watermarks(crtc); if (old_pri_state) { struct intel_plane_state *primary_state = @@ -5126,7 +5125,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) if (old_crtc_state->base.active) { intel_set_memory_cxsr(dev_priv, false); dev_priv->wm.vlv.cxsr = false; - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } } @@ -5139,7 +5138,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) */ if (pipe_config->disable_lp_wm) { ilk_disable_lp_wm(dev); - intel_wait_for_vblank(dev, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); } /* @@ -5166,7 +5165,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state) if (dev_priv->display.initial_watermarks != NULL) dev_priv->display.initial_watermarks(pipe_config); else if (pipe_config->update_wm_pre) - intel_update_watermarks(&crtc->base); + intel_update_watermarks(crtc); } static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask) @@ -5394,7 +5393,7 @@ static void ironlake_crtc_enable(struct intel_crtc_state *pipe_config, /* Must wait for vblank to avoid spurious PCH FIFO underruns */ if (intel_crtc->config->has_pch_encoder) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true); } @@ -5486,7 +5485,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, if (dev_priv->display.initial_watermarks != NULL) dev_priv->display.initial_watermarks(pipe_config); else - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); /* XXX: Do the pipe assertions at the right place for BXT DSI. */ if (!transcoder_is_dsi(cpu_transcoder)) @@ -5504,8 +5503,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, intel_encoders_enable(crtc, pipe_config, old_state); if (intel_crtc->config->has_pch_encoder) { - intel_wait_for_vblank(dev, pipe); - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A, true); @@ -5515,8 +5514,8 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config, * to change the workaround. */ hsw_workaround_pipe = pipe_config->hsw_workaround_pipe; if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) { - intel_wait_for_vblank(dev, hsw_workaround_pipe); - intel_wait_for_vblank(dev, hsw_workaround_pipe); + intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); + intel_wait_for_vblank(dev_priv, hsw_workaround_pipe); } } @@ -5837,10 +5836,8 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv) static int skl_calc_cdclk(int max_pixclk, int vco); -static void intel_update_max_cdclk(struct drm_device *dev) +static void intel_update_max_cdclk(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK; int max_cdclk, vco; @@ -5898,11 +5895,9 @@ static void intel_update_max_cdclk(struct drm_device *dev) dev_priv->max_dotclk_freq); } -static void intel_update_cdclk(struct drm_device *dev) +static void intel_update_cdclk(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev); + dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev_priv); if (INTEL_GEN(dev_priv) >= 9) DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n", @@ -6063,14 +6058,14 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int cdclk) return; } - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); } static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv) { u32 cdctl, expected; - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); if (dev_priv->cdclk_pll.vco == 0 || dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref) @@ -6203,7 +6198,7 @@ void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco) dev_priv->skl_preferred_vco_freq = vco; if (changed) - intel_update_max_cdclk(&dev_priv->drm); + intel_update_max_cdclk(dev_priv); } static void @@ -6289,7 +6284,6 @@ static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv) static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco) { - struct drm_device *dev = &dev_priv->drm; u32 freq_select, pcu_ack; WARN_ON((cdclk == 24000) != (vco == 0)); @@ -6340,7 +6334,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco) sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack); mutex_unlock(&dev_priv->rps.hw_lock); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); } static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv); @@ -6387,7 +6381,7 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv) if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0) goto sanitize; - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); /* Is PLL enabled and locked ? */ if (dev_priv->cdclk_pll.vco == 0 || dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref) @@ -6421,7 +6415,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) struct drm_i915_private *dev_priv = to_i915(dev); u32 val, cmd; - WARN_ON(dev_priv->display.get_display_clock_speed(dev) + WARN_ON(dev_priv->display.get_display_clock_speed(dev_priv) != dev_priv->cdclk_freq); if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */ @@ -6478,7 +6472,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk) mutex_unlock(&dev_priv->sb_lock); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); } static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) @@ -6486,7 +6480,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) struct drm_i915_private *dev_priv = to_i915(dev); u32 val, cmd; - WARN_ON(dev_priv->display.get_display_clock_speed(dev) + WARN_ON(dev_priv->display.get_display_clock_speed(dev_priv) != dev_priv->cdclk_freq); switch (cdclk) { @@ -6519,7 +6513,7 @@ static void cherryview_set_cdclk(struct drm_device *dev, int cdclk) } mutex_unlock(&dev_priv->rps.hw_lock); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); } static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv, @@ -6739,7 +6733,7 @@ static void valleyview_crtc_enable(struct intel_crtc_state *pipe_config, intel_color_load_luts(&pipe_config->base); - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); intel_enable_pipe(intel_crtc); assert_vblank_disabled(crtc); @@ -6792,7 +6786,7 @@ static void i9xx_crtc_enable(struct intel_crtc_state *pipe_config, intel_color_load_luts(&pipe_config->base); - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); intel_enable_pipe(intel_crtc); assert_vblank_disabled(crtc); @@ -6830,7 +6824,7 @@ static void i9xx_crtc_disable(struct intel_crtc_state *old_crtc_state, * wait for planes to fully turn off before disabling the pipe. */ if (IS_GEN2(dev_priv)) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); intel_encoders_disable(crtc, old_crtc_state, old_state); @@ -6908,7 +6902,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc) encoder->base.crtc = NULL; intel_fbc_disable(intel_crtc); - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); intel_disable_shared_dpll(intel_crtc); domains = intel_crtc->enabled_power_domains; @@ -7068,7 +7062,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, if (pipe_config->fdi_lanes <= 2) return 0; - other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C)); + other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_C); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -7087,7 +7081,7 @@ static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe, return -EINVAL; } - other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B)); + other_crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_B); other_crtc_state = intel_atomic_get_crtc_state(state, other_crtc); if (IS_ERR(other_crtc_state)) @@ -7245,10 +7239,9 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc, return 0; } -static int skylake_get_display_clock_speed(struct drm_device *dev) +static int skylake_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - uint32_t cdctl; + u32 cdctl; skl_dpll0_update(dev_priv); @@ -7307,9 +7300,8 @@ static void bxt_de_pll_update(struct drm_i915_private *dev_priv) dev_priv->cdclk_pll.ref; } -static int broxton_get_display_clock_speed(struct drm_device *dev) +static int broxton_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 divider; int div, vco; @@ -7342,9 +7334,8 @@ static int broxton_get_display_clock_speed(struct drm_device *dev) return DIV_ROUND_CLOSEST(vco, div); } -static int broadwell_get_display_clock_speed(struct drm_device *dev) +static int broadwell_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; @@ -7362,9 +7353,8 @@ static int broadwell_get_display_clock_speed(struct drm_device *dev) return 675000; } -static int haswell_get_display_clock_speed(struct drm_device *dev) +static int haswell_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t lcpll = I915_READ(LCPLL_CTL); uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK; @@ -7380,35 +7370,35 @@ static int haswell_get_display_clock_speed(struct drm_device *dev) return 540000; } -static int valleyview_get_display_clock_speed(struct drm_device *dev) +static int valleyview_get_display_clock_speed(struct drm_i915_private *dev_priv) { - return vlv_get_cck_clock_hpll(to_i915(dev), "cdclk", + return vlv_get_cck_clock_hpll(dev_priv, "cdclk", CCK_DISPLAY_CLOCK_CONTROL); } -static int ilk_get_display_clock_speed(struct drm_device *dev) +static int ilk_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 450000; } -static int i945_get_display_clock_speed(struct drm_device *dev) +static int i945_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 400000; } -static int i915_get_display_clock_speed(struct drm_device *dev) +static int i915_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 333333; } -static int i9xx_misc_get_display_clock_speed(struct drm_device *dev) +static int i9xx_misc_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 200000; } -static int pnv_get_display_clock_speed(struct drm_device *dev) +static int pnv_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; u16 gcfgc = 0; pci_read_config_word(pdev, GCFGC, &gcfgc); @@ -7431,9 +7421,9 @@ static int pnv_get_display_clock_speed(struct drm_device *dev) } } -static int i915gm_get_display_clock_speed(struct drm_device *dev) +static int i915gm_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; u16 gcfgc = 0; pci_read_config_word(pdev, GCFGC, &gcfgc); @@ -7451,14 +7441,14 @@ static int i915gm_get_display_clock_speed(struct drm_device *dev) } } -static int i865_get_display_clock_speed(struct drm_device *dev) +static int i865_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 266667; } -static int i85x_get_display_clock_speed(struct drm_device *dev) +static int i85x_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; u16 hpllcc = 0; /* @@ -7494,14 +7484,13 @@ static int i85x_get_display_clock_speed(struct drm_device *dev) return 0; } -static int i830_get_display_clock_speed(struct drm_device *dev) +static int i830_get_display_clock_speed(struct drm_i915_private *dev_priv) { return 133333; } -static unsigned int intel_hpll_vco(struct drm_device *dev) +static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); static const unsigned int blb_vco[8] = { [0] = 3200000, [1] = 4000000, @@ -7548,16 +7537,16 @@ static unsigned int intel_hpll_vco(struct drm_device *dev) vco_table = ctg_vco; else if (IS_G4X(dev_priv)) vco_table = elk_vco; - else if (IS_CRESTLINE(dev)) + else if (IS_CRESTLINE(dev_priv)) vco_table = cl_vco; - else if (IS_PINEVIEW(dev)) + else if (IS_PINEVIEW(dev_priv)) vco_table = pnv_vco; - else if (IS_G33(dev)) + else if (IS_G33(dev_priv)) vco_table = blb_vco; else return 0; - tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO); + tmp = I915_READ(IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO); vco = vco_table[tmp & 0x7]; if (vco == 0) @@ -7568,10 +7557,10 @@ static unsigned int intel_hpll_vco(struct drm_device *dev) return vco; } -static int gm45_get_display_clock_speed(struct drm_device *dev) +static int gm45_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; - unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + struct pci_dev *pdev = dev_priv->drm.pdev; + unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv); uint16_t tmp = 0; pci_read_config_word(pdev, GCFGC, &tmp); @@ -7591,14 +7580,14 @@ static int gm45_get_display_clock_speed(struct drm_device *dev) } } -static int i965gm_get_display_clock_speed(struct drm_device *dev) +static int i965gm_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; static const uint8_t div_3200[] = { 16, 10, 8 }; static const uint8_t div_4000[] = { 20, 12, 10 }; static const uint8_t div_5333[] = { 24, 16, 14 }; const uint8_t *div_table; - unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv); uint16_t tmp = 0; pci_read_config_word(pdev, GCFGC, &tmp); @@ -7629,15 +7618,15 @@ fail: return 200000; } -static int g33_get_display_clock_speed(struct drm_device *dev) +static int g33_get_display_clock_speed(struct drm_i915_private *dev_priv) { - struct pci_dev *pdev = dev->pdev; + struct pci_dev *pdev = dev_priv->drm.pdev; static const uint8_t div_3200[] = { 12, 10, 8, 7, 5, 16 }; static const uint8_t div_4000[] = { 14, 12, 10, 8, 6, 20 }; static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 }; static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 }; const uint8_t *div_table; - unsigned int cdclk_sel, vco = intel_hpll_vco(dev); + unsigned int cdclk_sel, vco = intel_hpll_vco(dev_priv); uint16_t tmp = 0; pci_read_config_word(pdev, GCFGC, &tmp); @@ -7726,10 +7715,10 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, struct dpll *reduced_clock) { - struct drm_device *dev = crtc->base.dev; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 fp, fp2 = 0; - if (IS_PINEVIEW(dev)) { + if (IS_PINEVIEW(dev_priv)) { fp = pnv_dpll_compute_fp(&crtc_state->dpll); if (reduced_clock) fp2 = pnv_dpll_compute_fp(reduced_clock); @@ -8099,11 +8088,10 @@ static void chv_prepare_pll(struct intel_crtc *crtc, * in cases where we need the PLL enabled even when @pipe is not going to * be enabled. */ -int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, +int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll) { - struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_crtc_state *pipe_config; pipe_config = kzalloc(sizeof(*pipe_config), GFP_KERNEL); @@ -8114,7 +8102,7 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, pipe_config->pixel_multiplier = 1; pipe_config->dpll = *dpll; - if (IS_CHERRYVIEW(to_i915(dev))) { + if (IS_CHERRYVIEW(dev_priv)) { chv_compute_dpll(crtc, pipe_config); chv_prepare_pll(crtc, pipe_config); chv_enable_pll(crtc, pipe_config); @@ -8137,20 +8125,19 @@ int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, * Disable the PLL for @pipe. To be used in cases where we need * the PLL enabled even when @pipe is not going to be enabled. */ -void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe) +void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe) { - if (IS_CHERRYVIEW(to_i915(dev))) - chv_disable_pll(to_i915(dev), pipe); + if (IS_CHERRYVIEW(dev_priv)) + chv_disable_pll(dev_priv, pipe); else - vlv_disable_pll(to_i915(dev), pipe); + vlv_disable_pll(dev_priv, pipe); } static void i9xx_compute_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state, struct dpll *reduced_clock) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 dpll; struct dpll *clock = &crtc_state->dpll; @@ -8176,7 +8163,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_SDVO_HIGH_SPEED; /* compute bitmask from p1 value */ - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW; else { dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT; @@ -8197,7 +8184,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14; break; } - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_GEN(dev_priv) >= 4) dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT); if (crtc_state->sdvo_tv_clock) @@ -8211,7 +8198,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc, dpll |= DPLL_VCO_ENABLE; crtc_state->dpll_hw_state.dpll = dpll; - if (INTEL_INFO(dev)->gen >= 4) { + if (INTEL_GEN(dev_priv) >= 4) { u32 dpll_md = (crtc_state->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT; crtc_state->dpll_hw_state.dpll_md = dpll_md; @@ -10184,7 +10171,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv) } intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); - intel_update_cdclk(&dev_priv->drm); + intel_update_cdclk(dev_priv); } /* @@ -10254,6 +10241,29 @@ static void bxt_modeset_commit_cdclk(struct drm_atomic_state *old_state) bxt_set_cdclk(to_i915(dev), req_cdclk); } +static int bdw_adjust_min_pipe_pixel_rate(struct intel_crtc_state *crtc_state, + int pixel_rate) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->base.crtc->dev); + + /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ + if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled) + pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95); + + /* BSpec says "Do not use DisplayPort with CDCLK less than + * 432 MHz, audio enabled, port width x4, and link rate + * HBR2 (5.4 GHz), or else there may be audio corruption or + * screen corruption." + */ + if (intel_crtc_has_dp_encoder(crtc_state) && + crtc_state->has_audio && + crtc_state->port_clock >= 540000 && + crtc_state->lane_count == 4) + pixel_rate = max(432000, pixel_rate); + + return pixel_rate; +} + /* compute the max rate for new configuration */ static int ilk_max_pixel_rate(struct drm_atomic_state *state) { @@ -10279,9 +10289,9 @@ static int ilk_max_pixel_rate(struct drm_atomic_state *state) pixel_rate = ilk_pipe_pixel_rate(crtc_state); - /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */ - if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled) - pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95); + if (IS_BROADWELL(dev_priv) || IS_GEN9(dev_priv)) + pixel_rate = bdw_adjust_min_pipe_pixel_rate(crtc_state, + pixel_rate); intel_state->min_pixclk[i] = pixel_rate; } @@ -10364,7 +10374,7 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk) I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); WARN(cdclk != dev_priv->cdclk_freq, "cdclk requested %d kHz but got %d kHz\n", @@ -10729,10 +10739,8 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc, I915_READ(GAMMA_MODE(crtc->pipe)) & GAMMA_MODE_MODE_MASK; if (INTEL_INFO(dev)->gen >= 9) { - skl_init_scalers(dev, crtc, pipe_config); - } + skl_init_scalers(dev_priv, crtc, pipe_config); - if (INTEL_INFO(dev)->gen >= 9) { pipe_config->scaler_state.scaler_id = -1; pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX); } @@ -11044,7 +11052,7 @@ intel_framebuffer_create_for_mode(struct drm_device *dev, fb = intel_framebuffer_create(dev, &mode_cmd, obj); if (IS_ERR(fb)) - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return fb; } @@ -11129,6 +11137,7 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector, struct drm_encoder *encoder = &intel_encoder->base; struct drm_crtc *crtc = NULL; struct drm_device *dev = encoder->dev; + struct drm_i915_private *dev_priv = to_i915(dev); struct drm_framebuffer *fb; struct drm_mode_config *config = &dev->mode_config; struct drm_atomic_state *state = NULL, *restore_state = NULL; @@ -11281,7 +11290,7 @@ found: old->restore_state = restore_state; /* let the connector get through one full cycle before testing */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); return true; fail: @@ -11360,7 +11369,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, fp = pipe_config->dpll_hw_state.fp1; clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT; - if (IS_PINEVIEW(dev)) { + if (IS_PINEVIEW(dev_priv)) { clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1; clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT; } else { @@ -11369,7 +11378,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, } if (!IS_GEN2(dev_priv)) { - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >> DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW); else @@ -11391,7 +11400,7 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc, return; } - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) port_clock = pnv_calc_dpll_params(refclk, &clock); else port_clock = i9xx_calc_dpll_params(refclk, &clock); @@ -11660,8 +11669,7 @@ static bool pageflip_finished(struct intel_crtc *crtc, void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; unsigned long flags; @@ -11674,12 +11682,12 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) * lost pageflips) so needs the full irqsave spinlocks. */ spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && !is_mmio_work(work) && - pageflip_finished(intel_crtc, work)) - page_flip_completed(intel_crtc); + pageflip_finished(crtc, work)) + page_flip_completed(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); } @@ -11687,8 +11695,7 @@ void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe) void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; unsigned long flags; @@ -11701,12 +11708,12 @@ void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe) * lost pageflips) so needs the full irqsave spinlocks. */ spin_lock_irqsave(&dev->event_lock, flags); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && is_mmio_work(work) && - pageflip_finished(intel_crtc, work)) - page_flip_completed(intel_crtc); + pageflip_finished(crtc, work)) + page_flip_completed(crtc); spin_unlock_irqrestore(&dev->event_lock, flags); } @@ -11960,8 +11967,6 @@ static int intel_gen7_queue_flip(struct drm_device *dev, static bool use_mmio_flip(struct intel_engine_cs *engine, struct drm_i915_gem_object *obj) { - struct reservation_object *resv; - /* * This is not being used for older platforms, because * non-availability of flip done interrupt forces us to use @@ -11983,12 +11988,7 @@ static bool use_mmio_flip(struct intel_engine_cs *engine, else if (i915.enable_execlists) return true; - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv && !reservation_object_test_signaled_rcu(resv, false)) - return true; - - return engine != i915_gem_active_get_engine(&obj->last_write, - &obj->base.dev->struct_mutex); + return engine != i915_gem_object_last_write_engine(obj); } static void skl_do_mmio_flip(struct intel_crtc *intel_crtc, @@ -12061,17 +12061,8 @@ static void intel_mmio_flip_work_func(struct work_struct *w) struct intel_framebuffer *intel_fb = to_intel_framebuffer(crtc->base.primary->fb); struct drm_i915_gem_object *obj = intel_fb->obj; - struct reservation_object *resv; - if (work->flip_queued_req) - WARN_ON(i915_wait_request(work->flip_queued_req, - 0, NULL, NO_WAITBOOST)); - - /* For framebuffer backed by dmabuf, wait for fence */ - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) - WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false, - MAX_SCHEDULE_TIMEOUT) < 0); + WARN_ON(i915_gem_object_wait(obj, 0, MAX_SCHEDULE_TIMEOUT, NULL) < 0); intel_pipe_update_start(crtc); @@ -12134,8 +12125,7 @@ static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv, void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) { struct drm_device *dev = &dev_priv->drm; - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); struct intel_flip_work *work; WARN_ON(!in_interrupt()); @@ -12144,19 +12134,19 @@ void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe) return; spin_lock(&dev->event_lock); - work = intel_crtc->flip_work; + work = crtc->flip_work; if (work != NULL && !is_mmio_work(work) && - __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) { + __pageflip_stall_check_cs(dev_priv, crtc, work)) { WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n", - work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc)); - page_flip_completed(intel_crtc); + work->flip_queued_vblank, intel_crtc_get_vblank_counter(crtc)); + page_flip_completed(crtc); work = NULL; } if (work != NULL && !is_mmio_work(work) && - intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1) + intel_crtc_get_vblank_counter(crtc) - work->flip_queued_vblank > 1) intel_queue_rps_boost_for_request(work->flip_queued_req); spin_unlock(&dev->event_lock); } @@ -12272,8 +12262,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, } else if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) { engine = dev_priv->engine[BCS]; } else if (INTEL_INFO(dev)->gen >= 7) { - engine = i915_gem_active_get_engine(&obj->last_write, - &obj->base.dev->struct_mutex); + engine = i915_gem_object_last_write_engine(obj); if (engine == NULL || engine->id != RCS) engine = dev_priv->engine[BCS]; } else { @@ -12305,9 +12294,6 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, if (mmio_flip) { INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func); - - work->flip_queued_req = i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); queue_work(system_unbound_wq, &work->mmio_work); } else { request = i915_gem_request_alloc(engine, engine->last_context); @@ -12353,7 +12339,7 @@ cleanup: crtc->primary->fb = old_fb; update_state_fb(crtc->primary); - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); drm_framebuffer_unreference(work->old_fb); spin_lock_irq(&dev->event_lock); @@ -13472,13 +13458,13 @@ static void verify_wm_state(struct drm_crtc *crtc, return; skl_pipe_wm_get_hw_state(crtc, &hw_wm); - sw_wm = &intel_crtc->wm.active.skl; + sw_wm = &to_intel_crtc_state(new_state)->wm.skl.optimal; skl_ddb_get_hw_state(dev_priv, &hw_ddb); sw_ddb = &dev_priv->wm.skl_hw.ddb; /* planes */ - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { hw_plane_wm = &hw_wm.planes[plane]; sw_plane_wm = &sw_wm->planes[plane]; @@ -14147,13 +14133,10 @@ static int intel_atomic_check(struct drm_device *dev, } static int intel_atomic_prepare_commit(struct drm_device *dev, - struct drm_atomic_state *state, - bool nonblock) + struct drm_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_plane_state *plane_state; struct drm_crtc_state *crtc_state; - struct drm_plane *plane; struct drm_crtc *crtc; int i, ret; @@ -14176,28 +14159,6 @@ static int intel_atomic_prepare_commit(struct drm_device *dev, ret = drm_atomic_helper_prepare_planes(dev, state); mutex_unlock(&dev->struct_mutex); - if (!ret && !nonblock) { - for_each_plane_in_state(state, plane, plane_state, i) { - struct intel_plane_state *intel_plane_state = - to_intel_plane_state(plane_state); - - if (!intel_plane_state->wait_req) - continue; - - ret = i915_wait_request(intel_plane_state->wait_req, - I915_WAIT_INTERRUPTIBLE, - NULL, NULL); - if (ret) { - /* Any hang should be swallowed by the wait */ - WARN_ON(ret == -EIO); - mutex_lock(&dev->struct_mutex); - drm_atomic_helper_cleanup_planes(dev, state); - mutex_unlock(&dev->struct_mutex); - break; - } - } - } - return ret; } @@ -14223,22 +14184,24 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, return; for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); if (!((1 << pipe) & crtc_mask)) continue; - ret = drm_crtc_vblank_get(crtc); + ret = drm_crtc_vblank_get(&crtc->base); if (WARN_ON(ret != 0)) { crtc_mask &= ~(1 << pipe); continue; } - last_vblank_count[pipe] = drm_crtc_vblank_count(crtc); + last_vblank_count[pipe] = drm_crtc_vblank_count(&crtc->base); } for_each_pipe(dev_priv, pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + pipe); long lret; if (!((1 << pipe) & crtc_mask)) @@ -14246,12 +14209,12 @@ static void intel_atomic_wait_for_vblanks(struct drm_device *dev, lret = wait_event_timeout(dev->vblank[pipe].queue, last_vblank_count[pipe] != - drm_crtc_vblank_count(crtc), + drm_crtc_vblank_count(&crtc->base), msecs_to_jiffies(50)); WARN(!lret, "pipe %c vblank wait timed out\n", pipe_name(pipe)); - drm_crtc_vblank_put(crtc); + drm_crtc_vblank_put(&crtc->base); } } @@ -14325,7 +14288,7 @@ static void intel_update_crtcs(struct drm_atomic_state *state, static void skl_update_crtcs(struct drm_atomic_state *state, unsigned int *crtc_vblank_mask) { - struct drm_device *dev = state->dev; + struct drm_i915_private *dev_priv = to_i915(state->dev); struct intel_atomic_state *intel_state = to_intel_atomic_state(state); struct drm_crtc *crtc; struct intel_crtc *intel_crtc; @@ -14376,7 +14339,7 @@ static void skl_update_crtcs(struct drm_atomic_state *state, crtc_vblank_mask); if (vbl_wait) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); progress = true; } @@ -14391,26 +14354,10 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) struct drm_crtc_state *old_crtc_state; struct drm_crtc *crtc; struct intel_crtc_state *intel_cstate; - struct drm_plane *plane; - struct drm_plane_state *plane_state; bool hw_check = intel_state->modeset; unsigned long put_domains[I915_MAX_PIPES] = {}; unsigned crtc_vblank_mask = 0; - int i, ret; - - for_each_plane_in_state(state, plane, plane_state, i) { - struct intel_plane_state *intel_plane_state = - to_intel_plane_state(plane_state); - - if (!intel_plane_state->wait_req) - continue; - - ret = i915_wait_request(intel_plane_state->wait_req, - 0, NULL, NULL); - /* EIO should be eaten, and we can't get interrupted in the - * worker, and blocking commits have waited already. */ - WARN_ON(ret); - } + int i; drm_atomic_helper_wait_for_dependencies(state); @@ -14455,7 +14402,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) intel_check_pch_fifo_underruns(dev_priv); if (!crtc->state->active) - intel_update_watermarks(crtc); + intel_update_watermarks(intel_crtc); } } @@ -14565,12 +14512,33 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state) static void intel_atomic_commit_work(struct work_struct *work) { - struct drm_atomic_state *state = container_of(work, - struct drm_atomic_state, - commit_work); + struct drm_atomic_state *state = + container_of(work, struct drm_atomic_state, commit_work); + intel_atomic_commit_tail(state); } +static int __i915_sw_fence_call +intel_atomic_commit_ready(struct i915_sw_fence *fence, + enum i915_sw_fence_notify notify) +{ + struct intel_atomic_state *state = + container_of(fence, struct intel_atomic_state, commit_ready); + + switch (notify) { + case FENCE_COMPLETE: + if (state->base.commit_work.func) + queue_work(system_unbound_wq, &state->base.commit_work); + break; + + case FENCE_FREE: + drm_atomic_state_put(&state->base); + break; + } + + return NOTIFY_DONE; +} + static void intel_atomic_track_fbs(struct drm_atomic_state *state) { struct drm_plane_state *old_plane_state; @@ -14616,11 +14584,14 @@ static int intel_atomic_commit(struct drm_device *dev, if (ret) return ret; - INIT_WORK(&state->commit_work, intel_atomic_commit_work); + drm_atomic_state_get(state); + i915_sw_fence_init(&intel_state->commit_ready, + intel_atomic_commit_ready); - ret = intel_atomic_prepare_commit(dev, state, nonblock); + ret = intel_atomic_prepare_commit(dev, state); if (ret) { DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret); + i915_sw_fence_commit(&intel_state->commit_ready); return ret; } @@ -14631,10 +14602,14 @@ static int intel_atomic_commit(struct drm_device *dev, intel_atomic_track_fbs(state); drm_atomic_state_get(state); - if (nonblock) - queue_work(system_unbound_wq, &state->commit_work); - else + INIT_WORK(&state->commit_work, + nonblock ? intel_atomic_commit_work : NULL); + + i915_sw_fence_commit(&intel_state->commit_ready); + if (!nonblock) { + i915_sw_fence_wait(&intel_state->commit_ready); intel_atomic_commit_tail(state); + } return 0; } @@ -14746,20 +14721,22 @@ int intel_prepare_plane_fb(struct drm_plane *plane, struct drm_plane_state *new_state) { + struct intel_atomic_state *intel_state = + to_intel_atomic_state(new_state->state); struct drm_device *dev = plane->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_framebuffer *fb = new_state->fb; struct drm_i915_gem_object *obj = intel_fb_obj(fb); struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb); - struct reservation_object *resv; - int ret = 0; + int ret; if (!obj && !old_obj) return 0; if (old_obj) { struct drm_crtc_state *crtc_state = - drm_atomic_get_existing_crtc_state(new_state->state, plane->state->crtc); + drm_atomic_get_existing_crtc_state(new_state->state, + plane->state->crtc); /* Big Hammer, we also need to ensure that any pending * MI_WAIT_FOR_EVENT inside a user batch buffer on the @@ -14772,52 +14749,56 @@ intel_prepare_plane_fb(struct drm_plane *plane, * This should only fail upon a hung GPU, in which case we * can safely continue. */ - if (needs_modeset(crtc_state)) - ret = i915_gem_object_wait_rendering(old_obj, true); - if (ret) { - /* GPU hangs should have been swallowed by the wait */ - WARN_ON(ret == -EIO); - return ret; + if (needs_modeset(crtc_state)) { + ret = i915_sw_fence_await_reservation(&intel_state->commit_ready, + old_obj->resv, NULL, + false, 0, + GFP_KERNEL); + if (ret < 0) + return ret; } } + if (new_state->fence) { /* explicit fencing */ + ret = i915_sw_fence_await_dma_fence(&intel_state->commit_ready, + new_state->fence, + I915_FENCE_TIMEOUT, + GFP_KERNEL); + if (ret < 0) + return ret; + } + if (!obj) return 0; - /* For framebuffer backed by dmabuf, wait for fence */ - resv = i915_gem_object_get_dmabuf_resv(obj); - if (resv) { - long lret; - - lret = reservation_object_wait_timeout_rcu(resv, false, true, - MAX_SCHEDULE_TIMEOUT); - if (lret == -ERESTARTSYS) - return lret; - - WARN(lret < 0, "waiting returns %li\n", lret); + if (!new_state->fence) { /* implicit fencing */ + ret = i915_sw_fence_await_reservation(&intel_state->commit_ready, + obj->resv, NULL, + false, I915_FENCE_TIMEOUT, + GFP_KERNEL); + if (ret < 0) + return ret; } if (plane->type == DRM_PLANE_TYPE_CURSOR && INTEL_INFO(dev)->cursor_needs_physical) { int align = IS_I830(dev_priv) ? 16 * 1024 : 256; ret = i915_gem_object_attach_phys(obj, align); - if (ret) + if (ret) { DRM_DEBUG_KMS("failed to attach phys object\n"); + return ret; + } } else { struct i915_vma *vma; vma = intel_pin_and_fence_fb_obj(fb, new_state->rotation); - if (IS_ERR(vma)) - ret = PTR_ERR(vma); - } - - if (ret == 0) { - to_intel_plane_state(new_state)->wait_req = - i915_gem_active_get(&obj->last_write, - &obj->base.dev->struct_mutex); + if (IS_ERR(vma)) { + DRM_DEBUG_KMS("failed to pin object\n"); + return PTR_ERR(vma); + } } - return ret; + return 0; } /** @@ -14835,7 +14816,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane, { struct drm_device *dev = plane->dev; struct intel_plane_state *old_intel_state; - struct intel_plane_state *intel_state = to_intel_plane_state(plane->state); struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb); struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb); @@ -14847,9 +14827,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane, if (old_obj && (plane->type != DRM_PLANE_TYPE_CURSOR || !INTEL_INFO(dev)->cursor_needs_physical)) intel_unpin_fb_obj(old_state->fb, old_state->rotation); - - i915_gem_request_assign(&intel_state->wait_req, NULL); - i915_gem_request_assign(&old_intel_state->wait_req, NULL); } int @@ -14969,9 +14946,6 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc, */ void intel_plane_destroy(struct drm_plane *plane) { - if (!plane) - return; - drm_plane_cleanup(plane); kfree(to_intel_plane(plane)); } @@ -14985,13 +14959,11 @@ const struct drm_plane_funcs intel_plane_funcs = { .atomic_set_property = intel_plane_atomic_set_property, .atomic_duplicate_state = intel_plane_duplicate_state, .atomic_destroy_state = intel_plane_destroy_state, - }; -static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, - int pipe) +static struct intel_plane * +intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *primary = NULL; struct intel_plane_state *state = NULL; const uint32_t *intel_primary_formats; @@ -15000,17 +14972,22 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, int ret; primary = kzalloc(sizeof(*primary), GFP_KERNEL); - if (!primary) + if (!primary) { + ret = -ENOMEM; goto fail; + } state = intel_create_plane_state(&primary->base); - if (!state) + if (!state) { + ret = -ENOMEM; goto fail; + } + primary->base.state = &state->base; primary->can_scale = false; primary->max_downscale = 1; - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { primary->can_scale = true; state->scaler_id = -1; } @@ -15018,10 +14995,10 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->plane = pipe; primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe); primary->check_plane = intel_check_primary_plane; - if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) + if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4) primary->plane = !pipe; - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { intel_primary_formats = skl_primary_formats; num_formats = ARRAY_SIZE(skl_primary_formats); @@ -15033,7 +15010,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->update_plane = ironlake_update_primary_plane; primary->disable_plane = i9xx_disable_primary_plane; - } else if (INTEL_INFO(dev)->gen >= 4) { + } else if (INTEL_GEN(dev_priv) >= 4) { intel_primary_formats = i965_primary_formats; num_formats = ARRAY_SIZE(i965_primary_formats); @@ -15047,21 +15024,21 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, primary->disable_plane = i9xx_disable_primary_plane; } - if (INTEL_INFO(dev)->gen >= 9) - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, + if (INTEL_GEN(dev_priv) >= 9) + ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, + 0, &intel_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY, "plane 1%c", pipe_name(pipe)); else if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, + 0, &intel_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY, "primary %c", pipe_name(pipe)); else - ret = drm_universal_plane_init(dev, &primary->base, 0, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &primary->base, + 0, &intel_plane_funcs, intel_primary_formats, num_formats, DRM_PLANE_TYPE_PRIMARY, "plane %c", plane_name(primary->plane)); @@ -15086,13 +15063,13 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev, drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs); - return &primary->base; + return primary; fail: kfree(state); kfree(primary); - return NULL; + return ERR_PTR(ret); } static int @@ -15188,21 +15165,25 @@ intel_update_cursor_plane(struct drm_plane *plane, intel_crtc_update_cursor(crtc, state); } -static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, - int pipe) +static struct intel_plane * +intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *cursor = NULL; struct intel_plane_state *state = NULL; int ret; cursor = kzalloc(sizeof(*cursor), GFP_KERNEL); - if (!cursor) + if (!cursor) { + ret = -ENOMEM; goto fail; + } state = intel_create_plane_state(&cursor->base); - if (!state) + if (!state) { + ret = -ENOMEM; goto fail; + } + cursor->base.state = &state->base; cursor->can_scale = false; @@ -15214,8 +15195,8 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, cursor->update_plane = intel_update_cursor_plane; cursor->disable_plane = intel_disable_cursor_plane; - ret = drm_universal_plane_init(dev, &cursor->base, 0, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base, + 0, &intel_plane_funcs, intel_cursor_formats, ARRAY_SIZE(intel_cursor_formats), DRM_PLANE_TYPE_CURSOR, @@ -15229,76 +15210,94 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev, DRM_ROTATE_0 | DRM_ROTATE_180); - if (INTEL_INFO(dev)->gen >=9) + if (INTEL_GEN(dev_priv) >= 9) state->scaler_id = -1; drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs); - return &cursor->base; + return cursor; fail: kfree(state); kfree(cursor); - return NULL; + return ERR_PTR(ret); } -static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc, - struct intel_crtc_state *crtc_state) +static void skl_init_scalers(struct drm_i915_private *dev_priv, + struct intel_crtc *crtc, + struct intel_crtc_state *crtc_state) { + struct intel_crtc_scaler_state *scaler_state = + &crtc_state->scaler_state; int i; - struct intel_scaler *intel_scaler; - struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state; - for (i = 0; i < intel_crtc->num_scalers; i++) { - intel_scaler = &scaler_state->scalers[i]; - intel_scaler->in_use = 0; - intel_scaler->mode = PS_SCALER_MODE_DYN; + for (i = 0; i < crtc->num_scalers; i++) { + struct intel_scaler *scaler = &scaler_state->scalers[i]; + + scaler->in_use = 0; + scaler->mode = PS_SCALER_MODE_DYN; } scaler_state->scaler_id = -1; } -static void intel_crtc_init(struct drm_device *dev, int pipe) +static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_crtc *intel_crtc; struct intel_crtc_state *crtc_state = NULL; - struct drm_plane *primary = NULL; - struct drm_plane *cursor = NULL; - int ret; + struct intel_plane *primary = NULL; + struct intel_plane *cursor = NULL; + int sprite, ret; intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL); - if (intel_crtc == NULL) - return; + if (!intel_crtc) + return -ENOMEM; crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL); - if (!crtc_state) + if (!crtc_state) { + ret = -ENOMEM; goto fail; + } intel_crtc->config = crtc_state; intel_crtc->base.state = &crtc_state->base; crtc_state->base.crtc = &intel_crtc->base; /* initialize shared scalers */ - if (INTEL_INFO(dev)->gen >= 9) { + if (INTEL_GEN(dev_priv) >= 9) { if (pipe == PIPE_C) intel_crtc->num_scalers = 1; else intel_crtc->num_scalers = SKL_NUM_SCALERS; - skl_init_scalers(dev, intel_crtc, crtc_state); + skl_init_scalers(dev_priv, intel_crtc, crtc_state); } - primary = intel_primary_plane_create(dev, pipe); - if (!primary) + primary = intel_primary_plane_create(dev_priv, pipe); + if (IS_ERR(primary)) { + ret = PTR_ERR(primary); goto fail; + } + + for_each_sprite(dev_priv, pipe, sprite) { + struct intel_plane *plane; - cursor = intel_cursor_plane_create(dev, pipe); - if (!cursor) + plane = intel_sprite_plane_create(dev_priv, pipe, sprite); + if (!plane) { + ret = PTR_ERR(plane); + goto fail; + } + } + + cursor = intel_cursor_plane_create(dev_priv, pipe); + if (!cursor) { + ret = PTR_ERR(cursor); goto fail; + } - ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary, - cursor, &intel_crtc_funcs, + ret = drm_crtc_init_with_planes(&dev_priv->drm, &intel_crtc->base, + &primary->base, &cursor->base, + &intel_crtc_funcs, "pipe %c", pipe_name(pipe)); if (ret) goto fail; @@ -15308,8 +15307,8 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) * is hooked to pipe B. Hence we want plane A feeding pipe B. */ intel_crtc->pipe = pipe; - intel_crtc->plane = pipe; - if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) { + intel_crtc->plane = (enum plane) pipe; + if (HAS_FBC(dev_priv) && INTEL_GEN(dev_priv) < 4) { DRM_DEBUG_KMS("swapping pipes & planes for FBC\n"); intel_crtc->plane = !pipe; } @@ -15322,21 +15321,26 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) || dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL); - dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; - dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base; + dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = intel_crtc; + dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = intel_crtc; drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs); intel_color_init(&intel_crtc->base); WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe); - return; + + return 0; fail: - intel_plane_destroy(primary); - intel_plane_destroy(cursor); + /* + * drm_mode_config_cleanup() will free up any + * crtcs/planes already initialized. + */ kfree(crtc_state); kfree(intel_crtc); + + return ret; } enum pipe intel_get_pipe_from_connector(struct intel_connector *connector) @@ -15386,11 +15390,9 @@ static int intel_encoder_clones(struct intel_encoder *encoder) return index_mask; } -static bool has_edp_a(struct drm_device *dev) +static bool has_edp_a(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - if (!IS_MOBILE(dev)) + if (!IS_MOBILE(dev_priv)) return false; if ((I915_READ(DP_A) & DP_DETECTED) == 0) @@ -15530,7 +15532,7 @@ static void intel_setup_outputs(struct drm_device *dev) int found; dpd_is_edp = intel_dp_is_edp(dev, PORT_D); - if (has_edp_a(dev)) + if (has_edp_a(dev_priv)) intel_dp_init(dev, DP_A, PORT_A); if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) { @@ -15922,7 +15924,7 @@ intel_user_framebuffer_create(struct drm_device *dev, fb = intel_framebuffer_create(dev, &mode_cmd, obj); if (IS_ERR(fb)) - i915_gem_object_put_unlocked(obj); + i915_gem_object_put(obj); return fb; } @@ -16325,11 +16327,11 @@ void intel_modeset_init_hw(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq; - intel_init_clock_gating(dev); + intel_init_clock_gating(dev_priv); } /* @@ -16413,11 +16415,10 @@ fail: drm_modeset_acquire_fini(&ctx); } -void intel_modeset_init(struct drm_device *dev) +int intel_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = to_i915(dev); struct i915_ggtt *ggtt = &dev_priv->ggtt; - int sprite, ret; enum pipe pipe; struct intel_crtc *crtc; @@ -16435,10 +16436,10 @@ void intel_modeset_init(struct drm_device *dev) intel_init_quirks(dev); - intel_init_pm(dev); + intel_init_pm(dev_priv); if (INTEL_INFO(dev)->num_pipes == 0) - return; + return 0; /* * There may be no VBT; and if the BIOS enabled SSC we can @@ -16487,22 +16488,22 @@ void intel_modeset_init(struct drm_device *dev) INTEL_INFO(dev)->num_pipes > 1 ? "s" : ""); for_each_pipe(dev_priv, pipe) { - intel_crtc_init(dev, pipe); - for_each_sprite(dev_priv, pipe, sprite) { - ret = intel_plane_init(dev, pipe, sprite); - if (ret) - DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n", - pipe_name(pipe), sprite_name(pipe, sprite), ret); + int ret; + + ret = intel_crtc_init(dev_priv, pipe); + if (ret) { + drm_mode_config_cleanup(dev); + return ret; } } intel_update_czclk(dev_priv); - intel_update_cdclk(dev); + intel_update_cdclk(dev_priv); intel_shared_dpll_init(dev); if (dev_priv->max_cdclk_freq == 0) - intel_update_max_cdclk(dev); + intel_update_max_cdclk(dev_priv); /* Just disable it once at startup */ i915_disable_vga(dev); @@ -16541,6 +16542,8 @@ void intel_modeset_init(struct drm_device *dev) * since the watermark calculation done here will use pstate->fb. */ sanitize_watermarks(dev); + + return 0; } static void intel_enable_pipe_a(struct drm_device *dev) @@ -16870,7 +16873,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) pipe = 0; if (encoder->get_hw_state(encoder, &pipe)) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + encoder->base.crtc = &crtc->base; crtc->config->output_types |= 1 << encoder->type; encoder->get_config(encoder, crtc->config); @@ -16971,7 +16975,8 @@ intel_modeset_setup_hw_state(struct drm_device *dev) } for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + intel_sanitize_crtc(crtc); intel_dump_pipe_config(crtc, crtc->config, "[setup_hw_state]"); diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 3c2293bd24bf..9df331b3305b 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -213,6 +213,81 @@ intel_dp_downstream_max_dotclock(struct intel_dp *intel_dp) return max_dotclk; } +static int +intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates) +{ + if (intel_dp->num_sink_rates) { + *sink_rates = intel_dp->sink_rates; + return intel_dp->num_sink_rates; + } + + *sink_rates = default_rates; + + return (intel_dp_max_link_bw(intel_dp) >> 3) + 1; +} + +static int +intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates) +{ + struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); + int size; + + if (IS_BROXTON(dev_priv)) { + *source_rates = bxt_rates; + size = ARRAY_SIZE(bxt_rates); + } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { + *source_rates = skl_rates; + size = ARRAY_SIZE(skl_rates); + } else { + *source_rates = default_rates; + size = ARRAY_SIZE(default_rates); + } + + /* This depends on the fact that 5.4 is last value in the array */ + if (!intel_dp_source_supports_hbr2(intel_dp)) + size--; + + return size; +} + +static int intersect_rates(const int *source_rates, int source_len, + const int *sink_rates, int sink_len, + int *common_rates) +{ + int i = 0, j = 0, k = 0; + + while (i < source_len && j < sink_len) { + if (source_rates[i] == sink_rates[j]) { + if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES)) + return k; + common_rates[k] = source_rates[i]; + ++k; + ++i; + ++j; + } else if (source_rates[i] < sink_rates[j]) { + ++i; + } else { + ++j; + } + } + return k; +} + +static int intel_dp_common_rates(struct intel_dp *intel_dp, + int *common_rates) +{ + const int *source_rates, *sink_rates; + int source_len, sink_len; + + sink_len = intel_dp_sink_rates(intel_dp, &sink_rates); + source_len = intel_dp_source_rates(intel_dp, &source_rates); + + return intersect_rates(source_rates, source_len, + sink_rates, sink_len, + common_rates); +} + static enum drm_mode_status intel_dp_mode_valid(struct drm_connector *connector, struct drm_display_mode *mode) @@ -320,8 +395,7 @@ static void vlv_power_sequencer_kick(struct intel_dp *intel_dp) { struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = intel_dig_port->base.base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); enum pipe pipe = intel_dp->pps_pipe; bool pll_enabled, release_cl_override = false; enum dpio_phy phy = DPIO_PHY(pipe); @@ -359,7 +433,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) release_cl_override = IS_CHERRYVIEW(dev_priv) && !chv_phy_powergate_ch(dev_priv, phy, ch, true); - if (vlv_force_pll_on(dev, pipe, IS_CHERRYVIEW(dev_priv) ? + if (vlv_force_pll_on(dev_priv, pipe, IS_CHERRYVIEW(dev_priv) ? &chv_dpll[0].dpll : &vlv_dpll[0].dpll)) { DRM_ERROR("Failed to force on pll for pipe %c!\n", pipe_name(pipe)); @@ -383,7 +457,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) POSTING_READ(intel_dp->output_reg); if (!pll_enabled) { - vlv_force_pll_off(dev, pipe); + vlv_force_pll_off(dev_priv, pipe); if (release_cl_override) chv_phy_powergate_ch(dev_priv, phy, ch, false); @@ -1291,19 +1365,6 @@ intel_dp_aux_init(struct intel_dp *intel_dp) intel_dp->aux.transfer = intel_dp_aux_transfer; } -static int -intel_dp_sink_rates(struct intel_dp *intel_dp, const int **sink_rates) -{ - if (intel_dp->num_sink_rates) { - *sink_rates = intel_dp->sink_rates; - return intel_dp->num_sink_rates; - } - - *sink_rates = default_rates; - - return (intel_dp_max_link_bw(intel_dp) >> 3) + 1; -} - bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); @@ -1316,31 +1377,6 @@ bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp) return false; } -static int -intel_dp_source_rates(struct intel_dp *intel_dp, const int **source_rates) -{ - struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - int size; - - if (IS_BROXTON(dev_priv)) { - *source_rates = bxt_rates; - size = ARRAY_SIZE(bxt_rates); - } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) { - *source_rates = skl_rates; - size = ARRAY_SIZE(skl_rates); - } else { - *source_rates = default_rates; - size = ARRAY_SIZE(default_rates); - } - - /* This depends on the fact that 5.4 is last value in the array */ - if (!intel_dp_source_supports_hbr2(intel_dp)) - size--; - - return size; -} - static void intel_dp_set_clock(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) @@ -1375,43 +1411,6 @@ intel_dp_set_clock(struct intel_encoder *encoder, } } -static int intersect_rates(const int *source_rates, int source_len, - const int *sink_rates, int sink_len, - int *common_rates) -{ - int i = 0, j = 0, k = 0; - - while (i < source_len && j < sink_len) { - if (source_rates[i] == sink_rates[j]) { - if (WARN_ON(k >= DP_MAX_SUPPORTED_RATES)) - return k; - common_rates[k] = source_rates[i]; - ++k; - ++i; - ++j; - } else if (source_rates[i] < sink_rates[j]) { - ++i; - } else { - ++j; - } - } - return k; -} - -static int intel_dp_common_rates(struct intel_dp *intel_dp, - int *common_rates) -{ - const int *source_rates, *sink_rates; - int source_len, sink_len; - - sink_len = intel_dp_sink_rates(intel_dp, &sink_rates); - source_len = intel_dp_source_rates(intel_dp, &source_rates); - - return intersect_rates(source_rates, source_len, - sink_rates, sink_len, - common_rates); -} - static void snprintf_int_array(char *str, size_t len, const int *array, int nelem) { @@ -1451,40 +1450,35 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp) DRM_DEBUG_KMS("common rates: %s\n", str); } -static void intel_dp_print_hw_revision(struct intel_dp *intel_dp) +bool +__intel_dp_read_desc(struct intel_dp *intel_dp, struct intel_dp_desc *desc) { - uint8_t rev; - int len; - - if ((drm_debug & DRM_UT_KMS) == 0) - return; + u32 base = drm_dp_is_branch(intel_dp->dpcd) ? DP_BRANCH_OUI : + DP_SINK_OUI; - if (!drm_dp_is_branch(intel_dp->dpcd)) - return; - - len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_HW_REV, &rev, 1); - if (len < 0) - return; - - DRM_DEBUG_KMS("sink hw revision: %d.%d\n", (rev & 0xf0) >> 4, rev & 0xf); + return drm_dp_dpcd_read(&intel_dp->aux, base, desc, sizeof(*desc)) == + sizeof(*desc); } -static void intel_dp_print_sw_revision(struct intel_dp *intel_dp) +bool intel_dp_read_desc(struct intel_dp *intel_dp) { - uint8_t rev[2]; - int len; + struct intel_dp_desc *desc = &intel_dp->desc; + bool oui_sup = intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & + DP_OUI_SUPPORT; + int dev_id_len; - if ((drm_debug & DRM_UT_KMS) == 0) - return; - - if (!drm_dp_is_branch(intel_dp->dpcd)) - return; + if (!__intel_dp_read_desc(intel_dp, desc)) + return false; - len = drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_SW_REV, &rev, 2); - if (len < 0) - return; + dev_id_len = strnlen(desc->device_id, sizeof(desc->device_id)); + DRM_DEBUG_KMS("DP %s: OUI %*phD%s dev-ID %*pE HW-rev %d.%d SW-rev %d.%d\n", + drm_dp_is_branch(intel_dp->dpcd) ? "branch" : "sink", + (int)sizeof(desc->oui), desc->oui, oui_sup ? "" : "(NS)", + dev_id_len, desc->device_id, + desc->hw_rev >> 4, desc->hw_rev & 0xf, + desc->sw_major_rev, desc->sw_minor_rev); - DRM_DEBUG_KMS("sink sw revision: %d.%d\n", rev[0], rev[1]); + return true; } static int rate_to_index(int find, const int *rates) @@ -2369,7 +2363,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp, * 2. Program DP PLL enable */ if (IS_GEN5(dev_priv)) - intel_wait_for_vblank_if_active(&dev_priv->drm, !crtc->pipe); + intel_wait_for_vblank_if_active(dev_priv, !crtc->pipe); intel_dp->DP |= DP_PLL_ENABLE; @@ -3492,7 +3486,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) I915_WRITE(intel_dp->output_reg, DP); POSTING_READ(intel_dp->output_reg); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -3502,7 +3496,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) intel_dp->DP = DP; } -static bool +bool intel_dp_read_dpcd(struct intel_dp *intel_dp) { if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd, @@ -3526,6 +3520,8 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) if (!intel_dp_read_dpcd(intel_dp)) return false; + intel_dp_read_desc(intel_dp); + if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11) dev_priv->no_aux_handshake = intel_dp->dpcd[DP_MAX_DOWNSPREAD] & DP_NO_AUX_HANDSHAKE_LINK_TRAINING; @@ -3627,23 +3623,6 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) return true; } -static void -intel_dp_probe_oui(struct intel_dp *intel_dp) -{ - u8 buf[3]; - - if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT)) - return; - - if (drm_dp_dpcd_read(&intel_dp->aux, DP_SINK_OUI, buf, 3) == 3) - DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n", - buf[0], buf[1], buf[2]); - - if (drm_dp_dpcd_read(&intel_dp->aux, DP_BRANCH_OUI, buf, 3) == 3) - DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n", - buf[0], buf[1], buf[2]); -} - static bool intel_dp_can_mst(struct intel_dp *intel_dp) { @@ -3687,7 +3666,7 @@ intel_dp_configure_mst(struct intel_dp *intel_dp) static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int ret = 0; @@ -3708,7 +3687,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) } do { - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { @@ -3731,7 +3710,7 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp) static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int ret; @@ -3759,14 +3738,14 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp) return -EIO; } - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); return 0; } int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); - struct drm_device *dev = dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc); u8 buf; int count, ret; @@ -3777,7 +3756,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc) return ret; do { - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, &buf) < 0) { @@ -4010,7 +3989,7 @@ intel_dp_retrain_link(struct intel_dp *intel_dp) intel_dp_stop_link_train(intel_dp); /* Keep underrun reporting disabled until things are stable */ - intel_wait_for_vblank(&dev_priv->drm, crtc->pipe); + intel_wait_for_vblank(dev_priv, crtc->pipe); intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); if (crtc->config->has_pch_encoder) @@ -4422,10 +4401,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector) intel_dp_print_rates(intel_dp); - intel_dp_probe_oui(intel_dp); - - intel_dp_print_hw_revision(intel_dp); - intel_dp_print_sw_revision(intel_dp); + intel_dp_read_desc(intel_dp); intel_dp_configure_mst(intel_dp); @@ -4489,21 +4465,11 @@ static enum drm_connector_status intel_dp_detect(struct drm_connector *connector, bool force) { struct intel_dp *intel_dp = intel_attached_dp(connector); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct intel_encoder *intel_encoder = &intel_dig_port->base; enum drm_connector_status status = connector->status; DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); - if (intel_dp->is_mst) { - /* MST devices are disconnected from a monitor POV */ - intel_dp_unset_edid(intel_dp); - if (intel_encoder->type != INTEL_OUTPUT_EDP) - intel_encoder->type = INTEL_OUTPUT_DP; - return connector_status_disconnected; - } - /* If full detect is not performed yet, do a full detect */ if (!intel_dp->detect_done) status = intel_dp_long_pulse(intel_dp->attached_connector); diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c index 047f48748944..7a8e82dabbf2 100644 --- a/drivers/gpu/drm/i915/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/intel_dpio_phy.c @@ -23,6 +23,565 @@ #include "intel_drv.h" +/** + * DOC: DPIO + * + * VLV, CHV and BXT have slightly peculiar display PHYs for driving DP/HDMI + * ports. DPIO is the name given to such a display PHY. These PHYs + * don't follow the standard programming model using direct MMIO + * registers, and instead their registers must be accessed trough IOSF + * sideband. VLV has one such PHY for driving ports B and C, and CHV + * adds another PHY for driving port D. Each PHY responds to specific + * IOSF-SB port. + * + * Each display PHY is made up of one or two channels. Each channel + * houses a common lane part which contains the PLL and other common + * logic. CH0 common lane also contains the IOSF-SB logic for the + * Common Register Interface (CRI) ie. the DPIO registers. CRI clock + * must be running when any DPIO registers are accessed. + * + * In addition to having their own registers, the PHYs are also + * controlled through some dedicated signals from the display + * controller. These include PLL reference clock enable, PLL enable, + * and CRI clock selection, for example. + * + * Eeach channel also has two splines (also called data lanes), and + * each spline is made up of one Physical Access Coding Sub-Layer + * (PCS) block and two TX lanes. So each channel has two PCS blocks + * and four TX lanes. The TX lanes are used as DP lanes or TMDS + * data/clock pairs depending on the output type. + * + * Additionally the PHY also contains an AUX lane with AUX blocks + * for each channel. This is used for DP AUX communication, but + * this fact isn't really relevant for the driver since AUX is + * controlled from the display controller side. No DPIO registers + * need to be accessed during AUX communication, + * + * Generally on VLV/CHV the common lane corresponds to the pipe and + * the spline (PCS/TX) corresponds to the port. + * + * For dual channel PHY (VLV/CHV): + * + * pipe A == CMN/PLL/REF CH0 + * + * pipe B == CMN/PLL/REF CH1 + * + * port B == PCS/TX CH0 + * + * port C == PCS/TX CH1 + * + * This is especially important when we cross the streams + * ie. drive port B with pipe B, or port C with pipe A. + * + * For single channel PHY (CHV): + * + * pipe C == CMN/PLL/REF CH0 + * + * port D == PCS/TX CH0 + * + * On BXT the entire PHY channel corresponds to the port. That means + * the PLL is also now associated with the port rather than the pipe, + * and so the clock needs to be routed to the appropriate transcoder. + * Port A PLL is directly connected to transcoder EDP and port B/C + * PLLs can be routed to any transcoder A/B/C. + * + * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is + * digital port D (CHV) or port A (BXT). :: + * + * + * Dual channel PHY (VLV/CHV/BXT) + * --------------------------------- + * | CH0 | CH1 | + * | CMN/PLL/REF | CMN/PLL/REF | + * |---------------|---------------| Display PHY + * | PCS01 | PCS23 | PCS01 | PCS23 | + * |-------|-------|-------|-------| + * |TX0|TX1|TX2|TX3|TX0|TX1|TX2|TX3| + * --------------------------------- + * | DDI0 | DDI1 | DP/HDMI ports + * --------------------------------- + * + * Single channel PHY (CHV/BXT) + * ----------------- + * | CH0 | + * | CMN/PLL/REF | + * |---------------| Display PHY + * | PCS01 | PCS23 | + * |-------|-------| + * |TX0|TX1|TX2|TX3| + * ----------------- + * | DDI2 | DP/HDMI port + * ----------------- + */ + +/** + * struct bxt_ddi_phy_info - Hold info for a broxton DDI phy + */ +struct bxt_ddi_phy_info { + /** + * @dual_channel: true if this phy has a second channel. + */ + bool dual_channel; + + /** + * @rcomp_phy: If -1, indicates this phy has its own rcomp resistor. + * Otherwise the GRC value will be copied from the phy indicated by + * this field. + */ + enum dpio_phy rcomp_phy; + + /** + * @channel: struct containing per channel information. + */ + struct { + /** + * @port: which port maps to this channel. + */ + enum port port; + } channel[2]; +}; + +static const struct bxt_ddi_phy_info bxt_ddi_phy_info[] = { + [DPIO_PHY0] = { + .dual_channel = true, + .rcomp_phy = DPIO_PHY1, + + .channel = { + [DPIO_CH0] = { .port = PORT_B }, + [DPIO_CH1] = { .port = PORT_C }, + } + }, + [DPIO_PHY1] = { + .dual_channel = false, + .rcomp_phy = -1, + + .channel = { + [DPIO_CH0] = { .port = PORT_A }, + } + }, +}; + +static u32 bxt_phy_port_mask(const struct bxt_ddi_phy_info *phy_info) +{ + return (phy_info->dual_channel * BIT(phy_info->channel[DPIO_CH1].port)) | + BIT(phy_info->channel[DPIO_CH0].port); +} + +void bxt_port_to_phy_channel(enum port port, + enum dpio_phy *phy, enum dpio_channel *ch) +{ + const struct bxt_ddi_phy_info *phy_info; + int i; + + for (i = 0; i < ARRAY_SIZE(bxt_ddi_phy_info); i++) { + phy_info = &bxt_ddi_phy_info[i]; + + if (port == phy_info->channel[DPIO_CH0].port) { + *phy = i; + *ch = DPIO_CH0; + return; + } + + if (phy_info->dual_channel && + port == phy_info->channel[DPIO_CH1].port) { + *phy = i; + *ch = DPIO_CH1; + return; + } + } + + WARN(1, "PHY not found for PORT %c", port_name(port)); + *phy = DPIO_PHY0; + *ch = DPIO_CH0; +} + +void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, + enum port port, u32 margin, u32 scale, + u32 enable, u32 deemphasis) +{ + u32 val; + enum dpio_phy phy; + enum dpio_channel ch; + + bxt_port_to_phy_channel(port, &phy, &ch); + + /* + * While we write to the group register to program all lanes at once we + * can read only lane registers and we pick lanes 0/1 for that. + */ + val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); + val &= ~(TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT); + I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_TX_DW2_LN0(phy, ch)); + val &= ~(MARGIN_000 | UNIQ_TRANS_SCALE); + val |= margin << MARGIN_000_SHIFT | scale << UNIQ_TRANS_SCALE_SHIFT; + I915_WRITE(BXT_PORT_TX_DW2_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_TX_DW3_LN0(phy, ch)); + val &= ~SCALE_DCOMP_METHOD; + if (enable) + val |= SCALE_DCOMP_METHOD; + + if ((val & UNIQUE_TRANGE_EN_METHOD) && !(val & SCALE_DCOMP_METHOD)) + DRM_ERROR("Disabled scaling while ouniqetrangenmethod was set"); + + I915_WRITE(BXT_PORT_TX_DW3_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_TX_DW4_LN0(phy, ch)); + val &= ~DE_EMPHASIS; + val |= deemphasis << DEEMPH_SHIFT; + I915_WRITE(BXT_PORT_TX_DW4_GRP(phy, ch), val); + + val = I915_READ(BXT_PORT_PCS_DW10_LN01(phy, ch)); + val |= TX2_SWING_CALC_INIT | TX1_SWING_CALC_INIT; + I915_WRITE(BXT_PORT_PCS_DW10_GRP(phy, ch), val); +} + +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + enum port port; + + if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy))) + return false; + + if ((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & + (PHY_POWER_GOOD | PHY_RESERVED)) != PHY_POWER_GOOD) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but power hasn't settled\n", + phy); + + return false; + } + + if (phy_info->rcomp_phy == -1 && + !(I915_READ(BXT_PORT_REF_DW3(phy)) & GRC_DONE)) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but GRC isn't done\n", + phy); + + return false; + } + + if (!(I915_READ(BXT_PHY_CTL_FAMILY(phy)) & COMMON_RESET_DIS)) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but still in reset\n", + phy); + + return false; + } + + for_each_port_masked(port, bxt_phy_port_mask(phy_info)) { + u32 tmp = I915_READ(BXT_PHY_CTL(port)); + + if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) { + DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane " + "for port %c powered down " + "(PHY_CTL %08x)\n", + phy, port_name(port), tmp); + + return false; + } + } + + return true; +} + +static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy) +{ + u32 val = I915_READ(BXT_PORT_REF_DW6(phy)); + + return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT; +} + +static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + if (intel_wait_for_register(dev_priv, + BXT_PORT_REF_DW3(phy), + GRC_DONE, GRC_DONE, + 10)) + DRM_ERROR("timeout waiting for PHY%d GRC\n", phy); +} + +static void _bxt_ddi_phy_init(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + u32 val; + + if (bxt_ddi_phy_is_enabled(dev_priv, phy)) { + /* Still read out the GRC value for state verification */ + if (phy_info->rcomp_phy != -1) + dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy); + + if (bxt_ddi_phy_verify_state(dev_priv, phy)) { + DRM_DEBUG_DRIVER("DDI PHY %d already enabled, " + "won't reprogram it\n", phy); + + return; + } + + DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, " + "force reprogramming it\n", phy); + } + + val = I915_READ(BXT_P_CR_GT_DISP_PWRON); + val |= GT_DISPLAY_POWER_ON(phy); + I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); + + /* + * The PHY registers start out inaccessible and respond to reads with + * all 1s. Eventually they become accessible as they power up, then + * the reserved bit will give the default 0. Poll on the reserved bit + * becoming 0 to find when the PHY is accessible. + * HW team confirmed that the time to reach phypowergood status is + * anywhere between 50 us and 100us. + */ + if (wait_for_us(((I915_READ(BXT_PORT_CL1CM_DW0(phy)) & + (PHY_RESERVED | PHY_POWER_GOOD)) == PHY_POWER_GOOD), 100)) { + DRM_ERROR("timeout during PHY%d power on\n", phy); + } + + /* Program PLL Rcomp code offset */ + val = I915_READ(BXT_PORT_CL1CM_DW9(phy)); + val &= ~IREF0RC_OFFSET_MASK; + val |= 0xE4 << IREF0RC_OFFSET_SHIFT; + I915_WRITE(BXT_PORT_CL1CM_DW9(phy), val); + + val = I915_READ(BXT_PORT_CL1CM_DW10(phy)); + val &= ~IREF1RC_OFFSET_MASK; + val |= 0xE4 << IREF1RC_OFFSET_SHIFT; + I915_WRITE(BXT_PORT_CL1CM_DW10(phy), val); + + /* Program power gating */ + val = I915_READ(BXT_PORT_CL1CM_DW28(phy)); + val |= OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | + SUS_CLK_CONFIG; + I915_WRITE(BXT_PORT_CL1CM_DW28(phy), val); + + if (phy_info->dual_channel) { + val = I915_READ(BXT_PORT_CL2CM_DW6(phy)); + val |= DW6_OLDO_DYN_PWR_DOWN_EN; + I915_WRITE(BXT_PORT_CL2CM_DW6(phy), val); + } + + if (phy_info->rcomp_phy != -1) { + uint32_t grc_code; + /* + * PHY0 isn't connected to an RCOMP resistor so copy over + * the corresponding calibrated value from PHY1, and disable + * the automatic calibration on PHY0. + */ + val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, + phy_info->rcomp_phy); + grc_code = val << GRC_CODE_FAST_SHIFT | + val << GRC_CODE_SLOW_SHIFT | + val; + I915_WRITE(BXT_PORT_REF_DW6(phy), grc_code); + + val = I915_READ(BXT_PORT_REF_DW8(phy)); + val |= GRC_DIS | GRC_RDY_OVRD; + I915_WRITE(BXT_PORT_REF_DW8(phy), val); + } + + val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); + val |= COMMON_RESET_DIS; + I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); + + if (phy_info->rcomp_phy == -1) + bxt_phy_wait_grc_done(dev_priv, phy); + +} + +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy) +{ + uint32_t val; + + val = I915_READ(BXT_PHY_CTL_FAMILY(phy)); + val &= ~COMMON_RESET_DIS; + I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val); + + val = I915_READ(BXT_P_CR_GT_DISP_PWRON); + val &= ~GT_DISPLAY_POWER_ON(phy); + I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val); +} + +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + enum dpio_phy rcomp_phy = phy_info->rcomp_phy; + bool was_enabled; + + lockdep_assert_held(&dev_priv->power_domains.lock); + + if (rcomp_phy != -1) { + was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy); + + /* + * We need to copy the GRC calibration value from rcomp_phy, + * so make sure it's powered up. + */ + if (!was_enabled) + _bxt_ddi_phy_init(dev_priv, rcomp_phy); + } + + _bxt_ddi_phy_init(dev_priv, phy); + + if (rcomp_phy != -1 && !was_enabled) + bxt_ddi_phy_uninit(dev_priv, phy_info->rcomp_phy); +} + +static bool __printf(6, 7) +__phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy, + i915_reg_t reg, u32 mask, u32 expected, + const char *reg_fmt, ...) +{ + struct va_format vaf; + va_list args; + u32 val; + + val = I915_READ(reg); + if ((val & mask) == expected) + return true; + + va_start(args, reg_fmt); + vaf.fmt = reg_fmt; + vaf.va = &args; + + DRM_DEBUG_DRIVER("DDI PHY %d reg %pV [%08x] state mismatch: " + "current %08x, expected %08x (mask %08x)\n", + phy, &vaf, reg.reg, val, (val & ~mask) | expected, + mask); + + va_end(args); + + return false; +} + +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy) +{ + const struct bxt_ddi_phy_info *phy_info = &bxt_ddi_phy_info[phy]; + uint32_t mask; + bool ok; + +#define _CHK(reg, mask, exp, fmt, ...) \ + __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt, \ + ## __VA_ARGS__) + + if (!bxt_ddi_phy_is_enabled(dev_priv, phy)) + return false; + + ok = true; + + /* PLL Rcomp code offset */ + ok &= _CHK(BXT_PORT_CL1CM_DW9(phy), + IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT, + "BXT_PORT_CL1CM_DW9(%d)", phy); + ok &= _CHK(BXT_PORT_CL1CM_DW10(phy), + IREF1RC_OFFSET_MASK, 0xe4 << IREF1RC_OFFSET_SHIFT, + "BXT_PORT_CL1CM_DW10(%d)", phy); + + /* Power gating */ + mask = OCL1_POWER_DOWN_EN | DW28_OLDO_DYN_PWR_DOWN_EN | SUS_CLK_CONFIG; + ok &= _CHK(BXT_PORT_CL1CM_DW28(phy), mask, mask, + "BXT_PORT_CL1CM_DW28(%d)", phy); + + if (phy_info->dual_channel) + ok &= _CHK(BXT_PORT_CL2CM_DW6(phy), + DW6_OLDO_DYN_PWR_DOWN_EN, DW6_OLDO_DYN_PWR_DOWN_EN, + "BXT_PORT_CL2CM_DW6(%d)", phy); + + if (phy_info->rcomp_phy != -1) { + u32 grc_code = dev_priv->bxt_phy_grc; + + grc_code = grc_code << GRC_CODE_FAST_SHIFT | + grc_code << GRC_CODE_SLOW_SHIFT | + grc_code; + mask = GRC_CODE_FAST_MASK | GRC_CODE_SLOW_MASK | + GRC_CODE_NOM_MASK; + ok &= _CHK(BXT_PORT_REF_DW6(phy), mask, grc_code, + "BXT_PORT_REF_DW6(%d)", phy); + + mask = GRC_DIS | GRC_RDY_OVRD; + ok &= _CHK(BXT_PORT_REF_DW8(phy), mask, mask, + "BXT_PORT_REF_DW8(%d)", phy); + } + + return ok; +#undef _CHK +} + +uint8_t +bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, + uint8_t lane_count) +{ + switch (lane_count) { + case 1: + return 0; + case 2: + return BIT(2) | BIT(0); + case 4: + return BIT(3) | BIT(2) | BIT(0); + default: + MISSING_CASE(lane_count); + + return 0; + } +} + +void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, + uint8_t lane_lat_optim_mask) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); + enum port port = dport->port; + enum dpio_phy phy; + enum dpio_channel ch; + int lane; + + bxt_port_to_phy_channel(port, &phy, &ch); + + for (lane = 0; lane < 4; lane++) { + u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); + + /* + * Note that on CHV this flag is called UPAR, but has + * the same function. + */ + val &= ~LATENCY_OPTIM; + if (lane_lat_optim_mask & BIT(lane)) + val |= LATENCY_OPTIM; + + I915_WRITE(BXT_PORT_TX_DW14_LN(phy, ch, lane), val); + } +} + +uint8_t +bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) +{ + struct intel_digital_port *dport = enc_to_dig_port(&encoder->base); + struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); + enum port port = dport->port; + enum dpio_phy phy; + enum dpio_channel ch; + int lane; + uint8_t mask; + + bxt_port_to_phy_channel(port, &phy, &ch); + + mask = 0; + for (lane = 0; lane < 4; lane++) { + u32 val = I915_READ(BXT_PORT_TX_DW14_LN(phy, ch, lane)); + + if (val & LATENCY_OPTIM) + mask |= BIT(lane); + } + + return mask; +} + + void chv_set_phy_signal_level(struct intel_encoder *encoder, u32 deemph_reg_value, u32 margin_reg_value, bool uniq_trans_scale) diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c index 605d0b509f24..21853a17b6d9 100644 --- a/drivers/gpu/drm/i915/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c @@ -1371,6 +1371,10 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, { uint32_t temp; enum port port = (enum port)pll->id; /* 1:1 port->PLL mapping */ + enum dpio_phy phy; + enum dpio_channel ch; + + bxt_port_to_phy_channel(port, &phy, &ch); /* Non-SSC reference */ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); @@ -1378,72 +1382,72 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, I915_WRITE(BXT_PORT_PLL_ENABLE(port), temp); /* Disable 10 bit clock */ - temp = I915_READ(BXT_PORT_PLL_EBB_4(port)); + temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); temp &= ~PORT_PLL_10BIT_CLK_ENABLE; - I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); /* Write P1 & P2 */ - temp = I915_READ(BXT_PORT_PLL_EBB_0(port)); + temp = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch)); temp &= ~(PORT_PLL_P1_MASK | PORT_PLL_P2_MASK); temp |= pll->config.hw_state.ebb0; - I915_WRITE(BXT_PORT_PLL_EBB_0(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_0(phy, ch), temp); /* Write M2 integer */ - temp = I915_READ(BXT_PORT_PLL(port, 0)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 0)); temp &= ~PORT_PLL_M2_MASK; temp |= pll->config.hw_state.pll0; - I915_WRITE(BXT_PORT_PLL(port, 0), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 0), temp); /* Write N */ - temp = I915_READ(BXT_PORT_PLL(port, 1)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 1)); temp &= ~PORT_PLL_N_MASK; temp |= pll->config.hw_state.pll1; - I915_WRITE(BXT_PORT_PLL(port, 1), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 1), temp); /* Write M2 fraction */ - temp = I915_READ(BXT_PORT_PLL(port, 2)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 2)); temp &= ~PORT_PLL_M2_FRAC_MASK; temp |= pll->config.hw_state.pll2; - I915_WRITE(BXT_PORT_PLL(port, 2), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 2), temp); /* Write M2 fraction enable */ - temp = I915_READ(BXT_PORT_PLL(port, 3)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 3)); temp &= ~PORT_PLL_M2_FRAC_ENABLE; temp |= pll->config.hw_state.pll3; - I915_WRITE(BXT_PORT_PLL(port, 3), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 3), temp); /* Write coeff */ - temp = I915_READ(BXT_PORT_PLL(port, 6)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 6)); temp &= ~PORT_PLL_PROP_COEFF_MASK; temp &= ~PORT_PLL_INT_COEFF_MASK; temp &= ~PORT_PLL_GAIN_CTL_MASK; temp |= pll->config.hw_state.pll6; - I915_WRITE(BXT_PORT_PLL(port, 6), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 6), temp); /* Write calibration val */ - temp = I915_READ(BXT_PORT_PLL(port, 8)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 8)); temp &= ~PORT_PLL_TARGET_CNT_MASK; temp |= pll->config.hw_state.pll8; - I915_WRITE(BXT_PORT_PLL(port, 8), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 8), temp); - temp = I915_READ(BXT_PORT_PLL(port, 9)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 9)); temp &= ~PORT_PLL_LOCK_THRESHOLD_MASK; temp |= pll->config.hw_state.pll9; - I915_WRITE(BXT_PORT_PLL(port, 9), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 9), temp); - temp = I915_READ(BXT_PORT_PLL(port, 10)); + temp = I915_READ(BXT_PORT_PLL(phy, ch, 10)); temp &= ~PORT_PLL_DCO_AMP_OVR_EN_H; temp &= ~PORT_PLL_DCO_AMP_MASK; temp |= pll->config.hw_state.pll10; - I915_WRITE(BXT_PORT_PLL(port, 10), temp); + I915_WRITE(BXT_PORT_PLL(phy, ch, 10), temp); /* Recalibrate with new settings */ - temp = I915_READ(BXT_PORT_PLL_EBB_4(port)); + temp = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); temp |= PORT_PLL_RECALIBRATE; - I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); temp &= ~PORT_PLL_10BIT_CLK_ENABLE; temp |= pll->config.hw_state.ebb4; - I915_WRITE(BXT_PORT_PLL_EBB_4(port), temp); + I915_WRITE(BXT_PORT_PLL_EBB_4(phy, ch), temp); /* Enable PLL */ temp = I915_READ(BXT_PORT_PLL_ENABLE(port)); @@ -1459,11 +1463,11 @@ static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, * While we write to the group register to program all lanes at once we * can read only lane registers and we pick lanes 0/1 for that. */ - temp = I915_READ(BXT_PORT_PCS_DW12_LN01(port)); + temp = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch)); temp &= ~LANE_STAGGER_MASK; temp &= ~LANESTAGGER_STRAP_OVRD; temp |= pll->config.hw_state.pcsdw12; - I915_WRITE(BXT_PORT_PCS_DW12_GRP(port), temp); + I915_WRITE(BXT_PORT_PCS_DW12_GRP(phy, ch), temp); } static void bxt_ddi_pll_disable(struct drm_i915_private *dev_priv, @@ -1485,6 +1489,10 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, enum port port = (enum port)pll->id; /* 1:1 port->PLL mapping */ uint32_t val; bool ret; + enum dpio_phy phy; + enum dpio_channel ch; + + bxt_port_to_phy_channel(port, &phy, &ch); if (!intel_display_power_get_if_enabled(dev_priv, POWER_DOMAIN_PLLS)) return false; @@ -1495,36 +1503,36 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, if (!(val & PORT_PLL_ENABLE)) goto out; - hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(port)); + hw_state->ebb0 = I915_READ(BXT_PORT_PLL_EBB_0(phy, ch)); hw_state->ebb0 &= PORT_PLL_P1_MASK | PORT_PLL_P2_MASK; - hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(port)); + hw_state->ebb4 = I915_READ(BXT_PORT_PLL_EBB_4(phy, ch)); hw_state->ebb4 &= PORT_PLL_10BIT_CLK_ENABLE; - hw_state->pll0 = I915_READ(BXT_PORT_PLL(port, 0)); + hw_state->pll0 = I915_READ(BXT_PORT_PLL(phy, ch, 0)); hw_state->pll0 &= PORT_PLL_M2_MASK; - hw_state->pll1 = I915_READ(BXT_PORT_PLL(port, 1)); + hw_state->pll1 = I915_READ(BXT_PORT_PLL(phy, ch, 1)); hw_state->pll1 &= PORT_PLL_N_MASK; - hw_state->pll2 = I915_READ(BXT_PORT_PLL(port, 2)); + hw_state->pll2 = I915_READ(BXT_PORT_PLL(phy, ch, 2)); hw_state->pll2 &= PORT_PLL_M2_FRAC_MASK; - hw_state->pll3 = I915_READ(BXT_PORT_PLL(port, 3)); + hw_state->pll3 = I915_READ(BXT_PORT_PLL(phy, ch, 3)); hw_state->pll3 &= PORT_PLL_M2_FRAC_ENABLE; - hw_state->pll6 = I915_READ(BXT_PORT_PLL(port, 6)); + hw_state->pll6 = I915_READ(BXT_PORT_PLL(phy, ch, 6)); hw_state->pll6 &= PORT_PLL_PROP_COEFF_MASK | PORT_PLL_INT_COEFF_MASK | PORT_PLL_GAIN_CTL_MASK; - hw_state->pll8 = I915_READ(BXT_PORT_PLL(port, 8)); + hw_state->pll8 = I915_READ(BXT_PORT_PLL(phy, ch, 8)); hw_state->pll8 &= PORT_PLL_TARGET_CNT_MASK; - hw_state->pll9 = I915_READ(BXT_PORT_PLL(port, 9)); + hw_state->pll9 = I915_READ(BXT_PORT_PLL(phy, ch, 9)); hw_state->pll9 &= PORT_PLL_LOCK_THRESHOLD_MASK; - hw_state->pll10 = I915_READ(BXT_PORT_PLL(port, 10)); + hw_state->pll10 = I915_READ(BXT_PORT_PLL(phy, ch, 10)); hw_state->pll10 &= PORT_PLL_DCO_AMP_OVR_EN_H | PORT_PLL_DCO_AMP_MASK; @@ -1533,11 +1541,11 @@ static bool bxt_ddi_pll_get_hw_state(struct drm_i915_private *dev_priv, * can read only lane registers. We configure all lanes the same way, so * here just read out lanes 0/1 and output a note if lanes 2/3 differ. */ - hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(port)); - if (I915_READ(BXT_PORT_PCS_DW12_LN23(port)) != hw_state->pcsdw12) + hw_state->pcsdw12 = I915_READ(BXT_PORT_PCS_DW12_LN01(phy, ch)); + if (I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch)) != hw_state->pcsdw12) DRM_DEBUG_DRIVER("lane stagger config different for lane 01 (%08x) and 23 (%08x)\n", hw_state->pcsdw12, - I915_READ(BXT_PORT_PCS_DW12_LN23(port))); + I915_READ(BXT_PORT_PCS_DW12_LN23(phy, ch))); hw_state->pcsdw12 &= LANE_STAGGER_MASK | LANESTAGGER_STRAP_OVRD; ret = true; diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index 4e90b0760f3f..398195bf6dd1 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -365,6 +365,8 @@ struct intel_atomic_state { /* Gen9+ only */ struct skl_wm_values wm_results; + + struct i915_sw_fence commit_ready; }; struct intel_plane_state { @@ -401,9 +403,6 @@ struct intel_plane_state { int scaler_id; struct drm_intel_sprite_colorkey ckey; - - /* async flip related structures */ - struct drm_i915_gem_request *wait_req; }; struct intel_initial_plane_config { @@ -501,14 +500,6 @@ struct intel_crtc_wm_state { /* gen9+ only needs 1-step wm programming */ struct skl_pipe_wm optimal; struct skl_ddb_entry ddb; - - /* cached plane data rate */ - unsigned plane_data_rate[I915_MAX_PLANES]; - unsigned plane_y_data_rate[I915_MAX_PLANES]; - - /* minimum block allocation */ - uint16_t minimum_blocks[I915_MAX_PLANES]; - uint16_t minimum_y_blocks[I915_MAX_PLANES]; } skl; }; @@ -731,7 +722,6 @@ struct intel_crtc { /* watermarks currently being used */ union { struct intel_pipe_wm ilk; - struct skl_pipe_wm skl; } active; /* allow CxSR on this pipe */ @@ -883,6 +873,14 @@ enum link_m_n_set { M2_N2 }; +struct intel_dp_desc { + u8 oui[3]; + u8 device_id[6]; + u8 hw_rev; + u8 sw_major_rev; + u8 sw_minor_rev; +} __packed; + struct intel_dp { i915_reg_t output_reg; i915_reg_t aux_ch_ctl_reg; @@ -905,6 +903,8 @@ struct intel_dp { /* sink rates as reported by DP_SUPPORTED_LINK_RATES */ uint8_t num_sink_rates; int sink_rates[DP_MAX_SUPPORTED_RATES]; + /* sink or branch descriptor */ + struct intel_dp_desc desc; struct drm_dp_aux aux; uint8_t train_set[4]; int panel_power_up_delay; @@ -964,7 +964,7 @@ struct intel_dp { struct intel_lspcon { bool active; enum drm_lspcon_mode mode; - struct drm_dp_aux *aux; + bool desc_valid; }; struct intel_digital_port { @@ -1028,17 +1028,15 @@ vlv_pipe_to_channel(enum pipe pipe) } } -static inline struct drm_crtc * -intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) +static inline struct intel_crtc * +intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->pipe_to_crtc_mapping[pipe]; } -static inline struct drm_crtc * -intel_get_crtc_for_plane(struct drm_device *dev, int plane) +static inline struct intel_crtc * +intel_get_crtc_for_plane(struct drm_i915_private *dev_priv, enum plane plane) { - struct drm_i915_private *dev_priv = to_i915(dev); return dev_priv->plane_to_crtc_mapping[plane]; } @@ -1098,15 +1096,6 @@ hdmi_to_dig_port(struct intel_hdmi *intel_hdmi) return container_of(intel_hdmi, struct intel_digital_port, hdmi); } -/* - * Returns the number of planes for this pipe, ie the number of sprites + 1 - * (primary plane). This doesn't count the cursor plane then. - */ -static inline unsigned int intel_num_planes(struct intel_crtc *crtc) -{ - return INTEL_INFO(crtc->base.dev)->num_sprites[crtc->pipe] + 1; -} - /* intel_fifo_underrun.c */ bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable); @@ -1123,6 +1112,9 @@ void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv); /* i915_irq.c */ void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask); +void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 mask); +void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); +void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask); void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask); void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv); @@ -1145,6 +1137,9 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, unsigned int pipe_mask); void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, unsigned int pipe_mask); +void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv); +void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv); +void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv); /* intel_crt.c */ void intel_crt_init(struct drm_device *dev); @@ -1247,18 +1242,17 @@ intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state) (1 << INTEL_OUTPUT_EDP)); } static inline void -intel_wait_for_vblank(struct drm_device *dev, int pipe) +intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) { - drm_wait_one_vblank(dev, pipe); + drm_wait_one_vblank(&dev_priv->drm, pipe); } static inline void -intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe) +intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, int pipe) { - const struct intel_crtc *crtc = - to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe)); + const struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->active) - intel_wait_for_vblank(dev, pipe); + intel_wait_for_vblank(dev_priv, pipe); } u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc); @@ -1305,9 +1299,9 @@ unsigned int intel_tile_height(const struct drm_i915_private *dev_priv, void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv, enum pipe pipe); -int vlv_force_pll_on(struct drm_device *dev, enum pipe pipe, +int vlv_force_pll_on(struct drm_i915_private *dev_priv, enum pipe pipe, const struct dpll *dpll); -void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe); +void vlv_force_pll_off(struct drm_i915_private *dev_priv, enum pipe pipe); int lpt_get_iclkip(struct drm_i915_private *dev_priv); /* modesetting asserts */ @@ -1335,12 +1329,6 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv); void hsw_disable_pc8(struct drm_i915_private *dev_priv); void bxt_init_cdclk(struct drm_i915_private *dev_priv); void bxt_uninit_cdclk(struct drm_i915_private *dev_priv); -void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); -void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); -bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, - enum dpio_phy phy); -bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, - enum dpio_phy phy); void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv); void bxt_enable_dc9(struct drm_i915_private *dev_priv); void bxt_disable_dc9(struct drm_i915_private *dev_priv); @@ -1358,7 +1346,7 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock, struct dpll *best_clock); int chv_calc_dpll_params(int refclk, struct dpll *pll_clock); -bool intel_crtc_active(struct drm_crtc *crtc); +bool intel_crtc_active(struct intel_crtc *crtc); void hsw_enable_ips(struct intel_crtc *crtc); void hsw_disable_ips(struct intel_crtc *crtc); enum intel_display_power_domain @@ -1451,6 +1439,11 @@ static inline unsigned int intel_dp_unused_lane_mask(int lane_count) return ~((1 << lane_count) - 1) & 0xf; } +bool intel_dp_read_dpcd(struct intel_dp *intel_dp); +bool __intel_dp_read_desc(struct intel_dp *intel_dp, + struct intel_dp_desc *desc); +bool intel_dp_read_desc(struct intel_dp *intel_dp); + /* intel_dp_aux_backlight.c */ int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector); @@ -1655,23 +1648,6 @@ assert_rpm_wakelock_held(struct drm_i915_private *dev_priv) DRM_DEBUG_DRIVER("RPM wakelock ref not held during HW access"); } -static inline int -assert_rpm_atomic_begin(struct drm_i915_private *dev_priv) -{ - int seq = atomic_read(&dev_priv->pm.atomic_seq); - - assert_rpm_wakelock_held(dev_priv); - - return seq; -} - -static inline void -assert_rpm_atomic_end(struct drm_i915_private *dev_priv, int begin_seq) -{ - WARN_ONCE(atomic_read(&dev_priv->pm.atomic_seq) != begin_seq, - "HW access outside of RPM atomic section\n"); -} - /** * disable_rpm_wakeref_asserts - disable the RPM assert checks * @dev_priv: i915 device instance @@ -1727,11 +1703,11 @@ bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, /* intel_pm.c */ -void intel_init_clock_gating(struct drm_device *dev); -void intel_suspend_hw(struct drm_device *dev); +void intel_init_clock_gating(struct drm_i915_private *dev_priv); +void intel_suspend_hw(struct drm_i915_private *dev_priv); int ilk_wm_max_level(const struct drm_i915_private *dev_priv); -void intel_update_watermarks(struct drm_crtc *crtc); -void intel_init_pm(struct drm_device *dev); +void intel_update_watermarks(struct intel_crtc *crtc); +void intel_init_pm(struct drm_i915_private *dev_priv); void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv); void intel_pm_setup(struct drm_device *dev); void intel_gpu_ips_init(struct drm_i915_private *dev_priv); @@ -1790,7 +1766,8 @@ bool intel_sdvo_init(struct drm_device *dev, /* intel_sprite.c */ int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode, int usecs); -int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane); +struct intel_plane *intel_sprite_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, int plane); int intel_sprite_set_colorkey(struct drm_device *dev, void *data, struct drm_file *file_priv); void intel_pipe_update_start(struct intel_crtc *crtc); diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c index cd574900cd8d..708645443046 100644 --- a/drivers/gpu/drm/i915/intel_dvo.c +++ b/drivers/gpu/drm/i915/intel_dvo.c @@ -393,12 +393,12 @@ intel_dvo_get_current_mode(struct drm_connector *connector) * its timings to get how the BIOS set up the panel. */ if (dvo_val & DVO_ENABLE) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; int pipe = (dvo_val & DVO_PIPE_B_SELECT) ? 1 : 0; - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc) { - mode = intel_crtc_mode_get(dev, crtc); + mode = intel_crtc_mode_get(dev, &crtc->base); if (mode) { mode->type |= DRM_MODE_TYPE_PREFERRED; if (dvo_val & DVO_HSYNC_ACTIVE_HIGH) diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c index 8cceb345aa0f..841f8d1e1410 100644 --- a/drivers/gpu/drm/i915/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/intel_engine_cs.c @@ -174,7 +174,7 @@ cleanup: return ret; } -void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) +void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno) { struct drm_i915_private *dev_priv = engine->i915; @@ -204,13 +204,13 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) I915_NUM_ENGINES * gen8_semaphore_seqno_size); kunmap(page); } - memset(engine->semaphore.sync_seqno, 0, - sizeof(engine->semaphore.sync_seqno)); intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno); if (engine->irq_seqno_barrier) engine->irq_seqno_barrier(engine); - engine->last_submitted_seqno = seqno; + + GEM_BUG_ON(i915_gem_active_isset(&engine->timeline->last_request)); + engine->timeline->last_submitted_seqno = seqno; engine->hangcheck.seqno = seqno; @@ -220,15 +220,9 @@ void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno) intel_engine_wakeup(engine); } -void intel_engine_init_hangcheck(struct intel_engine_cs *engine) -{ - memset(&engine->hangcheck, 0, sizeof(engine->hangcheck)); -} - -static void intel_engine_init_requests(struct intel_engine_cs *engine) +static void intel_engine_init_timeline(struct intel_engine_cs *engine) { - init_request_active(&engine->last_request, NULL); - INIT_LIST_HEAD(&engine->request_list); + engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id]; } /** @@ -245,9 +239,7 @@ void intel_engine_setup_common(struct intel_engine_cs *engine) INIT_LIST_HEAD(&engine->execlist_queue); spin_lock_init(&engine->execlist_lock); - engine->fence_context = dma_fence_context_alloc(1); - - intel_engine_init_requests(engine); + intel_engine_init_timeline(engine); intel_engine_init_hangcheck(engine); i915_gem_batch_pool_init(engine, &engine->batch_pool); @@ -264,7 +256,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) obj = i915_gem_object_create_stolen(&engine->i915->drm, size); if (!obj) - obj = i915_gem_object_create(&engine->i915->drm, size); + obj = i915_gem_object_create_internal(engine->i915, size); if (IS_ERR(obj)) { DRM_ERROR("Failed to allocate scratch page\n"); return PTR_ERR(obj); @@ -314,6 +306,10 @@ int intel_engine_init_common(struct intel_engine_cs *engine) if (ret) return ret; + ret = i915_gem_render_state_init(engine); + if (ret) + return ret; + return 0; } @@ -328,6 +324,7 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) { intel_engine_cleanup_scratch(engine); + i915_gem_render_state_fini(engine); intel_engine_fini_breadcrumbs(engine); intel_engine_cleanup_cmd_parser(engine); i915_gem_batch_pool_fini(&engine->batch_pool); diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c index 49048d9d7895..e230d480c5e6 100644 --- a/drivers/gpu/drm/i915/intel_fbc.c +++ b/drivers/gpu/drm/i915/intel_fbc.c @@ -104,8 +104,10 @@ static int intel_fbc_calculate_cfb_size(struct drm_i915_private *dev_priv, int lines; intel_fbc_get_plane_source_size(cache, NULL, &lines); - if (INTEL_INFO(dev_priv)->gen >= 7) + if (INTEL_GEN(dev_priv) == 7) lines = min(lines, 2048); + else if (INTEL_GEN(dev_priv) >= 8) + lines = min(lines, 2560); /* Hardware needs the full buffer stride, not just the active area. */ return lines * cache->fb.stride; @@ -1304,7 +1306,7 @@ void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv) return; for_each_intel_crtc(&dev_priv->drm, crtc) - if (intel_crtc_active(&crtc->base) && + if (intel_crtc_active(crtc) && to_intel_plane_state(crtc->base.primary->state)->base.visible) dev_priv->fbc.visible_pipes_mask |= (1 << crtc->pipe); } diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c index 3018f4f589c8..e660d8b4bbc3 100644 --- a/drivers/gpu/drm/i915/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c @@ -57,7 +57,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->cpu_fifo_underrun_disabled) return false; @@ -75,7 +75,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev) assert_spin_locked(&dev_priv->irq_lock); for_each_pipe(dev_priv, pipe) { - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc->pch_fifo_underrun_disabled) return false; @@ -245,14 +245,13 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev, enum pipe pipe, bool enable) { struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); bool old; assert_spin_locked(&dev_priv->irq_lock); - old = !intel_crtc->cpu_fifo_underrun_disabled; - intel_crtc->cpu_fifo_underrun_disabled = !enable; + old = !crtc->cpu_fifo_underrun_disabled; + crtc->cpu_fifo_underrun_disabled = !enable; if (HAS_GMCH_DISPLAY(dev_priv)) i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old); @@ -314,8 +313,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum transcoder pch_transcoder, bool enable) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder]; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = + intel_get_crtc_for_pipe(dev_priv, (enum pipe) pch_transcoder); unsigned long flags; bool old; @@ -330,8 +329,8 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, spin_lock_irqsave(&dev_priv->irq_lock, flags); - old = !intel_crtc->pch_fifo_underrun_disabled; - intel_crtc->pch_fifo_underrun_disabled = !enable; + old = !crtc->pch_fifo_underrun_disabled; + crtc->pch_fifo_underrun_disabled = !enable; if (HAS_PCH_IBX(dev_priv)) ibx_set_fifo_underrun_reporting(&dev_priv->drm, @@ -358,7 +357,7 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv, void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, enum pipe pipe) { - struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); /* We may be called too early in init, thanks BIOS! */ if (crtc == NULL) @@ -366,7 +365,7 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, /* GMCH can't disable fifo underruns, filter them. */ if (HAS_GMCH_DISPLAY(dev_priv) && - to_intel_crtc(crtc)->cpu_fifo_underrun_disabled) + crtc->cpu_fifo_underrun_disabled) return; if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false)) diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h index 5cdf7aa75be5..0053258e03d3 100644 --- a/drivers/gpu/drm/i915/intel_guc.h +++ b/drivers/gpu/drm/i915/intel_guc.h @@ -64,7 +64,7 @@ struct drm_i915_gem_request; */ struct i915_guc_client { struct i915_vma *vma; - void *client_base; /* first page (only) of above */ + void *vaddr; struct i915_gem_context *owner; struct intel_guc *guc; @@ -123,10 +123,28 @@ struct intel_guc_fw { uint32_t ucode_offset; }; +struct intel_guc_log { + uint32_t flags; + struct i915_vma *vma; + void *buf_addr; + struct workqueue_struct *flush_wq; + struct work_struct flush_work; + struct rchan *relay_chan; + + /* logging related stats */ + u32 capture_miss_count; + u32 flush_interrupt_count; + u32 prev_overflow_count[GUC_MAX_LOG_BUFFER]; + u32 total_overflow_count[GUC_MAX_LOG_BUFFER]; + u32 flush_count[GUC_MAX_LOG_BUFFER]; +}; + struct intel_guc { struct intel_guc_fw guc_fw; - uint32_t log_flags; - struct i915_vma *log_vma; + struct intel_guc_log log; + + /* GuC2Host interrupt related state */ + bool interrupts_enabled; struct i915_vma *ads_vma; struct i915_vma *ctx_pool_vma; @@ -146,6 +164,9 @@ struct intel_guc { uint64_t submissions[I915_NUM_ENGINES]; uint32_t last_seqno[I915_NUM_ENGINES]; + + /* To serialize the Host2GuC actions */ + struct mutex action_lock; }; /* intel_guc_loader.c */ @@ -163,5 +184,10 @@ int i915_guc_wq_reserve(struct drm_i915_gem_request *rq); void i915_guc_wq_unreserve(struct drm_i915_gem_request *request); void i915_guc_submission_disable(struct drm_i915_private *dev_priv); void i915_guc_submission_fini(struct drm_i915_private *dev_priv); +void i915_guc_capture_logs(struct drm_i915_private *dev_priv); +void i915_guc_flush_logs(struct drm_i915_private *dev_priv); +void i915_guc_register(struct drm_i915_private *dev_priv); +void i915_guc_unregister(struct drm_i915_private *dev_priv); +int i915_guc_log_control(struct drm_i915_private *dev_priv, u64 control_val); #endif diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h index e40db2d2ae99..324ea902558b 100644 --- a/drivers/gpu/drm/i915/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/intel_guc_fwif.h @@ -104,9 +104,9 @@ #define GUC_LOG_ALLOC_IN_MEGABYTE (1 << 3) #define GUC_LOG_CRASH_PAGES 1 #define GUC_LOG_CRASH_SHIFT 4 -#define GUC_LOG_DPC_PAGES 3 +#define GUC_LOG_DPC_PAGES 7 #define GUC_LOG_DPC_SHIFT 6 -#define GUC_LOG_ISR_PAGES 3 +#define GUC_LOG_ISR_PAGES 7 #define GUC_LOG_ISR_SHIFT 9 #define GUC_LOG_BUF_ADDR_SHIFT 12 @@ -419,15 +419,87 @@ struct guc_ads { u32 reserved2[4]; } __packed; +/* GuC logging structures */ + +enum guc_log_buffer_type { + GUC_ISR_LOG_BUFFER, + GUC_DPC_LOG_BUFFER, + GUC_CRASH_DUMP_LOG_BUFFER, + GUC_MAX_LOG_BUFFER +}; + +/** + * DOC: GuC Log buffer Layout + * + * Page0 +-------------------------------+ + * | ISR state header (32 bytes) | + * | DPC state header | + * | Crash dump state header | + * Page1 +-------------------------------+ + * | ISR logs | + * Page9 +-------------------------------+ + * | DPC logs | + * Page17 +-------------------------------+ + * | Crash Dump logs | + * +-------------------------------+ + * + * Below state structure is used for coordination of retrieval of GuC firmware + * logs. Separate state is maintained for each log buffer type. + * read_ptr points to the location where i915 read last in log buffer and + * is read only for GuC firmware. write_ptr is incremented by GuC with number + * of bytes written for each log entry and is read only for i915. + * When any type of log buffer becomes half full, GuC sends a flush interrupt. + * GuC firmware expects that while it is writing to 2nd half of the buffer, + * first half would get consumed by Host and then get a flush completed + * acknowledgment from Host, so that it does not end up doing any overwrite + * causing loss of logs. So when buffer gets half filled & i915 has requested + * for interrupt, GuC will set flush_to_file field, set the sampled_write_ptr + * to the value of write_ptr and raise the interrupt. + * On receiving the interrupt i915 should read the buffer, clear flush_to_file + * field and also update read_ptr with the value of sample_write_ptr, before + * sending an acknowledgment to GuC. marker & version fields are for internal + * usage of GuC and opaque to i915. buffer_full_cnt field is incremented every + * time GuC detects the log buffer overflow. + */ +struct guc_log_buffer_state { + u32 marker[2]; + u32 read_ptr; + u32 write_ptr; + u32 size; + u32 sampled_write_ptr; + union { + struct { + u32 flush_to_file:1; + u32 buffer_full_cnt:4; + u32 reserved:27; + }; + u32 flags; + }; + u32 version; +} __packed; + +union guc_log_control { + struct { + u32 logging_enabled:1; + u32 reserved1:3; + u32 verbosity:4; + u32 reserved2:24; + }; + u32 value; +} __packed; + /* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */ enum host2guc_action { HOST2GUC_ACTION_DEFAULT = 0x0, HOST2GUC_ACTION_SAMPLE_FORCEWAKE = 0x6, HOST2GUC_ACTION_ALLOCATE_DOORBELL = 0x10, HOST2GUC_ACTION_DEALLOCATE_DOORBELL = 0x20, + HOST2GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE = 0x30, + HOST2GUC_ACTION_FORCE_LOG_BUFFER_FLUSH = 0x302, HOST2GUC_ACTION_ENTER_S_STATE = 0x501, HOST2GUC_ACTION_EXIT_S_STATE = 0x502, HOST2GUC_ACTION_SLPC_REQUEST = 0x3003, + HOST2GUC_ACTION_UK_LOG_ENABLE_LOGGING = 0x0E000, HOST2GUC_ACTION_LIMIT }; @@ -449,4 +521,10 @@ enum guc2host_status { GUC2HOST_STATUS_GENERIC_FAIL = GUC2HOST_STATUS(0x0000F000) }; +/* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ +enum guc2host_message { + GUC2HOST_MSG_CRASH_DUMP_POSTED = (1 << 1), + GUC2HOST_MSG_FLUSH_LOG_BUFFER = (1 << 3) +}; + #endif diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c index 3c8eaae13732..1aa85236b788 100644 --- a/drivers/gpu/drm/i915/intel_guc_loader.c +++ b/drivers/gpu/drm/i915/intel_guc_loader.c @@ -211,11 +211,13 @@ static void guc_params_init(struct drm_i915_private *dev_priv) params[GUC_CTL_FEATURE] |= GUC_CTL_DISABLE_SCHEDULER | GUC_CTL_VCS2_ENABLED; + params[GUC_CTL_LOG_PARAMS] = guc->log.flags; + if (i915.guc_log_level >= 0) { - params[GUC_CTL_LOG_PARAMS] = guc->log_flags; params[GUC_CTL_DEBUG] = i915.guc_log_level << GUC_LOG_VERBOSITY_SHIFT; - } + } else + params[GUC_CTL_DEBUG] = GUC_LOG_DISABLED; if (guc->ads_vma) { u32 ads = i915_ggtt_offset(guc->ads_vma) >> PAGE_SHIFT; @@ -483,6 +485,7 @@ int intel_guc_setup(struct drm_device *dev) } guc_interrupts_release(dev_priv); + gen9_reset_guc_interrupts(dev_priv); guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING; @@ -527,6 +530,9 @@ int intel_guc_setup(struct drm_device *dev) intel_guc_fw_status_repr(guc_fw->guc_fw_load_status)); if (i915.enable_guc_submission) { + if (i915.guc_log_level >= 0) + gen9_enable_guc_interrupts(dev_priv); + err = i915_guc_submission_enable(dev_priv); if (err) goto fail; diff --git a/drivers/gpu/drm/i915/intel_hangcheck.c b/drivers/gpu/drm/i915/intel_hangcheck.c new file mode 100644 index 000000000000..53df5b11bff4 --- /dev/null +++ b/drivers/gpu/drm/i915/intel_hangcheck.c @@ -0,0 +1,450 @@ +/* + * Copyright © 2016 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "i915_drv.h" + +static bool +ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr) +{ + if (INTEL_GEN(engine->i915) >= 8) { + return (ipehr >> 23) == 0x1c; + } else { + ipehr &= ~MI_SEMAPHORE_SYNC_MASK; + return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | + MI_SEMAPHORE_REGISTER); + } +} + +static struct intel_engine_cs * +semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr, + u64 offset) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_engine_cs *signaller; + enum intel_engine_id id; + + if (INTEL_GEN(dev_priv) >= 8) { + for_each_engine(signaller, dev_priv, id) { + if (engine == signaller) + continue; + + if (offset == signaller->semaphore.signal_ggtt[engine->hw_id]) + return signaller; + } + } else { + u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK; + + for_each_engine(signaller, dev_priv, id) { + if(engine == signaller) + continue; + + if (sync_bits == signaller->semaphore.mbox.wait[engine->hw_id]) + return signaller; + } + } + + DRM_DEBUG_DRIVER("No signaller ring found for %s, ipehr 0x%08x, offset 0x%016llx\n", + engine->name, ipehr, offset); + + return ERR_PTR(-ENODEV); +} + +static struct intel_engine_cs * +semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno) +{ + struct drm_i915_private *dev_priv = engine->i915; + void __iomem *vaddr; + u32 cmd, ipehr, head; + u64 offset = 0; + int i, backwards; + + /* + * This function does not support execlist mode - any attempt to + * proceed further into this function will result in a kernel panic + * when dereferencing ring->buffer, which is not set up in execlist + * mode. + * + * The correct way of doing it would be to derive the currently + * executing ring buffer from the current context, which is derived + * from the currently running request. Unfortunately, to get the + * current request we would have to grab the struct_mutex before doing + * anything else, which would be ill-advised since some other thread + * might have grabbed it already and managed to hang itself, causing + * the hang checker to deadlock. + * + * Therefore, this function does not support execlist mode in its + * current form. Just return NULL and move on. + */ + if (engine->buffer == NULL) + return NULL; + + ipehr = I915_READ(RING_IPEHR(engine->mmio_base)); + if (!ipehr_is_semaphore_wait(engine, ipehr)) + return NULL; + + /* + * HEAD is likely pointing to the dword after the actual command, + * so scan backwards until we find the MBOX. But limit it to just 3 + * or 4 dwords depending on the semaphore wait command size. + * Note that we don't care about ACTHD here since that might + * point at at batch, and semaphores are always emitted into the + * ringbuffer itself. + */ + head = I915_READ_HEAD(engine) & HEAD_ADDR; + backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4; + vaddr = (void __iomem *)engine->buffer->vaddr; + + for (i = backwards; i; --i) { + /* + * Be paranoid and presume the hw has gone off into the wild - + * our ring is smaller than what the hardware (and hence + * HEAD_ADDR) allows. Also handles wrap-around. + */ + head &= engine->buffer->size - 1; + + /* This here seems to blow up */ + cmd = ioread32(vaddr + head); + if (cmd == ipehr) + break; + + head -= 4; + } + + if (!i) + return NULL; + + *seqno = ioread32(vaddr + head + 4) + 1; + if (INTEL_GEN(dev_priv) >= 8) { + offset = ioread32(vaddr + head + 12); + offset <<= 32; + offset |= ioread32(vaddr + head + 8); + } + return semaphore_wait_to_signaller_ring(engine, ipehr, offset); +} + +static int semaphore_passed(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_engine_cs *signaller; + u32 seqno; + + engine->hangcheck.deadlock++; + + signaller = semaphore_waits_for(engine, &seqno); + if (signaller == NULL) + return -1; + + if (IS_ERR(signaller)) + return 0; + + /* Prevent pathological recursion due to driver bugs */ + if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES) + return -1; + + if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno)) + return 1; + + /* cursory check for an unkickable deadlock */ + if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE && + semaphore_passed(signaller) < 0) + return -1; + + return 0; +} + +static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, dev_priv, id) + engine->hangcheck.deadlock = 0; +} + +static bool instdone_unchanged(u32 current_instdone, u32 *old_instdone) +{ + u32 tmp = current_instdone | *old_instdone; + bool unchanged; + + unchanged = tmp == *old_instdone; + *old_instdone |= tmp; + + return unchanged; +} + +static bool subunits_stuck(struct intel_engine_cs *engine) +{ + struct drm_i915_private *dev_priv = engine->i915; + struct intel_instdone instdone; + struct intel_instdone *accu_instdone = &engine->hangcheck.instdone; + bool stuck; + int slice; + int subslice; + + if (engine->id != RCS) + return true; + + intel_engine_get_instdone(engine, &instdone); + + /* There might be unstable subunit states even when + * actual head is not moving. Filter out the unstable ones by + * accumulating the undone -> done transitions and only + * consider those as progress. + */ + stuck = instdone_unchanged(instdone.instdone, + &accu_instdone->instdone); + stuck &= instdone_unchanged(instdone.slice_common, + &accu_instdone->slice_common); + + for_each_instdone_slice_subslice(dev_priv, slice, subslice) { + stuck &= instdone_unchanged(instdone.sampler[slice][subslice], + &accu_instdone->sampler[slice][subslice]); + stuck &= instdone_unchanged(instdone.row[slice][subslice], + &accu_instdone->row[slice][subslice]); + } + + return stuck; +} + +static enum intel_engine_hangcheck_action +head_stuck(struct intel_engine_cs *engine, u64 acthd) +{ + if (acthd != engine->hangcheck.acthd) { + + /* Clear subunit states on head movement */ + memset(&engine->hangcheck.instdone, 0, + sizeof(engine->hangcheck.instdone)); + + return HANGCHECK_ACTIVE; + } + + if (!subunits_stuck(engine)) + return HANGCHECK_ACTIVE; + + return HANGCHECK_HUNG; +} + +static enum intel_engine_hangcheck_action +engine_stuck(struct intel_engine_cs *engine, u64 acthd) +{ + struct drm_i915_private *dev_priv = engine->i915; + enum intel_engine_hangcheck_action ha; + u32 tmp; + + ha = head_stuck(engine, acthd); + if (ha != HANGCHECK_HUNG) + return ha; + + if (IS_GEN2(dev_priv)) + return HANGCHECK_HUNG; + + /* Is the chip hanging on a WAIT_FOR_EVENT? + * If so we can simply poke the RB_WAIT bit + * and break the hang. This should work on + * all but the second generation chipsets. + */ + tmp = I915_READ_CTL(engine); + if (tmp & RING_WAIT) { + i915_handle_error(dev_priv, 0, + "Kicking stuck wait on %s", + engine->name); + I915_WRITE_CTL(engine, tmp); + return HANGCHECK_KICK; + } + + if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) { + switch (semaphore_passed(engine)) { + default: + return HANGCHECK_HUNG; + case 1: + i915_handle_error(dev_priv, 0, + "Kicking stuck semaphore on %s", + engine->name); + I915_WRITE_CTL(engine, tmp); + return HANGCHECK_KICK; + case 0: + return HANGCHECK_WAIT; + } + } + + return HANGCHECK_HUNG; +} + +/* + * This is called when the chip hasn't reported back with completed + * batchbuffers in a long time. We keep track per ring seqno progress and + * if there are no progress, hangcheck score for that ring is increased. + * Further, acthd is inspected to see if the ring is stuck. On stuck case + * we kick the ring. If we see no progress on three subsequent calls + * we assume chip is wedged and try to fix it by resetting the chip. + */ +static void i915_hangcheck_elapsed(struct work_struct *work) +{ + struct drm_i915_private *dev_priv = + container_of(work, typeof(*dev_priv), + gpu_error.hangcheck_work.work); + struct intel_engine_cs *engine; + enum intel_engine_id id; + unsigned int hung = 0, stuck = 0; + int busy_count = 0; +#define BUSY 1 +#define KICK 5 +#define HUNG 20 +#define ACTIVE_DECAY 15 + + if (!i915.enable_hangcheck) + return; + + if (!READ_ONCE(dev_priv->gt.awake)) + return; + + /* As enabling the GPU requires fairly extensive mmio access, + * periodically arm the mmio checker to see if we are triggering + * any invalid access. + */ + intel_uncore_arm_unclaimed_mmio_detection(dev_priv); + + for_each_engine(engine, dev_priv, id) { + bool busy = intel_engine_has_waiter(engine); + u64 acthd; + u32 seqno; + u32 submit; + + semaphore_clear_deadlocks(dev_priv); + + /* We don't strictly need an irq-barrier here, as we are not + * serving an interrupt request, be paranoid in case the + * barrier has side-effects (such as preventing a broken + * cacheline snoop) and so be sure that we can see the seqno + * advance. If the seqno should stick, due to a stale + * cacheline, we would erroneously declare the GPU hung. + */ + if (engine->irq_seqno_barrier) + engine->irq_seqno_barrier(engine); + + acthd = intel_engine_get_active_head(engine); + seqno = intel_engine_get_seqno(engine); + submit = intel_engine_last_submit(engine); + + if (engine->hangcheck.seqno == seqno) { + if (i915_seqno_passed(seqno, submit)) { + engine->hangcheck.action = HANGCHECK_IDLE; + } else { + /* We always increment the hangcheck score + * if the engine is busy and still processing + * the same request, so that no single request + * can run indefinitely (such as a chain of + * batches). The only time we do not increment + * the hangcheck score on this ring, if this + * engine is in a legitimate wait for another + * engine. In that case the waiting engine is a + * victim and we want to be sure we catch the + * right culprit. Then every time we do kick + * the ring, add a small increment to the + * score so that we can catch a batch that is + * being repeatedly kicked and so responsible + * for stalling the machine. + */ + engine->hangcheck.action = + engine_stuck(engine, acthd); + + switch (engine->hangcheck.action) { + case HANGCHECK_IDLE: + case HANGCHECK_WAIT: + break; + case HANGCHECK_ACTIVE: + engine->hangcheck.score += BUSY; + break; + case HANGCHECK_KICK: + engine->hangcheck.score += KICK; + break; + case HANGCHECK_HUNG: + engine->hangcheck.score += HUNG; + break; + } + } + + if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) { + hung |= intel_engine_flag(engine); + if (engine->hangcheck.action != HANGCHECK_HUNG) + stuck |= intel_engine_flag(engine); + } + } else { + engine->hangcheck.action = HANGCHECK_ACTIVE; + + /* Gradually reduce the count so that we catch DoS + * attempts across multiple batches. + */ + if (engine->hangcheck.score > 0) + engine->hangcheck.score -= ACTIVE_DECAY; + if (engine->hangcheck.score < 0) + engine->hangcheck.score = 0; + + /* Clear head and subunit states on seqno movement */ + acthd = 0; + + memset(&engine->hangcheck.instdone, 0, + sizeof(engine->hangcheck.instdone)); + } + + engine->hangcheck.seqno = seqno; + engine->hangcheck.acthd = acthd; + busy_count += busy; + } + + if (hung) { + char msg[80]; + unsigned int tmp; + int len; + + /* If some rings hung but others were still busy, only + * blame the hanging rings in the synopsis. + */ + if (stuck != hung) + hung &= ~stuck; + len = scnprintf(msg, sizeof(msg), + "%s on ", stuck == hung ? "No progress" : "Hang"); + for_each_engine_masked(engine, dev_priv, hung, tmp) + len += scnprintf(msg + len, sizeof(msg) - len, + "%s, ", engine->name); + msg[len-2] = '\0'; + + return i915_handle_error(dev_priv, hung, msg); + } + + /* Reset timer in case GPU hangs without another request being added */ + if (busy_count) + i915_queue_hangcheck(dev_priv); +} + +void intel_engine_init_hangcheck(struct intel_engine_cs *engine) +{ + memset(&engine->hangcheck, 0, sizeof(engine->hangcheck)); +} + +void intel_hangcheck_init(struct drm_i915_private *i915) +{ + INIT_DELAYED_WORK(&i915->gpu_error.hangcheck_work, + i915_hangcheck_elapsed); +} diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index af8715f31807..35ada4e1c6cf 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -1164,7 +1164,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder, I915_WRITE(intel_hdmi->hdmi_reg, temp); POSTING_READ(intel_hdmi->hdmi_reg); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c index bc86585b9fbb..dde04b7643b1 100644 --- a/drivers/gpu/drm/i915/intel_lrc.c +++ b/drivers/gpu/drm/i915/intel_lrc.c @@ -365,7 +365,7 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq) struct i915_hw_ppgtt *ppgtt = rq->ctx->ppgtt; u32 *reg_state = ce->lrc_reg_state; - reg_state[CTX_RING_TAIL+1] = intel_ring_offset(rq->ring, rq->tail); + reg_state[CTX_RING_TAIL+1] = rq->tail; /* True 32b PPGTT with dynamic page allocation: update PDP * registers and point the unallocated PDPs to scratch page. @@ -440,7 +440,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (last) /* WaIdleLiteRestore:bdw,skl * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL - * as we resubmit the request. See gen8_emit_request() + * as we resubmit the request. See gen8_emit_breadcrumb() * for where we prepare the padding after the end of the * request. */ @@ -522,6 +522,28 @@ static bool execlists_elsp_idle(struct intel_engine_cs *engine) return !engine->execlist_port[0].request; } +/** + * intel_execlists_idle() - Determine if all engine submission ports are idle + * @dev_priv: i915 device private + * + * Return true if there are no requests pending on any of the submission ports + * of any engines. + */ +bool intel_execlists_idle(struct drm_i915_private *dev_priv) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + if (!i915.enable_execlists) + return true; + + for_each_engine(engine, dev_priv, id) + if (!execlists_elsp_idle(engine)) + return false; + + return true; +} + static bool execlists_elsp_ready(struct intel_engine_cs *engine) { int port; @@ -599,6 +621,15 @@ static void execlists_submit_request(struct drm_i915_gem_request *request) spin_lock_irqsave(&engine->execlist_lock, flags); + /* We keep the previous context alive until we retire the following + * request. This ensures that any the context object is still pinned + * for any residual writes the HW makes into it on the context switch + * into the next object following the breadcrumb. Otherwise, we may + * retire the context too early. + */ + request->previous_context = engine->last_context; + engine->last_context = request->ctx; + list_add_tail(&request->execlist_link, &engine->execlist_queue); if (execlists_elsp_idle(engine)) tasklet_hi_schedule(&engine->irq_tasklet); @@ -671,46 +702,6 @@ err_unpin: return ret; } -/* - * intel_logical_ring_advance() - advance the tail and prepare for submission - * @request: Request to advance the logical ringbuffer of. - * - * The tail is updated in our logical ringbuffer struct, not in the actual context. What - * really happens during submission is that the context and current tail will be placed - * on a queue waiting for the ELSP to be ready to accept a new context submission. At that - * point, the tail *inside* the context is updated and the ELSP written to. - */ -static int -intel_logical_ring_advance(struct drm_i915_gem_request *request) -{ - struct intel_ring *ring = request->ring; - struct intel_engine_cs *engine = request->engine; - - intel_ring_advance(ring); - request->tail = ring->tail; - - /* - * Here we add two extra NOOPs as padding to avoid - * lite restore of a context with HEAD==TAIL. - * - * Caller must reserve WA_TAIL_DWORDS for us! - */ - intel_ring_emit(ring, MI_NOOP); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - request->wa_tail = ring->tail; - - /* We keep the previous context alive until we retire the following - * request. This ensures that any the context object is still pinned - * for any residual writes the HW makes into it on the context switch - * into the next object following the breadcrumb. Otherwise, we may - * retire the context too early. - */ - request->previous_context = engine->last_context; - engine->last_context = request->ctx; - return 0; -} - static int intel_lr_context_pin(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { @@ -744,7 +735,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx, ce->lrc_reg_state[CTX_RING_BUFFER_START+1] = i915_ggtt_offset(ce->ring->vma); - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true; /* Invalidate GuC TLB. */ if (i915.enable_guc_submission) { @@ -1566,39 +1557,35 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine) * used as a workaround for not being allowed to do lite * restore with HEAD==TAIL (WaIdleLiteRestore). */ - -static int gen8_emit_request(struct drm_i915_gem_request *request) +static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *out) { - struct intel_ring *ring = request->ring; - int ret; - - ret = intel_ring_begin(request, 6 + WA_TAIL_DWORDS); - if (ret) - return ret; + *out++ = MI_NOOP; + *out++ = MI_NOOP; + request->wa_tail = intel_ring_offset(request->ring, out); +} +static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, + u32 *out) +{ /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */ BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5)); - intel_ring_emit(ring, (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); - intel_ring_emit(ring, - intel_hws_seqno_address(request->engine) | - MI_FLUSH_DW_USE_GTT); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, request->fence.seqno); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_emit(ring, MI_NOOP); - return intel_logical_ring_advance(request); + *out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; + *out++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT; + *out++ = 0; + *out++ = request->global_seqno; + *out++ = MI_USER_INTERRUPT; + *out++ = MI_NOOP; + request->tail = intel_ring_offset(request->ring, out); + + gen8_emit_wa_tail(request, out); } -static int gen8_emit_request_render(struct drm_i915_gem_request *request) -{ - struct intel_ring *ring = request->ring; - int ret; - - ret = intel_ring_begin(request, 8 + WA_TAIL_DWORDS); - if (ret) - return ret; +static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS; +static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request, + u32 *out) +{ /* We're using qword write, seqno should be aligned to 8 bytes. */ BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1); @@ -1606,21 +1593,24 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request) * need a prior CS_STALL, which is emitted by the flush * following the batch. */ - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, - (PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_QW_WRITE)); - intel_ring_emit(ring, intel_hws_seqno_address(request->engine)); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(request)); + *out++ = GFX_OP_PIPE_CONTROL(6); + *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_QW_WRITE); + *out++ = intel_hws_seqno_address(request->engine); + *out++ = 0; + *out++ = request->global_seqno; /* We're thrashing one dword of HWS. */ - intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_emit(ring, MI_NOOP); - return intel_logical_ring_advance(request); + *out++ = 0; + *out++ = MI_USER_INTERRUPT; + *out++ = MI_NOOP; + request->tail = intel_ring_offset(request->ring, out); + + gen8_emit_wa_tail(request, out); } +static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS; + static int gen8_init_rcs_context(struct drm_i915_gem_request *req) { int ret; @@ -1637,7 +1627,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req) if (ret) DRM_ERROR("MOCS failed to program: expect performance issues.\n"); - return i915_gem_render_state_init(req); + return i915_gem_render_state_emit(req); } /** @@ -1694,7 +1684,8 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->init_hw = gen8_init_common_ring; engine->reset_hw = reset_common_ring; engine->emit_flush = gen8_emit_flush; - engine->emit_request = gen8_emit_request; + engine->emit_breadcrumb = gen8_emit_breadcrumb; + engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz; engine->submit_request = execlists_submit_request; engine->irq_enable = gen8_logical_ring_enable_irq; @@ -1816,7 +1807,8 @@ int logical_render_ring_init(struct intel_engine_cs *engine) engine->init_hw = gen8_init_render_ring; engine->init_context = gen8_init_rcs_context; engine->emit_flush = gen8_emit_flush_render; - engine->emit_request = gen8_emit_request_render; + engine->emit_breadcrumb = gen8_emit_breadcrumb_render; + engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz; ret = intel_engine_create_scratch(engine, 4096); if (ret) @@ -2042,7 +2034,7 @@ populate_lr_context(struct i915_gem_context *ctx, DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret); return ret; } - ctx_obj->dirty = true; + ctx_obj->mm.dirty = true; /* The second page of the context object contains some fields which must * be set up prior to the first execution. */ @@ -2180,7 +2172,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv) reg[CTX_RING_HEAD+1] = 0; reg[CTX_RING_TAIL+1] = 0; - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true; i915_gem_object_unpin_map(ce->state->obj); ce->ring->head = ce->ring->tail = 0; diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h index 4fed8165f98a..c1f546180ba2 100644 --- a/drivers/gpu/drm/i915/intel_lrc.h +++ b/drivers/gpu/drm/i915/intel_lrc.h @@ -95,5 +95,6 @@ uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx, int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists); void intel_execlists_enable_submission(struct drm_i915_private *dev_priv); +bool intel_execlists_idle(struct drm_i915_private *dev_priv); #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/intel_lspcon.c b/drivers/gpu/drm/i915/intel_lspcon.c index 632149c6b3ad..daa523410953 100644 --- a/drivers/gpu/drm/i915/intel_lspcon.c +++ b/drivers/gpu/drm/i915/intel_lspcon.c @@ -27,10 +27,18 @@ #include <drm/drm_dp_dual_mode_helper.h> #include "intel_drv.h" +static struct intel_dp *lspcon_to_intel_dp(struct intel_lspcon *lspcon) +{ + struct intel_digital_port *dig_port = + container_of(lspcon, struct intel_digital_port, lspcon); + + return &dig_port->dp; +} + static enum drm_lspcon_mode lspcon_get_current_mode(struct intel_lspcon *lspcon) { enum drm_lspcon_mode current_mode = DRM_LSPCON_MODE_INVALID; - struct i2c_adapter *adapter = &lspcon->aux->ddc; + struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc; if (drm_lspcon_get_mode(adapter, ¤t_mode)) DRM_ERROR("Error reading LSPCON mode\n"); @@ -45,7 +53,7 @@ static int lspcon_change_mode(struct intel_lspcon *lspcon, { int err; enum drm_lspcon_mode current_mode; - struct i2c_adapter *adapter = &lspcon->aux->ddc; + struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc; err = drm_lspcon_get_mode(adapter, ¤t_mode); if (err) { @@ -72,7 +80,7 @@ static int lspcon_change_mode(struct intel_lspcon *lspcon, static bool lspcon_probe(struct intel_lspcon *lspcon) { enum drm_dp_dual_mode_type adaptor_type; - struct i2c_adapter *adapter = &lspcon->aux->ddc; + struct i2c_adapter *adapter = &lspcon_to_intel_dp(lspcon)->aux.ddc; /* Lets probe the adaptor and check its type */ adaptor_type = drm_dp_dual_mode_detect(adapter); @@ -89,8 +97,43 @@ static bool lspcon_probe(struct intel_lspcon *lspcon) return true; } +static void lspcon_resume_in_pcon_wa(struct intel_lspcon *lspcon) +{ + struct intel_dp *intel_dp = lspcon_to_intel_dp(lspcon); + unsigned long start = jiffies; + + if (!lspcon->desc_valid) + return; + + while (1) { + struct intel_dp_desc desc; + + /* + * The w/a only applies in PCON mode and we don't expect any + * AUX errors. + */ + if (!__intel_dp_read_desc(intel_dp, &desc)) + return; + + if (!memcmp(&intel_dp->desc, &desc, sizeof(desc))) { + DRM_DEBUG_KMS("LSPCON recovering in PCON mode after %u ms\n", + jiffies_to_msecs(jiffies - start)); + return; + } + + if (time_after(jiffies, start + msecs_to_jiffies(1000))) + break; + + usleep_range(10000, 15000); + } + + DRM_DEBUG_KMS("LSPCON DP descriptor mismatch after resume\n"); +} + void lspcon_resume(struct intel_lspcon *lspcon) { + lspcon_resume_in_pcon_wa(lspcon); + if (lspcon_change_mode(lspcon, DRM_LSPCON_MODE_PCON, true)) DRM_ERROR("LSPCON resume failed\n"); else @@ -111,7 +154,6 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port) lspcon->active = false; lspcon->mode = DRM_LSPCON_MODE_INVALID; - lspcon->aux = &dp->aux; if (!lspcon_probe(lspcon)) { DRM_ERROR("Failed to probe lspcon\n"); @@ -131,6 +173,13 @@ bool lspcon_init(struct intel_digital_port *intel_dig_port) } } + if (!intel_dp_read_dpcd(dp)) { + DRM_ERROR("LSPCON DPCD read failed\n"); + return false; + } + + lspcon->desc_valid = intel_dp_read_desc(dp); + DRM_DEBUG_KMS("Success: LSPCON init\n"); return true; } diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 199b90c7907a..de7b3e6ed477 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -985,7 +985,7 @@ void intel_lvds_init(struct drm_device *dev) struct drm_display_mode *fixed_mode = NULL; struct drm_display_mode *downclock_mode = NULL; struct edid *edid; - struct drm_crtc *crtc; + struct intel_crtc *crtc; i915_reg_t lvds_reg; u32 lvds; int pipe; @@ -1163,10 +1163,10 @@ void intel_lvds_init(struct drm_device *dev) goto failed; pipe = (lvds & LVDS_PIPEB_SELECT) ? 1 : 0; - crtc = intel_get_crtc_for_pipe(dev, pipe); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); if (crtc && (lvds & LVDS_PORT_EN)) { - fixed_mode = intel_crtc_mode_get(dev, crtc); + fixed_mode = intel_crtc_mode_get(dev, &crtc->base); if (fixed_mode) { DRM_DEBUG_KMS("using current (BIOS) mode: "); drm_mode_debug_printmodeline(fixed_mode); diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index 25bcd4a178d3..fd0e4dac7cc1 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -1222,7 +1222,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, out_unlock: mutex_unlock(&dev->struct_mutex); drm_modeset_unlock_all(dev); - i915_gem_object_put_unlocked(new_bo); + i915_gem_object_put(new_bo); out_free: kfree(params); @@ -1466,7 +1466,7 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv) * hardware should be off already */ WARN_ON(dev_priv->overlay->active); - i915_gem_object_put_unlocked(dev_priv->overlay->reg_bo); + i915_gem_object_put(dev_priv->overlay->reg_bo); kfree(dev_priv->overlay); } diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index 560fc7af8267..cc9e0c0f445f 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -31,6 +31,7 @@ #include "intel_drv.h" #include "../../../platform/x86/intel_ips.h" #include <linux/module.h> +#include <drm/drm_atomic_helper.h> /** * DOC: RC6 @@ -55,10 +56,8 @@ #define INTEL_RC6p_ENABLE (1<<1) #define INTEL_RC6pp_ENABLE (1<<2) -static void gen9_init_clock_gating(struct drm_device *dev) +static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl */ I915_WRITE(CHICKEN_PAR1_1, I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP); @@ -81,11 +80,9 @@ static void gen9_init_clock_gating(struct drm_device *dev) ILK_DPFC_DISABLE_DUMMY0); } -static void bxt_init_clock_gating(struct drm_device *dev) +static void bxt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - gen9_init_clock_gating(dev); + gen9_init_clock_gating(dev_priv); /* WaDisableSDEUnitClockGating:bxt */ I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) | @@ -107,9 +104,8 @@ static void bxt_init_clock_gating(struct drm_device *dev) PWM1_GATING_DIS | PWM2_GATING_DIS); } -static void i915_pineview_get_mem_freq(struct drm_device *dev) +static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 tmp; tmp = I915_READ(CLKCFG); @@ -146,9 +142,8 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev) dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0; } -static void i915_ironlake_get_mem_freq(struct drm_device *dev) +static void i915_ironlake_get_mem_freq(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u16 ddrpll, csipll; ddrpll = I915_READ16(DDRMPLL1); @@ -319,7 +314,6 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable) void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) { - struct drm_device *dev = &dev_priv->drm; u32 val; if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { @@ -329,7 +323,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable) } else if (IS_G4X(dev_priv) || IS_CRESTLINE(dev_priv)) { I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0); POSTING_READ(FW_BLC_SELF); - } else if (IS_PINEVIEW(dev)) { + } else if (IS_PINEVIEW(dev_priv)) { val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN; val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0; I915_WRITE(DSPFW3, val); @@ -377,10 +371,9 @@ static const int pessimal_latency_ns = 5000; #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \ ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8)) -static int vlv_get_fifo_size(struct drm_device *dev, +static int vlv_get_fifo_size(struct drm_i915_private *dev_priv, enum pipe pipe, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); int sprite0_start, sprite1_start, size; switch (pipe) { @@ -429,9 +422,8 @@ static int vlv_get_fifo_size(struct drm_device *dev, return size; } -static int i9xx_get_fifo_size(struct drm_device *dev, int plane) +static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -445,9 +437,8 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane) return size; } -static int i830_get_fifo_size(struct drm_device *dev, int plane) +static int i830_get_fifo_size(struct drm_i915_private *dev_priv, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -462,9 +453,8 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane) return size; } -static int i845_get_fifo_size(struct drm_device *dev, int plane) +static int i845_get_fifo_size(struct drm_i915_private *dev_priv, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dsparb = I915_READ(DSPARB); int size; @@ -624,11 +614,11 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz, return wm_size; } -static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) +static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv) { - struct drm_crtc *crtc, *enabled = NULL; + struct intel_crtc *crtc, *enabled = NULL; - for_each_crtc(dev, crtc) { + for_each_intel_crtc(&dev_priv->drm, crtc) { if (intel_crtc_active(crtc)) { if (enabled) return NULL; @@ -639,11 +629,10 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev) return enabled; } -static void pineview_update_wm(struct drm_crtc *unused_crtc) +static void pineview_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); + struct intel_crtc *crtc; const struct cxsr_latency *latency; u32 reg; unsigned long wm; @@ -658,10 +647,13 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) return; } - crtc = single_enabled_crtc(dev); + crtc = single_enabled_crtc(dev_priv); if (crtc) { - const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; + int cpp = drm_format_plane_cpp(fb->pixel_format, 0); int clock = adjusted_mode->crtc_clock; /* Display SR */ @@ -708,7 +700,7 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc) } } -static bool g4x_compute_wm0(struct drm_device *dev, +static bool g4x_compute_wm0(struct drm_i915_private *dev_priv, int plane, const struct intel_watermark_params *display, int display_latency_ns, @@ -717,24 +709,26 @@ static bool g4x_compute_wm0(struct drm_device *dev, int *plane_wm, int *cursor_wm) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; + const struct drm_framebuffer *fb; int htotal, hdisplay, clock, cpp; int line_time_us, line_count; int entries, tlb_miss; - crtc = intel_get_crtc_for_plane(dev, plane); + crtc = intel_get_crtc_for_plane(dev_priv, plane); if (!intel_crtc_active(crtc)) { *cursor_wm = cursor->guard_size; *plane_wm = display->guard_size; return false; } - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + adjusted_mode = &crtc->config->base.adjusted_mode; + fb = crtc->base.primary->state->fb; clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; - hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; - cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + hdisplay = crtc->config->pipe_src_w; + cpp = drm_format_plane_cpp(fb->pixel_format, 0); /* Use the small buffer method to calculate plane watermark */ entries = ((clock * cpp / 1000) * display_latency_ns) / 1000; @@ -749,7 +743,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, /* Use the large buffer method to calculate cursor watermark */ line_time_us = max(htotal * 1000 / clock, 1); line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; - entries = line_count * crtc->cursor->state->crtc_w * cpp; + entries = line_count * crtc->base.cursor->state->crtc_w * cpp; tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; if (tlb_miss > 0) entries += tlb_miss; @@ -768,7 +762,7 @@ static bool g4x_compute_wm0(struct drm_device *dev, * can be programmed into the associated watermark register, that watermark * must be disabled. */ -static bool g4x_check_srwm(struct drm_device *dev, +static bool g4x_check_srwm(struct drm_i915_private *dev_priv, int display_wm, int cursor_wm, const struct intel_watermark_params *display, const struct intel_watermark_params *cursor) @@ -796,15 +790,16 @@ static bool g4x_check_srwm(struct drm_device *dev, return true; } -static bool g4x_compute_srwm(struct drm_device *dev, +static bool g4x_compute_srwm(struct drm_i915_private *dev_priv, int plane, int latency_ns, const struct intel_watermark_params *display, const struct intel_watermark_params *cursor, int *display_wm, int *cursor_wm) { - struct drm_crtc *crtc; + struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; + const struct drm_framebuffer *fb; int hdisplay, htotal, cpp, clock; unsigned long line_time_us; int line_count, line_size; @@ -816,12 +811,13 @@ static bool g4x_compute_srwm(struct drm_device *dev, return false; } - crtc = intel_get_crtc_for_plane(dev, plane); - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + crtc = intel_get_crtc_for_plane(dev_priv, plane); + adjusted_mode = &crtc->config->base.adjusted_mode; + fb = crtc->base.primary->state->fb; clock = adjusted_mode->crtc_clock; htotal = adjusted_mode->crtc_htotal; - hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; - cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + hdisplay = crtc->config->pipe_src_w; + cpp = drm_format_plane_cpp(fb->pixel_format, 0); line_time_us = max(htotal * 1000 / clock, 1); line_count = (latency_ns / line_time_us + 1000) / 1000; @@ -835,11 +831,11 @@ static bool g4x_compute_srwm(struct drm_device *dev, *display_wm = entries + display->guard_size; /* calculate the self-refresh watermark for display cursor */ - entries = line_count * cpp * crtc->cursor->state->crtc_w; + entries = line_count * cpp * crtc->base.cursor->state->crtc_w; entries = DIV_ROUND_UP(entries, cursor->cacheline_size); *cursor_wm = entries + cursor->guard_size; - return g4x_check_srwm(dev, + return g4x_check_srwm(dev_priv, *display_wm, *cursor_wm, display, cursor); } @@ -939,10 +935,8 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate, return ret; } -static void vlv_setup_wm_latency(struct drm_device *dev) +static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* all latencies in usec */ dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3; @@ -1329,20 +1323,19 @@ static void vlv_merge_wm(struct drm_device *dev, } } -static void vlv_update_wm(struct drm_crtc *crtc) +static void vlv_update_wm(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; + struct drm_device *dev = crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - enum pipe pipe = intel_crtc->pipe; + enum pipe pipe = crtc->pipe; struct vlv_wm_values wm = {}; - vlv_compute_wm(intel_crtc); + vlv_compute_wm(crtc); vlv_merge_wm(dev, &wm); if (memcmp(&dev_priv->wm.vlv, &wm, sizeof(wm)) == 0) { /* FIXME should be part of crtc atomic commit */ - vlv_pipe_set_fifo_size(intel_crtc); + vlv_pipe_set_fifo_size(crtc); return; } @@ -1358,9 +1351,9 @@ static void vlv_update_wm(struct drm_crtc *crtc) intel_set_memory_cxsr(dev_priv, false); /* FIXME should be part of crtc atomic commit */ - vlv_pipe_set_fifo_size(intel_crtc); + vlv_pipe_set_fifo_size(crtc); - vlv_write_wm_values(intel_crtc, &wm); + vlv_write_wm_values(crtc, &wm); DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, " "sprite0=%d, sprite1=%d, SR: plane=%d, cursor=%d level=%d cxsr=%d\n", @@ -1384,30 +1377,29 @@ static void vlv_update_wm(struct drm_crtc *crtc) #define single_plane_enabled(mask) is_power_of_2(mask) -static void g4x_update_wm(struct drm_crtc *crtc) +static void g4x_update_wm(struct intel_crtc *crtc) { - struct drm_device *dev = crtc->dev; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); static const int sr_latency_ns = 12000; - struct drm_i915_private *dev_priv = to_i915(dev); int planea_wm, planeb_wm, cursora_wm, cursorb_wm; int plane_sr, cursor_sr; unsigned int enabled = 0; bool cxsr_enabled; - if (g4x_compute_wm0(dev, PIPE_A, + if (g4x_compute_wm0(dev_priv, PIPE_A, &g4x_wm_info, pessimal_latency_ns, &g4x_cursor_wm_info, pessimal_latency_ns, &planea_wm, &cursora_wm)) enabled |= 1 << PIPE_A; - if (g4x_compute_wm0(dev, PIPE_B, + if (g4x_compute_wm0(dev_priv, PIPE_B, &g4x_wm_info, pessimal_latency_ns, &g4x_cursor_wm_info, pessimal_latency_ns, &planeb_wm, &cursorb_wm)) enabled |= 1 << PIPE_B; if (single_plane_enabled(enabled) && - g4x_compute_srwm(dev, ffs(enabled) - 1, + g4x_compute_srwm(dev_priv, ffs(enabled) - 1, sr_latency_ns, &g4x_wm_info, &g4x_cursor_wm_info, @@ -1442,25 +1434,27 @@ static void g4x_update_wm(struct drm_crtc *crtc) intel_set_memory_cxsr(dev_priv, true); } -static void i965_update_wm(struct drm_crtc *unused_crtc) +static void i965_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); + struct intel_crtc *crtc; int srwm = 1; int cursor_sr = 16; bool cxsr_enabled; /* Calc sr entries for one plane configs */ - crtc = single_enabled_crtc(dev); + crtc = single_enabled_crtc(dev_priv); if (crtc) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 12000; - const struct drm_display_mode *adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; - int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + int hdisplay = crtc->config->pipe_src_w; + int cpp = drm_format_plane_cpp(fb->pixel_format, 0); unsigned long line_time_us; int entries; @@ -1478,7 +1472,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) entries, srwm); entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) * - cpp * crtc->cursor->state->crtc_w; + cpp * crtc->base.cursor->state->crtc_w; entries = DIV_ROUND_UP(entries, i965_cursor_wm_info.cacheline_size); cursor_sr = i965_cursor_wm_info.fifo_size - @@ -1516,34 +1510,38 @@ static void i965_update_wm(struct drm_crtc *unused_crtc) #undef FW_WM -static void i9xx_update_wm(struct drm_crtc *unused_crtc) +static void i9xx_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); const struct intel_watermark_params *wm_info; uint32_t fwater_lo; uint32_t fwater_hi; int cwm, srwm = 1; int fifo_size; int planea_wm, planeb_wm; - struct drm_crtc *crtc, *enabled = NULL; + struct intel_crtc *crtc, *enabled = NULL; - if (IS_I945GM(dev)) + if (IS_I945GM(dev_priv)) wm_info = &i945_wm_info; else if (!IS_GEN2(dev_priv)) wm_info = &i915_wm_info; else wm_info = &i830_a_wm_info; - fifo_size = dev_priv->display.get_fifo_size(dev, 0); - crtc = intel_get_crtc_for_plane(dev, 0); + fifo_size = dev_priv->display.get_fifo_size(dev_priv, 0); + crtc = intel_get_crtc_for_plane(dev_priv, 0); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; + int cpp; + if (IS_GEN2(dev_priv)) cpp = 4; + else + cpp = drm_format_plane_cpp(fb->pixel_format, 0); - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); @@ -1557,15 +1555,20 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) if (IS_GEN2(dev_priv)) wm_info = &i830_bc_wm_info; - fifo_size = dev_priv->display.get_fifo_size(dev, 1); - crtc = intel_get_crtc_for_plane(dev, 1); + fifo_size = dev_priv->display.get_fifo_size(dev_priv, 1); + crtc = intel_get_crtc_for_plane(dev_priv, 1); if (intel_crtc_active(crtc)) { - const struct drm_display_mode *adjusted_mode; - int cpp = drm_format_plane_cpp(crtc->primary->state->fb->pixel_format, 0); + const struct drm_display_mode *adjusted_mode = + &crtc->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + crtc->base.primary->state->fb; + int cpp; + if (IS_GEN2(dev_priv)) cpp = 4; + else + cpp = drm_format_plane_cpp(fb->pixel_format, 0); - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock, wm_info, fifo_size, cpp, pessimal_latency_ns); @@ -1584,7 +1587,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) if (IS_I915GM(dev_priv) && enabled) { struct drm_i915_gem_object *obj; - obj = intel_fb_obj(enabled->primary->state->fb); + obj = intel_fb_obj(enabled->base.primary->state->fb); /* self-refresh seems busted with untiled */ if (!i915_gem_object_is_tiled(obj)) @@ -1600,19 +1603,24 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) intel_set_memory_cxsr(dev_priv, false); /* Calc sr entries for one plane configs */ - if (HAS_FW_BLC(dev) && enabled) { + if (HAS_FW_BLC(dev_priv) && enabled) { /* self-refresh has much higher latency */ static const int sr_latency_ns = 6000; - const struct drm_display_mode *adjusted_mode = &to_intel_crtc(enabled)->config->base.adjusted_mode; + const struct drm_display_mode *adjusted_mode = + &enabled->config->base.adjusted_mode; + const struct drm_framebuffer *fb = + enabled->base.primary->state->fb; int clock = adjusted_mode->crtc_clock; int htotal = adjusted_mode->crtc_htotal; - int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w; - int cpp = drm_format_plane_cpp(enabled->primary->state->fb->pixel_format, 0); + int hdisplay = enabled->config->pipe_src_w; + int cpp; unsigned long line_time_us; int entries; if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv)) cpp = 4; + else + cpp = drm_format_plane_cpp(fb->pixel_format, 0); line_time_us = max(htotal * 1000 / clock, 1); @@ -1649,23 +1657,22 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc) intel_set_memory_cxsr(dev_priv, true); } -static void i845_update_wm(struct drm_crtc *unused_crtc) +static void i845_update_wm(struct intel_crtc *unused_crtc) { - struct drm_device *dev = unused_crtc->dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct drm_crtc *crtc; + struct drm_i915_private *dev_priv = to_i915(unused_crtc->base.dev); + struct intel_crtc *crtc; const struct drm_display_mode *adjusted_mode; uint32_t fwater_lo; int planea_wm; - crtc = single_enabled_crtc(dev); + crtc = single_enabled_crtc(dev_priv); if (crtc == NULL) return; - adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode; + adjusted_mode = &crtc->config->base.adjusted_mode; planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock, &i845_wm_info, - dev_priv->display.get_fifo_size(dev, 0), + dev_priv->display.get_fifo_size(dev_priv, 0), 4, pessimal_latency_ns); fwater_lo = I915_READ(FW_BLC) & ~0xfff; fwater_lo |= (3<<8) | planea_wm; @@ -2078,10 +2085,9 @@ hsw_compute_linetime_wm(const struct intel_crtc_state *cstate) PIPE_WM_LINETIME_TIME(linetime); } -static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) +static void intel_read_wm_latency(struct drm_i915_private *dev_priv, + uint16_t wm[8]) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (IS_GEN9(dev_priv)) { uint32_t val; int ret, i; @@ -2167,14 +2173,14 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8]) wm[2] = (sskpd >> 12) & 0xFF; wm[3] = (sskpd >> 20) & 0x1FF; wm[4] = (sskpd >> 32) & 0x1FF; - } else if (INTEL_INFO(dev)->gen >= 6) { + } else if (INTEL_GEN(dev_priv) >= 6) { uint32_t sskpd = I915_READ(MCH_SSKPD); wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK; wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK; wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK; wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK; - } else if (INTEL_INFO(dev)->gen >= 5) { + } else if (INTEL_GEN(dev_priv) >= 5) { uint32_t mltr = I915_READ(MLTR_ILK); /* ILK primary LP0 latency is 700 ns */ @@ -2262,9 +2268,8 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv, return true; } -static void snb_wm_latency_quirk(struct drm_device *dev) +static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); bool changed; /* @@ -2284,11 +2289,9 @@ static void snb_wm_latency_quirk(struct drm_device *dev) intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); } -static void ilk_setup_wm_latency(struct drm_device *dev) +static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - intel_read_wm_latency(dev, dev_priv->wm.pri_latency); + intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency); memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency, sizeof(dev_priv->wm.pri_latency)); @@ -2303,14 +2306,12 @@ static void ilk_setup_wm_latency(struct drm_device *dev) intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency); if (IS_GEN6(dev_priv)) - snb_wm_latency_quirk(dev); + snb_wm_latency_quirk(dev_priv); } -static void skl_setup_wm_latency(struct drm_device *dev) +static void skl_setup_wm_latency(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - intel_read_wm_latency(dev, dev_priv->wm.skl_latency); + intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency); intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency); } @@ -3041,7 +3042,7 @@ bool intel_can_enable_sagv(struct drm_atomic_state *state) /* Since we're now guaranteed to only have one active CRTC... */ pipe = ffs(intel_state->active_crtcs) - 1; - crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]); + crtc = intel_get_crtc_for_pipe(dev_priv, pipe); cstate = to_intel_crtc_state(crtc->base.state); if (crtc->base.state->adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) @@ -3160,7 +3161,7 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv, if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) continue; - for_each_plane(dev_priv, pipe, plane) { + for_each_universal_plane(dev_priv, pipe, plane) { val = I915_READ(PLANE_BUF_CFG(pipe, plane)); skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane], val); @@ -3262,49 +3263,39 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate, * 3 * 4096 * 8192 * 4 < 2^32 */ static unsigned int -skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate) +skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate, + unsigned *plane_data_rate, + unsigned *plane_y_data_rate) { struct drm_crtc_state *cstate = &intel_cstate->base; struct drm_atomic_state *state = cstate->state; - struct drm_crtc *crtc = cstate->crtc; - struct drm_device *dev = crtc->dev; - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - const struct drm_plane *plane; + struct drm_plane *plane; const struct intel_plane *intel_plane; - struct drm_plane_state *pstate; + const struct drm_plane_state *pstate; unsigned int rate, total_data_rate = 0; int id; - int i; if (WARN_ON(!state)) return 0; /* Calculate and cache data rate for each plane */ - for_each_plane_in_state(state, plane, pstate, i) { + drm_atomic_crtc_state_for_each_plane_state(plane, pstate, cstate) { id = skl_wm_plane_id(to_intel_plane(plane)); intel_plane = to_intel_plane(plane); - if (intel_plane->pipe != intel_crtc->pipe) - continue; - /* packed/uv */ rate = skl_plane_relative_data_rate(intel_cstate, pstate, 0); - intel_cstate->wm.skl.plane_data_rate[id] = rate; + plane_data_rate[id] = rate; + + total_data_rate += rate; /* y-plane */ rate = skl_plane_relative_data_rate(intel_cstate, pstate, 1); - intel_cstate->wm.skl.plane_y_data_rate[id] = rate; - } + plane_y_data_rate[id] = rate; - /* Calculate CRTC's total data rate from cached values */ - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { - int id = skl_wm_plane_id(intel_plane); - - /* packed/uv */ - total_data_rate += intel_cstate->wm.skl.plane_data_rate[id]; - total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id]; + total_data_rate += rate; } return total_data_rate; @@ -3373,6 +3364,30 @@ skl_ddb_min_alloc(const struct drm_plane_state *pstate, return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3; } +static void +skl_ddb_calc_min(const struct intel_crtc_state *cstate, int num_active, + uint16_t *minimum, uint16_t *y_minimum) +{ + const struct drm_plane_state *pstate; + struct drm_plane *plane; + + drm_atomic_crtc_state_for_each_plane_state(plane, pstate, &cstate->base) { + struct intel_plane *intel_plane = to_intel_plane(plane); + int id = skl_wm_plane_id(intel_plane); + + if (id == PLANE_CURSOR) + continue; + + if (!pstate->visible) + continue; + + minimum[id] = skl_ddb_min_alloc(pstate, 0); + y_minimum[id] = skl_ddb_min_alloc(pstate, 1); + } + + minimum[PLANE_CURSOR] = skl_cursor_allocation(num_active); +} + static int skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, struct skl_ddb_allocation *ddb /* out */) @@ -3381,17 +3396,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, struct drm_crtc *crtc = cstate->base.crtc; struct drm_device *dev = crtc->dev; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct intel_plane *intel_plane; - struct drm_plane *plane; - struct drm_plane_state *pstate; enum pipe pipe = intel_crtc->pipe; struct skl_ddb_entry *alloc = &cstate->wm.skl.ddb; - uint16_t alloc_size, start, cursor_blocks; - uint16_t *minimum = cstate->wm.skl.minimum_blocks; - uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks; + uint16_t alloc_size, start; + uint16_t minimum[I915_MAX_PLANES] = {}; + uint16_t y_minimum[I915_MAX_PLANES] = {}; unsigned int total_data_rate; int num_active; int id, i; + unsigned plane_data_rate[I915_MAX_PLANES] = {}; + unsigned plane_y_data_rate[I915_MAX_PLANES] = {}; /* Clear the partitioning for disabled planes. */ memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe])); @@ -3412,57 +3426,43 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, return 0; } - cursor_blocks = skl_cursor_allocation(num_active); - ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks; - ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; - - alloc_size -= cursor_blocks; - - /* 1. Allocate the mininum required blocks for each active plane */ - for_each_plane_in_state(state, plane, pstate, i) { - intel_plane = to_intel_plane(plane); - id = skl_wm_plane_id(intel_plane); - - if (intel_plane->pipe != pipe) - continue; - - if (!to_intel_plane_state(pstate)->base.visible) { - minimum[id] = 0; - y_minimum[id] = 0; - continue; - } - if (plane->type == DRM_PLANE_TYPE_CURSOR) { - minimum[id] = 0; - y_minimum[id] = 0; - continue; - } + skl_ddb_calc_min(cstate, num_active, minimum, y_minimum); - minimum[id] = skl_ddb_min_alloc(pstate, 0); - y_minimum[id] = skl_ddb_min_alloc(pstate, 1); - } + /* + * 1. Allocate the mininum required blocks for each active plane + * and allocate the cursor, it doesn't require extra allocation + * proportional to the data rate. + */ - for (i = 0; i < PLANE_CURSOR; i++) { + for (i = 0; i < I915_MAX_PLANES; i++) { alloc_size -= minimum[i]; alloc_size -= y_minimum[i]; } + ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - minimum[PLANE_CURSOR]; + ddb->plane[pipe][PLANE_CURSOR].end = alloc->end; + /* * 2. Distribute the remaining space in proportion to the amount of * data each plane needs to fetch from memory. * * FIXME: we may not allocate every single block here. */ - total_data_rate = skl_get_total_relative_data_rate(cstate); + total_data_rate = skl_get_total_relative_data_rate(cstate, + plane_data_rate, + plane_y_data_rate); if (total_data_rate == 0) return 0; start = alloc->start; - for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { + for (id = 0; id < I915_MAX_PLANES; id++) { unsigned int data_rate, y_data_rate; uint16_t plane_blocks, y_plane_blocks = 0; - int id = skl_wm_plane_id(intel_plane); - data_rate = cstate->wm.skl.plane_data_rate[id]; + if (id == PLANE_CURSOR) + continue; + + data_rate = plane_data_rate[id]; /* * allocation for (packed formats) or (uv-plane part of planar format): @@ -3484,7 +3484,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate, /* * allocation for y_plane part of planar format: */ - y_data_rate = cstate->wm.skl.plane_y_data_rate[id]; + y_data_rate = plane_y_data_rate[id]; y_plane_blocks = y_minimum[id]; y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate, @@ -3930,11 +3930,11 @@ bool skl_ddb_allocation_overlaps(struct drm_atomic_state *state, } static int skl_update_pipe_wm(struct drm_crtc_state *cstate, - struct skl_ddb_allocation *ddb, /* out */ + const struct skl_pipe_wm *old_pipe_wm, struct skl_pipe_wm *pipe_wm, /* out */ + struct skl_ddb_allocation *ddb, /* out */ bool *changed /* out */) { - struct intel_crtc *intel_crtc = to_intel_crtc(cstate->crtc); struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate); int ret; @@ -3942,7 +3942,7 @@ static int skl_update_pipe_wm(struct drm_crtc_state *cstate, if (ret) return ret; - if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm))) + if (!memcmp(old_pipe_wm, pipe_wm, sizeof(*pipe_wm))) *changed = false; else *changed = true; @@ -3981,7 +3981,7 @@ skl_ddb_add_affected_planes(struct intel_crtc_state *cstate) WARN_ON(!drm_atomic_get_existing_crtc_state(state, crtc)); - drm_for_each_plane_mask(plane, dev, crtc->state->plane_mask) { + drm_for_each_plane_mask(plane, dev, cstate->base.plane_mask) { id = skl_wm_plane_id(to_intel_plane(plane)); if (skl_ddb_entry_equal(&cur_ddb->plane[pipe][id], @@ -4096,45 +4096,31 @@ skl_print_wm_changes(const struct drm_atomic_state *state) to_intel_atomic_state(state); const struct drm_crtc *crtc; const struct drm_crtc_state *cstate; - const struct drm_plane *plane; const struct intel_plane *intel_plane; - const struct drm_plane_state *pstate; const struct skl_ddb_allocation *old_ddb = &dev_priv->wm.skl_hw.ddb; const struct skl_ddb_allocation *new_ddb = &intel_state->wm_results.ddb; - enum pipe pipe; int id; - int i, j; + int i; for_each_crtc_in_state(state, crtc, cstate, i) { - pipe = to_intel_crtc(crtc)->pipe; + const struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + enum pipe pipe = intel_crtc->pipe; - for_each_plane_in_state(state, plane, pstate, j) { + for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) { const struct skl_ddb_entry *old, *new; - intel_plane = to_intel_plane(plane); id = skl_wm_plane_id(intel_plane); old = &old_ddb->plane[pipe][id]; new = &new_ddb->plane[pipe][id]; - if (intel_plane->pipe != pipe) - continue; - if (skl_ddb_entry_equal(old, new)) continue; - if (id != PLANE_CURSOR) { - DRM_DEBUG_ATOMIC("[PLANE:%d:plane %d%c] ddb (%d - %d) -> (%d - %d)\n", - plane->base.id, id + 1, - pipe_name(pipe), - old->start, old->end, - new->start, new->end); - } else { - DRM_DEBUG_ATOMIC("[PLANE:%d:cursor %c] ddb (%d - %d) -> (%d - %d)\n", - plane->base.id, - pipe_name(pipe), - old->start, old->end, - new->start, new->end); - } + DRM_DEBUG_ATOMIC("[PLANE:%d:%s] ddb (%d - %d) -> (%d - %d)\n", + intel_plane->base.base.id, + intel_plane->base.name, + old->start, old->end, + new->start, new->end); } } } @@ -4183,10 +4169,12 @@ skl_compute_wm(struct drm_atomic_state *state) for_each_crtc_in_state(state, crtc, cstate, i) { struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate); + const struct skl_pipe_wm *old_pipe_wm = + &to_intel_crtc_state(crtc->state)->wm.skl.optimal; pipe_wm = &intel_cstate->wm.skl.optimal; - ret = skl_update_pipe_wm(cstate, &results->ddb, pipe_wm, - &changed); + ret = skl_update_pipe_wm(cstate, old_pipe_wm, pipe_wm, + &results->ddb, &changed); if (ret) return ret; @@ -4205,22 +4193,19 @@ skl_compute_wm(struct drm_atomic_state *state) return 0; } -static void skl_update_wm(struct drm_crtc *crtc) +static void skl_update_wm(struct intel_crtc *intel_crtc) { - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_device *dev = crtc->dev; + struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); struct skl_wm_values *results = &dev_priv->wm.skl_results; struct skl_wm_values *hw_vals = &dev_priv->wm.skl_hw; - struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state); + struct intel_crtc_state *cstate = to_intel_crtc_state(intel_crtc->base.state); struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal; enum pipe pipe = intel_crtc->pipe; - if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0) + if ((results->dirty_pipes & drm_crtc_mask(&intel_crtc->base)) == 0) return; - intel_crtc->wm.active.skl = *pipe_wm; - mutex_lock(&dev_priv->wm.wm_mutex); /* @@ -4229,10 +4214,10 @@ static void skl_update_wm(struct drm_crtc *crtc) * the pipe's shut off, just do so here. Already active pipes will have * their watermarks updated once we update their planes. */ - if (crtc->state->active_changed) { + if (intel_crtc->base.state->active_changed) { int plane; - for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) + for_each_universal_plane(dev_priv, pipe, plane) skl_write_plane_wm(intel_crtc, &pipe_wm->planes[plane], &results->ddb, plane); @@ -4388,10 +4373,8 @@ void skl_wm_get_hw_state(struct drm_device *dev) skl_pipe_wm_get_hw_state(crtc, &cstate->wm.skl.optimal); - if (intel_crtc->active) { + if (intel_crtc->active) hw->dirty_pipes |= drm_crtc_mask(crtc); - intel_crtc->wm.active.skl = cstate->wm.skl.optimal; - } } if (dev_priv->active_crtcs) { @@ -4553,11 +4536,11 @@ void vlv_wm_get_hw_state(struct drm_device *dev) plane->wm.fifo_size = 63; break; case DRM_PLANE_TYPE_PRIMARY: - plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, 0); + plane->wm.fifo_size = vlv_get_fifo_size(dev_priv, plane->pipe, 0); break; case DRM_PLANE_TYPE_OVERLAY: sprite = plane->plane; - plane->wm.fifo_size = vlv_get_fifo_size(dev, plane->pipe, sprite + 1); + plane->wm.fifo_size = vlv_get_fifo_size(dev_priv, plane->pipe, sprite + 1); break; } } @@ -4670,9 +4653,9 @@ void ilk_wm_get_hw_state(struct drm_device *dev) * We don't use the sprite, so we can ignore that. And on Crestline we have * to set the non-SR watermarks to 8. */ -void intel_update_watermarks(struct drm_crtc *crtc) +void intel_update_watermarks(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->dev); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); if (dev_priv->display.update_wm) dev_priv->display.update_wm(crtc); @@ -5878,7 +5861,7 @@ static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv) if (WARN_ON(!dev_priv->vlv_pctx)) return; - i915_gem_object_put_unlocked(dev_priv->vlv_pctx); + i915_gem_object_put(dev_priv->vlv_pctx); dev_priv->vlv_pctx = NULL; } @@ -6862,10 +6845,8 @@ void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv) } } -static void ibx_init_clock_gating(struct drm_device *dev) +static void ibx_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* * On Ibex Peak and Cougar Point, we need to disable clock * gating for the panel power sequencer or it will fail to @@ -6874,9 +6855,8 @@ static void ibx_init_clock_gating(struct drm_device *dev) I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE); } -static void g4x_disable_trickle_feed(struct drm_device *dev) +static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; for_each_pipe(dev_priv, pipe) { @@ -6889,10 +6869,8 @@ static void g4x_disable_trickle_feed(struct drm_device *dev) } } -static void ilk_init_lp_watermarks(struct drm_device *dev) +static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN); I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN); I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN); @@ -6903,9 +6881,8 @@ static void ilk_init_lp_watermarks(struct drm_device *dev) */ } -static void ironlake_init_clock_gating(struct drm_device *dev) +static void ironlake_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; /* @@ -6937,7 +6914,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev) (I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS)); - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); /* * Based on the document from hardware guys the following bits @@ -6972,14 +6949,13 @@ static void ironlake_init_clock_gating(struct drm_device *dev) /* WaDisable_RenderCache_OperationalFlush:ilk */ I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); - ibx_init_clock_gating(dev); + ibx_init_clock_gating(dev_priv); } -static void cpt_init_clock_gating(struct drm_device *dev) +static void cpt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); int pipe; uint32_t val; @@ -7014,9 +6990,8 @@ static void cpt_init_clock_gating(struct drm_device *dev) } } -static void gen6_check_mch_setup(struct drm_device *dev) +static void gen6_check_mch_setup(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t tmp; tmp = I915_READ(MCH_SSKPD); @@ -7025,9 +7000,8 @@ static void gen6_check_mch_setup(struct drm_device *dev) tmp); } -static void gen6_init_clock_gating(struct drm_device *dev) +static void gen6_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE; I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate); @@ -7054,7 +7028,7 @@ static void gen6_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_GT_MODE, _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4)); - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB)); @@ -7115,11 +7089,11 @@ static void gen6_init_clock_gating(struct drm_device *dev) ILK_DPARBUNIT_CLOCK_GATE_ENABLE | ILK_DPFDUNIT_CLOCK_GATE_ENABLE); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); - cpt_init_clock_gating(dev); + cpt_init_clock_gating(dev_priv); - gen6_check_mch_setup(dev); + gen6_check_mch_setup(dev_priv); } static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) @@ -7140,10 +7114,8 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv) I915_WRITE(GEN7_FF_THREAD_MODE, reg); } -static void lpt_init_clock_gating(struct drm_device *dev) +static void lpt_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* * TODO: this bit should only be enabled when really needed, then * disabled when not needed anymore in order to save power. @@ -7159,10 +7131,8 @@ static void lpt_init_clock_gating(struct drm_device *dev) TRANS_CHICKEN1_DP0UNIT_GC_DISABLE); } -static void lpt_suspend_hw(struct drm_device *dev) +static void lpt_suspend_hw(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - if (HAS_PCH_LPT_LP(dev_priv)) { uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D); @@ -7194,11 +7164,9 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv, I915_WRITE(GEN7_MISCCPCTL, misccpctl); } -static void kabylake_init_clock_gating(struct drm_device *dev) +static void kabylake_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - gen9_init_clock_gating(dev); + gen9_init_clock_gating(dev_priv); /* WaDisableSDEUnitClockGating:kbl */ if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0)) @@ -7215,11 +7183,9 @@ static void kabylake_init_clock_gating(struct drm_device *dev) ILK_DPFC_NUKE_ON_ANY_MODIFICATION); } -static void skylake_init_clock_gating(struct drm_device *dev) +static void skylake_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; - - gen9_init_clock_gating(dev); + gen9_init_clock_gating(dev_priv); /* WAC6entrylatency:skl */ I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) | @@ -7230,12 +7196,11 @@ static void skylake_init_clock_gating(struct drm_device *dev) ILK_DPFC_NUKE_ON_ANY_MODIFICATION); } -static void broadwell_init_clock_gating(struct drm_device *dev) +static void broadwell_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); enum pipe pipe; - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); /* WaSwitchSolVfFArbitrationPriority:bdw */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL); @@ -7278,14 +7243,12 @@ static void broadwell_init_clock_gating(struct drm_device *dev) I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1) | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT); - lpt_init_clock_gating(dev); + lpt_init_clock_gating(dev_priv); } -static void haswell_init_clock_gating(struct drm_device *dev) +static void haswell_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); /* L3 caching of data atomics doesn't work -- disable it. */ I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE); @@ -7334,15 +7297,14 @@ static void haswell_init_clock_gating(struct drm_device *dev) I915_WRITE(CHICKEN_PAR1_1, I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES); - lpt_init_clock_gating(dev); + lpt_init_clock_gating(dev_priv); } -static void ivybridge_init_clock_gating(struct drm_device *dev) +static void ivybridge_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t snpcr; - ilk_init_lp_watermarks(dev); + ilk_init_lp_watermarks(dev_priv); I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE); @@ -7399,7 +7361,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) | GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); gen7_setup_fixed_func_scheduler(dev_priv); @@ -7430,15 +7392,13 @@ static void ivybridge_init_clock_gating(struct drm_device *dev) I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr); if (!HAS_PCH_NOP(dev_priv)) - cpt_init_clock_gating(dev); + cpt_init_clock_gating(dev_priv); - gen6_check_mch_setup(dev); + gen6_check_mch_setup(dev_priv); } -static void valleyview_init_clock_gating(struct drm_device *dev) +static void valleyview_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* WaDisableEarlyCull:vlv */ I915_WRITE(_3D_CHICKEN3, _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL)); @@ -7517,10 +7477,8 @@ static void valleyview_init_clock_gating(struct drm_device *dev) I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS); } -static void cherryview_init_clock_gating(struct drm_device *dev) +static void cherryview_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - /* WaVSRefCountFullforceMissDisable:chv */ /* WaDSRefCountFullforceMissDisable:chv */ I915_WRITE(GEN7_FF_THREAD_MODE, @@ -7553,9 +7511,8 @@ static void cherryview_init_clock_gating(struct drm_device *dev) I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL); } -static void g4x_init_clock_gating(struct drm_device *dev) +static void g4x_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); uint32_t dspclk_gate; I915_WRITE(RENCLK_GATE_D1, 0); @@ -7577,13 +7534,11 @@ static void g4x_init_clock_gating(struct drm_device *dev) /* WaDisable_RenderCache_OperationalFlush:g4x */ I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); - g4x_disable_trickle_feed(dev); + g4x_disable_trickle_feed(dev_priv); } -static void crestline_init_clock_gating(struct drm_device *dev) +static void crestline_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE); I915_WRITE(RENCLK_GATE_D2, 0); I915_WRITE(DSPCLK_GATE_D, 0); @@ -7596,10 +7551,8 @@ static void crestline_init_clock_gating(struct drm_device *dev) I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } -static void broadwater_init_clock_gating(struct drm_device *dev) +static void broadwater_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE | I965_RCC_CLOCK_GATE_DISABLE | I965_RCPB_CLOCK_GATE_DISABLE | @@ -7613,16 +7566,15 @@ static void broadwater_init_clock_gating(struct drm_device *dev) I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE)); } -static void gen3_init_clock_gating(struct drm_device *dev) +static void gen3_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); u32 dstate = I915_READ(D_STATE); dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING | DSTATE_DOT_CLOCK_GATING; I915_WRITE(D_STATE, dstate); - if (IS_PINEVIEW(dev)) + if (IS_PINEVIEW(dev_priv)) I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY)); /* IIR "flip pending" means done if this bit is set */ @@ -7638,10 +7590,8 @@ static void gen3_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE)); } -static void i85x_init_clock_gating(struct drm_device *dev) +static void i85x_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE); /* interrupts should cause a wake up from C3 */ @@ -7652,10 +7602,8 @@ static void i85x_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE)); } -static void i830_init_clock_gating(struct drm_device *dev) +static void i830_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); I915_WRITE(MEM_MODE, @@ -7663,20 +7611,18 @@ static void i830_init_clock_gating(struct drm_device *dev) _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE)); } -void intel_init_clock_gating(struct drm_device *dev) +void intel_init_clock_gating(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - - dev_priv->display.init_clock_gating(dev); + dev_priv->display.init_clock_gating(dev_priv); } -void intel_suspend_hw(struct drm_device *dev) +void intel_suspend_hw(struct drm_i915_private *dev_priv) { - if (HAS_PCH_LPT(to_i915(dev))) - lpt_suspend_hw(dev); + if (HAS_PCH_LPT(dev_priv)) + lpt_suspend_hw(dev_priv); } -static void nop_init_clock_gating(struct drm_device *dev) +static void nop_init_clock_gating(struct drm_i915_private *dev_priv) { DRM_DEBUG_KMS("No clock gating settings or workarounds applied.\n"); } @@ -7731,25 +7677,23 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv) } /* Set up chip specific power management-related functions */ -void intel_init_pm(struct drm_device *dev) +void intel_init_pm(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = to_i915(dev); - intel_fbc_init(dev_priv); /* For cxsr */ - if (IS_PINEVIEW(dev)) - i915_pineview_get_mem_freq(dev); + if (IS_PINEVIEW(dev_priv)) + i915_pineview_get_mem_freq(dev_priv); else if (IS_GEN5(dev_priv)) - i915_ironlake_get_mem_freq(dev); + i915_ironlake_get_mem_freq(dev_priv); /* For FIFO watermark updates */ - if (INTEL_INFO(dev)->gen >= 9) { - skl_setup_wm_latency(dev); + if (INTEL_GEN(dev_priv) >= 9) { + skl_setup_wm_latency(dev_priv); dev_priv->display.update_wm = skl_update_wm; dev_priv->display.compute_global_watermarks = skl_compute_wm; } else if (HAS_PCH_SPLIT(dev_priv)) { - ilk_setup_wm_latency(dev); + ilk_setup_wm_latency(dev_priv); if ((IS_GEN5(dev_priv) && dev_priv->wm.pri_latency[1] && dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) || @@ -7767,12 +7711,12 @@ void intel_init_pm(struct drm_device *dev) "Disable CxSR\n"); } } else if (IS_CHERRYVIEW(dev_priv)) { - vlv_setup_wm_latency(dev); + vlv_setup_wm_latency(dev_priv); dev_priv->display.update_wm = vlv_update_wm; } else if (IS_VALLEYVIEW(dev_priv)) { - vlv_setup_wm_latency(dev); + vlv_setup_wm_latency(dev_priv); dev_priv->display.update_wm = vlv_update_wm; - } else if (IS_PINEVIEW(dev)) { + } else if (IS_PINEVIEW(dev_priv)) { if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev_priv), dev_priv->is_ddr3, dev_priv->fsb_freq, @@ -7795,7 +7739,7 @@ void intel_init_pm(struct drm_device *dev) dev_priv->display.update_wm = i9xx_update_wm; dev_priv->display.get_fifo_size = i9xx_get_fifo_size; } else if (IS_GEN2(dev_priv)) { - if (INTEL_INFO(dev)->num_pipes == 1) { + if (INTEL_INFO(dev_priv)->num_pipes == 1) { dev_priv->display.update_wm = i845_update_wm; dev_priv->display.get_fifo_size = i845_get_fifo_size; } else { @@ -8043,5 +7987,4 @@ void intel_pm_setup(struct drm_device *dev) dev_priv->pm.suspended = false; atomic_set(&dev_priv->pm.wakeref_count, 0); - atomic_set(&dev_priv->pm.atomic_seq, 0); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 32786ba199b9..700e93d80616 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -648,7 +648,7 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req) if (ret != 0) return ret; - ret = i915_gem_render_state_init(req); + ret = i915_gem_render_state_emit(req); if (ret) return ret; @@ -1213,90 +1213,62 @@ static void render_ring_cleanup(struct intel_engine_cs *engine) i915_vma_unpin_and_release(&dev_priv->semaphore); } -static int gen8_rcs_signal(struct drm_i915_gem_request *req) +static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *out) { - struct intel_ring *ring = req->ring; struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *waiter; enum intel_engine_id id; - int ret, num_rings; - - num_rings = INTEL_INFO(dev_priv)->num_rings; - ret = intel_ring_begin(req, (num_rings-1) * 8); - if (ret) - return ret; for_each_engine(waiter, dev_priv, id) { u64 gtt_offset = req->engine->semaphore.signal_ggtt[id]; if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, - PIPE_CONTROL_GLOBAL_GTT_IVB | - PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_CS_STALL); - intel_ring_emit(ring, lower_32_bits(gtt_offset)); - intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, - MI_SEMAPHORE_SIGNAL | - MI_SEMAPHORE_TARGET(waiter->hw_id)); - intel_ring_emit(ring, 0); + *out++ = GFX_OP_PIPE_CONTROL(6); + *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_CS_STALL); + *out++ = lower_32_bits(gtt_offset); + *out++ = upper_32_bits(gtt_offset); + *out++ = req->global_seqno; + *out++ = 0; + *out++ = (MI_SEMAPHORE_SIGNAL | + MI_SEMAPHORE_TARGET(waiter->hw_id)); + *out++ = 0; } - intel_ring_advance(ring); - return 0; + return out; } -static int gen8_xcs_signal(struct drm_i915_gem_request *req) +static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *out) { - struct intel_ring *ring = req->ring; struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *waiter; enum intel_engine_id id; - int ret, num_rings; - - num_rings = INTEL_INFO(dev_priv)->num_rings; - ret = intel_ring_begin(req, (num_rings-1) * 6); - if (ret) - return ret; for_each_engine(waiter, dev_priv, id) { u64 gtt_offset = req->engine->semaphore.signal_ggtt[id]; if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID) continue; - intel_ring_emit(ring, - (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW); - intel_ring_emit(ring, - lower_32_bits(gtt_offset) | - MI_FLUSH_DW_USE_GTT); - intel_ring_emit(ring, upper_32_bits(gtt_offset)); - intel_ring_emit(ring, req->fence.seqno); - intel_ring_emit(ring, - MI_SEMAPHORE_SIGNAL | - MI_SEMAPHORE_TARGET(waiter->hw_id)); - intel_ring_emit(ring, 0); + *out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; + *out++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT; + *out++ = upper_32_bits(gtt_offset); + *out++ = req->global_seqno; + *out++ = (MI_SEMAPHORE_SIGNAL | + MI_SEMAPHORE_TARGET(waiter->hw_id)); + *out++ = 0; } - intel_ring_advance(ring); - return 0; + return out; } -static int gen6_signal(struct drm_i915_gem_request *req) +static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *out) { - struct intel_ring *ring = req->ring; struct drm_i915_private *dev_priv = req->i915; struct intel_engine_cs *engine; enum intel_engine_id id; - int ret, num_rings; - - num_rings = INTEL_INFO(dev_priv)->num_rings; - ret = intel_ring_begin(req, round_up((num_rings-1) * 3, 2)); - if (ret) - return ret; + int num_rings = 0; for_each_engine(engine, dev_priv, id) { i915_reg_t mbox_reg; @@ -1306,101 +1278,78 @@ static int gen6_signal(struct drm_i915_gem_request *req) mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id]; if (i915_mmio_reg_valid(mbox_reg)) { - intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); - intel_ring_emit_reg(ring, mbox_reg); - intel_ring_emit(ring, req->fence.seqno); + *out++ = MI_LOAD_REGISTER_IMM(1); + *out++ = i915_mmio_reg_offset(mbox_reg); + *out++ = req->global_seqno; + num_rings++; } } + if (num_rings & 1) + *out++ = MI_NOOP; - /* If num_dwords was rounded, make sure the tail pointer is correct */ - if (num_rings % 2 == 0) - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); - - return 0; + return out; } static void i9xx_submit_request(struct drm_i915_gem_request *request) { struct drm_i915_private *dev_priv = request->i915; - I915_WRITE_TAIL(request->engine, - intel_ring_offset(request->ring, request->tail)); + I915_WRITE_TAIL(request->engine, request->tail); } -static int i9xx_emit_request(struct drm_i915_gem_request *req) +static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req, + u32 *out) { - struct intel_ring *ring = req->ring; - int ret; + *out++ = MI_STORE_DWORD_INDEX; + *out++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT; + *out++ = req->global_seqno; + *out++ = MI_USER_INTERRUPT; - ret = intel_ring_begin(req, 4); - if (ret) - return ret; - - intel_ring_emit(ring, MI_STORE_DWORD_INDEX); - intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(ring, req->fence.seqno); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_advance(ring); - - req->tail = ring->tail; - - return 0; + req->tail = intel_ring_offset(req->ring, out); } +static const int i9xx_emit_breadcrumb_sz = 4; + /** - * gen6_sema_emit_request - Update the semaphore mailbox registers + * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers * * @request - request to write to the ring * * Update the mailbox registers in the *other* rings with the current seqno. * This acts like a signal in the canonical semaphore. */ -static int gen6_sema_emit_request(struct drm_i915_gem_request *req) +static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req, + u32 *out) { - int ret; - - ret = req->engine->semaphore.signal(req); - if (ret) - return ret; - - return i9xx_emit_request(req); + return i9xx_emit_breadcrumb(req, + req->engine->semaphore.signal(req, out)); } -static int gen8_render_emit_request(struct drm_i915_gem_request *req) +static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req, + u32 *out) { struct intel_engine_cs *engine = req->engine; - struct intel_ring *ring = req->ring; - int ret; - if (engine->semaphore.signal) { - ret = engine->semaphore.signal(req); - if (ret) - return ret; - } + if (engine->semaphore.signal) + out = engine->semaphore.signal(req, out); - ret = intel_ring_begin(req, 8); - if (ret) - return ret; - - intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6)); - intel_ring_emit(ring, (PIPE_CONTROL_GLOBAL_GTT_IVB | + *out++ = GFX_OP_PIPE_CONTROL(6); + *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_QW_WRITE)); - intel_ring_emit(ring, intel_hws_seqno_address(engine)); - intel_ring_emit(ring, 0); - intel_ring_emit(ring, i915_gem_request_get_seqno(req)); + PIPE_CONTROL_QW_WRITE); + *out++ = intel_hws_seqno_address(engine); + *out++ = 0; + *out++ = req->global_seqno; /* We're thrashing one dword of HWS. */ - intel_ring_emit(ring, 0); - intel_ring_emit(ring, MI_USER_INTERRUPT); - intel_ring_emit(ring, MI_NOOP); - intel_ring_advance(ring); + *out++ = 0; + *out++ = MI_USER_INTERRUPT; + *out++ = MI_NOOP; - req->tail = ring->tail; - - return 0; + req->tail = intel_ring_offset(req->ring, out); } +static const int gen8_render_emit_breadcrumb_sz = 8; + /** * intel_ring_sync - sync the waiter to the signaller on seqno * @@ -1427,7 +1376,7 @@ gen8_ring_sync_to(struct drm_i915_gem_request *req, MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT | MI_SEMAPHORE_SAD_GTE_SDD); - intel_ring_emit(ring, signal->fence.seqno); + intel_ring_emit(ring, signal->global_seqno); intel_ring_emit(ring, lower_32_bits(offset)); intel_ring_emit(ring, upper_32_bits(offset)); intel_ring_advance(ring); @@ -1465,7 +1414,7 @@ gen6_ring_sync_to(struct drm_i915_gem_request *req, * seqno is >= the last seqno executed. However for hardware the * comparison is strictly greater than. */ - intel_ring_emit(ring, signal->fence.seqno - 1); + intel_ring_emit(ring, signal->global_seqno - 1); intel_ring_emit(ring, 0); intel_ring_emit(ring, MI_NOOP); intel_ring_advance(ring); @@ -1608,7 +1557,7 @@ hsw_vebox_irq_enable(struct intel_engine_cs *engine) struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_IMR(engine, ~engine->irq_enable_mask); - gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask); + gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask); } static void @@ -1617,7 +1566,7 @@ hsw_vebox_irq_disable(struct intel_engine_cs *engine) struct drm_i915_private *dev_priv = engine->i915; I915_WRITE_IMR(engine, ~0); - gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask); + gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask); } static void @@ -1762,14 +1711,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine) static void cleanup_status_page(struct intel_engine_cs *engine) { struct i915_vma *vma; + struct drm_i915_gem_object *obj; vma = fetch_and_zero(&engine->status_page.vma); if (!vma) return; + obj = vma->obj; + i915_vma_unpin(vma); - i915_gem_object_unpin_map(vma->obj); - i915_vma_put(vma); + i915_vma_close(vma); + + i915_gem_object_unpin_map(obj); + __i915_gem_object_release_unless_active(obj); } static int init_status_page(struct intel_engine_cs *engine) @@ -1777,9 +1731,10 @@ static int init_status_page(struct intel_engine_cs *engine) struct drm_i915_gem_object *obj; struct i915_vma *vma; unsigned int flags; + void *vaddr; int ret; - obj = i915_gem_object_create(&engine->i915->drm, 4096); + obj = i915_gem_object_create_internal(engine->i915, 4096); if (IS_ERR(obj)) { DRM_ERROR("Failed to allocate status page\n"); return PTR_ERR(obj); @@ -1812,15 +1767,22 @@ static int init_status_page(struct intel_engine_cs *engine) if (ret) goto err; + vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(vaddr)) { + ret = PTR_ERR(vaddr); + goto err_unpin; + } + engine->status_page.vma = vma; engine->status_page.ggtt_offset = i915_ggtt_offset(vma); - engine->status_page.page_addr = - i915_gem_object_pin_map(obj, I915_MAP_WB); + engine->status_page.page_addr = memset(vaddr, 0, 4096); DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", engine->name, i915_ggtt_offset(vma)); return 0; +err_unpin: + i915_vma_unpin(vma); err: i915_gem_object_put(obj); return ret; @@ -1967,7 +1929,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size) void intel_ring_free(struct intel_ring *ring) { - i915_vma_put(ring->vma); + struct drm_i915_gem_object *obj = ring->vma->obj; + + i915_vma_close(ring->vma); + __i915_gem_object_release_unless_active(obj); + kfree(ring); } @@ -1983,14 +1949,13 @@ static int intel_ring_context_pin(struct i915_gem_context *ctx, return 0; if (ce->state) { - ret = i915_gem_object_set_to_gtt_domain(ce->state->obj, false); - if (ret) - goto error; + struct i915_vma *vma; - ret = i915_vma_pin(ce->state, 0, ctx->ggtt_alignment, - PIN_GLOBAL | PIN_HIGH); - if (ret) + vma = i915_gem_context_pin_legacy(ctx, PIN_HIGH); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); goto error; + } } /* The kernel context is only used as a placeholder for flushing the @@ -2037,9 +2002,6 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine) intel_engine_setup_common(engine); - memset(engine->semaphore.sync_seqno, 0, - sizeof(engine->semaphore.sync_seqno)); - ret = intel_engine_init_common(engine); if (ret) goto error; @@ -2155,7 +2117,9 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes) { struct intel_ring *ring = req->ring; struct drm_i915_gem_request *target; - int ret; + long timeout; + + lockdep_assert_held(&req->i915->drm.struct_mutex); intel_ring_update_space(ring); if (ring->space >= bytes) @@ -2185,11 +2149,11 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes) if (WARN_ON(&target->ring_link == &ring->request_list)) return -ENOSPC; - ret = i915_wait_request(target, - I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, - NULL, NO_WAITBOOST); - if (ret) - return ret; + timeout = i915_wait_request(target, + I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT); + if (timeout < 0) + return timeout; i915_gem_request_retire_upto(target); @@ -2618,9 +2582,22 @@ static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv, engine->init_hw = init_ring_common; engine->reset_hw = reset_ring_common; - engine->emit_request = i9xx_emit_request; - if (i915.semaphores) - engine->emit_request = gen6_sema_emit_request; + engine->emit_breadcrumb = i9xx_emit_breadcrumb; + engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz; + if (i915.semaphores) { + int num_rings; + + engine->emit_breadcrumb = gen6_sema_emit_breadcrumb; + + num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1; + if (INTEL_GEN(dev_priv) >= 8) { + engine->emit_breadcrumb_sz += num_rings * 6; + } else { + engine->emit_breadcrumb_sz += num_rings * 3; + if (num_rings & 1) + engine->emit_breadcrumb_sz++; + } + } engine->submit_request = i9xx_submit_request; if (INTEL_GEN(dev_priv) >= 8) @@ -2647,10 +2624,18 @@ int intel_init_render_ring_buffer(struct intel_engine_cs *engine) if (INTEL_GEN(dev_priv) >= 8) { engine->init_context = intel_rcs_ctx_init; - engine->emit_request = gen8_render_emit_request; + engine->emit_breadcrumb = gen8_render_emit_breadcrumb; + engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz; engine->emit_flush = gen8_render_ring_flush; - if (i915.semaphores) + if (i915.semaphores) { + int num_rings; + engine->semaphore.signal = gen8_rcs_signal; + + num_rings = + hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1; + engine->emit_breadcrumb_sz += num_rings * 6; + } } else if (INTEL_GEN(dev_priv) >= 6) { engine->init_context = intel_rcs_ctx_init; engine->emit_flush = gen7_render_ring_flush; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index 32b2e6332ccf..642b54692d0d 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -4,6 +4,7 @@ #include <linux/hashtable.h> #include "i915_gem_batch_pool.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" #define I915_CMD_HASH_ORDER 9 @@ -157,6 +158,7 @@ struct i915_ctx_workarounds { }; struct drm_i915_gem_request; +struct intel_render_state; struct intel_engine_cs { struct drm_i915_private *i915; @@ -168,7 +170,6 @@ struct intel_engine_cs { VCS2, /* Keep instances of the same type engine together. */ VECS } id; -#define I915_NUM_ENGINES 5 #define _VCS(n) (VCS + (n)) unsigned int exec_id; enum intel_engine_hw_id { @@ -179,10 +180,12 @@ struct intel_engine_cs { VCS2_HW } hw_id; enum intel_engine_hw_id guc_id; /* XXX same as hw_id? */ - u64 fence_context; u32 mmio_base; unsigned int irq_shift; struct intel_ring *buffer; + struct intel_timeline *timeline; + + struct intel_render_state *render_state; /* Rather than have every client wait upon all user interrupts, * with the herd waking after every interrupt and each doing the @@ -204,7 +207,7 @@ struct intel_engine_cs { struct task_struct __rcu *irq_seqno_bh; /* bh for interrupts */ bool irq_posted; - spinlock_t lock; /* protects the lists of requests */ + spinlock_t lock; /* protects the lists of requests; irqsafe */ struct rb_root waiters; /* sorted by retirement, priority */ struct rb_root signals; /* sorted by retirement */ struct intel_wait *first_wait; /* oldest waiter by retirement */ @@ -252,7 +255,9 @@ struct intel_engine_cs { #define I915_DISPATCH_SECURE BIT(0) #define I915_DISPATCH_PINNED BIT(1) #define I915_DISPATCH_RS BIT(2) - int (*emit_request)(struct drm_i915_gem_request *req); + void (*emit_breadcrumb)(struct drm_i915_gem_request *req, + u32 *out); + int emit_breadcrumb_sz; /* Pass the request to the hardware queue (e.g. directly into * the legacy ringbuffer or to the end of an execlist). @@ -309,8 +314,6 @@ struct intel_engine_cs { * ie. transpose of f(x, y) */ struct { - u32 sync_seqno[I915_NUM_ENGINES-1]; - union { #define GEN6_SEMAPHORE_LAST VECS_HW #define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1) @@ -327,7 +330,7 @@ struct intel_engine_cs { /* AKA wait() */ int (*sync_to)(struct drm_i915_gem_request *req, struct drm_i915_gem_request *signal); - int (*signal)(struct drm_i915_gem_request *req); + u32 *(*signal)(struct drm_i915_gem_request *req, u32 *out); } semaphore; /* Execlists */ @@ -343,27 +346,6 @@ struct intel_engine_cs { bool preempt_wa; u32 ctx_desc_template; - /** - * List of breadcrumbs associated with GPU requests currently - * outstanding. - */ - struct list_head request_list; - - /** - * Seqno of request most recently submitted to request_list. - * Used exclusively by hang checker to avoid grabbing lock while - * inspecting request list. - */ - u32 last_submitted_seqno; - u32 last_pending_seqno; - - /* An RCU guarded pointer to the last request. No reference is - * held to the request, users must carefully acquire a reference to - * the request using i915_gem_active_get_rcu(), or hold the - * struct_mutex. - */ - struct i915_gem_active last_request; - struct i915_gem_context *last_context; struct intel_engine_hangcheck hangcheck; @@ -401,27 +383,6 @@ intel_engine_flag(const struct intel_engine_cs *engine) return 1 << engine->id; } -static inline u32 -intel_engine_sync_index(struct intel_engine_cs *engine, - struct intel_engine_cs *other) -{ - int idx; - - /* - * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2; - * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs; - * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs; - * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs; - * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs; - */ - - idx = (other->id - engine->id) - 1; - if (idx < 0) - idx += I915_NUM_ENGINES; - - return idx; -} - static inline void intel_flush_status_page(struct intel_engine_cs *engine, int reg) { @@ -504,30 +465,23 @@ static inline void intel_ring_advance(struct intel_ring *ring) */ } -static inline u32 intel_ring_offset(struct intel_ring *ring, u32 value) +static inline u32 intel_ring_offset(struct intel_ring *ring, void *addr) { /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */ - return value & (ring->size - 1); + u32 offset = addr - ring->vaddr; + return offset & (ring->size - 1); } int __intel_ring_space(int head, int tail, int size); void intel_ring_update_space(struct intel_ring *ring); -void intel_engine_init_seqno(struct intel_engine_cs *engine, u32 seqno); +void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno); void intel_engine_setup_common(struct intel_engine_cs *engine); int intel_engine_init_common(struct intel_engine_cs *engine); int intel_engine_create_scratch(struct intel_engine_cs *engine, int size); void intel_engine_cleanup_common(struct intel_engine_cs *engine); -static inline int intel_engine_idle(struct intel_engine_cs *engine, - unsigned int flags) -{ - /* Wait upon the last request to be completed */ - return i915_gem_active_wait_unlocked(&engine->last_request, - flags, NULL, NULL); -} - int intel_init_render_ring_buffer(struct intel_engine_cs *engine); int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine); int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine); @@ -542,6 +496,18 @@ static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine) return intel_read_status_page(engine, I915_GEM_HWS_INDEX); } +static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine) +{ + /* We are only peeking at the tail of the submit queue (and not the + * queue itself) in order to gain a hint as to the current active + * state of the engine. Callers are not expected to be taking + * engine->timeline->lock, nor are they expected to be concerned + * wtih serialising this hint with anything, so document it as + * a hint and nothing more. + */ + return READ_ONCE(engine->timeline->last_submitted_seqno); +} + int init_workarounds_ring(struct intel_engine_cs *engine); void intel_engine_get_instdone(struct intel_engine_cs *engine, @@ -615,9 +581,4 @@ void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); unsigned int intel_kick_waiters(struct drm_i915_private *i915); unsigned int intel_kick_signalers(struct drm_i915_private *i915); -static inline bool intel_engine_is_active(struct intel_engine_cs *engine) -{ - return i915_gem_active_isset(&engine->last_request); -} - #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index ee56a8756c07..05994083e161 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -330,7 +330,7 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv, * sure vgacon can keep working normally without triggering interrupts * and error messages. */ - if (power_well->data == SKL_DISP_PW_2) { + if (power_well->id == SKL_DISP_PW_2) { vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO); outb(inb(VGA_MSR_READ), VGA_MSR_WRITE); vga_put(pdev, VGA_RSRC_LEGACY_IO); @@ -343,7 +343,7 @@ static void skl_power_well_post_enable(struct drm_i915_private *dev_priv, static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - if (power_well->data == SKL_DISP_PW_2) + if (power_well->id == SKL_DISP_PW_2) gen8_irq_power_well_pre_disable(dev_priv, 1 << PIPE_C | 1 << PIPE_B); } @@ -658,7 +658,7 @@ static void gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum skl_disp_power_wells power_well_id = power_well->data; + enum skl_disp_power_wells power_well_id = power_well->id; u32 val; u32 mask; @@ -703,7 +703,7 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, tmp = I915_READ(HSW_PWR_WELL_DRIVER); fuse_status = I915_READ(SKL_FUSE_STATUS); - switch (power_well->data) { + switch (power_well->id) { case SKL_DISP_PW_1: if (intel_wait_for_register(dev_priv, SKL_FUSE_STATUS, @@ -727,13 +727,13 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, case SKL_DISP_PW_MISC_IO: break; default: - WARN(1, "Unknown power well %lu\n", power_well->data); + WARN(1, "Unknown power well %lu\n", power_well->id); return; } - req_mask = SKL_POWER_WELL_REQ(power_well->data); + req_mask = SKL_POWER_WELL_REQ(power_well->id); enable_requested = tmp & req_mask; - state_mask = SKL_POWER_WELL_STATE(power_well->data); + state_mask = SKL_POWER_WELL_STATE(power_well->id); is_enabled = tmp & state_mask; if (!enable && enable_requested) @@ -769,14 +769,14 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv, power_well->name, enable ? "enable" : "disable"); if (check_fuse_status) { - if (power_well->data == SKL_DISP_PW_1) { + if (power_well->id == SKL_DISP_PW_1) { if (intel_wait_for_register(dev_priv, SKL_FUSE_STATUS, SKL_FUSE_PG1_DIST_STATUS, SKL_FUSE_PG1_DIST_STATUS, 1)) DRM_ERROR("PG1 distributing status timeout\n"); - } else if (power_well->data == SKL_DISP_PW_2) { + } else if (power_well->id == SKL_DISP_PW_2) { if (intel_wait_for_register(dev_priv, SKL_FUSE_STATUS, SKL_FUSE_PG2_DIST_STATUS, @@ -818,8 +818,8 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv, static bool skl_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - uint32_t mask = SKL_POWER_WELL_REQ(power_well->data) | - SKL_POWER_WELL_STATE(power_well->data); + uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) | + SKL_POWER_WELL_STATE(power_well->id); return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask; } @@ -845,45 +845,22 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv, skl_set_power_well(dev_priv, power_well, false); } -static enum dpio_phy bxt_power_well_to_phy(struct i915_power_well *power_well) -{ - enum skl_disp_power_wells power_well_id = power_well->data; - - return power_well_id == BXT_DPIO_CMN_A ? DPIO_PHY1 : DPIO_PHY0; -} - static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum skl_disp_power_wells power_well_id = power_well->data; - struct i915_power_well *cmn_a_well = NULL; - - if (power_well_id == BXT_DPIO_CMN_BC) { - /* - * We need to copy the GRC calibration value from the eDP PHY, - * so make sure it's powered up. - */ - cmn_a_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A); - intel_power_well_get(dev_priv, cmn_a_well); - } - - bxt_ddi_phy_init(dev_priv, bxt_power_well_to_phy(power_well)); - - if (cmn_a_well) - intel_power_well_put(dev_priv, cmn_a_well); + bxt_ddi_phy_init(dev_priv, power_well->data); } static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - bxt_ddi_phy_uninit(dev_priv, bxt_power_well_to_phy(power_well)); + bxt_ddi_phy_uninit(dev_priv, power_well->data); } static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - return bxt_ddi_phy_is_enabled(dev_priv, - bxt_power_well_to_phy(power_well)); + return bxt_ddi_phy_is_enabled(dev_priv, power_well->data); } static void bxt_dpio_cmn_power_well_sync_hw(struct drm_i915_private *dev_priv, @@ -902,13 +879,11 @@ static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv) power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A); if (power_well->count > 0) - bxt_ddi_phy_verify_state(dev_priv, - bxt_power_well_to_phy(power_well)); + bxt_ddi_phy_verify_state(dev_priv, power_well->data); power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC); if (power_well->count > 0) - bxt_ddi_phy_verify_state(dev_priv, - bxt_power_well_to_phy(power_well)); + bxt_ddi_phy_verify_state(dev_priv, power_well->data); } static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv, @@ -932,7 +907,7 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv, gen9_set_dc_state(dev_priv, DC_STATE_DISABLE); WARN_ON(dev_priv->cdclk_freq != - dev_priv->display.get_display_clock_speed(&dev_priv->drm)); + dev_priv->display.get_display_clock_speed(dev_priv)); gen9_assert_dbuf_enabled(dev_priv); @@ -975,7 +950,7 @@ static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv, static void vlv_set_power_well(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool enable) { - enum punit_power_well power_well_id = power_well->data; + enum punit_power_well power_well_id = power_well->id; u32 mask; u32 state; u32 ctrl; @@ -1029,7 +1004,7 @@ static void vlv_power_well_disable(struct drm_i915_private *dev_priv, static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - int power_well_id = power_well->data; + int power_well_id = power_well->id; bool enabled = false; u32 mask; u32 state; @@ -1138,13 +1113,15 @@ static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv) intel_power_sequencer_reset(dev_priv); - intel_hpd_poll_init(dev_priv); + /* Prevent us from re-enabling polling on accident in late suspend */ + if (!dev_priv->drm.dev->power.is_suspended) + intel_hpd_poll_init(dev_priv); } static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D); vlv_set_power_well(dev_priv, power_well, true); @@ -1154,7 +1131,7 @@ static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv, static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D); vlv_display_power_well_deinit(dev_priv); @@ -1164,7 +1141,7 @@ static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv, static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC); /* since ref/cri clock was enabled */ udelay(1); /* >10ns for cmnreset, >0ns for sidereset */ @@ -1190,7 +1167,7 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, { enum pipe pipe; - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC); for_each_pipe(dev_priv, pipe) assert_pll_disabled(dev_priv, pipe); @@ -1213,7 +1190,7 @@ static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_pr struct i915_power_well *power_well; power_well = &power_domains->power_wells[i]; - if (power_well->data == power_well_id) + if (power_well->id == power_well_id) return power_well; } @@ -1337,10 +1314,10 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, enum pipe pipe; uint32_t tmp; - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && - power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC && + power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D); - if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) { pipe = PIPE_A; phy = DPIO_PHY0; } else { @@ -1368,7 +1345,7 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, DPIO_SUS_CLK_CONFIG_GATE_CLKREQ; vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp); - if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) { tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1); tmp |= DPIO_DYNPWRDOWNEN_CH1; vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp); @@ -1399,10 +1376,10 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, { enum dpio_phy phy; - WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC && - power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D); + WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC && + power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D); - if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) { + if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) { phy = DPIO_PHY0; assert_pll_disabled(dev_priv, PIPE_A); assert_pll_disabled(dev_priv, PIPE_B); @@ -1551,7 +1528,7 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder, static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - enum pipe pipe = power_well->data; + enum pipe pipe = power_well->id; bool enabled; u32 state, ctrl; @@ -1581,7 +1558,7 @@ static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv, struct i915_power_well *power_well, bool enable) { - enum pipe pipe = power_well->data; + enum pipe pipe = power_well->id; u32 state; u32 ctrl; @@ -1614,7 +1591,7 @@ out: static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A); + WARN_ON_ONCE(power_well->id != PIPE_A); chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0); } @@ -1622,7 +1599,7 @@ static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv, static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A); + WARN_ON_ONCE(power_well->id != PIPE_A); chv_set_pipe_power_well(dev_priv, power_well, true); @@ -1632,7 +1609,7 @@ static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv, static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN_ON_ONCE(power_well->data != PIPE_A); + WARN_ON_ONCE(power_well->id != PIPE_A); vlv_display_power_well_deinit(dev_priv); @@ -1976,12 +1953,12 @@ static struct i915_power_well vlv_power_wells[] = { .always_on = 1, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, - .data = PUNIT_POWER_WELL_ALWAYS_ON, + .id = PUNIT_POWER_WELL_ALWAYS_ON, }, { .name = "display", .domains = VLV_DISPLAY_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DISP2D, + .id = PUNIT_POWER_WELL_DISP2D, .ops = &vlv_display_power_well_ops, }, { @@ -1991,7 +1968,7 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, + .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01, }, { .name = "dpio-tx-b-23", @@ -2000,7 +1977,7 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, + .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23, }, { .name = "dpio-tx-c-01", @@ -2009,7 +1986,7 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, + .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01, }, { .name = "dpio-tx-c-23", @@ -2018,12 +1995,12 @@ static struct i915_power_well vlv_power_wells[] = { VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS | VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS, .ops = &vlv_dpio_power_well_ops, - .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, + .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23, }, { .name = "dpio-common", .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .id = PUNIT_POWER_WELL_DPIO_CMN_BC, .ops = &vlv_dpio_cmn_power_well_ops, }, }; @@ -2043,19 +2020,19 @@ static struct i915_power_well chv_power_wells[] = { * required for any pipe to work. */ .domains = CHV_DISPLAY_POWER_DOMAINS, - .data = PIPE_A, + .id = PIPE_A, .ops = &chv_pipe_power_well_ops, }, { .name = "dpio-common-bc", .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_BC, + .id = PUNIT_POWER_WELL_DPIO_CMN_BC, .ops = &chv_dpio_cmn_power_well_ops, }, { .name = "dpio-common-d", .domains = CHV_DPIO_CMN_D_POWER_DOMAINS, - .data = PUNIT_POWER_WELL_DPIO_CMN_D, + .id = PUNIT_POWER_WELL_DPIO_CMN_D, .ops = &chv_dpio_cmn_power_well_ops, }, }; @@ -2078,57 +2055,57 @@ static struct i915_power_well skl_power_wells[] = { .always_on = 1, .domains = POWER_DOMAIN_MASK, .ops = &i9xx_always_on_power_well_ops, - .data = SKL_DISP_PW_ALWAYS_ON, + .id = SKL_DISP_PW_ALWAYS_ON, }, { .name = "power well 1", /* Handled by the DMC firmware */ .domains = 0, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_1, + .id = SKL_DISP_PW_1, }, { .name = "MISC IO power well", /* Handled by the DMC firmware */ .domains = 0, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_MISC_IO, + .id = SKL_DISP_PW_MISC_IO, }, { .name = "DC off", .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS, .ops = &gen9_dc_off_power_well_ops, - .data = SKL_DISP_PW_DC_OFF, + .id = SKL_DISP_PW_DC_OFF, }, { .name = "power well 2", .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_2, + .id = SKL_DISP_PW_2, }, { .name = "DDI A/E power well", .domains = SKL_DISPLAY_DDI_A_E_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_A_E, + .id = SKL_DISP_PW_DDI_A_E, }, { .name = "DDI B power well", .domains = SKL_DISPLAY_DDI_B_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_B, + .id = SKL_DISP_PW_DDI_B, }, { .name = "DDI C power well", .domains = SKL_DISPLAY_DDI_C_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_C, + .id = SKL_DISP_PW_DDI_C, }, { .name = "DDI D power well", .domains = SKL_DISPLAY_DDI_D_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_DDI_D, + .id = SKL_DISP_PW_DDI_D, }, }; @@ -2143,31 +2120,33 @@ static struct i915_power_well bxt_power_wells[] = { .name = "power well 1", .domains = 0, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_1, + .id = SKL_DISP_PW_1, }, { .name = "DC off", .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS, .ops = &gen9_dc_off_power_well_ops, - .data = SKL_DISP_PW_DC_OFF, + .id = SKL_DISP_PW_DC_OFF, }, { .name = "power well 2", .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS, .ops = &skl_power_well_ops, - .data = SKL_DISP_PW_2, + .id = SKL_DISP_PW_2, }, { .name = "dpio-common-a", .domains = BXT_DPIO_CMN_A_POWER_DOMAINS, .ops = &bxt_dpio_cmn_power_well_ops, - .data = BXT_DPIO_CMN_A, + .id = BXT_DPIO_CMN_A, + .data = DPIO_PHY1, }, { .name = "dpio-common-bc", .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS, .ops = &bxt_dpio_cmn_power_well_ops, - .data = BXT_DPIO_CMN_BC, + .id = BXT_DPIO_CMN_BC, + .data = DPIO_PHY0, }, }; @@ -2736,8 +2715,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv) struct device *kdev = &pdev->dev; assert_rpm_wakelock_held(dev_priv); - if (atomic_dec_and_test(&dev_priv->pm.wakeref_count)) - atomic_inc(&dev_priv->pm.atomic_seq); + atomic_dec(&dev_priv->pm.wakeref_count); pm_runtime_mark_last_busy(kdev); pm_runtime_put_autosuspend(kdev); diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 49fb95d03d74..3990c805a5b5 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -1472,7 +1472,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder, temp &= ~SDVO_ENABLE; intel_sdvo_write_sdvox(intel_sdvo, temp); - intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A); + intel_wait_for_vblank_if_active(dev_priv, PIPE_A); intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } @@ -1509,7 +1509,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder, intel_sdvo_write_sdvox(intel_sdvo, temp); for (i = 0; i < 2; i++) - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); success = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2); /* Warn if the device reported failure to sync. @@ -2411,10 +2411,10 @@ static void intel_sdvo_add_hdmi_properties(struct intel_sdvo *intel_sdvo, struct intel_sdvo_connector *connector) { - struct drm_device *dev = connector->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(connector->base.base.dev); intel_attach_force_audio_property(&connector->base.base); - if (INTEL_INFO(dev)->gen >= 4 && IS_MOBILE(dev)) { + if (INTEL_GEN(dev_priv) >= 4 && IS_MOBILE(dev_priv)) { intel_attach_broadcast_rgb_property(&connector->base.base); intel_sdvo->color_range_auto = true; } diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c index da9e8b309dd2..0e9aac2e3eae 100644 --- a/drivers/gpu/drm/i915/intel_sprite.c +++ b/drivers/gpu/drm/i915/intel_sprite.c @@ -1035,10 +1035,10 @@ static uint32_t skl_plane_formats[] = { DRM_FORMAT_VYUY, }; -int -intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) +struct intel_plane * +intel_sprite_plane_create(struct drm_i915_private *dev_priv, + enum pipe pipe, int plane) { - struct drm_i915_private *dev_priv = to_i915(dev); struct intel_plane *intel_plane = NULL; struct intel_plane_state *state = NULL; unsigned long possible_crtcs; @@ -1047,9 +1047,6 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) int num_plane_formats; int ret; - if (INTEL_INFO(dev)->gen < 5) - return -ENODEV; - intel_plane = kzalloc(sizeof(*intel_plane), GFP_KERNEL); if (!intel_plane) { ret = -ENOMEM; @@ -1063,25 +1060,25 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) } intel_plane->base.state = &state->base; - switch (INTEL_INFO(dev)->gen) { - case 5: - case 6: + if (INTEL_GEN(dev_priv) >= 9) { intel_plane->can_scale = true; - intel_plane->max_downscale = 16; - intel_plane->update_plane = ilk_update_plane; - intel_plane->disable_plane = ilk_disable_plane; + state->scaler_id = -1; - if (IS_GEN6(dev_priv)) { - plane_formats = snb_plane_formats; - num_plane_formats = ARRAY_SIZE(snb_plane_formats); - } else { - plane_formats = ilk_plane_formats; - num_plane_formats = ARRAY_SIZE(ilk_plane_formats); - } - break; + intel_plane->update_plane = skl_update_plane; + intel_plane->disable_plane = skl_disable_plane; - case 7: - case 8: + plane_formats = skl_plane_formats; + num_plane_formats = ARRAY_SIZE(skl_plane_formats); + } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { + intel_plane->can_scale = false; + intel_plane->max_downscale = 1; + + intel_plane->update_plane = vlv_update_plane; + intel_plane->disable_plane = vlv_disable_plane; + + plane_formats = vlv_plane_formats; + num_plane_formats = ARRAY_SIZE(vlv_plane_formats); + } else if (INTEL_GEN(dev_priv) >= 7) { if (IS_IVYBRIDGE(dev_priv)) { intel_plane->can_scale = true; intel_plane->max_downscale = 2; @@ -1090,33 +1087,25 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) intel_plane->max_downscale = 1; } - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - intel_plane->update_plane = vlv_update_plane; - intel_plane->disable_plane = vlv_disable_plane; + intel_plane->update_plane = ivb_update_plane; + intel_plane->disable_plane = ivb_disable_plane; - plane_formats = vlv_plane_formats; - num_plane_formats = ARRAY_SIZE(vlv_plane_formats); - } else { - intel_plane->update_plane = ivb_update_plane; - intel_plane->disable_plane = ivb_disable_plane; + plane_formats = snb_plane_formats; + num_plane_formats = ARRAY_SIZE(snb_plane_formats); + } else { + intel_plane->can_scale = true; + intel_plane->max_downscale = 16; + + intel_plane->update_plane = ilk_update_plane; + intel_plane->disable_plane = ilk_disable_plane; + if (IS_GEN6(dev_priv)) { plane_formats = snb_plane_formats; num_plane_formats = ARRAY_SIZE(snb_plane_formats); + } else { + plane_formats = ilk_plane_formats; + num_plane_formats = ARRAY_SIZE(ilk_plane_formats); } - break; - case 9: - intel_plane->can_scale = true; - intel_plane->update_plane = skl_update_plane; - intel_plane->disable_plane = skl_disable_plane; - state->scaler_id = -1; - - plane_formats = skl_plane_formats; - num_plane_formats = ARRAY_SIZE(skl_plane_formats); - break; - default: - MISSING_CASE(INTEL_INFO(dev)->gen); - ret = -ENODEV; - goto fail; } if (INTEL_GEN(dev_priv) >= 9) { @@ -1135,15 +1124,15 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) possible_crtcs = (1 << pipe); - if (INTEL_INFO(dev)->gen >= 9) - ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, - &intel_plane_funcs, + if (INTEL_GEN(dev_priv) >= 9) + ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base, + possible_crtcs, &intel_plane_funcs, plane_formats, num_plane_formats, DRM_PLANE_TYPE_OVERLAY, "plane %d%c", plane + 2, pipe_name(pipe)); else - ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs, - &intel_plane_funcs, + ret = drm_universal_plane_init(&dev_priv->drm, &intel_plane->base, + possible_crtcs, &intel_plane_funcs, plane_formats, num_plane_formats, DRM_PLANE_TYPE_OVERLAY, "sprite %c", sprite_name(pipe, plane)); @@ -1156,11 +1145,11 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane) drm_plane_helper_add(&intel_plane->base, &intel_plane_helper_funcs); - return 0; + return intel_plane; fail: kfree(state); kfree(intel_plane); - return ret; + return ERR_PTR(ret); } diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 7118fb55f57f..9212f00d5752 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -856,7 +856,7 @@ intel_enable_tv(struct intel_encoder *encoder, struct drm_i915_private *dev_priv = to_i915(dev); /* Prevents vblank waits from timing out in intel_tv_detect_type() */ - intel_wait_for_vblank(encoder->base.dev, + intel_wait_for_vblank(dev_priv, to_intel_crtc(encoder->base.crtc)->pipe); I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE); @@ -1238,7 +1238,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, I915_WRITE(TV_DAC, tv_dac); POSTING_READ(TV_DAC); - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); type = -1; tv_dac = I915_READ(TV_DAC); @@ -1268,7 +1268,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv, POSTING_READ(TV_CTL); /* For unknown reasons the hw barfs if we don't do this vblank wait. */ - intel_wait_for_vblank(dev, intel_crtc->pipe); + intel_wait_for_vblank(dev_priv, intel_crtc->pipe); /* Restore interrupt config */ if (connector->polled & DRM_CONNECTOR_POLL_HPD) { diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c index 07fe9556c7aa..a16e8b7df120 100644 --- a/drivers/gpu/drm/imx/imx-drm-core.c +++ b/drivers/gpu/drm/imx/imx-drm-core.c @@ -359,8 +359,8 @@ static int imx_drm_bind(struct device *dev) int ret; drm = drm_dev_alloc(&imx_drm_driver, dev); - if (!drm) - return -ENOMEM; + if (IS_ERR(drm)) + return PTR_ERR(drm); imxdrm = devm_kzalloc(dev, sizeof(*imxdrm), GFP_KERNEL); if (!imxdrm) { @@ -438,9 +438,11 @@ static int imx_drm_bind(struct device *dev) err_fbhelper: drm_kms_helper_poll_fini(drm); +#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) if (imxdrm->fbhelper) drm_fbdev_cma_fini(imxdrm->fbhelper); err_unbind: +#endif component_unbind_all(drm->dev, drm); err_vblank: drm_vblank_cleanup(drm); diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c index ce22d0a0ddc8..d5864ed4d772 100644 --- a/drivers/gpu/drm/imx/ipuv3-plane.c +++ b/drivers/gpu/drm/imx/ipuv3-plane.c @@ -103,11 +103,11 @@ drm_plane_state_to_vbo(struct drm_plane_state *state) (state->src_x >> 16) / 2 - eba; } -static void ipu_plane_atomic_set_base(struct ipu_plane *ipu_plane, - struct drm_plane_state *old_state) +static void ipu_plane_atomic_set_base(struct ipu_plane *ipu_plane) { struct drm_plane *plane = &ipu_plane->base; struct drm_plane_state *state = plane->state; + struct drm_crtc_state *crtc_state = state->crtc->state; struct drm_framebuffer *fb = state->fb; unsigned long eba, ubo, vbo; int active; @@ -117,7 +117,7 @@ static void ipu_plane_atomic_set_base(struct ipu_plane *ipu_plane, switch (fb->pixel_format) { case DRM_FORMAT_YUV420: case DRM_FORMAT_YVU420: - if (old_state->fb) + if (!drm_atomic_crtc_needs_modeset(crtc_state)) break; /* @@ -149,7 +149,7 @@ static void ipu_plane_atomic_set_base(struct ipu_plane *ipu_plane, break; } - if (old_state->fb) { + if (!drm_atomic_crtc_needs_modeset(crtc_state)) { active = ipu_idmac_get_current_buffer(ipu_plane->ipu_ch); ipu_cpmem_set_buffer(ipu_plane->ipu_ch, !active, eba); ipu_idmac_select_buffer(ipu_plane->ipu_ch, !active); @@ -259,6 +259,7 @@ static int ipu_plane_atomic_check(struct drm_plane *plane, struct drm_framebuffer *fb = state->fb; struct drm_framebuffer *old_fb = old_state->fb; unsigned long eba, ubo, vbo, old_ubo, old_vbo; + int hsub, vsub; /* Ok to disable */ if (!fb) @@ -355,7 +356,9 @@ static int ipu_plane_atomic_check(struct drm_plane *plane, if ((ubo > 0xfffff8) || (vbo > 0xfffff8)) return -EINVAL; - if (old_fb) { + if (old_fb && + (old_fb->pixel_format == DRM_FORMAT_YUV420 || + old_fb->pixel_format == DRM_FORMAT_YVU420)) { old_ubo = drm_plane_state_to_ubo(old_state); old_vbo = drm_plane_state_to_vbo(old_state); if (ubo != old_ubo || vbo != old_vbo) @@ -370,6 +373,16 @@ static int ipu_plane_atomic_check(struct drm_plane *plane, if (old_fb && old_fb->pitches[1] != fb->pitches[1]) crtc_state->mode_changed = true; + + /* + * The x/y offsets must be even in case of horizontal/vertical + * chroma subsampling. + */ + hsub = drm_format_horz_chroma_subsampling(fb->pixel_format); + vsub = drm_format_vert_chroma_subsampling(fb->pixel_format); + if (((state->src_x >> 16) & (hsub - 1)) || + ((state->src_y >> 16) & (vsub - 1))) + return -EINVAL; } return 0; @@ -392,7 +405,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, struct drm_crtc_state *crtc_state = state->crtc->state; if (!drm_atomic_crtc_needs_modeset(crtc_state)) { - ipu_plane_atomic_set_base(ipu_plane, old_state); + ipu_plane_atomic_set_base(ipu_plane); return; } } @@ -424,6 +437,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, ipu_dp_set_global_alpha(ipu_plane->dp, false, 0, false); break; default: + ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true); break; } } @@ -437,7 +451,7 @@ static void ipu_plane_atomic_update(struct drm_plane *plane, ipu_cpmem_set_high_priority(ipu_plane->ipu_ch); ipu_idmac_set_double_buffer(ipu_plane->ipu_ch, 1); ipu_cpmem_set_stride(ipu_plane->ipu_ch, state->fb->pitches[0]); - ipu_plane_atomic_set_base(ipu_plane, old_state); + ipu_plane_atomic_set_base(ipu_plane); ipu_plane_enable(ipu_plane); } diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c index 83272b456329..5e20220ef4c6 100644 --- a/drivers/gpu/drm/mgag200/mgag200_ttm.c +++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c @@ -267,6 +267,9 @@ int mgag200_mm_init(struct mga_device *mdev) return ret; } + arch_io_reserve_memtype_wc(pci_resource_start(dev->pdev, 0), + pci_resource_len(dev->pdev, 0)); + mdev->fb_mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 0), pci_resource_len(dev->pdev, 0)); @@ -275,10 +278,14 @@ int mgag200_mm_init(struct mga_device *mdev) void mgag200_mm_fini(struct mga_device *mdev) { + struct drm_device *dev = mdev->dev; + ttm_bo_device_release(&mdev->ttm.bdev); mgag200_ttm_global_release(mdev); + arch_io_free_memtype_wc(pci_resource_start(dev->pdev, 0), + pci_resource_len(dev->pdev, 0)); arch_phys_wc_del(mdev->fb_mtrr); mdev->fb_mtrr = 0; } diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c index ec444eac6258..a79514d440b3 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/overlay.c +++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c @@ -33,7 +33,7 @@ #include "nouveau_connector.h" #include "nouveau_display.h" #include "nvreg.h" - +#include "disp.h" struct nouveau_plane { struct drm_plane base; diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index d15c296b5f33..ae49dfd1f97b 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -34,6 +34,8 @@ struct nv50_disp_mthd_v1 { #define NV50_DISP_MTHD_V1_SOR_HDMI_PWR 0x22 #define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT 0x23 #define NV50_DISP_MTHD_V1_SOR_DP_PWR 0x24 +#define NV50_DISP_MTHD_V1_SOR_DP_MST_LINK 0x25 +#define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI 0x26 #define NV50_DISP_MTHD_V1_PIOR_PWR 0x30 __u8 method; __u16 hasht; @@ -90,6 +92,21 @@ struct nv50_disp_sor_dp_pwr_v0 { __u8 pad02[6]; }; +struct nv50_disp_sor_dp_mst_link_v0 { + __u8 version; + __u8 state; + __u8 pad02[6]; +}; + +struct nv50_disp_sor_dp_mst_vcpi_v0 { + __u8 version; + __u8 pad01[1]; + __u8 start_slot; + __u8 num_slots; + __u16 pbn; + __u16 aligned_pbn; +}; + struct nv50_disp_pior_pwr_v0 { __u8 version; __u8 state; diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h index 8d815967767f..9e58b305b020 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/object.h +++ b/drivers/gpu/drm/nouveau/include/nvif/object.h @@ -66,6 +66,35 @@ void nvif_object_unmap(struct nvif_object *); #define nvif_mthd(a,b,c,d) nvif_object_mthd((a), (b), (c), (d)) +struct nvif_mclass { + s32 oclass; + int version; +}; + +#define nvif_mclass(o,m) ({ \ + struct nvif_object *object = (o); \ + struct nvif_sclass *sclass; \ + const typeof(m[0]) *mclass = (m); \ + int ret = -ENODEV; \ + int cnt, i, j; \ + \ + cnt = nvif_object_sclass_get(object, &sclass); \ + if (cnt >= 0) { \ + for (i = 0; ret < 0 && mclass[i].oclass; i++) { \ + for (j = 0; j < cnt; j++) { \ + if (mclass[i].oclass == sclass[j].oclass && \ + mclass[i].version >= sclass[j].minver && \ + mclass[i].version <= sclass[j].maxver) { \ + ret = i; \ + break; \ + } \ + } \ + } \ + nvif_object_sclass_put(&sclass); \ + } \ + ret; \ +}) + /*XXX*/ #include <core/object.h> #define nvxx_object(a) ({ \ diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index 3a410275fa71..65ce79a85d37 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -93,6 +93,7 @@ int gk104_fb_new(struct nvkm_device *, int, struct nvkm_fb **); int gk20a_fb_new(struct nvkm_device *, int, struct nvkm_fb **); int gm107_fb_new(struct nvkm_device *, int, struct nvkm_fb **); int gm200_fb_new(struct nvkm_device *, int, struct nvkm_fb **); +int gm20b_fb_new(struct nvkm_device *, int, struct nvkm_fb **); int gp100_fb_new(struct nvkm_device *, int, struct nvkm_fb **); int gp104_fb_new(struct nvkm_device *, int, struct nvkm_fb **); @@ -156,4 +157,6 @@ struct nvkm_ram_func { int (*prog)(struct nvkm_ram *); void (*tidy)(struct nvkm_ram *); }; + +extern const u8 gf100_pte_storage_type_map[256]; #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c index dc57b628e074..193573d191e5 100644 --- a/drivers/gpu/drm/nouveau/nouveau_acpi.c +++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c @@ -240,7 +240,8 @@ static bool nouveau_pr3_present(struct pci_dev *pdev) if (!parent_adev) return false; - return acpi_has_method(parent_adev->handle, "_PR3"); + return parent_adev->power.flags.power_resources && + acpi_has_method(parent_adev->handle, "_PR3"); } static void nouveau_dsm_pci_probe(struct pci_dev *pdev, acpi_handle *dhandle_out, diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c index a1570b109434..23ffe8571a99 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bios.c +++ b/drivers/gpu/drm/nouveau/nouveau_bios.c @@ -333,6 +333,9 @@ get_fp_strap(struct drm_device *dev, struct nvbios *bios) if (bios->major_version < 5 && bios->data[0x48] & 0x4) return NVReadVgaCrtc5758(dev, 0, 0xf) & 0xf; + if (drm->device.info.family >= NV_DEVICE_INFO_V0_MAXWELL) + return nvif_rd32(device, 0x001800) & 0x0000000f; + else if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) return (nvif_rd32(device, NV_PEXTDEV_BOOT_0) >> 24) & 0xf; else diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index c1084088f9e4..947c200655b4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -30,6 +30,7 @@ #include <linux/vga_switcheroo.h> #include <drm/drmP.h> +#include <drm/drm_atomic_helper.h> #include <drm/drm_edid.h> #include <drm/drm_crtc_helper.h> @@ -47,6 +48,301 @@ #include <nvif/cl0046.h> #include <nvif/event.h> +struct drm_display_mode * +nouveau_conn_native_mode(struct drm_connector *connector) +{ + const struct drm_connector_helper_funcs *helper = connector->helper_private; + struct nouveau_drm *drm = nouveau_drm(connector->dev); + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode, *largest = NULL; + int high_w = 0, high_h = 0, high_v = 0; + + list_for_each_entry(mode, &connector->probed_modes, head) { + mode->vrefresh = drm_mode_vrefresh(mode); + if (helper->mode_valid(connector, mode) != MODE_OK || + (mode->flags & DRM_MODE_FLAG_INTERLACE)) + continue; + + /* Use preferred mode if there is one.. */ + if (mode->type & DRM_MODE_TYPE_PREFERRED) { + NV_DEBUG(drm, "native mode from preferred\n"); + return drm_mode_duplicate(dev, mode); + } + + /* Otherwise, take the resolution with the largest width, then + * height, then vertical refresh + */ + if (mode->hdisplay < high_w) + continue; + + if (mode->hdisplay == high_w && mode->vdisplay < high_h) + continue; + + if (mode->hdisplay == high_w && mode->vdisplay == high_h && + mode->vrefresh < high_v) + continue; + + high_w = mode->hdisplay; + high_h = mode->vdisplay; + high_v = mode->vrefresh; + largest = mode; + } + + NV_DEBUG(drm, "native mode from largest: %dx%d@%d\n", + high_w, high_h, high_v); + return largest ? drm_mode_duplicate(dev, largest) : NULL; +} + +int +nouveau_conn_atomic_get_property(struct drm_connector *connector, + const struct drm_connector_state *state, + struct drm_property *property, u64 *val) +{ + struct nouveau_conn_atom *asyc = nouveau_conn_atom(state); + struct nouveau_display *disp = nouveau_display(connector->dev); + struct drm_device *dev = connector->dev; + + if (property == dev->mode_config.scaling_mode_property) + *val = asyc->scaler.mode; + else if (property == disp->underscan_property) + *val = asyc->scaler.underscan.mode; + else if (property == disp->underscan_hborder_property) + *val = asyc->scaler.underscan.hborder; + else if (property == disp->underscan_vborder_property) + *val = asyc->scaler.underscan.vborder; + else if (property == disp->dithering_mode) + *val = asyc->dither.mode; + else if (property == disp->dithering_depth) + *val = asyc->dither.depth; + else if (property == disp->vibrant_hue_property) + *val = asyc->procamp.vibrant_hue; + else if (property == disp->color_vibrance_property) + *val = asyc->procamp.color_vibrance; + else + return -EINVAL; + + return 0; +} + +int +nouveau_conn_atomic_set_property(struct drm_connector *connector, + struct drm_connector_state *state, + struct drm_property *property, u64 val) +{ + struct drm_device *dev = connector->dev; + struct nouveau_conn_atom *asyc = nouveau_conn_atom(state); + struct nouveau_display *disp = nouveau_display(dev); + + if (property == dev->mode_config.scaling_mode_property) { + switch (val) { + case DRM_MODE_SCALE_NONE: + /* We allow 'None' for EDID modes, even on a fixed + * panel (some exist with support for lower refresh + * rates, which people might want to use for power- + * saving purposes). + * + * Non-EDID modes will force the use of GPU scaling + * to the native mode regardless of this setting. + */ + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_LVDS: + case DRM_MODE_CONNECTOR_eDP: + /* ... except prior to G80, where the code + * doesn't support such things. + */ + if (disp->disp.oclass < NV50_DISP) + return -EINVAL; + break; + default: + break; + } + case DRM_MODE_SCALE_FULLSCREEN: + case DRM_MODE_SCALE_CENTER: + case DRM_MODE_SCALE_ASPECT: + break; + default: + return -EINVAL; + } + + if (asyc->scaler.mode != val) { + asyc->scaler.mode = val; + asyc->set.scaler = true; + } + } else + if (property == disp->underscan_property) { + if (asyc->scaler.underscan.mode != val) { + asyc->scaler.underscan.mode = val; + asyc->set.scaler = true; + } + } else + if (property == disp->underscan_hborder_property) { + if (asyc->scaler.underscan.hborder != val) { + asyc->scaler.underscan.hborder = val; + asyc->set.scaler = true; + } + } else + if (property == disp->underscan_vborder_property) { + if (asyc->scaler.underscan.vborder != val) { + asyc->scaler.underscan.vborder = val; + asyc->set.scaler = true; + } + } else + if (property == disp->dithering_mode) { + if (asyc->dither.mode != val) { + asyc->dither.mode = val; + asyc->set.dither = true; + } + } else + if (property == disp->dithering_depth) { + if (asyc->dither.mode != val) { + asyc->dither.depth = val; + asyc->set.dither = true; + } + } else + if (property == disp->vibrant_hue_property) { + if (asyc->procamp.vibrant_hue != val) { + asyc->procamp.vibrant_hue = val; + asyc->set.procamp = true; + } + } else + if (property == disp->color_vibrance_property) { + if (asyc->procamp.color_vibrance != val) { + asyc->procamp.color_vibrance = val; + asyc->set.procamp = true; + } + } else { + return -EINVAL; + } + + return 0; +} + +void +nouveau_conn_atomic_destroy_state(struct drm_connector *connector, + struct drm_connector_state *state) +{ + struct nouveau_conn_atom *asyc = nouveau_conn_atom(state); + __drm_atomic_helper_connector_destroy_state(&asyc->state); + kfree(asyc); +} + +struct drm_connector_state * +nouveau_conn_atomic_duplicate_state(struct drm_connector *connector) +{ + struct nouveau_conn_atom *armc = nouveau_conn_atom(connector->state); + struct nouveau_conn_atom *asyc; + if (!(asyc = kmalloc(sizeof(*asyc), GFP_KERNEL))) + return NULL; + __drm_atomic_helper_connector_duplicate_state(connector, &asyc->state); + asyc->dither = armc->dither; + asyc->scaler = armc->scaler; + asyc->procamp = armc->procamp; + asyc->set.mask = 0; + return &asyc->state; +} + +void +nouveau_conn_reset(struct drm_connector *connector) +{ + struct nouveau_conn_atom *asyc; + + if (WARN_ON(!(asyc = kzalloc(sizeof(*asyc), GFP_KERNEL)))) + return; + + if (connector->state) + __drm_atomic_helper_connector_destroy_state(connector->state); + __drm_atomic_helper_connector_reset(connector, &asyc->state); + asyc->dither.mode = DITHERING_MODE_AUTO; + asyc->dither.depth = DITHERING_DEPTH_AUTO; + asyc->scaler.mode = DRM_MODE_SCALE_NONE; + asyc->scaler.underscan.mode = UNDERSCAN_OFF; + asyc->procamp.color_vibrance = 150; + asyc->procamp.vibrant_hue = 90; + + if (nouveau_display(connector->dev)->disp.oclass < NV50_DISP) { + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_LVDS: + /* See note in nouveau_conn_atomic_set_property(). */ + asyc->scaler.mode = DRM_MODE_SCALE_FULLSCREEN; + break; + default: + break; + } + } +} + +void +nouveau_conn_attach_properties(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct nouveau_conn_atom *armc = nouveau_conn_atom(connector->state); + struct nouveau_display *disp = nouveau_display(dev); + + /* Init DVI-I specific properties. */ + if (connector->connector_type == DRM_MODE_CONNECTOR_DVII) + drm_object_attach_property(&connector->base, dev->mode_config. + dvi_i_subconnector_property, 0); + + /* Add overscan compensation options to digital outputs. */ + if (disp->underscan_property && + (connector->connector_type == DRM_MODE_CONNECTOR_DVID || + connector->connector_type == DRM_MODE_CONNECTOR_DVII || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || + connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort)) { + drm_object_attach_property(&connector->base, + disp->underscan_property, + UNDERSCAN_OFF); + drm_object_attach_property(&connector->base, + disp->underscan_hborder_property, 0); + drm_object_attach_property(&connector->base, + disp->underscan_vborder_property, 0); + } + + /* Add hue and saturation options. */ + if (disp->vibrant_hue_property) + drm_object_attach_property(&connector->base, + disp->vibrant_hue_property, + armc->procamp.vibrant_hue); + if (disp->color_vibrance_property) + drm_object_attach_property(&connector->base, + disp->color_vibrance_property, + armc->procamp.color_vibrance); + + /* Scaling mode property. */ + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_TV: + break; + case DRM_MODE_CONNECTOR_VGA: + if (disp->disp.oclass < NV50_DISP) + break; /* Can only scale on DFPs. */ + /* Fall-through. */ + default: + drm_object_attach_property(&connector->base, dev->mode_config. + scaling_mode_property, + armc->scaler.mode); + break; + } + + /* Dithering properties. */ + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_TV: + case DRM_MODE_CONNECTOR_VGA: + break; + default: + if (disp->dithering_mode) { + drm_object_attach_property(&connector->base, + disp->dithering_mode, + armc->dither.mode); + } + if (disp->dithering_depth) { + drm_object_attach_property(&connector->base, + disp->dithering_depth, + armc->dither.depth); + } + break; + } +} + MODULE_PARM_DESC(tv_disable, "Disable TV-out detection"); int nouveau_tv_disable = 0; module_param_named(tv_disable, nouveau_tv_disable, int, 0400); @@ -151,7 +447,9 @@ nouveau_connector_ddc_detect(struct drm_connector *connector) if (nv_encoder->dcb->type == DCB_OUTPUT_DP) { int ret = nouveau_dp_detect(nv_encoder); - if (ret == 0) + if (ret == NOUVEAU_DP_MST) + return NULL; + if (ret == NOUVEAU_DP_SST) break; } else if ((vga_switcheroo_handler_flags() & @@ -465,199 +763,39 @@ static int nouveau_connector_set_property(struct drm_connector *connector, struct drm_property *property, uint64_t value) { - struct nouveau_display *disp = nouveau_display(connector->dev); + struct nouveau_conn_atom *asyc = nouveau_conn_atom(connector->state); struct nouveau_connector *nv_connector = nouveau_connector(connector); struct nouveau_encoder *nv_encoder = nv_connector->detected_encoder; struct drm_encoder *encoder = to_drm_encoder(nv_encoder); - struct drm_device *dev = connector->dev; - struct nouveau_crtc *nv_crtc; int ret; - nv_crtc = NULL; - if (connector->encoder && connector->encoder->crtc) - nv_crtc = nouveau_crtc(connector->encoder->crtc); - - /* Scaling mode */ - if (property == dev->mode_config.scaling_mode_property) { - bool modeset = false; - - switch (value) { - case DRM_MODE_SCALE_NONE: - /* We allow 'None' for EDID modes, even on a fixed - * panel (some exist with support for lower refresh - * rates, which people might want to use for power - * saving purposes). - * - * Non-EDID modes will force the use of GPU scaling - * to the native mode regardless of this setting. - */ - switch (nv_connector->type) { - case DCB_CONNECTOR_LVDS: - case DCB_CONNECTOR_LVDS_SPWG: - case DCB_CONNECTOR_eDP: - /* ... except prior to G80, where the code - * doesn't support such things. - */ - if (disp->disp.oclass < NV50_DISP) - return -EINVAL; - break; - default: - break; - } - break; - case DRM_MODE_SCALE_FULLSCREEN: - case DRM_MODE_SCALE_CENTER: - case DRM_MODE_SCALE_ASPECT: - break; - default: - return -EINVAL; - } - - /* Changing between GPU and panel scaling requires a full - * modeset - */ - if ((nv_connector->scaling_mode == DRM_MODE_SCALE_NONE) || - (value == DRM_MODE_SCALE_NONE)) - modeset = true; - nv_connector->scaling_mode = value; - - if (!nv_crtc) - return 0; - - if (modeset || !nv_crtc->set_scale) { - ret = drm_crtc_helper_set_mode(&nv_crtc->base, - &nv_crtc->base.mode, - nv_crtc->base.x, - nv_crtc->base.y, NULL); - if (!ret) - return -EINVAL; - } else { - ret = nv_crtc->set_scale(nv_crtc, true); - if (ret) - return ret; - } - - return 0; - } - - /* Underscan */ - if (property == disp->underscan_property) { - if (nv_connector->underscan != value) { - nv_connector->underscan = value; - if (!nv_crtc || !nv_crtc->set_scale) - return 0; - - return nv_crtc->set_scale(nv_crtc, true); - } - - return 0; - } - - if (property == disp->underscan_hborder_property) { - if (nv_connector->underscan_hborder != value) { - nv_connector->underscan_hborder = value; - if (!nv_crtc || !nv_crtc->set_scale) - return 0; - - return nv_crtc->set_scale(nv_crtc, true); - } - - return 0; - } - - if (property == disp->underscan_vborder_property) { - if (nv_connector->underscan_vborder != value) { - nv_connector->underscan_vborder = value; - if (!nv_crtc || !nv_crtc->set_scale) - return 0; - - return nv_crtc->set_scale(nv_crtc, true); - } - - return 0; - } + if (connector->dev->mode_config.funcs->atomic_commit) + return drm_atomic_helper_connector_set_property(connector, property, value); - /* Dithering */ - if (property == disp->dithering_mode) { - nv_connector->dithering_mode = value; - if (!nv_crtc || !nv_crtc->set_dither) - return 0; - - return nv_crtc->set_dither(nv_crtc, true); - } - - if (property == disp->dithering_depth) { - nv_connector->dithering_depth = value; - if (!nv_crtc || !nv_crtc->set_dither) - return 0; - - return nv_crtc->set_dither(nv_crtc, true); - } - - if (nv_crtc && nv_crtc->set_color_vibrance) { - /* Hue */ - if (property == disp->vibrant_hue_property) { - nv_crtc->vibrant_hue = value - 90; - return nv_crtc->set_color_vibrance(nv_crtc, true); - } - /* Saturation */ - if (property == disp->color_vibrance_property) { - nv_crtc->color_vibrance = value - 100; - return nv_crtc->set_color_vibrance(nv_crtc, true); - } + ret = connector->funcs->atomic_set_property(&nv_connector->base, + &asyc->state, + property, value); + if (ret) { + if (nv_encoder && nv_encoder->dcb->type == DCB_OUTPUT_TV) + return get_slave_funcs(encoder)->set_property( + encoder, connector, property, value); + return ret; } - if (nv_encoder && nv_encoder->dcb->type == DCB_OUTPUT_TV) - return get_slave_funcs(encoder)->set_property( - encoder, connector, property, value); - - return -EINVAL; -} - -static struct drm_display_mode * -nouveau_connector_native_mode(struct drm_connector *connector) -{ - const struct drm_connector_helper_funcs *helper = connector->helper_private; - struct nouveau_drm *drm = nouveau_drm(connector->dev); - struct nouveau_connector *nv_connector = nouveau_connector(connector); - struct drm_device *dev = connector->dev; - struct drm_display_mode *mode, *largest = NULL; - int high_w = 0, high_h = 0, high_v = 0; + nv_connector->scaling_mode = asyc->scaler.mode; + nv_connector->dithering_mode = asyc->dither.mode; - list_for_each_entry(mode, &nv_connector->base.probed_modes, head) { - mode->vrefresh = drm_mode_vrefresh(mode); - if (helper->mode_valid(connector, mode) != MODE_OK || - (mode->flags & DRM_MODE_FLAG_INTERLACE)) - continue; - - /* Use preferred mode if there is one.. */ - if (mode->type & DRM_MODE_TYPE_PREFERRED) { - NV_DEBUG(drm, "native mode from preferred\n"); - return drm_mode_duplicate(dev, mode); - } - - /* Otherwise, take the resolution with the largest width, then - * height, then vertical refresh - */ - if (mode->hdisplay < high_w) - continue; - - if (mode->hdisplay == high_w && mode->vdisplay < high_h) - continue; - - if (mode->hdisplay == high_w && mode->vdisplay == high_h && - mode->vrefresh < high_v) - continue; - - high_w = mode->hdisplay; - high_h = mode->vdisplay; - high_v = mode->vrefresh; - largest = mode; + if (connector->encoder && connector->encoder->crtc) { + ret = drm_crtc_helper_set_mode(connector->encoder->crtc, + &connector->encoder->crtc->mode, + connector->encoder->crtc->x, + connector->encoder->crtc->y, + NULL); + if (!ret) + return -EINVAL; } - NV_DEBUG(drm, "native mode from largest: %dx%d@%d\n", - high_w, high_h, high_v); - return largest ? drm_mode_duplicate(dev, largest) : NULL; + return 0; } struct moderec { @@ -805,8 +943,7 @@ nouveau_connector_get_modes(struct drm_connector *connector) * the list of modes. */ if (!nv_connector->native_mode) - nv_connector->native_mode = - nouveau_connector_native_mode(connector); + nv_connector->native_mode = nouveau_conn_native_mode(connector); if (ret == 0 && nv_connector->native_mode) { struct drm_display_mode *mode; @@ -934,56 +1071,42 @@ nouveau_connector_helper_funcs = { .best_encoder = nouveau_connector_best_encoder, }; +static int +nouveau_connector_dpms(struct drm_connector *connector, int mode) +{ + if (connector->dev->mode_config.funcs->atomic_commit) + return drm_atomic_helper_connector_dpms(connector, mode); + return drm_helper_connector_dpms(connector, mode); +} + static const struct drm_connector_funcs nouveau_connector_funcs = { - .dpms = drm_helper_connector_dpms, + .dpms = nouveau_connector_dpms, + .reset = nouveau_conn_reset, .detect = nouveau_connector_detect, - .destroy = nouveau_connector_destroy, + .force = nouveau_connector_force, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = nouveau_connector_set_property, - .force = nouveau_connector_force + .destroy = nouveau_connector_destroy, + .atomic_duplicate_state = nouveau_conn_atomic_duplicate_state, + .atomic_destroy_state = nouveau_conn_atomic_destroy_state, + .atomic_set_property = nouveau_conn_atomic_set_property, + .atomic_get_property = nouveau_conn_atomic_get_property, }; static const struct drm_connector_funcs nouveau_connector_funcs_lvds = { - .dpms = drm_helper_connector_dpms, + .dpms = nouveau_connector_dpms, + .reset = nouveau_conn_reset, .detect = nouveau_connector_detect_lvds, - .destroy = nouveau_connector_destroy, + .force = nouveau_connector_force, .fill_modes = drm_helper_probe_single_connector_modes, .set_property = nouveau_connector_set_property, - .force = nouveau_connector_force -}; - -static int -nouveau_connector_dp_dpms(struct drm_connector *connector, int mode) -{ - struct nouveau_encoder *nv_encoder = NULL; - - if (connector->encoder) - nv_encoder = nouveau_encoder(connector->encoder); - if (nv_encoder && nv_encoder->dcb && - nv_encoder->dcb->type == DCB_OUTPUT_DP) { - if (mode == DRM_MODE_DPMS_ON) { - u8 data = DP_SET_POWER_D0; - nvkm_wraux(nv_encoder->aux, DP_SET_POWER, &data, 1); - usleep_range(1000, 2000); - } else { - u8 data = DP_SET_POWER_D3; - nvkm_wraux(nv_encoder->aux, DP_SET_POWER, &data, 1); - } - } - - return drm_helper_connector_dpms(connector, mode); -} - -static const struct drm_connector_funcs -nouveau_connector_funcs_dp = { - .dpms = nouveau_connector_dp_dpms, - .detect = nouveau_connector_detect, .destroy = nouveau_connector_destroy, - .fill_modes = drm_helper_probe_single_connector_modes, - .set_property = nouveau_connector_set_property, - .force = nouveau_connector_force + .atomic_duplicate_state = nouveau_conn_atomic_duplicate_state, + .atomic_destroy_state = nouveau_conn_atomic_destroy_state, + .atomic_set_property = nouveau_conn_atomic_set_property, + .atomic_get_property = nouveau_conn_atomic_get_property, }; static int @@ -995,19 +1118,20 @@ nouveau_connector_hotplug(struct nvif_notify *notify) struct nouveau_drm *drm = nouveau_drm(connector->dev); const struct nvif_notify_conn_rep_v0 *rep = notify->data; const char *name = connector->name; + struct nouveau_encoder *nv_encoder; if (rep->mask & NVIF_NOTIFY_CONN_V0_IRQ) { + NV_DEBUG(drm, "service %s\n", name); + if ((nv_encoder = find_encoder(connector, DCB_OUTPUT_DP))) + nv50_mstm_service(nv_encoder->dp.mstm); } else { bool plugged = (rep->mask != NVIF_NOTIFY_CONN_V0_UNPLUG); NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", name); - - mutex_lock(&drm->dev->mode_config.mutex); - if (plugged) - drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); - else - drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); - mutex_unlock(&drm->dev->mode_config.mutex); + if ((nv_encoder = find_encoder(connector, DCB_OUTPUT_DP))) { + if (!plugged) + nv50_mstm_remove(nv_encoder->dp.mstm); + } drm_helper_hpd_irq_event(connector->dev); } @@ -1188,7 +1312,7 @@ nouveau_connector_create(struct drm_device *dev, int index) return ERR_PTR(ret); } - funcs = &nouveau_connector_funcs_dp; + funcs = &nouveau_connector_funcs; break; default: funcs = &nouveau_connector_funcs; @@ -1202,38 +1326,10 @@ nouveau_connector_create(struct drm_device *dev, int index) drm_connector_init(dev, connector, funcs, type); drm_connector_helper_add(connector, &nouveau_connector_helper_funcs); - /* Init DVI-I specific properties */ - if (nv_connector->type == DCB_CONNECTOR_DVI_I) - drm_object_attach_property(&connector->base, dev->mode_config.dvi_i_subconnector_property, 0); + connector->funcs->reset(connector); + nouveau_conn_attach_properties(connector); - /* Add overscan compensation options to digital outputs */ - if (disp->underscan_property && - (type == DRM_MODE_CONNECTOR_DVID || - type == DRM_MODE_CONNECTOR_DVII || - type == DRM_MODE_CONNECTOR_HDMIA || - type == DRM_MODE_CONNECTOR_DisplayPort)) { - drm_object_attach_property(&connector->base, - disp->underscan_property, - UNDERSCAN_OFF); - drm_object_attach_property(&connector->base, - disp->underscan_hborder_property, - 0); - drm_object_attach_property(&connector->base, - disp->underscan_vborder_property, - 0); - } - - /* Add hue and saturation options */ - if (disp->vibrant_hue_property) - drm_object_attach_property(&connector->base, - disp->vibrant_hue_property, - 90); - if (disp->color_vibrance_property) - drm_object_attach_property(&connector->base, - disp->color_vibrance_property, - 150); - - /* default scaling mode */ + /* Default scaling mode */ switch (nv_connector->type) { case DCB_CONNECTOR_LVDS: case DCB_CONNECTOR_LVDS_SPWG: @@ -1250,23 +1346,6 @@ nouveau_connector_create(struct drm_device *dev, int index) break; } - /* scaling mode property */ - switch (nv_connector->type) { - case DCB_CONNECTOR_TV_0: - case DCB_CONNECTOR_TV_1: - case DCB_CONNECTOR_TV_3: - break; - case DCB_CONNECTOR_VGA: - if (disp->disp.oclass < NV50_DISP) - break; /* can only scale on DFPs */ - /* fall-through */ - default: - drm_object_attach_property(&connector->base, dev->mode_config. - scaling_mode_property, - nv_connector->scaling_mode); - break; - } - /* dithering properties */ switch (nv_connector->type) { case DCB_CONNECTOR_TV_0: @@ -1275,20 +1354,7 @@ nouveau_connector_create(struct drm_device *dev, int index) case DCB_CONNECTOR_VGA: break; default: - if (disp->dithering_mode) { - nv_connector->dithering_mode = DITHERING_MODE_AUTO; - drm_object_attach_property(&connector->base, - disp->dithering_mode, - nv_connector-> - dithering_mode); - } - if (disp->dithering_depth) { - nv_connector->dithering_depth = DITHERING_DEPTH_AUTO; - drm_object_attach_property(&connector->base, - disp->dithering_depth, - nv_connector-> - dithering_depth); - } + nv_connector->dithering_mode = DITHERING_MODE_AUTO; break; } diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h index 7446ee66ea04..096983c42a1f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.h +++ b/drivers/gpu/drm/nouveau/nouveau_connector.h @@ -35,30 +35,6 @@ struct nvkm_i2c_port; -enum nouveau_underscan_type { - UNDERSCAN_OFF, - UNDERSCAN_ON, - UNDERSCAN_AUTO, -}; - -/* the enum values specifically defined here match nv50/nvd0 hw values, and - * the code relies on this - */ -enum nouveau_dithering_mode { - DITHERING_MODE_OFF = 0x00, - DITHERING_MODE_ON = 0x01, - DITHERING_MODE_DYNAMIC2X2 = 0x10 | DITHERING_MODE_ON, - DITHERING_MODE_STATIC2X2 = 0x18 | DITHERING_MODE_ON, - DITHERING_MODE_TEMPORAL = 0x20 | DITHERING_MODE_ON, - DITHERING_MODE_AUTO -}; - -enum nouveau_dithering_depth { - DITHERING_DEPTH_6BPC = 0x00, - DITHERING_DEPTH_8BPC = 0x02, - DITHERING_DEPTH_AUTO -}; - struct nouveau_connector { struct drm_connector base; enum dcb_connector_type type; @@ -70,12 +46,7 @@ struct nouveau_connector { struct drm_dp_aux aux; int dithering_mode; - int dithering_depth; int scaling_mode; - bool scaling_full; - enum nouveau_underscan_type underscan; - u32 underscan_hborder; - u32 underscan_vborder; struct nouveau_encoder *detected_encoder; struct edid *edid; @@ -109,5 +80,74 @@ nouveau_connector_create(struct drm_device *, int index); extern int nouveau_tv_disable; extern int nouveau_ignorelid; extern int nouveau_duallink; +extern int nouveau_hdmimhz; + +#include <drm/drm_crtc.h> +#define nouveau_conn_atom(p) \ + container_of((p), struct nouveau_conn_atom, state) + +struct nouveau_conn_atom { + struct drm_connector_state state; + + struct { + /* The enum values specifically defined here match nv50/gf119 + * hw values, and the code relies on this. + */ + enum { + DITHERING_MODE_OFF = 0x00, + DITHERING_MODE_ON = 0x01, + DITHERING_MODE_DYNAMIC2X2 = 0x10 | DITHERING_MODE_ON, + DITHERING_MODE_STATIC2X2 = 0x18 | DITHERING_MODE_ON, + DITHERING_MODE_TEMPORAL = 0x20 | DITHERING_MODE_ON, + DITHERING_MODE_AUTO + } mode; + enum { + DITHERING_DEPTH_6BPC = 0x00, + DITHERING_DEPTH_8BPC = 0x02, + DITHERING_DEPTH_AUTO + } depth; + } dither; + + struct { + int mode; /* DRM_MODE_SCALE_* */ + struct { + enum { + UNDERSCAN_OFF, + UNDERSCAN_ON, + UNDERSCAN_AUTO, + } mode; + u32 hborder; + u32 vborder; + } underscan; + bool full; + } scaler; + + struct { + int color_vibrance; + int vibrant_hue; + } procamp; + + union { + struct { + bool dither:1; + bool scaler:1; + bool procamp:1; + }; + u8 mask; + } set; +}; +void nouveau_conn_attach_properties(struct drm_connector *); +void nouveau_conn_reset(struct drm_connector *); +struct drm_connector_state * +nouveau_conn_atomic_duplicate_state(struct drm_connector *); +void nouveau_conn_atomic_destroy_state(struct drm_connector *, + struct drm_connector_state *); +int nouveau_conn_atomic_set_property(struct drm_connector *, + struct drm_connector_state *, + struct drm_property *, u64); +int nouveau_conn_atomic_get_property(struct drm_connector *, + const struct drm_connector_state *, + struct drm_property *, u64 *); +struct drm_display_mode *nouveau_conn_native_mode(struct drm_connector *); #endif /* __NOUVEAU_CONNECTOR_H__ */ diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h index 863f10b8d818..050fcf30a0d2 100644 --- a/drivers/gpu/drm/nouveau/nouveau_crtc.h +++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h @@ -38,8 +38,6 @@ struct nouveau_crtc { uint32_t dpms_saved_fp_control; uint32_t fp_users; int saturation; - int color_vibrance; - int vibrant_hue; int sharpness; int last_dpms; @@ -54,7 +52,6 @@ struct nouveau_crtc { struct { struct nouveau_bo *nvbo; - bool visible; uint32_t offset; void (*set_offset)(struct nouveau_crtc *, uint32_t offset); void (*set_pos)(struct nouveau_crtc *, int x, int y); @@ -70,10 +67,6 @@ struct nouveau_crtc { int depth; } lut; - int (*set_dither)(struct nouveau_crtc *crtc, bool update); - int (*set_scale)(struct nouveau_crtc *crtc, bool update); - int (*set_color_vibrance)(struct nouveau_crtc *crtc, bool update); - void (*save)(struct drm_crtc *crtc); void (*restore)(struct drm_crtc *crtc); }; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index afbf557b23d4..75c90a8da18a 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -25,6 +25,8 @@ */ #include <drm/drmP.h> +#include <drm/drm_atomic.h> +#include <drm/drm_atomic_helper.h> #include <drm/drm_crtc_helper.h> #include <nvif/class.h> @@ -92,7 +94,7 @@ calc(int blanks, int blanke, int total, int line) return line; } -int +static int nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos, ktime_t *stime, ktime_t *etime) { @@ -158,9 +160,13 @@ nouveau_display_vblstamp(struct drm_device *dev, unsigned int pipe, list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { if (nouveau_crtc(crtc)->index == pipe) { + struct drm_display_mode *mode; + if (dev->mode_config.funcs->atomic_commit) + mode = &crtc->state->adjusted_mode; + else + mode = &crtc->hwmode; return drm_calc_vbltimestamp_from_scanoutpos(dev, - pipe, max_error, time, flags, - &crtc->hwmode); + pipe, max_error, time, flags, mode); } } @@ -217,10 +223,6 @@ static void nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb) { struct nouveau_framebuffer *fb = nouveau_framebuffer(drm_fb); - struct nouveau_display *disp = nouveau_display(drm_fb->dev); - - if (disp->fb_dtor) - disp->fb_dtor(drm_fb); if (fb->nvbo) drm_gem_object_unreference_unlocked(&fb->nvbo->gem); @@ -245,57 +247,45 @@ static const struct drm_framebuffer_funcs nouveau_framebuffer_funcs = { }; int -nouveau_framebuffer_init(struct drm_device *dev, - struct nouveau_framebuffer *nv_fb, - const struct drm_mode_fb_cmd2 *mode_cmd, - struct nouveau_bo *nvbo) +nouveau_framebuffer_new(struct drm_device *dev, + const struct drm_mode_fb_cmd2 *mode_cmd, + struct nouveau_bo *nvbo, + struct nouveau_framebuffer **pfb) { - struct nouveau_display *disp = nouveau_display(dev); - struct drm_framebuffer *fb = &nv_fb->base; + struct nouveau_framebuffer *fb; int ret; - drm_helper_mode_fill_fb_struct(fb, mode_cmd); - nv_fb->nvbo = nvbo; - - ret = drm_framebuffer_init(dev, fb, &nouveau_framebuffer_funcs); - if (ret) - return ret; + if (!(fb = *pfb = kzalloc(sizeof(*fb), GFP_KERNEL))) + return -ENOMEM; - if (disp->fb_ctor) { - ret = disp->fb_ctor(fb); - if (ret) - disp->fb_dtor(fb); - } + drm_helper_mode_fill_fb_struct(&fb->base, mode_cmd); + fb->nvbo = nvbo; + ret = drm_framebuffer_init(dev, &fb->base, &nouveau_framebuffer_funcs); + if (ret) + kfree(fb); return ret; } -static struct drm_framebuffer * +struct drm_framebuffer * nouveau_user_framebuffer_create(struct drm_device *dev, struct drm_file *file_priv, const struct drm_mode_fb_cmd2 *mode_cmd) { - struct nouveau_framebuffer *nouveau_fb; + struct nouveau_framebuffer *fb; + struct nouveau_bo *nvbo; struct drm_gem_object *gem; - int ret = -ENOMEM; + int ret; gem = drm_gem_object_lookup(file_priv, mode_cmd->handles[0]); if (!gem) return ERR_PTR(-ENOENT); + nvbo = nouveau_gem_object(gem); - nouveau_fb = kzalloc(sizeof(struct nouveau_framebuffer), GFP_KERNEL); - if (!nouveau_fb) - goto err_unref; - - ret = nouveau_framebuffer_init(dev, nouveau_fb, mode_cmd, nouveau_gem_object(gem)); - if (ret) - goto err; - - return &nouveau_fb->base; + ret = nouveau_framebuffer_new(dev, mode_cmd, nvbo, &fb); + if (ret == 0) + return &fb->base; -err: - kfree(nouveau_fb); -err_unref: drm_gem_object_unreference_unlocked(gem); return ERR_PTR(ret); } @@ -385,13 +375,16 @@ nouveau_display_init(struct drm_device *dev) } void -nouveau_display_fini(struct drm_device *dev) +nouveau_display_fini(struct drm_device *dev, bool suspend) { struct nouveau_display *disp = nouveau_display(dev); struct nouveau_drm *drm = nouveau_drm(dev); struct drm_connector *connector; int head; + if (!suspend) + drm_crtc_force_disable_all(dev); + /* Make sure that drm and hw vblank irqs get properly disabled. */ for (head = 0; head < dev->mode_config.num_crtc; head++) drm_vblank_off(dev, head); @@ -530,6 +523,8 @@ nouveau_display_create(struct drm_device *dev) if (ret) goto disp_create_err; + drm_mode_config_reset(dev); + if (dev->mode_config.num_crtc) { ret = nouveau_display_vblank_init(dev); if (ret) @@ -556,7 +551,6 @@ nouveau_display_destroy(struct drm_device *dev) nouveau_display_vblank_fini(dev); drm_kms_helper_poll_fini(dev); - drm_crtc_force_disable_all(dev); drm_mode_config_cleanup(dev); if (disp->dtor) @@ -568,12 +562,138 @@ nouveau_display_destroy(struct drm_device *dev) kfree(disp); } +static int +nouveau_atomic_disable_connector(struct drm_atomic_state *state, + struct drm_connector *connector) +{ + struct drm_connector_state *connector_state; + struct drm_crtc *crtc; + struct drm_crtc_state *crtc_state; + struct drm_plane_state *plane_state; + struct drm_plane *plane; + int ret; + + if (!(crtc = connector->state->crtc)) + return 0; + + connector_state = drm_atomic_get_connector_state(state, connector); + if (IS_ERR(connector_state)) + return PTR_ERR(connector_state); + + ret = drm_atomic_set_crtc_for_connector(connector_state, NULL); + if (ret) + return ret; + + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) + return PTR_ERR(crtc_state); + + ret = drm_atomic_set_mode_for_crtc(crtc_state, NULL); + if (ret) + return ret; + + crtc_state->active = false; + + drm_for_each_plane_mask(plane, connector->dev, crtc_state->plane_mask) { + plane_state = drm_atomic_get_plane_state(state, plane); + if (IS_ERR(plane_state)) + return PTR_ERR(plane_state); + + ret = drm_atomic_set_crtc_for_plane(plane_state, NULL); + if (ret) + return ret; + + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + + return 0; +} + +static int +nouveau_atomic_disable(struct drm_device *dev, + struct drm_modeset_acquire_ctx *ctx) +{ + struct drm_atomic_state *state; + struct drm_connector *connector; + int ret; + + state = drm_atomic_state_alloc(dev); + if (!state) + return -ENOMEM; + + state->acquire_ctx = ctx; + + drm_for_each_connector(connector, dev) { + ret = nouveau_atomic_disable_connector(state, connector); + if (ret) + break; + } + + if (ret == 0) + ret = drm_atomic_commit(state); + drm_atomic_state_put(state); + return ret; +} + +static struct drm_atomic_state * +nouveau_atomic_suspend(struct drm_device *dev) +{ + struct drm_modeset_acquire_ctx ctx; + struct drm_atomic_state *state; + int ret; + + drm_modeset_acquire_init(&ctx, 0); + +retry: + ret = drm_modeset_lock_all_ctx(dev, &ctx); + if (ret < 0) { + state = ERR_PTR(ret); + goto unlock; + } + + state = drm_atomic_helper_duplicate_state(dev, &ctx); + if (IS_ERR(state)) + goto unlock; + + ret = nouveau_atomic_disable(dev, &ctx); + if (ret < 0) { + drm_atomic_state_put(state); + state = ERR_PTR(ret); + goto unlock; + } + +unlock: + if (PTR_ERR(state) == -EDEADLK) { + drm_modeset_backoff(&ctx); + goto retry; + } + + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + return state; +} + int nouveau_display_suspend(struct drm_device *dev, bool runtime) { + struct nouveau_display *disp = nouveau_display(dev); struct drm_crtc *crtc; - nouveau_display_fini(dev); + if (dev->mode_config.funcs->atomic_commit) { + if (!runtime) { + disp->suspend = nouveau_atomic_suspend(dev); + if (IS_ERR(disp->suspend)) { + int ret = PTR_ERR(disp->suspend); + disp->suspend = NULL; + return ret; + } + } + + nouveau_display_fini(dev, true); + return 0; + } + + nouveau_display_fini(dev, true); list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { struct nouveau_framebuffer *nouveau_fb; @@ -600,10 +720,20 @@ nouveau_display_suspend(struct drm_device *dev, bool runtime) void nouveau_display_resume(struct drm_device *dev, bool runtime) { + struct nouveau_display *disp = nouveau_display(dev); struct nouveau_drm *drm = nouveau_drm(dev); struct drm_crtc *crtc; int ret, head; + if (dev->mode_config.funcs->atomic_commit) { + nouveau_display_init(dev); + if (disp->suspend) { + drm_atomic_helper_resume(dev, disp->suspend); + disp->suspend = NULL; + } + return; + } + /* re-pin fb/cursors */ list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { struct nouveau_framebuffer *nouveau_fb; @@ -692,10 +822,7 @@ nouveau_page_flip_emit(struct nouveau_channel *chan, if (ret) goto fail; - if (drm->device.info.family < NV_DEVICE_INFO_V0_FERMI) - BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1); - else - BEGIN_NVC0(chan, FermiSw, NV_SW_PAGE_FLIP, 1); + BEGIN_NV04(chan, NvSubSw, NV_SW_PAGE_FLIP, 1); OUT_RING (chan, 0x00000000); FIRE_RING (chan); @@ -724,6 +851,8 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, struct nouveau_channel *chan; struct nouveau_cli *cli; struct nouveau_fence *fence; + struct nv04_display *dispnv04 = nv04_display(dev); + int head = nouveau_crtc(crtc)->index; int ret; chan = drm->channel; @@ -770,32 +899,23 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, drm_crtc_vblank_get(crtc); /* Emit a page flip */ - if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nv50_display_flip_next(crtc, fb, chan, swap_interval); + if (swap_interval) { + ret = RING_SPACE(chan, 8); if (ret) goto fail_unreserve; - } else { - struct nv04_display *dispnv04 = nv04_display(dev); - int head = nouveau_crtc(crtc)->index; - - if (swap_interval) { - ret = RING_SPACE(chan, 8); - if (ret) - goto fail_unreserve; - - BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1); - OUT_RING (chan, 0); - BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1); - OUT_RING (chan, head); - BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1); - OUT_RING (chan, 0); - BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1); - OUT_RING (chan, 0); - } - nouveau_bo_ref(new_bo, &dispnv04->image[head]); + BEGIN_NV04(chan, NvSubImageBlit, 0x012c, 1); + OUT_RING (chan, 0); + BEGIN_NV04(chan, NvSubImageBlit, 0x0134, 1); + OUT_RING (chan, head); + BEGIN_NV04(chan, NvSubImageBlit, 0x0100, 1); + OUT_RING (chan, 0); + BEGIN_NV04(chan, NvSubImageBlit, 0x0130, 1); + OUT_RING (chan, 0); } + nouveau_bo_ref(new_bo, &dispnv04->image[head]); + ret = nouveau_page_flip_emit(chan, old_bo, new_bo, s, &fence); if (ret) goto fail_unreserve; @@ -843,16 +963,8 @@ nouveau_finish_page_flip(struct nouveau_channel *chan, s = list_first_entry(&fctx->flip, struct nouveau_page_flip_state, head); if (s->event) { - if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { - drm_crtc_arm_vblank_event(s->crtc, s->event); - } else { - drm_crtc_send_vblank_event(s->crtc, s->event); - - /* Give up ownership of vblank for page-flipped crtc */ - drm_crtc_vblank_put(s->crtc); - } - } - else { + drm_crtc_arm_vblank_event(s->crtc, s->event); + } else { /* Give up ownership of vblank for page-flipped crtc */ drm_crtc_vblank_put(s->crtc); } @@ -874,12 +986,10 @@ nouveau_flip_complete(struct nvif_notify *notify) struct nouveau_page_flip_state state; if (!nouveau_finish_page_flip(chan, &state)) { - if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) { - nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc), - state.offset + state.crtc->y * - state.pitch + state.crtc->x * - state.bpp / 8); - } + nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc), + state.offset + state.crtc->y * + state.pitch + state.crtc->x * + state.bpp / 8); } return NVIF_NOTIFY_KEEP; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h index 0420ee861ea4..330fe0fc5c11 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.h +++ b/drivers/gpu/drm/nouveau/nouveau_display.h @@ -22,8 +22,9 @@ nouveau_framebuffer(struct drm_framebuffer *fb) return container_of(fb, struct nouveau_framebuffer, base); } -int nouveau_framebuffer_init(struct drm_device *, struct nouveau_framebuffer *, - const struct drm_mode_fb_cmd2 *, struct nouveau_bo *); +int nouveau_framebuffer_new(struct drm_device *, + const struct drm_mode_fb_cmd2 *, + struct nouveau_bo *, struct nouveau_framebuffer **); struct nouveau_page_flip_state { struct list_head head; @@ -39,9 +40,6 @@ struct nouveau_display { int (*init)(struct drm_device *); void (*fini)(struct drm_device *); - int (*fb_ctor)(struct drm_framebuffer *); - void (*fb_dtor)(struct drm_framebuffer *); - struct nvif_object disp; struct drm_property *dithering_mode; @@ -52,6 +50,8 @@ struct nouveau_display { /* not really hue and saturation: */ struct drm_property *vibrant_hue_property; struct drm_property *color_vibrance_property; + + struct drm_atomic_state *suspend; }; static inline struct nouveau_display * @@ -63,7 +63,7 @@ nouveau_display(struct drm_device *dev) int nouveau_display_create(struct drm_device *dev); void nouveau_display_destroy(struct drm_device *dev); int nouveau_display_init(struct drm_device *dev); -void nouveau_display_fini(struct drm_device *dev); +void nouveau_display_fini(struct drm_device *dev, bool suspend); int nouveau_display_suspend(struct drm_device *dev, bool runtime); void nouveau_display_resume(struct drm_device *dev, bool runtime); int nouveau_display_vblank_enable(struct drm_device *, unsigned int); @@ -103,4 +103,7 @@ nouveau_backlight_exit(struct drm_device *dev) { } #endif +struct drm_framebuffer * +nouveau_user_framebuffer_create(struct drm_device *, struct drm_file *, + const struct drm_mode_fb_cmd2 *); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c index 87d52d36f4fc..0d052e1660f8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dp.c +++ b/drivers/gpu/drm/nouveau/nouveau_dp.c @@ -30,6 +30,13 @@ #include "nouveau_encoder.h" #include "nouveau_crtc.h" +#include <nvif/class.h> +#include <nvif/cl5070.h> + +MODULE_PARM_DESC(mst, "Enable DisplayPort multi-stream (default: enabled)"); +static int nouveau_mst = 1; +module_param_named(mst, nouveau_mst, int, 0400); + static void nouveau_dp_probe_oui(struct drm_device *dev, struct nvkm_i2c_aux *aux, u8 *dpcd) { @@ -55,14 +62,14 @@ nouveau_dp_detect(struct nouveau_encoder *nv_encoder) struct drm_device *dev = nv_encoder->base.base.dev; struct nouveau_drm *drm = nouveau_drm(dev); struct nvkm_i2c_aux *aux; - u8 *dpcd = nv_encoder->dp.dpcd; + u8 dpcd[8]; int ret; aux = nv_encoder->aux; if (!aux) return -ENODEV; - ret = nvkm_rdaux(aux, DP_DPCD_REV, dpcd, 8); + ret = nvkm_rdaux(aux, DP_DPCD_REV, dpcd, sizeof(dpcd)); if (ret) return ret; @@ -84,5 +91,11 @@ nouveau_dp_detect(struct nouveau_encoder *nv_encoder) nv_encoder->dp.link_nr, nv_encoder->dp.link_bw); nouveau_dp_probe_oui(dev, aux, dpcd); - return 0; + + ret = nv50_mstm_detect(nv_encoder->dp.mstm, dpcd, nouveau_mst); + if (ret == 1) + return NOUVEAU_DP_MST; + if (ret == 0) + return NOUVEAU_DP_SST; + return ret; } diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 6adf94789417..9876e6fcfcf0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -519,7 +519,7 @@ nouveau_drm_unload(struct drm_device *dev) nouveau_debugfs_fini(drm); if (dev->mode_config.num_crtc) - nouveau_display_fini(dev); + nouveau_display_fini(dev, false); nouveau_display_destroy(dev); nouveau_bios_takedown(dev); @@ -1037,6 +1037,7 @@ static void nouveau_display_options(void) DRM_DEBUG_DRIVER("... modeset : %d\n", nouveau_modeset); DRM_DEBUG_DRIVER("... runpm : %d\n", nouveau_runtime_pm); DRM_DEBUG_DRIVER("... vram_pushbuf : %d\n", nouveau_vram_pushbuf); + DRM_DEBUG_DRIVER("... hdmimhz : %d\n", nouveau_hdmimhz); } static const struct dev_pm_ops nouveau_pm_ops = { diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index c0e2b3207503..4cd47bae73c7 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -204,6 +204,10 @@ void nouveau_drm_device_remove(struct drm_device *dev); if (unlikely(drm_debug & DRM_UT_DRIVER)) \ NV_PRINTK(info, &(drm)->client, f, ##a); \ } while(0) +#define NV_ATOMIC(drm,f,a...) do { \ + if (unlikely(drm_debug & DRM_UT_ATOMIC)) \ + NV_PRINTK(info, &(drm)->client, f, ##a); \ +} while(0) extern int nouveau_modeset; diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h index ee6a6d3fc80f..198e5f27682f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_encoder.h +++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h @@ -30,6 +30,7 @@ #include <subdev/bios/dcb.h> #include <drm/drm_encoder_slave.h> +#include <drm/drm_dp_mst_helper.h> #include "dispnv04/disp.h" #define NV_DPMS_CLEARED 0x80 @@ -57,15 +58,16 @@ struct nouveau_encoder { union { struct { - u8 dpcd[8]; + struct nv50_mstm *mstm; int link_nr; int link_bw; - u32 datarate; } dp; }; void (*enc_save)(struct drm_encoder *encoder); void (*enc_restore)(struct drm_encoder *encoder); + void (*update)(struct nouveau_encoder *, u8 head, + struct drm_display_mode *, u8 proto, u8 depth); }; struct nouveau_encoder * @@ -90,9 +92,17 @@ get_slave_funcs(struct drm_encoder *enc) } /* nouveau_dp.c */ +enum nouveau_dp_status { + NOUVEAU_DP_SST, + NOUVEAU_DP_MST, +}; + int nouveau_dp_detect(struct nouveau_encoder *); struct nouveau_connector * nouveau_encoder_connector_get(struct nouveau_encoder *encoder); +int nv50_mstm_detect(struct nv50_mstm *, u8 dpcd[8], int allow); +void nv50_mstm_remove(struct nv50_mstm *); +void nv50_mstm_service(struct nv50_mstm *); #endif /* __NOUVEAU_ENCODER_H__ */ diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c index 9f5692726c16..cc8c58337981 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c +++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c @@ -58,7 +58,7 @@ static void nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); struct nvif_device *device = &drm->device; int ret; @@ -90,7 +90,7 @@ static void nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); struct nvif_device *device = &drm->device; int ret; @@ -122,7 +122,7 @@ static void nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); struct nvif_device *device = &drm->device; int ret; @@ -154,7 +154,7 @@ static int nouveau_fbcon_sync(struct fb_info *info) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -181,7 +181,7 @@ static int nouveau_fbcon_open(struct fb_info *info, int user) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); int ret = pm_runtime_get_sync(drm->dev->dev); if (ret < 0 && ret != -EACCES) return ret; @@ -192,7 +192,7 @@ static int nouveau_fbcon_release(struct fb_info *info, int user) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); pm_runtime_put(drm->dev->dev); return 0; } @@ -333,16 +333,15 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, { struct nouveau_fbdev *fbcon = container_of(helper, struct nouveau_fbdev, helper); - struct drm_device *dev = fbcon->dev; + struct drm_device *dev = fbcon->helper.dev; struct nouveau_drm *drm = nouveau_drm(dev); struct nvif_device *device = &drm->device; struct fb_info *info; - struct drm_framebuffer *fb; - struct nouveau_framebuffer *nouveau_fb; + struct nouveau_framebuffer *fb; struct nouveau_channel *chan; struct nouveau_bo *nvbo; struct drm_mode_fb_cmd2 mode_cmd; - int size, ret; + int ret; mode_cmd.width = sizes->surface_width; mode_cmd.height = sizes->surface_height; @@ -353,16 +352,17 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, sizes->surface_depth); - size = mode_cmd.pitches[0] * mode_cmd.height; - size = roundup(size, PAGE_SIZE); - - ret = nouveau_gem_new(dev, size, 0, NOUVEAU_GEM_DOMAIN_VRAM, - 0, 0x0000, &nvbo); + ret = nouveau_gem_new(dev, mode_cmd.pitches[0] * mode_cmd.height, + 0, NOUVEAU_GEM_DOMAIN_VRAM, 0, 0x0000, &nvbo); if (ret) { NV_ERROR(drm, "failed to allocate framebuffer\n"); goto out; } + ret = nouveau_framebuffer_new(dev, &mode_cmd, nvbo, &fb); + if (ret) + goto out_unref; + ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, false); if (ret) { NV_ERROR(drm, "failed to pin fb: %d\n", ret); @@ -377,8 +377,7 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, chan = nouveau_nofbaccel ? NULL : drm->channel; if (chan && device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_bo_vma_add(nvbo, drm->client.vm, - &fbcon->nouveau_fb.vma); + ret = nouveau_bo_vma_add(nvbo, drm->client.vm, &fb->vma); if (ret) { NV_ERROR(drm, "failed to map fb into chan: %d\n", ret); chan = NULL; @@ -394,13 +393,8 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, info->par = fbcon; - nouveau_framebuffer_init(dev, &fbcon->nouveau_fb, &mode_cmd, nvbo); - - nouveau_fb = &fbcon->nouveau_fb; - fb = &nouveau_fb->base; - /* setup helper */ - fbcon->helper.fb = fb; + fbcon->helper.fb = &fb->base; strcpy(info->fix.id, "nouveaufb"); if (!chan) @@ -411,14 +405,14 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, FBINFO_HWACCEL_IMAGEBLIT; info->flags |= FBINFO_CAN_FORCE_OUTPUT; info->fbops = &nouveau_fbcon_sw_ops; - info->fix.smem_start = nvbo->bo.mem.bus.base + - nvbo->bo.mem.bus.offset; - info->fix.smem_len = size; + info->fix.smem_start = fb->nvbo->bo.mem.bus.base + + fb->nvbo->bo.mem.bus.offset; + info->fix.smem_len = fb->nvbo->bo.mem.num_pages << PAGE_SHIFT; - info->screen_base = nvbo_kmap_obj_iovirtual(nouveau_fb->nvbo); - info->screen_size = size; + info->screen_base = nvbo_kmap_obj_iovirtual(fb->nvbo); + info->screen_size = fb->nvbo->bo.mem.num_pages << PAGE_SHIFT; - drm_fb_helper_fill_fix(info, fb->pitches[0], fb->depth); + drm_fb_helper_fill_fix(info, fb->base.pitches[0], fb->base.depth); drm_fb_helper_fill_var(info, &fbcon->helper, sizes->fb_width, sizes->fb_height); /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ @@ -429,20 +423,19 @@ nouveau_fbcon_create(struct drm_fb_helper *helper, /* To allow resizeing without swapping buffers */ NV_INFO(drm, "allocated %dx%d fb: 0x%llx, bo %p\n", - nouveau_fb->base.width, nouveau_fb->base.height, - nvbo->bo.offset, nvbo); + fb->base.width, fb->base.height, fb->nvbo->bo.offset, nvbo); vga_switcheroo_client_fb_set(dev->pdev, info); return 0; out_unlock: if (chan) - nouveau_bo_vma_del(nvbo, &fbcon->nouveau_fb.vma); - nouveau_bo_unmap(nvbo); + nouveau_bo_vma_del(fb->nvbo, &fb->vma); + nouveau_bo_unmap(fb->nvbo); out_unpin: - nouveau_bo_unpin(nvbo); + nouveau_bo_unpin(fb->nvbo); out_unref: - nouveau_bo_ref(NULL, &nvbo); + nouveau_bo_ref(NULL, &fb->nvbo); out: return ret; } @@ -458,28 +451,26 @@ nouveau_fbcon_output_poll_changed(struct drm_device *dev) static int nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon) { - struct nouveau_framebuffer *nouveau_fb = &fbcon->nouveau_fb; + struct nouveau_framebuffer *nouveau_fb = nouveau_framebuffer(fbcon->helper.fb); drm_fb_helper_unregister_fbi(&fbcon->helper); drm_fb_helper_release_fbi(&fbcon->helper); + drm_fb_helper_fini(&fbcon->helper); if (nouveau_fb->nvbo) { - nouveau_bo_unmap(nouveau_fb->nvbo); nouveau_bo_vma_del(nouveau_fb->nvbo, &nouveau_fb->vma); + nouveau_bo_unmap(nouveau_fb->nvbo); nouveau_bo_unpin(nouveau_fb->nvbo); - drm_gem_object_unreference_unlocked(&nouveau_fb->nvbo->gem); - nouveau_fb->nvbo = NULL; + drm_framebuffer_unreference(&nouveau_fb->base); } - drm_fb_helper_fini(&fbcon->helper); - drm_framebuffer_unregister_private(&nouveau_fb->base); - drm_framebuffer_cleanup(&nouveau_fb->base); + return 0; } void nouveau_fbcon_gpu_lockup(struct fb_info *info) { struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->dev); + struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); NV_ERROR(drm, "GPU lockup - switching to software fbcon\n"); info->flags |= FBINFO_HWACCEL_DISABLED; @@ -522,7 +513,6 @@ nouveau_fbcon_init(struct drm_device *dev) if (!fbcon) return -ENOMEM; - fbcon->dev = dev; drm->fbcon = fbcon; drm_fb_helper_prepare(dev, &fbcon->helper, &nouveau_fbcon_helper_funcs); @@ -545,7 +535,8 @@ nouveau_fbcon_init(struct drm_device *dev) preferred_bpp = 32; /* disable all the possible outputs/crtcs before entering KMS mode */ - drm_helper_disable_unused_functions(dev); + if (!dev->mode_config.funcs->atomic_commit) + drm_helper_disable_unused_functions(dev); ret = drm_fb_helper_initial_config(&fbcon->helper, preferred_bpp); if (ret) diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h index ca77ad001978..e2bca729721e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h +++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.h @@ -33,8 +33,6 @@ struct nouveau_fbdev { struct drm_fb_helper helper; - struct nouveau_framebuffer nouveau_fb; - struct drm_device *dev; unsigned int saved_flags; struct nvif_object surf2d; struct nvif_object clip; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index e9529ee6bc23..f2f348f0160c 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -586,5 +586,5 @@ static const struct dma_fence_ops nouveau_fence_ops_uevent = { .enable_signaling = nouveau_fence_enable_signaling, .signaled = nouveau_fence_is_signaled, .wait = dma_fence_default_wait, - .release = NULL + .release = nouveau_fence_release }; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index 41f3c019e534..ccdce1b4eec4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -92,7 +92,6 @@ struct nv84_fence_chan { struct nouveau_fence_chan base; struct nvkm_vma vma; struct nvkm_vma vma_gart; - struct nvkm_vma dispc_vma[4]; }; struct nv84_fence_priv { @@ -102,7 +101,6 @@ struct nv84_fence_priv { u32 *suspend; }; -u64 nv84_fence_crtc(struct nouveau_channel *, int); int nv84_fence_context_new(struct nouveau_channel *); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index 7f083c95f422..201b52b750dd 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -369,7 +369,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, { struct nouveau_cli *cli = nouveau_cli(file_priv); int trycnt = 0; - int ret, i; + int ret = -EINVAL, i; struct nouveau_bo *res_bo = NULL; LIST_HEAD(gart_list); LIST_HEAD(vram_list); diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index 1825dbc33192..a6dbe8258040 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -398,6 +398,9 @@ nouveau_ttm_init(struct nouveau_drm *drm) /* VRAM init */ drm->gem.vram_available = drm->device.info.ram_user; + arch_io_reserve_memtype_wc(device->func->resource_addr(device, 1), + device->func->resource_size(device, 1)); + ret = ttm_bo_init_mm(&drm->ttm.bdev, TTM_PL_VRAM, drm->gem.vram_available >> PAGE_SHIFT); if (ret) { @@ -430,6 +433,8 @@ nouveau_ttm_init(struct nouveau_drm *drm) void nouveau_ttm_fini(struct nouveau_drm *drm) { + struct nvkm_device *device = nvxx_device(&drm->device); + ttm_bo_clean_mm(&drm->ttm.bdev, TTM_PL_VRAM); ttm_bo_clean_mm(&drm->ttm.bdev, TTM_PL_TT); @@ -439,4 +444,7 @@ nouveau_ttm_fini(struct nouveau_drm *drm) arch_phys_wc_del(drm->ttm.mtrr); drm->ttm.mtrr = 0; + arch_io_free_memtype_wc(device->func->resource_addr(device, 1), + device->func->resource_size(device, 1)); + } diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c index da8fd5ff9d0f..6a2b187e3c3b 100644 --- a/drivers/gpu/drm/nouveau/nv04_fbcon.c +++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c @@ -30,7 +30,7 @@ int nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -50,7 +50,7 @@ int nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -77,7 +77,7 @@ int nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; uint32_t fg; uint32_t bg; @@ -133,7 +133,7 @@ int nv04_fbcon_accel_init(struct fb_info *info) { struct nouveau_fbdev *nfbdev = info->par; - struct drm_device *dev = nfbdev->dev; + struct drm_device *dev = nfbdev->helper.dev; struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_channel *chan = drm->channel; struct nvif_device *device = &drm->device; diff --git a/drivers/gpu/drm/nouveau/nv10_fence.c b/drivers/gpu/drm/nouveau/nv10_fence.c index f99fcf56928a..2998bde29211 100644 --- a/drivers/gpu/drm/nouveau/nv10_fence.c +++ b/drivers/gpu/drm/nouveau/nv10_fence.c @@ -57,16 +57,13 @@ void nv10_fence_context_del(struct nouveau_channel *chan) { struct nv10_fence_chan *fctx = chan->fence; - int i; nouveau_fence_context_del(&fctx->base); - for (i = 0; i < ARRAY_SIZE(fctx->head); i++) - nvif_object_fini(&fctx->head[i]); nvif_object_fini(&fctx->sema); chan->fence = NULL; nouveau_fence_context_free(&fctx->base); } -int +static int nv10_fence_context_new(struct nouveau_channel *chan) { struct nv10_fence_chan *fctx; diff --git a/drivers/gpu/drm/nouveau/nv10_fence.h b/drivers/gpu/drm/nouveau/nv10_fence.h index a87259f3983a..b7a508585304 100644 --- a/drivers/gpu/drm/nouveau/nv10_fence.h +++ b/drivers/gpu/drm/nouveau/nv10_fence.h @@ -7,7 +7,6 @@ struct nv10_fence_chan { struct nouveau_fence_chan base; struct nvif_object sema; - struct nvif_object head[4]; }; struct nv10_fence_priv { diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c index 7d0edcbcfca7..a9855a4ec532 100644 --- a/drivers/gpu/drm/nouveau/nv50_display.c +++ b/drivers/gpu/drm/nouveau/nv50_display.c @@ -25,10 +25,12 @@ #include <linux/dma-mapping.h> #include <drm/drmP.h> +#include <drm/drm_atomic.h> +#include <drm/drm_atomic_helper.h> #include <drm/drm_crtc_helper.h> -#include <drm/drm_plane_helper.h> #include <drm/drm_dp_helper.h> #include <drm/drm_fb_helper.h> +#include <drm/drm_plane_helper.h> #include <nvif/class.h> #include <nvif/cl0002.h> @@ -38,6 +40,7 @@ #include <nvif/cl507c.h> #include <nvif/cl507d.h> #include <nvif/cl507e.h> +#include <nvif/event.h> #include "nouveau_drv.h" #include "nouveau_dma.h" @@ -46,6 +49,7 @@ #include "nouveau_encoder.h" #include "nouveau_crtc.h" #include "nouveau_fence.h" +#include "nouveau_fbcon.h" #include "nv50_display.h" #define EVO_DMA_NR 9 @@ -61,6 +65,227 @@ #define EVO_MAST_NTFY EVO_SYNC( 0, 0x00) #define EVO_FLIP_SEM0(c) EVO_SYNC((c) + 1, 0x00) #define EVO_FLIP_SEM1(c) EVO_SYNC((c) + 1, 0x10) +#define EVO_FLIP_NTFY0(c) EVO_SYNC((c) + 1, 0x20) +#define EVO_FLIP_NTFY1(c) EVO_SYNC((c) + 1, 0x30) + +/****************************************************************************** + * Atomic state + *****************************************************************************/ +#define nv50_atom(p) container_of((p), struct nv50_atom, state) + +struct nv50_atom { + struct drm_atomic_state state; + + struct list_head outp; + bool lock_core; + bool flush_disable; +}; + +struct nv50_outp_atom { + struct list_head head; + + struct drm_encoder *encoder; + bool flush_disable; + + union { + struct { + bool ctrl:1; + }; + u8 mask; + } clr; + + union { + struct { + bool ctrl:1; + }; + u8 mask; + } set; +}; + +#define nv50_head_atom(p) container_of((p), struct nv50_head_atom, state) + +struct nv50_head_atom { + struct drm_crtc_state state; + + struct { + u16 iW; + u16 iH; + u16 oW; + u16 oH; + } view; + + struct nv50_head_mode { + bool interlace; + u32 clock; + struct { + u16 active; + u16 synce; + u16 blanke; + u16 blanks; + } h; + struct { + u32 active; + u16 synce; + u16 blanke; + u16 blanks; + u16 blank2s; + u16 blank2e; + u16 blankus; + } v; + } mode; + + struct { + u32 handle; + u64 offset:40; + } lut; + + struct { + bool visible; + u32 handle; + u64 offset:40; + u8 format; + u8 kind:7; + u8 layout:1; + u8 block:4; + u32 pitch:20; + u16 x; + u16 y; + u16 w; + u16 h; + } core; + + struct { + bool visible; + u32 handle; + u64 offset:40; + u8 layout:1; + u8 format:1; + } curs; + + struct { + u8 depth; + u8 cpp; + u16 x; + u16 y; + u16 w; + u16 h; + } base; + + struct { + u8 cpp; + } ovly; + + struct { + bool enable:1; + u8 bits:2; + u8 mode:4; + } dither; + + struct { + struct { + u16 cos:12; + u16 sin:12; + } sat; + } procamp; + + union { + struct { + bool core:1; + bool curs:1; + }; + u8 mask; + } clr; + + union { + struct { + bool core:1; + bool curs:1; + bool view:1; + bool mode:1; + bool base:1; + bool ovly:1; + bool dither:1; + bool procamp:1; + }; + u16 mask; + } set; +}; + +static inline struct nv50_head_atom * +nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc) +{ + struct drm_crtc_state *statec = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(statec)) + return (void *)statec; + return nv50_head_atom(statec); +} + +#define nv50_wndw_atom(p) container_of((p), struct nv50_wndw_atom, state) + +struct nv50_wndw_atom { + struct drm_plane_state state; + u8 interval; + + struct drm_rect clip; + + struct { + u32 handle; + u16 offset:12; + bool awaken:1; + } ntfy; + + struct { + u32 handle; + u16 offset:12; + u32 acquire; + u32 release; + } sema; + + struct { + u8 enable:2; + } lut; + + struct { + u8 mode:2; + u8 interval:4; + + u8 format; + u8 kind:7; + u8 layout:1; + u8 block:4; + u32 pitch:20; + u16 w; + u16 h; + + u32 handle; + u64 offset; + } image; + + struct { + u16 x; + u16 y; + } point; + + union { + struct { + bool ntfy:1; + bool sema:1; + bool image:1; + }; + u8 mask; + } clr; + + union { + struct { + bool ntfy:1; + bool sema:1; + bool image:1; + bool lut:1; + bool point:1; + }; + u8 mask; + } set; +}; /****************************************************************************** * EVO channel @@ -133,34 +358,6 @@ nv50_pioc_create(struct nvif_device *device, struct nvif_object *disp, } /****************************************************************************** - * Cursor Immediate - *****************************************************************************/ - -struct nv50_curs { - struct nv50_pioc base; -}; - -static int -nv50_curs_create(struct nvif_device *device, struct nvif_object *disp, - int head, struct nv50_curs *curs) -{ - struct nv50_disp_cursor_v0 args = { - .head = head, - }; - static const s32 oclass[] = { - GK104_DISP_CURSOR, - GF110_DISP_CURSOR, - GT214_DISP_CURSOR, - G82_DISP_CURSOR, - NV50_DISP_CURSOR, - 0 - }; - - return nv50_pioc_create(device, disp, oclass, head, &args, sizeof(args), - &curs->base); -} - -/****************************************************************************** * Overlay Immediate *****************************************************************************/ @@ -192,6 +389,11 @@ nv50_oimm_create(struct nvif_device *device, struct nvif_object *disp, * DMA EVO channel *****************************************************************************/ +struct nv50_dmac_ctxdma { + struct list_head head; + struct nvif_object object; +}; + struct nv50_dmac { struct nv50_chan base; dma_addr_t handle; @@ -199,6 +401,7 @@ struct nv50_dmac { struct nvif_object sync; struct nvif_object vram; + struct list_head ctxdma; /* Protects against concurrent pushbuf access to this channel, lock is * grabbed by evo_wait (if the pushbuf reservation is successful) and @@ -207,9 +410,82 @@ struct nv50_dmac { }; static void +nv50_dmac_ctxdma_del(struct nv50_dmac_ctxdma *ctxdma) +{ + nvif_object_fini(&ctxdma->object); + list_del(&ctxdma->head); + kfree(ctxdma); +} + +static struct nv50_dmac_ctxdma * +nv50_dmac_ctxdma_new(struct nv50_dmac *dmac, struct nouveau_framebuffer *fb) +{ + struct nouveau_drm *drm = nouveau_drm(fb->base.dev); + struct nv50_dmac_ctxdma *ctxdma; + const u8 kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8; + const u32 handle = 0xfb000000 | kind; + struct { + struct nv_dma_v0 base; + union { + struct nv50_dma_v0 nv50; + struct gf100_dma_v0 gf100; + struct gf119_dma_v0 gf119; + }; + } args = {}; + u32 argc = sizeof(args.base); + int ret; + + list_for_each_entry(ctxdma, &dmac->ctxdma, head) { + if (ctxdma->object.handle == handle) + return ctxdma; + } + + if (!(ctxdma = kzalloc(sizeof(*ctxdma), GFP_KERNEL))) + return ERR_PTR(-ENOMEM); + list_add(&ctxdma->head, &dmac->ctxdma); + + args.base.target = NV_DMA_V0_TARGET_VRAM; + args.base.access = NV_DMA_V0_ACCESS_RDWR; + args.base.start = 0; + args.base.limit = drm->device.info.ram_user - 1; + + if (drm->device.info.chipset < 0x80) { + args.nv50.part = NV50_DMA_V0_PART_256; + argc += sizeof(args.nv50); + } else + if (drm->device.info.chipset < 0xc0) { + args.nv50.part = NV50_DMA_V0_PART_256; + args.nv50.kind = kind; + argc += sizeof(args.nv50); + } else + if (drm->device.info.chipset < 0xd0) { + args.gf100.kind = kind; + argc += sizeof(args.gf100); + } else { + args.gf119.page = GF119_DMA_V0_PAGE_LP; + args.gf119.kind = kind; + argc += sizeof(args.gf119); + } + + ret = nvif_object_init(&dmac->base.user, handle, NV_DMA_IN_MEMORY, + &args, argc, &ctxdma->object); + if (ret) { + nv50_dmac_ctxdma_del(ctxdma); + return ERR_PTR(ret); + } + + return ctxdma; +} + +static void nv50_dmac_destroy(struct nv50_dmac *dmac, struct nvif_object *disp) { struct nvif_device *device = dmac->base.device; + struct nv50_dmac_ctxdma *ctxdma, *ctxtmp; + + list_for_each_entry_safe(ctxdma, ctxtmp, &dmac->ctxdma, head) { + nv50_dmac_ctxdma_del(ctxdma); + } nvif_object_fini(&dmac->vram); nvif_object_fini(&dmac->sync); @@ -278,6 +554,7 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp, if (ret) return ret; + INIT_LIST_HEAD(&dmac->ctxdma); return ret; } @@ -381,34 +658,23 @@ nv50_ovly_create(struct nvif_device *device, struct nvif_object *disp, struct nv50_head { struct nouveau_crtc base; - struct nouveau_bo *image; - struct nv50_curs curs; - struct nv50_sync sync; struct nv50_ovly ovly; struct nv50_oimm oimm; }; #define nv50_head(c) ((struct nv50_head *)nouveau_crtc(c)) -#define nv50_curs(c) (&nv50_head(c)->curs) -#define nv50_sync(c) (&nv50_head(c)->sync) #define nv50_ovly(c) (&nv50_head(c)->ovly) #define nv50_oimm(c) (&nv50_head(c)->oimm) #define nv50_chan(c) (&(c)->base.base) #define nv50_vers(c) nv50_chan(c)->user.oclass -struct nv50_fbdma { - struct list_head head; - struct nvif_object core; - struct nvif_object base[4]; -}; - struct nv50_disp { struct nvif_object *disp; struct nv50_mast mast; - struct list_head fbdma; - struct nouveau_bo *sync; + + struct mutex mutex; }; static struct nv50_disp * @@ -419,12 +685,6 @@ nv50_disp(struct drm_device *dev) #define nv50_mast(d) (&nv50_disp(d)->mast) -static struct drm_crtc * -nv50_display_crtc_get(struct drm_encoder *encoder) -{ - return nouveau_encoder(encoder)->crtc; -} - /****************************************************************************** * EVO channel helpers *****************************************************************************/ @@ -463,812 +723,1460 @@ evo_kick(u32 *push, void *evoc) mutex_unlock(&dmac->lock); } -#if 1 -#define evo_mthd(p,m,s) *((p)++) = (((s) << 18) | (m)) -#define evo_data(p,d) *((p)++) = (d) -#else #define evo_mthd(p,m,s) do { \ const u32 _m = (m), _s = (s); \ - printk(KERN_ERR "%04x %d %s\n", _m, _s, __func__); \ + if (drm_debug & DRM_UT_KMS) \ + printk(KERN_ERR "%04x %d %s\n", _m, _s, __func__); \ *((p)++) = ((_s << 18) | _m); \ } while(0) + #define evo_data(p,d) do { \ const u32 _d = (d); \ - printk(KERN_ERR "\t%08x\n", _d); \ + if (drm_debug & DRM_UT_KMS) \ + printk(KERN_ERR "\t%08x\n", _d); \ *((p)++) = _d; \ } while(0) -#endif -static bool -evo_sync_wait(void *data) +/****************************************************************************** + * Plane + *****************************************************************************/ +#define nv50_wndw(p) container_of((p), struct nv50_wndw, plane) + +struct nv50_wndw { + const struct nv50_wndw_func *func; + struct nv50_dmac *dmac; + + struct drm_plane plane; + + struct nvif_notify notify; + u16 ntfy; + u16 sema; + u32 data; +}; + +struct nv50_wndw_func { + void *(*dtor)(struct nv50_wndw *); + int (*acquire)(struct nv50_wndw *, struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh); + void (*release)(struct nv50_wndw *, struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh); + void (*prepare)(struct nv50_wndw *, struct nv50_head_atom *asyh, + struct nv50_wndw_atom *asyw); + + void (*sema_set)(struct nv50_wndw *, struct nv50_wndw_atom *); + void (*sema_clr)(struct nv50_wndw *); + void (*ntfy_set)(struct nv50_wndw *, struct nv50_wndw_atom *); + void (*ntfy_clr)(struct nv50_wndw *); + int (*ntfy_wait_begun)(struct nv50_wndw *, struct nv50_wndw_atom *); + void (*image_set)(struct nv50_wndw *, struct nv50_wndw_atom *); + void (*image_clr)(struct nv50_wndw *); + void (*lut)(struct nv50_wndw *, struct nv50_wndw_atom *); + void (*point)(struct nv50_wndw *, struct nv50_wndw_atom *); + + u32 (*update)(struct nv50_wndw *, u32 interlock); +}; + +static int +nv50_wndw_wait_armed(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) { - if (nouveau_bo_rd32(data, EVO_MAST_NTFY) != 0x00000000) - return true; - usleep_range(1, 2); - return false; + if (asyw->set.ntfy) + return wndw->func->ntfy_wait_begun(wndw, asyw); + return 0; } -static int -evo_sync(struct drm_device *dev) +static u32 +nv50_wndw_flush_clr(struct nv50_wndw *wndw, u32 interlock, bool flush, + struct nv50_wndw_atom *asyw) { - struct nvif_device *device = &nouveau_drm(dev)->device; - struct nv50_disp *disp = nv50_disp(dev); - struct nv50_mast *mast = nv50_mast(dev); - u32 *push = evo_wait(mast, 8); - if (push) { - nouveau_bo_wr32(disp->sync, EVO_MAST_NTFY, 0x00000000); - evo_mthd(push, 0x0084, 1); - evo_data(push, 0x80000000 | EVO_MAST_NTFY); - evo_mthd(push, 0x0080, 2); - evo_data(push, 0x00000000); - evo_data(push, 0x00000000); - evo_kick(push, mast); - if (nvif_msec(device, 2000, - if (evo_sync_wait(disp->sync)) - break; - ) >= 0) - return 0; + if (asyw->clr.sema && (!asyw->set.sema || flush)) + wndw->func->sema_clr(wndw); + if (asyw->clr.ntfy && (!asyw->set.ntfy || flush)) + wndw->func->ntfy_clr(wndw); + if (asyw->clr.image && (!asyw->set.image || flush)) + wndw->func->image_clr(wndw); + + return flush ? wndw->func->update(wndw, interlock) : 0; +} + +static u32 +nv50_wndw_flush_set(struct nv50_wndw *wndw, u32 interlock, + struct nv50_wndw_atom *asyw) +{ + if (interlock) { + asyw->image.mode = 0; + asyw->image.interval = 1; } - return -EBUSY; + if (asyw->set.sema ) wndw->func->sema_set (wndw, asyw); + if (asyw->set.ntfy ) wndw->func->ntfy_set (wndw, asyw); + if (asyw->set.image) wndw->func->image_set(wndw, asyw); + if (asyw->set.lut ) wndw->func->lut (wndw, asyw); + if (asyw->set.point) wndw->func->point (wndw, asyw); + + return wndw->func->update(wndw, interlock); } -/****************************************************************************** - * Page flipping channel - *****************************************************************************/ -struct nouveau_bo * -nv50_display_crtc_sema(struct drm_device *dev, int crtc) +static void +nv50_wndw_atomic_check_release(struct nv50_wndw *wndw, + struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh) { - return nv50_disp(dev)->sync; + struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev); + NV_ATOMIC(drm, "%s release\n", wndw->plane.name); + wndw->func->release(wndw, asyw, asyh); + asyw->ntfy.handle = 0; + asyw->sema.handle = 0; } -struct nv50_display_flip { - struct nv50_disp *disp; - struct nv50_sync *chan; -}; - -static bool -nv50_display_flip_wait(void *data) +static int +nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw, + struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh) { - struct nv50_display_flip *flip = data; - if (nouveau_bo_rd32(flip->disp->sync, flip->chan->addr / 4) == - flip->chan->data) - return true; - usleep_range(1, 2); - return false; + struct nouveau_framebuffer *fb = nouveau_framebuffer(asyw->state.fb); + struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev); + int ret; + + NV_ATOMIC(drm, "%s acquire\n", wndw->plane.name); + asyw->clip.x1 = 0; + asyw->clip.y1 = 0; + asyw->clip.x2 = asyh->state.mode.hdisplay; + asyw->clip.y2 = asyh->state.mode.vdisplay; + + asyw->image.w = fb->base.width; + asyw->image.h = fb->base.height; + asyw->image.kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8; + if (asyw->image.kind) { + asyw->image.layout = 0; + if (drm->device.info.chipset >= 0xc0) + asyw->image.block = fb->nvbo->tile_mode >> 4; + else + asyw->image.block = fb->nvbo->tile_mode; + asyw->image.pitch = (fb->base.pitches[0] / 4) << 4; + } else { + asyw->image.layout = 1; + asyw->image.block = 0; + asyw->image.pitch = fb->base.pitches[0]; + } + + ret = wndw->func->acquire(wndw, asyw, asyh); + if (ret) + return ret; + + if (asyw->set.image) { + if (!(asyw->image.mode = asyw->interval ? 0 : 1)) + asyw->image.interval = asyw->interval; + else + asyw->image.interval = 0; + } + + return 0; } -void -nv50_display_flip_stop(struct drm_crtc *crtc) +static int +nv50_wndw_atomic_check(struct drm_plane *plane, struct drm_plane_state *state) { - struct nvif_device *device = &nouveau_drm(crtc->dev)->device; - struct nv50_display_flip flip = { - .disp = nv50_disp(crtc->dev), - .chan = nv50_sync(crtc), - }; - u32 *push; + struct nouveau_drm *drm = nouveau_drm(plane->dev); + struct nv50_wndw *wndw = nv50_wndw(plane); + struct nv50_wndw_atom *armw = nv50_wndw_atom(wndw->plane.state); + struct nv50_wndw_atom *asyw = nv50_wndw_atom(state); + struct nv50_head_atom *harm = NULL, *asyh = NULL; + bool varm = false, asyv = false, asym = false; + int ret; - push = evo_wait(flip.chan, 8); - if (push) { - evo_mthd(push, 0x0084, 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x0094, 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x00c0, 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x0080, 1); - evo_data(push, 0x00000000); - evo_kick(push, flip.chan); + NV_ATOMIC(drm, "%s atomic_check\n", plane->name); + if (asyw->state.crtc) { + asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc); + if (IS_ERR(asyh)) + return PTR_ERR(asyh); + asym = drm_atomic_crtc_needs_modeset(&asyh->state); + asyv = asyh->state.active; } - nvif_msec(device, 2000, - if (nv50_display_flip_wait(&flip)) - break; - ); + if (armw->state.crtc) { + harm = nv50_head_atom_get(asyw->state.state, armw->state.crtc); + if (IS_ERR(harm)) + return PTR_ERR(harm); + varm = harm->state.crtc->state->active; + } + + if (asyv) { + asyw->point.x = asyw->state.crtc_x; + asyw->point.y = asyw->state.crtc_y; + if (memcmp(&armw->point, &asyw->point, sizeof(asyw->point))) + asyw->set.point = true; + + if (!varm || asym || armw->state.fb != asyw->state.fb) { + ret = nv50_wndw_atomic_check_acquire(wndw, asyw, asyh); + if (ret) + return ret; + } + } else + if (varm) { + nv50_wndw_atomic_check_release(wndw, asyw, harm); + } else { + return 0; + } + + if (!asyv || asym) { + asyw->clr.ntfy = armw->ntfy.handle != 0; + asyw->clr.sema = armw->sema.handle != 0; + if (wndw->func->image_clr) + asyw->clr.image = armw->image.handle != 0; + asyw->set.lut = wndw->func->lut && asyv; + } + + return 0; } -int -nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb, - struct nouveau_channel *chan, u32 swap_interval) +static void +nv50_wndw_cleanup_fb(struct drm_plane *plane, struct drm_plane_state *old_state) { - struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb); - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - struct nv50_head *head = nv50_head(crtc); - struct nv50_sync *sync = nv50_sync(crtc); - u32 *push; - int ret; + struct nouveau_framebuffer *fb = nouveau_framebuffer(old_state->fb); + struct nouveau_drm *drm = nouveau_drm(plane->dev); - if (crtc->primary->fb->width != fb->width || - crtc->primary->fb->height != fb->height) - return -EINVAL; + NV_ATOMIC(drm, "%s cleanup: %p\n", plane->name, old_state->fb); + if (!old_state->fb) + return; + + nouveau_bo_unpin(fb->nvbo); +} - swap_interval <<= 4; - if (swap_interval == 0) - swap_interval |= 0x100; - if (chan == NULL) - evo_sync(crtc->dev); +static int +nv50_wndw_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state) +{ + struct nouveau_framebuffer *fb = nouveau_framebuffer(state->fb); + struct nouveau_drm *drm = nouveau_drm(plane->dev); + struct nv50_wndw *wndw = nv50_wndw(plane); + struct nv50_wndw_atom *asyw = nv50_wndw_atom(state); + struct nv50_head_atom *asyh; + struct nv50_dmac_ctxdma *ctxdma; + int ret; - push = evo_wait(sync, 128); - if (unlikely(push == NULL)) - return -EBUSY; + NV_ATOMIC(drm, "%s prepare: %p\n", plane->name, state->fb); + if (!asyw->state.fb) + return 0; - if (chan && chan->user.oclass < G82_CHANNEL_GPFIFO) { - ret = RING_SPACE(chan, 8); - if (ret) - return ret; + ret = nouveau_bo_pin(fb->nvbo, TTM_PL_FLAG_VRAM, true); + if (ret) + return ret; - BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 2); - OUT_RING (chan, NvEvoSema0 + nv_crtc->index); - OUT_RING (chan, sync->addr ^ 0x10); - BEGIN_NV04(chan, 0, NV11_SUBCHAN_SEMAPHORE_RELEASE, 1); - OUT_RING (chan, sync->data + 1); - BEGIN_NV04(chan, 0, NV11_SUBCHAN_SEMAPHORE_OFFSET, 2); - OUT_RING (chan, sync->addr); - OUT_RING (chan, sync->data); - } else - if (chan && chan->user.oclass < FERMI_CHANNEL_GPFIFO) { - u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + sync->addr; - ret = RING_SPACE(chan, 12); - if (ret) - return ret; + ctxdma = nv50_dmac_ctxdma_new(wndw->dmac, fb); + if (IS_ERR(ctxdma)) { + nouveau_bo_unpin(fb->nvbo); + return PTR_ERR(ctxdma); + } - BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1); - OUT_RING (chan, chan->vram.handle); - BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); - OUT_RING (chan, upper_32_bits(addr ^ 0x10)); - OUT_RING (chan, lower_32_bits(addr ^ 0x10)); - OUT_RING (chan, sync->data + 1); - OUT_RING (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG); - BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); - OUT_RING (chan, upper_32_bits(addr)); - OUT_RING (chan, lower_32_bits(addr)); - OUT_RING (chan, sync->data); - OUT_RING (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL); - } else - if (chan) { - u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + sync->addr; - ret = RING_SPACE(chan, 10); - if (ret) - return ret; + asyw->state.fence = reservation_object_get_excl_rcu(fb->nvbo->bo.resv); + asyw->image.handle = ctxdma->object.handle; + asyw->image.offset = fb->nvbo->bo.offset; - BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); - OUT_RING (chan, upper_32_bits(addr ^ 0x10)); - OUT_RING (chan, lower_32_bits(addr ^ 0x10)); - OUT_RING (chan, sync->data + 1); - OUT_RING (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG | - NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD); - BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4); - OUT_RING (chan, upper_32_bits(addr)); - OUT_RING (chan, lower_32_bits(addr)); - OUT_RING (chan, sync->data); - OUT_RING (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL | - NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD); - } - - if (chan) { - sync->addr ^= 0x10; - sync->data++; - FIRE_RING (chan); - } - - /* queue the flip */ - evo_mthd(push, 0x0100, 1); - evo_data(push, 0xfffe0000); - evo_mthd(push, 0x0084, 1); - evo_data(push, swap_interval); - if (!(swap_interval & 0x00000100)) { - evo_mthd(push, 0x00e0, 1); - evo_data(push, 0x40000000); - } - evo_mthd(push, 0x0088, 4); - evo_data(push, sync->addr); - evo_data(push, sync->data++); - evo_data(push, sync->data); - evo_data(push, sync->base.sync.handle); - evo_mthd(push, 0x00a0, 2); - evo_data(push, 0x00000000); - evo_data(push, 0x00000000); - evo_mthd(push, 0x00c0, 1); - evo_data(push, nv_fb->r_handle); - evo_mthd(push, 0x0110, 2); - evo_data(push, 0x00000000); - evo_data(push, 0x00000000); - if (nv50_vers(sync) < GF110_DISP_BASE_CHANNEL_DMA) { - evo_mthd(push, 0x0800, 5); - evo_data(push, nv_fb->nvbo->bo.offset >> 8); - evo_data(push, 0); - evo_data(push, (fb->height << 16) | fb->width); - evo_data(push, nv_fb->r_pitch); - evo_data(push, nv_fb->r_format); - } else { - evo_mthd(push, 0x0400, 5); - evo_data(push, nv_fb->nvbo->bo.offset >> 8); - evo_data(push, 0); - evo_data(push, (fb->height << 16) | fb->width); - evo_data(push, nv_fb->r_pitch); - evo_data(push, nv_fb->r_format); + if (wndw->func->prepare) { + asyh = nv50_head_atom_get(asyw->state.state, asyw->state.crtc); + if (IS_ERR(asyh)) + return PTR_ERR(asyh); + + wndw->func->prepare(wndw, asyh, asyw); } - evo_mthd(push, 0x0080, 1); - evo_data(push, 0x00000000); - evo_kick(push, sync); - nouveau_bo_ref(nv_fb->nvbo, &head->image); + return 0; +} + +static const struct drm_plane_helper_funcs +nv50_wndw_helper = { + .prepare_fb = nv50_wndw_prepare_fb, + .cleanup_fb = nv50_wndw_cleanup_fb, + .atomic_check = nv50_wndw_atomic_check, +}; + +static void +nv50_wndw_atomic_destroy_state(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct nv50_wndw_atom *asyw = nv50_wndw_atom(state); + __drm_atomic_helper_plane_destroy_state(&asyw->state); + dma_fence_put(asyw->state.fence); + kfree(asyw); +} + +static struct drm_plane_state * +nv50_wndw_atomic_duplicate_state(struct drm_plane *plane) +{ + struct nv50_wndw_atom *armw = nv50_wndw_atom(plane->state); + struct nv50_wndw_atom *asyw; + if (!(asyw = kmalloc(sizeof(*asyw), GFP_KERNEL))) + return NULL; + __drm_atomic_helper_plane_duplicate_state(plane, &asyw->state); + asyw->state.fence = NULL; + asyw->interval = 1; + asyw->sema = armw->sema; + asyw->ntfy = armw->ntfy; + asyw->image = armw->image; + asyw->point = armw->point; + asyw->lut = armw->lut; + asyw->clr.mask = 0; + asyw->set.mask = 0; + return &asyw->state; +} + +static void +nv50_wndw_reset(struct drm_plane *plane) +{ + struct nv50_wndw_atom *asyw; + + if (WARN_ON(!(asyw = kzalloc(sizeof(*asyw), GFP_KERNEL)))) + return; + + if (plane->state) + plane->funcs->atomic_destroy_state(plane, plane->state); + plane->state = &asyw->state; + plane->state->plane = plane; + plane->state->rotation = DRM_ROTATE_0; +} + +static void +nv50_wndw_destroy(struct drm_plane *plane) +{ + struct nv50_wndw *wndw = nv50_wndw(plane); + void *data; + nvif_notify_fini(&wndw->notify); + data = wndw->func->dtor(wndw); + drm_plane_cleanup(&wndw->plane); + kfree(data); +} + +static const struct drm_plane_funcs +nv50_wndw = { + .update_plane = drm_atomic_helper_update_plane, + .disable_plane = drm_atomic_helper_disable_plane, + .destroy = nv50_wndw_destroy, + .reset = nv50_wndw_reset, + .set_property = drm_atomic_helper_plane_set_property, + .atomic_duplicate_state = nv50_wndw_atomic_duplicate_state, + .atomic_destroy_state = nv50_wndw_atomic_destroy_state, +}; + +static void +nv50_wndw_fini(struct nv50_wndw *wndw) +{ + nvif_notify_put(&wndw->notify); +} + +static void +nv50_wndw_init(struct nv50_wndw *wndw) +{ + nvif_notify_get(&wndw->notify); +} + +static int +nv50_wndw_ctor(const struct nv50_wndw_func *func, struct drm_device *dev, + enum drm_plane_type type, const char *name, int index, + struct nv50_dmac *dmac, const u32 *format, int nformat, + struct nv50_wndw *wndw) +{ + int ret; + + wndw->func = func; + wndw->dmac = dmac; + + ret = drm_universal_plane_init(dev, &wndw->plane, 0, &nv50_wndw, format, + nformat, type, "%s-%d", name, index); + if (ret) + return ret; + + drm_plane_helper_add(&wndw->plane, &nv50_wndw_helper); return 0; } /****************************************************************************** - * CRTC + * Cursor plane *****************************************************************************/ +#define nv50_curs(p) container_of((p), struct nv50_curs, wndw) + +struct nv50_curs { + struct nv50_wndw wndw; + struct nvif_object chan; +}; + +static u32 +nv50_curs_update(struct nv50_wndw *wndw, u32 interlock) +{ + struct nv50_curs *curs = nv50_curs(wndw); + nvif_wr32(&curs->chan, 0x0080, 0x00000000); + return 0; +} + +static void +nv50_curs_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) +{ + struct nv50_curs *curs = nv50_curs(wndw); + nvif_wr32(&curs->chan, 0x0084, (asyw->point.y << 16) | asyw->point.x); +} + +static void +nv50_curs_prepare(struct nv50_wndw *wndw, struct nv50_head_atom *asyh, + struct nv50_wndw_atom *asyw) +{ + asyh->curs.handle = nv50_disp(wndw->plane.dev)->mast.base.vram.handle; + asyh->curs.offset = asyw->image.offset; + asyh->set.curs = asyh->curs.visible; +} + +static void +nv50_curs_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh) +{ + asyh->curs.visible = false; +} + static int -nv50_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update) +nv50_curs_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); - struct nouveau_connector *nv_connector; - struct drm_connector *connector; - u32 *push, mode = 0x00; + int ret; - nv_connector = nouveau_crtc_connector_get(nv_crtc); - connector = &nv_connector->base; - if (nv_connector->dithering_mode == DITHERING_MODE_AUTO) { - if (nv_crtc->base.primary->fb->depth > connector->display_info.bpc * 3) - mode = DITHERING_MODE_DYNAMIC2X2; - } else { - mode = nv_connector->dithering_mode; - } + ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + true, true); + asyh->curs.visible = asyw->state.visible; + if (ret || !asyh->curs.visible) + return ret; - if (nv_connector->dithering_depth == DITHERING_DEPTH_AUTO) { - if (connector->display_info.bpc >= 8) - mode |= DITHERING_DEPTH_8BPC; - } else { - mode |= nv_connector->dithering_depth; + switch (asyw->state.fb->width) { + case 32: asyh->curs.layout = 0; break; + case 64: asyh->curs.layout = 1; break; + default: + return -EINVAL; } - push = evo_wait(mast, 4); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x08a0 + (nv_crtc->index * 0x0400), 1); - evo_data(push, mode); - } else - if (nv50_vers(mast) < GK104_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0490 + (nv_crtc->index * 0x0300), 1); - evo_data(push, mode); - } else { - evo_mthd(push, 0x04a0 + (nv_crtc->index * 0x0300), 1); - evo_data(push, mode); - } + if (asyw->state.fb->width != asyw->state.fb->height) + return -EINVAL; - if (update) { - evo_mthd(push, 0x0080, 1); - evo_data(push, 0x00000000); - } - evo_kick(push, mast); + switch (asyw->state.fb->pixel_format) { + case DRM_FORMAT_ARGB8888: asyh->curs.format = 1; break; + default: + WARN_ON(1); + return -EINVAL; } return 0; } +static void * +nv50_curs_dtor(struct nv50_wndw *wndw) +{ + struct nv50_curs *curs = nv50_curs(wndw); + nvif_object_fini(&curs->chan); + return curs; +} + +static const u32 +nv50_curs_format[] = { + DRM_FORMAT_ARGB8888, +}; + +static const struct nv50_wndw_func +nv50_curs = { + .dtor = nv50_curs_dtor, + .acquire = nv50_curs_acquire, + .release = nv50_curs_release, + .prepare = nv50_curs_prepare, + .point = nv50_curs_point, + .update = nv50_curs_update, +}; + static int -nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update) +nv50_curs_new(struct nouveau_drm *drm, struct nv50_head *head, + struct nv50_curs **pcurs) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); - struct drm_display_mode *omode, *umode = &nv_crtc->base.mode; - struct drm_crtc *crtc = &nv_crtc->base; - struct nouveau_connector *nv_connector; - int mode = DRM_MODE_SCALE_NONE; - u32 oX, oY, *push; + static const struct nvif_mclass curses[] = { + { GK104_DISP_CURSOR, 0 }, + { GF110_DISP_CURSOR, 0 }, + { GT214_DISP_CURSOR, 0 }, + { G82_DISP_CURSOR, 0 }, + { NV50_DISP_CURSOR, 0 }, + {} + }; + struct nv50_disp_cursor_v0 args = { + .head = head->base.index, + }; + struct nv50_disp *disp = nv50_disp(drm->dev); + struct nv50_curs *curs; + int cid, ret; + + cid = nvif_mclass(disp->disp, curses); + if (cid < 0) { + NV_ERROR(drm, "No supported cursor immediate class\n"); + return cid; + } - /* start off at the resolution we programmed the crtc for, this - * effectively handles NONE/FULL scaling - */ - nv_connector = nouveau_crtc_connector_get(nv_crtc); - if (nv_connector && nv_connector->native_mode) { - mode = nv_connector->scaling_mode; - if (nv_connector->scaling_full) /* non-EDID LVDS/eDP mode */ - mode = DRM_MODE_SCALE_FULLSCREEN; + if (!(curs = *pcurs = kzalloc(sizeof(*curs), GFP_KERNEL))) + return -ENOMEM; + + ret = nv50_wndw_ctor(&nv50_curs, drm->dev, DRM_PLANE_TYPE_CURSOR, + "curs", head->base.index, &disp->mast.base, + nv50_curs_format, ARRAY_SIZE(nv50_curs_format), + &curs->wndw); + if (ret) { + kfree(curs); + return ret; } - if (mode != DRM_MODE_SCALE_NONE) - omode = nv_connector->native_mode; - else - omode = umode; + ret = nvif_object_init(disp->disp, 0, curses[cid].oclass, &args, + sizeof(args), &curs->chan); + if (ret) { + NV_ERROR(drm, "curs%04x allocation failed: %d\n", + curses[cid].oclass, ret); + return ret; + } - oX = omode->hdisplay; - oY = omode->vdisplay; - if (omode->flags & DRM_MODE_FLAG_DBLSCAN) - oY *= 2; + return 0; +} - /* add overscan compensation if necessary, will keep the aspect - * ratio the same as the backend mode unless overridden by the - * user setting both hborder and vborder properties. - */ - if (nv_connector && ( nv_connector->underscan == UNDERSCAN_ON || - (nv_connector->underscan == UNDERSCAN_AUTO && - drm_detect_hdmi_monitor(nv_connector->edid)))) { - u32 bX = nv_connector->underscan_hborder; - u32 bY = nv_connector->underscan_vborder; - u32 aspect = (oY << 19) / oX; +/****************************************************************************** + * Primary plane + *****************************************************************************/ +#define nv50_base(p) container_of((p), struct nv50_base, wndw) - if (bX) { - oX -= (bX * 2); - if (bY) oY -= (bY * 2); - else oY = ((oX * aspect) + (aspect / 2)) >> 19; - } else { - oX -= (oX >> 4) + 32; - if (bY) oY -= (bY * 2); - else oY = ((oX * aspect) + (aspect / 2)) >> 19; - } +struct nv50_base { + struct nv50_wndw wndw; + struct nv50_sync chan; + int id; +}; + +static int +nv50_base_notify(struct nvif_notify *notify) +{ + return NVIF_NOTIFY_KEEP; +} + +static void +nv50_base_lut(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) +{ + struct nv50_base *base = nv50_base(wndw); + u32 *push; + if ((push = evo_wait(&base->chan, 2))) { + evo_mthd(push, 0x00e0, 1); + evo_data(push, asyw->lut.enable << 30); + evo_kick(push, &base->chan); } +} - /* handle CENTER/ASPECT scaling, taking into account the areas - * removed already for overscan compensation - */ - switch (mode) { - case DRM_MODE_SCALE_CENTER: - oX = min((u32)umode->hdisplay, oX); - oY = min((u32)umode->vdisplay, oY); - /* fall-through */ - case DRM_MODE_SCALE_ASPECT: - if (oY < oX) { - u32 aspect = (umode->hdisplay << 19) / umode->vdisplay; - oX = ((oY * aspect) + (aspect / 2)) >> 19; - } else { - u32 aspect = (umode->vdisplay << 19) / umode->hdisplay; - oY = ((oX * aspect) + (aspect / 2)) >> 19; - } - break; - default: - break; +static void +nv50_base_image_clr(struct nv50_wndw *wndw) +{ + struct nv50_base *base = nv50_base(wndw); + u32 *push; + if ((push = evo_wait(&base->chan, 4))) { + evo_mthd(push, 0x0084, 1); + evo_data(push, 0x00000000); + evo_mthd(push, 0x00c0, 1); + evo_data(push, 0x00000000); + evo_kick(push, &base->chan); } +} - push = evo_wait(mast, 8); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - /*XXX: SCALE_CTRL_ACTIVE??? */ - evo_mthd(push, 0x08d8 + (nv_crtc->index * 0x400), 2); - evo_data(push, (oY << 16) | oX); - evo_data(push, (oY << 16) | oX); - evo_mthd(push, 0x08a4 + (nv_crtc->index * 0x400), 1); +static void +nv50_base_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) +{ + struct nv50_base *base = nv50_base(wndw); + const s32 oclass = base->chan.base.base.user.oclass; + u32 *push; + if ((push = evo_wait(&base->chan, 10))) { + evo_mthd(push, 0x0084, 1); + evo_data(push, (asyw->image.mode << 8) | + (asyw->image.interval << 4)); + evo_mthd(push, 0x00c0, 1); + evo_data(push, asyw->image.handle); + if (oclass < G82_DISP_BASE_CHANNEL_DMA) { + evo_mthd(push, 0x0800, 5); + evo_data(push, asyw->image.offset >> 8); + evo_data(push, 0x00000000); + evo_data(push, (asyw->image.h << 16) | asyw->image.w); + evo_data(push, (asyw->image.layout << 20) | + asyw->image.pitch | + asyw->image.block); + evo_data(push, (asyw->image.kind << 16) | + (asyw->image.format << 8)); + } else + if (oclass < GF110_DISP_BASE_CHANNEL_DMA) { + evo_mthd(push, 0x0800, 5); + evo_data(push, asyw->image.offset >> 8); evo_data(push, 0x00000000); - evo_mthd(push, 0x08c8 + (nv_crtc->index * 0x400), 1); - evo_data(push, umode->vdisplay << 16 | umode->hdisplay); + evo_data(push, (asyw->image.h << 16) | asyw->image.w); + evo_data(push, (asyw->image.layout << 20) | + asyw->image.pitch | + asyw->image.block); + evo_data(push, asyw->image.format << 8); } else { - evo_mthd(push, 0x04c0 + (nv_crtc->index * 0x300), 3); - evo_data(push, (oY << 16) | oX); - evo_data(push, (oY << 16) | oX); - evo_data(push, (oY << 16) | oX); - evo_mthd(push, 0x0494 + (nv_crtc->index * 0x300), 1); + evo_mthd(push, 0x0400, 5); + evo_data(push, asyw->image.offset >> 8); evo_data(push, 0x00000000); - evo_mthd(push, 0x04b8 + (nv_crtc->index * 0x300), 1); - evo_data(push, umode->vdisplay << 16 | umode->hdisplay); + evo_data(push, (asyw->image.h << 16) | asyw->image.w); + evo_data(push, (asyw->image.layout << 24) | + asyw->image.pitch | + asyw->image.block); + evo_data(push, asyw->image.format << 8); } + evo_kick(push, &base->chan); + } +} - evo_kick(push, mast); +static void +nv50_base_ntfy_clr(struct nv50_wndw *wndw) +{ + struct nv50_base *base = nv50_base(wndw); + u32 *push; + if ((push = evo_wait(&base->chan, 2))) { + evo_mthd(push, 0x00a4, 1); + evo_data(push, 0x00000000); + evo_kick(push, &base->chan); + } +} - if (update) { - nv50_display_flip_stop(crtc); - nv50_display_flip_next(crtc, crtc->primary->fb, - NULL, 1); - } +static void +nv50_base_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) +{ + struct nv50_base *base = nv50_base(wndw); + u32 *push; + if ((push = evo_wait(&base->chan, 3))) { + evo_mthd(push, 0x00a0, 2); + evo_data(push, (asyw->ntfy.awaken << 30) | asyw->ntfy.offset); + evo_data(push, asyw->ntfy.handle); + evo_kick(push, &base->chan); } +} - return 0; +static void +nv50_base_sema_clr(struct nv50_wndw *wndw) +{ + struct nv50_base *base = nv50_base(wndw); + u32 *push; + if ((push = evo_wait(&base->chan, 2))) { + evo_mthd(push, 0x0094, 1); + evo_data(push, 0x00000000); + evo_kick(push, &base->chan); + } } -static int -nv50_crtc_set_raster_vblank_dmi(struct nouveau_crtc *nv_crtc, u32 usec) +static void +nv50_base_sema_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); + struct nv50_base *base = nv50_base(wndw); u32 *push; + if ((push = evo_wait(&base->chan, 5))) { + evo_mthd(push, 0x0088, 4); + evo_data(push, asyw->sema.offset); + evo_data(push, asyw->sema.acquire); + evo_data(push, asyw->sema.release); + evo_data(push, asyw->sema.handle); + evo_kick(push, &base->chan); + } +} - push = evo_wait(mast, 8); - if (!push) - return -ENOMEM; +static u32 +nv50_base_update(struct nv50_wndw *wndw, u32 interlock) +{ + struct nv50_base *base = nv50_base(wndw); + u32 *push; - evo_mthd(push, 0x0828 + (nv_crtc->index * 0x400), 1); - evo_data(push, usec); - evo_kick(push, mast); + if (!(push = evo_wait(&base->chan, 2))) + return 0; + evo_mthd(push, 0x0080, 1); + evo_data(push, interlock); + evo_kick(push, &base->chan); + + if (base->chan.base.base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA) + return interlock ? 2 << (base->id * 8) : 0; + return interlock ? 2 << (base->id * 4) : 0; +} + +static int +nv50_base_ntfy_wait_begun(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw) +{ + struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev); + struct nv50_disp *disp = nv50_disp(wndw->plane.dev); + if (nvif_msec(&drm->device, 2000ULL, + u32 data = nouveau_bo_rd32(disp->sync, asyw->ntfy.offset / 4); + if ((data & 0xc0000000) == 0x40000000) + break; + usleep_range(1, 2); + ) < 0) + return -ETIMEDOUT; return 0; } +static void +nv50_base_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh) +{ + asyh->base.cpp = 0; +} + static int -nv50_crtc_set_color_vibrance(struct nouveau_crtc *nv_crtc, bool update) +nv50_base_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw, + struct nv50_head_atom *asyh) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); - u32 *push, hue, vib; - int adj; + const u32 format = asyw->state.fb->pixel_format; + const struct drm_format_info *info; + int ret; - adj = (nv_crtc->color_vibrance > 0) ? 50 : 0; - vib = ((nv_crtc->color_vibrance * 2047 + adj) / 100) & 0xfff; - hue = ((nv_crtc->vibrant_hue * 2047) / 100) & 0xfff; + info = drm_format_info(format); + if (!info || !info->depth) + return -EINVAL; - push = evo_wait(mast, 16); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x08a8 + (nv_crtc->index * 0x400), 1); - evo_data(push, (hue << 20) | (vib << 8)); - } else { - evo_mthd(push, 0x0498 + (nv_crtc->index * 0x300), 1); - evo_data(push, (hue << 20) | (vib << 8)); - } + ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip, + DRM_PLANE_HELPER_NO_SCALING, + DRM_PLANE_HELPER_NO_SCALING, + false, true); + if (ret) + return ret; - if (update) { - evo_mthd(push, 0x0080, 1); - evo_data(push, 0x00000000); - } - evo_kick(push, mast); + asyh->base.depth = info->depth; + asyh->base.cpp = info->cpp[0]; + asyh->base.x = asyw->state.src.x1 >> 16; + asyh->base.y = asyw->state.src.y1 >> 16; + asyh->base.w = asyw->state.fb->width; + asyh->base.h = asyw->state.fb->height; + + switch (format) { + case DRM_FORMAT_C8 : asyw->image.format = 0x1e; break; + case DRM_FORMAT_RGB565 : asyw->image.format = 0xe8; break; + case DRM_FORMAT_XRGB1555 : + case DRM_FORMAT_ARGB1555 : asyw->image.format = 0xe9; break; + case DRM_FORMAT_XRGB8888 : + case DRM_FORMAT_ARGB8888 : asyw->image.format = 0xcf; break; + case DRM_FORMAT_XBGR2101010: + case DRM_FORMAT_ABGR2101010: asyw->image.format = 0xd1; break; + case DRM_FORMAT_XBGR8888 : + case DRM_FORMAT_ABGR8888 : asyw->image.format = 0xd5; break; + default: + WARN_ON(1); + return -EINVAL; } + asyw->lut.enable = 1; + asyw->set.image = true; return 0; } +static void * +nv50_base_dtor(struct nv50_wndw *wndw) +{ + struct nv50_disp *disp = nv50_disp(wndw->plane.dev); + struct nv50_base *base = nv50_base(wndw); + nv50_dmac_destroy(&base->chan.base, disp->disp); + return base; +} + +static const u32 +nv50_base_format[] = { + DRM_FORMAT_C8, + DRM_FORMAT_RGB565, + DRM_FORMAT_XRGB1555, + DRM_FORMAT_ARGB1555, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR2101010, + DRM_FORMAT_ABGR2101010, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, +}; + +static const struct nv50_wndw_func +nv50_base = { + .dtor = nv50_base_dtor, + .acquire = nv50_base_acquire, + .release = nv50_base_release, + .sema_set = nv50_base_sema_set, + .sema_clr = nv50_base_sema_clr, + .ntfy_set = nv50_base_ntfy_set, + .ntfy_clr = nv50_base_ntfy_clr, + .ntfy_wait_begun = nv50_base_ntfy_wait_begun, + .image_set = nv50_base_image_set, + .image_clr = nv50_base_image_clr, + .lut = nv50_base_lut, + .update = nv50_base_update, +}; + static int -nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb, - int x, int y, bool update) +nv50_base_new(struct nouveau_drm *drm, struct nv50_head *head, + struct nv50_base **pbase) +{ + struct nv50_disp *disp = nv50_disp(drm->dev); + struct nv50_base *base; + int ret; + + if (!(base = *pbase = kzalloc(sizeof(*base), GFP_KERNEL))) + return -ENOMEM; + base->id = head->base.index; + base->wndw.ntfy = EVO_FLIP_NTFY0(base->id); + base->wndw.sema = EVO_FLIP_SEM0(base->id); + base->wndw.data = 0x00000000; + + ret = nv50_wndw_ctor(&nv50_base, drm->dev, DRM_PLANE_TYPE_PRIMARY, + "base", base->id, &base->chan.base, + nv50_base_format, ARRAY_SIZE(nv50_base_format), + &base->wndw); + if (ret) { + kfree(base); + return ret; + } + + ret = nv50_base_create(&drm->device, disp->disp, base->id, + disp->sync->bo.offset, &base->chan); + if (ret) + return ret; + + return nvif_notify_init(&base->chan.base.base.user, nv50_base_notify, + false, + NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT, + &(struct nvif_notify_uevent_req) {}, + sizeof(struct nvif_notify_uevent_req), + sizeof(struct nvif_notify_uevent_rep), + &base->wndw.notify); +} + +/****************************************************************************** + * Head + *****************************************************************************/ +static void +nv50_head_procamp(struct nv50_head *head, struct nv50_head_atom *asyh) { - struct nouveau_framebuffer *nvfb = nouveau_framebuffer(fb); - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; u32 *push; + if ((push = evo_wait(core, 2))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) + evo_mthd(push, 0x08a8 + (head->base.index * 0x400), 1); + else + evo_mthd(push, 0x0498 + (head->base.index * 0x300), 1); + evo_data(push, (asyh->procamp.sat.sin << 20) | + (asyh->procamp.sat.cos << 8)); + evo_kick(push, core); + } +} - push = evo_wait(mast, 16); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0860 + (nv_crtc->index * 0x400), 1); - evo_data(push, nvfb->nvbo->bo.offset >> 8); - evo_mthd(push, 0x0868 + (nv_crtc->index * 0x400), 3); - evo_data(push, (fb->height << 16) | fb->width); - evo_data(push, nvfb->r_pitch); - evo_data(push, nvfb->r_format); - evo_mthd(push, 0x08c0 + (nv_crtc->index * 0x400), 1); - evo_data(push, (y << 16) | x); - if (nv50_vers(mast) > NV50_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1); - evo_data(push, nvfb->r_handle); - } - } else { - evo_mthd(push, 0x0460 + (nv_crtc->index * 0x300), 1); - evo_data(push, nvfb->nvbo->bo.offset >> 8); - evo_mthd(push, 0x0468 + (nv_crtc->index * 0x300), 4); - evo_data(push, (fb->height << 16) | fb->width); - evo_data(push, nvfb->r_pitch); - evo_data(push, nvfb->r_format); - evo_data(push, nvfb->r_handle); - evo_mthd(push, 0x04b0 + (nv_crtc->index * 0x300), 1); - evo_data(push, (y << 16) | x); - } +static void +nv50_head_dither(struct nv50_head *head, struct nv50_head_atom *asyh) +{ + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 *push; + if ((push = evo_wait(core, 2))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) + evo_mthd(push, 0x08a0 + (head->base.index * 0x0400), 1); + else + if (core->base.user.oclass < GK104_DISP_CORE_CHANNEL_DMA) + evo_mthd(push, 0x0490 + (head->base.index * 0x0300), 1); + else + evo_mthd(push, 0x04a0 + (head->base.index * 0x0300), 1); + evo_data(push, (asyh->dither.mode << 3) | + (asyh->dither.bits << 1) | + asyh->dither.enable); + evo_kick(push, core); + } +} - if (update) { - evo_mthd(push, 0x0080, 1); - evo_data(push, 0x00000000); +static void +nv50_head_ovly(struct nv50_head *head, struct nv50_head_atom *asyh) +{ + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 bounds = 0; + u32 *push; + + if (asyh->base.cpp) { + switch (asyh->base.cpp) { + case 8: bounds |= 0x00000500; break; + case 4: bounds |= 0x00000300; break; + case 2: bounds |= 0x00000100; break; + default: + WARN_ON(1); + break; } - evo_kick(push, mast); + bounds |= 0x00000001; } - nv_crtc->fb.handle = nvfb->r_handle; - return 0; + if ((push = evo_wait(core, 2))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) + evo_mthd(push, 0x0904 + head->base.index * 0x400, 1); + else + evo_mthd(push, 0x04d4 + head->base.index * 0x300, 1); + evo_data(push, bounds); + evo_kick(push, core); + } } static void -nv50_crtc_cursor_show(struct nouveau_crtc *nv_crtc) +nv50_head_base(struct nv50_head *head, struct nv50_head_atom *asyh) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); - u32 *push = evo_wait(mast, 16); - if (push) { - if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2); - evo_data(push, 0x85000000); - evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8); - } else - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 2); - evo_data(push, 0x85000000); - evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8); - evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1); - evo_data(push, mast->base.vram.handle); - } else { - evo_mthd(push, 0x0480 + (nv_crtc->index * 0x300), 2); - evo_data(push, 0x85000000); - evo_data(push, nv_crtc->cursor.nvbo->bo.offset >> 8); - evo_mthd(push, 0x048c + (nv_crtc->index * 0x300), 1); - evo_data(push, mast->base.vram.handle); + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 bounds = 0; + u32 *push; + + if (asyh->base.cpp) { + switch (asyh->base.cpp) { + case 8: bounds |= 0x00000500; break; + case 4: bounds |= 0x00000300; break; + case 2: bounds |= 0x00000100; break; + case 1: bounds |= 0x00000000; break; + default: + WARN_ON(1); + break; } - evo_kick(push, mast); + bounds |= 0x00000001; + } + + if ((push = evo_wait(core, 2))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) + evo_mthd(push, 0x0900 + head->base.index * 0x400, 1); + else + evo_mthd(push, 0x04d0 + head->base.index * 0x300, 1); + evo_data(push, bounds); + evo_kick(push, core); } - nv_crtc->cursor.visible = true; } static void -nv50_crtc_cursor_hide(struct nouveau_crtc *nv_crtc) +nv50_head_curs_clr(struct nv50_head *head) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); - u32 *push = evo_wait(mast, 16); - if (push) { - if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 1); + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 *push; + if ((push = evo_wait(core, 4))) { + if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0880 + head->base.index * 0x400, 1); evo_data(push, 0x05000000); } else - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0880 + (nv_crtc->index * 0x400), 1); + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0880 + head->base.index * 0x400, 1); evo_data(push, 0x05000000); - evo_mthd(push, 0x089c + (nv_crtc->index * 0x400), 1); + evo_mthd(push, 0x089c + head->base.index * 0x400, 1); evo_data(push, 0x00000000); } else { - evo_mthd(push, 0x0480 + (nv_crtc->index * 0x300), 1); + evo_mthd(push, 0x0480 + head->base.index * 0x300, 1); evo_data(push, 0x05000000); - evo_mthd(push, 0x048c + (nv_crtc->index * 0x300), 1); + evo_mthd(push, 0x048c + head->base.index * 0x300, 1); evo_data(push, 0x00000000); } - evo_kick(push, mast); + evo_kick(push, core); } - nv_crtc->cursor.visible = false; } static void -nv50_crtc_cursor_show_hide(struct nouveau_crtc *nv_crtc, bool show, bool update) +nv50_head_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh) { - struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev); - - if (show && nv_crtc->cursor.nvbo && nv_crtc->base.enabled) - nv50_crtc_cursor_show(nv_crtc); - else - nv50_crtc_cursor_hide(nv_crtc); - - if (update) { - u32 *push = evo_wait(mast, 2); - if (push) { - evo_mthd(push, 0x0080, 1); - evo_data(push, 0x00000000); - evo_kick(push, mast); + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 *push; + if ((push = evo_wait(core, 5))) { + if (core->base.user.oclass < G82_DISP_BASE_CHANNEL_DMA) { + evo_mthd(push, 0x0880 + head->base.index * 0x400, 2); + evo_data(push, 0x80000000 | (asyh->curs.layout << 26) | + (asyh->curs.format << 24)); + evo_data(push, asyh->curs.offset >> 8); + } else + if (core->base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA) { + evo_mthd(push, 0x0880 + head->base.index * 0x400, 2); + evo_data(push, 0x80000000 | (asyh->curs.layout << 26) | + (asyh->curs.format << 24)); + evo_data(push, asyh->curs.offset >> 8); + evo_mthd(push, 0x089c + head->base.index * 0x400, 1); + evo_data(push, asyh->curs.handle); + } else { + evo_mthd(push, 0x0480 + head->base.index * 0x300, 2); + evo_data(push, 0x80000000 | (asyh->curs.layout << 26) | + (asyh->curs.format << 24)); + evo_data(push, asyh->curs.offset >> 8); + evo_mthd(push, 0x048c + head->base.index * 0x300, 1); + evo_data(push, asyh->curs.handle); } + evo_kick(push, core); } } static void -nv50_crtc_dpms(struct drm_crtc *crtc, int mode) +nv50_head_core_clr(struct nv50_head *head) { + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 *push; + if ((push = evo_wait(core, 2))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) + evo_mthd(push, 0x0874 + head->base.index * 0x400, 1); + else + evo_mthd(push, 0x0474 + head->base.index * 0x300, 1); + evo_data(push, 0x00000000); + evo_kick(push, core); + } } static void -nv50_crtc_prepare(struct drm_crtc *crtc) +nv50_head_core_set(struct nv50_head *head, struct nv50_head_atom *asyh) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - struct nv50_mast *mast = nv50_mast(crtc->dev); + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; u32 *push; + if ((push = evo_wait(core, 9))) { + if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0860 + head->base.index * 0x400, 1); + evo_data(push, asyh->core.offset >> 8); + evo_mthd(push, 0x0868 + head->base.index * 0x400, 4); + evo_data(push, (asyh->core.h << 16) | asyh->core.w); + evo_data(push, asyh->core.layout << 20 | + (asyh->core.pitch >> 8) << 8 | + asyh->core.block); + evo_data(push, asyh->core.kind << 16 | + asyh->core.format << 8); + evo_data(push, asyh->core.handle); + evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1); + evo_data(push, (asyh->core.y << 16) | asyh->core.x); + } else + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0860 + head->base.index * 0x400, 1); + evo_data(push, asyh->core.offset >> 8); + evo_mthd(push, 0x0868 + head->base.index * 0x400, 4); + evo_data(push, (asyh->core.h << 16) | asyh->core.w); + evo_data(push, asyh->core.layout << 20 | + (asyh->core.pitch >> 8) << 8 | + asyh->core.block); + evo_data(push, asyh->core.format << 8); + evo_data(push, asyh->core.handle); + evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1); + evo_data(push, (asyh->core.y << 16) | asyh->core.x); + } else { + evo_mthd(push, 0x0460 + head->base.index * 0x300, 1); + evo_data(push, asyh->core.offset >> 8); + evo_mthd(push, 0x0468 + head->base.index * 0x300, 4); + evo_data(push, (asyh->core.h << 16) | asyh->core.w); + evo_data(push, asyh->core.layout << 24 | + (asyh->core.pitch >> 8) << 8 | + asyh->core.block); + evo_data(push, asyh->core.format << 8); + evo_data(push, asyh->core.handle); + evo_mthd(push, 0x04b0 + head->base.index * 0x300, 1); + evo_data(push, (asyh->core.y << 16) | asyh->core.x); + } + evo_kick(push, core); + } +} - nv50_display_flip_stop(crtc); - - push = evo_wait(mast, 6); - if (push) { - if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 1); +static void +nv50_head_lut_clr(struct nv50_head *head) +{ + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 *push; + if ((push = evo_wait(core, 4))) { + if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1); evo_data(push, 0x40000000); } else - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 1); + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1); evo_data(push, 0x40000000); - evo_mthd(push, 0x085c + (nv_crtc->index * 0x400), 1); + evo_mthd(push, 0x085c + (head->base.index * 0x400), 1); evo_data(push, 0x00000000); } else { - evo_mthd(push, 0x0474 + (nv_crtc->index * 0x300), 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x0440 + (nv_crtc->index * 0x300), 1); + evo_mthd(push, 0x0440 + (head->base.index * 0x300), 1); evo_data(push, 0x03000000); - evo_mthd(push, 0x045c + (nv_crtc->index * 0x300), 1); + evo_mthd(push, 0x045c + (head->base.index * 0x300), 1); evo_data(push, 0x00000000); } - - evo_kick(push, mast); + evo_kick(push, core); } - - nv50_crtc_cursor_show_hide(nv_crtc, false, false); } static void -nv50_crtc_commit(struct drm_crtc *crtc) +nv50_head_lut_set(struct nv50_head *head, struct nv50_head_atom *asyh) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - struct nv50_mast *mast = nv50_mast(crtc->dev); + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; u32 *push; - - push = evo_wait(mast, 32); - if (push) { - if (nv50_vers(mast) < G82_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1); - evo_data(push, nv_crtc->fb.handle); - evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 2); + if ((push = evo_wait(core, 7))) { + if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2); evo_data(push, 0xc0000000); - evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8); + evo_data(push, asyh->lut.offset >> 8); } else - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0874 + (nv_crtc->index * 0x400), 1); - evo_data(push, nv_crtc->fb.handle); - evo_mthd(push, 0x0840 + (nv_crtc->index * 0x400), 2); + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2); evo_data(push, 0xc0000000); - evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8); - evo_mthd(push, 0x085c + (nv_crtc->index * 0x400), 1); - evo_data(push, mast->base.vram.handle); + evo_data(push, asyh->lut.offset >> 8); + evo_mthd(push, 0x085c + (head->base.index * 0x400), 1); + evo_data(push, asyh->lut.handle); } else { - evo_mthd(push, 0x0474 + (nv_crtc->index * 0x300), 1); - evo_data(push, nv_crtc->fb.handle); - evo_mthd(push, 0x0440 + (nv_crtc->index * 0x300), 4); + evo_mthd(push, 0x0440 + (head->base.index * 0x300), 4); evo_data(push, 0x83000000); - evo_data(push, nv_crtc->lut.nvbo->bo.offset >> 8); + evo_data(push, asyh->lut.offset >> 8); evo_data(push, 0x00000000); evo_data(push, 0x00000000); - evo_mthd(push, 0x045c + (nv_crtc->index * 0x300), 1); - evo_data(push, mast->base.vram.handle); - evo_mthd(push, 0x0430 + (nv_crtc->index * 0x300), 1); + evo_mthd(push, 0x045c + (head->base.index * 0x300), 1); + evo_data(push, asyh->lut.handle); + } + evo_kick(push, core); + } +} + +static void +nv50_head_mode(struct nv50_head *head, struct nv50_head_atom *asyh) +{ + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + struct nv50_head_mode *m = &asyh->mode; + u32 *push; + if ((push = evo_wait(core, 14))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0804 + (head->base.index * 0x400), 2); + evo_data(push, 0x00800000 | m->clock); + evo_data(push, m->interlace ? 0x00000002 : 0x00000000); + evo_mthd(push, 0x0810 + (head->base.index * 0x400), 7); + evo_data(push, 0x00000000); + evo_data(push, (m->v.active << 16) | m->h.active ); + evo_data(push, (m->v.synce << 16) | m->h.synce ); + evo_data(push, (m->v.blanke << 16) | m->h.blanke ); + evo_data(push, (m->v.blanks << 16) | m->h.blanks ); + evo_data(push, (m->v.blank2e << 16) | m->v.blank2s); + evo_data(push, asyh->mode.v.blankus); + evo_mthd(push, 0x082c + (head->base.index * 0x400), 1); + evo_data(push, 0x00000000); + } else { + evo_mthd(push, 0x0410 + (head->base.index * 0x300), 6); + evo_data(push, 0x00000000); + evo_data(push, (m->v.active << 16) | m->h.active ); + evo_data(push, (m->v.synce << 16) | m->h.synce ); + evo_data(push, (m->v.blanke << 16) | m->h.blanke ); + evo_data(push, (m->v.blanks << 16) | m->h.blanks ); + evo_data(push, (m->v.blank2e << 16) | m->v.blank2s); + evo_mthd(push, 0x042c + (head->base.index * 0x300), 2); + evo_data(push, 0x00000000); /* ??? */ evo_data(push, 0xffffff00); + evo_mthd(push, 0x0450 + (head->base.index * 0x300), 3); + evo_data(push, m->clock * 1000); + evo_data(push, 0x00200000); /* ??? */ + evo_data(push, m->clock * 1000); } + evo_kick(push, core); + } +} - evo_kick(push, mast); +static void +nv50_head_view(struct nv50_head *head, struct nv50_head_atom *asyh) +{ + struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base; + u32 *push; + if ((push = evo_wait(core, 10))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x08a4 + (head->base.index * 0x400), 1); + evo_data(push, 0x00000000); + evo_mthd(push, 0x08c8 + (head->base.index * 0x400), 1); + evo_data(push, (asyh->view.iH << 16) | asyh->view.iW); + evo_mthd(push, 0x08d8 + (head->base.index * 0x400), 2); + evo_data(push, (asyh->view.oH << 16) | asyh->view.oW); + evo_data(push, (asyh->view.oH << 16) | asyh->view.oW); + } else { + evo_mthd(push, 0x0494 + (head->base.index * 0x300), 1); + evo_data(push, 0x00000000); + evo_mthd(push, 0x04b8 + (head->base.index * 0x300), 1); + evo_data(push, (asyh->view.iH << 16) | asyh->view.iW); + evo_mthd(push, 0x04c0 + (head->base.index * 0x300), 3); + evo_data(push, (asyh->view.oH << 16) | asyh->view.oW); + evo_data(push, (asyh->view.oH << 16) | asyh->view.oW); + evo_data(push, (asyh->view.oH << 16) | asyh->view.oW); + } + evo_kick(push, core); } +} - nv50_crtc_cursor_show_hide(nv_crtc, true, true); - nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1); +static void +nv50_head_flush_clr(struct nv50_head *head, struct nv50_head_atom *asyh, bool y) +{ + if (asyh->clr.core && (!asyh->set.core || y)) + nv50_head_lut_clr(head); + if (asyh->clr.core && (!asyh->set.core || y)) + nv50_head_core_clr(head); + if (asyh->clr.curs && (!asyh->set.curs || y)) + nv50_head_curs_clr(head); } -static bool -nv50_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +static void +nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh) { - drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V); - return true; + if (asyh->set.view ) nv50_head_view (head, asyh); + if (asyh->set.mode ) nv50_head_mode (head, asyh); + if (asyh->set.core ) nv50_head_lut_set (head, asyh); + if (asyh->set.core ) nv50_head_core_set(head, asyh); + if (asyh->set.curs ) nv50_head_curs_set(head, asyh); + if (asyh->set.base ) nv50_head_base (head, asyh); + if (asyh->set.ovly ) nv50_head_ovly (head, asyh); + if (asyh->set.dither ) nv50_head_dither (head, asyh); + if (asyh->set.procamp) nv50_head_procamp (head, asyh); } -static int -nv50_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb) +static void +nv50_head_atomic_check_procamp(struct nv50_head_atom *armh, + struct nv50_head_atom *asyh, + struct nouveau_conn_atom *asyc) { - struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->primary->fb); - struct nv50_head *head = nv50_head(crtc); - int ret; + const int vib = asyc->procamp.color_vibrance - 100; + const int hue = asyc->procamp.vibrant_hue - 90; + const int adj = (vib > 0) ? 50 : 0; + asyh->procamp.sat.cos = ((vib * 2047 + adj) / 100) & 0xfff; + asyh->procamp.sat.sin = ((hue * 2047) / 100) & 0xfff; + asyh->set.procamp = true; +} - ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM, true); - if (ret == 0) { - if (head->image) - nouveau_bo_unpin(head->image); - nouveau_bo_ref(nvfb->nvbo, &head->image); +static void +nv50_head_atomic_check_dither(struct nv50_head_atom *armh, + struct nv50_head_atom *asyh, + struct nouveau_conn_atom *asyc) +{ + struct drm_connector *connector = asyc->state.connector; + u32 mode = 0x00; + + if (asyc->dither.mode == DITHERING_MODE_AUTO) { + if (asyh->base.depth > connector->display_info.bpc * 3) + mode = DITHERING_MODE_DYNAMIC2X2; + } else { + mode = asyc->dither.mode; } - return ret; + if (asyc->dither.depth == DITHERING_DEPTH_AUTO) { + if (connector->display_info.bpc >= 8) + mode |= DITHERING_DEPTH_8BPC; + } else { + mode |= asyc->dither.depth; + } + + asyh->dither.enable = mode; + asyh->dither.bits = mode >> 1; + asyh->dither.mode = mode >> 3; + asyh->set.dither = true; } -static int -nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *umode, - struct drm_display_mode *mode, int x, int y, - struct drm_framebuffer *old_fb) +static void +nv50_head_atomic_check_view(struct nv50_head_atom *armh, + struct nv50_head_atom *asyh, + struct nouveau_conn_atom *asyc) { - struct nv50_mast *mast = nv50_mast(crtc->dev); - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - struct nouveau_connector *nv_connector; - u32 ilace = (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 2 : 1; - u32 vscan = (mode->flags & DRM_MODE_FLAG_DBLSCAN) ? 2 : 1; - u32 hactive, hsynce, hbackp, hfrontp, hblanke, hblanks; - u32 vactive, vsynce, vbackp, vfrontp, vblanke, vblanks; - u32 vblan2e = 0, vblan2s = 1, vblankus = 0; - u32 *push; - int ret; - - hactive = mode->htotal; - hsynce = mode->hsync_end - mode->hsync_start - 1; - hbackp = mode->htotal - mode->hsync_end; - hblanke = hsynce + hbackp; - hfrontp = mode->hsync_start - mode->hdisplay; - hblanks = mode->htotal - hfrontp - 1; - - vactive = mode->vtotal * vscan / ilace; - vsynce = ((mode->vsync_end - mode->vsync_start) * vscan / ilace) - 1; - vbackp = (mode->vtotal - mode->vsync_end) * vscan / ilace; - vblanke = vsynce + vbackp; - vfrontp = (mode->vsync_start - mode->vdisplay) * vscan / ilace; - vblanks = vactive - vfrontp - 1; - /* XXX: Safe underestimate, even "0" works */ - vblankus = (vactive - mode->vdisplay - 2) * hactive; - vblankus *= 1000; - vblankus /= mode->clock; + struct drm_connector *connector = asyc->state.connector; + struct drm_display_mode *omode = &asyh->state.adjusted_mode; + struct drm_display_mode *umode = &asyh->state.mode; + int mode = asyc->scaler.mode; + struct edid *edid; + + if (connector->edid_blob_ptr) + edid = (struct edid *)connector->edid_blob_ptr->data; + else + edid = NULL; - if (mode->flags & DRM_MODE_FLAG_INTERLACE) { - vblan2e = vactive + vsynce + vbackp; - vblan2s = vblan2e + (mode->vdisplay * vscan / ilace); - vactive = (vactive * 2) + 1; + if (!asyc->scaler.full) { + if (mode == DRM_MODE_SCALE_NONE) + omode = umode; + } else { + /* Non-EDID LVDS/eDP mode. */ + mode = DRM_MODE_SCALE_FULLSCREEN; } - ret = nv50_crtc_swap_fbs(crtc, old_fb); - if (ret) - return ret; + asyh->view.iW = umode->hdisplay; + asyh->view.iH = umode->vdisplay; + asyh->view.oW = omode->hdisplay; + asyh->view.oH = omode->vdisplay; + if (omode->flags & DRM_MODE_FLAG_DBLSCAN) + asyh->view.oH *= 2; - push = evo_wait(mast, 64); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0804 + (nv_crtc->index * 0x400), 2); - evo_data(push, 0x00800000 | mode->clock); - evo_data(push, (ilace == 2) ? 2 : 0); - evo_mthd(push, 0x0810 + (nv_crtc->index * 0x400), 6); - evo_data(push, 0x00000000); - evo_data(push, (vactive << 16) | hactive); - evo_data(push, ( vsynce << 16) | hsynce); - evo_data(push, (vblanke << 16) | hblanke); - evo_data(push, (vblanks << 16) | hblanks); - evo_data(push, (vblan2e << 16) | vblan2s); - evo_mthd(push, 0x082c + (nv_crtc->index * 0x400), 1); - evo_data(push, 0x00000000); - evo_mthd(push, 0x0900 + (nv_crtc->index * 0x400), 2); - evo_data(push, 0x00000311); - evo_data(push, 0x00000100); + /* Add overscan compensation if necessary, will keep the aspect + * ratio the same as the backend mode unless overridden by the + * user setting both hborder and vborder properties. + */ + if ((asyc->scaler.underscan.mode == UNDERSCAN_ON || + (asyc->scaler.underscan.mode == UNDERSCAN_AUTO && + drm_detect_hdmi_monitor(edid)))) { + u32 bX = asyc->scaler.underscan.hborder; + u32 bY = asyc->scaler.underscan.vborder; + u32 r = (asyh->view.oH << 19) / asyh->view.oW; + + if (bX) { + asyh->view.oW -= (bX * 2); + if (bY) asyh->view.oH -= (bY * 2); + else asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19; } else { - evo_mthd(push, 0x0410 + (nv_crtc->index * 0x300), 6); - evo_data(push, 0x00000000); - evo_data(push, (vactive << 16) | hactive); - evo_data(push, ( vsynce << 16) | hsynce); - evo_data(push, (vblanke << 16) | hblanke); - evo_data(push, (vblanks << 16) | hblanks); - evo_data(push, (vblan2e << 16) | vblan2s); - evo_mthd(push, 0x042c + (nv_crtc->index * 0x300), 1); - evo_data(push, 0x00000000); /* ??? */ - evo_mthd(push, 0x0450 + (nv_crtc->index * 0x300), 3); - evo_data(push, mode->clock * 1000); - evo_data(push, 0x00200000); /* ??? */ - evo_data(push, mode->clock * 1000); - evo_mthd(push, 0x04d0 + (nv_crtc->index * 0x300), 2); - evo_data(push, 0x00000311); - evo_data(push, 0x00000100); + asyh->view.oW -= (asyh->view.oW >> 4) + 32; + if (bY) asyh->view.oH -= (bY * 2); + else asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19; } + } - evo_kick(push, mast); + /* Handle CENTER/ASPECT scaling, taking into account the areas + * removed already for overscan compensation. + */ + switch (mode) { + case DRM_MODE_SCALE_CENTER: + asyh->view.oW = min((u16)umode->hdisplay, asyh->view.oW); + asyh->view.oH = min((u16)umode->vdisplay, asyh->view.oH); + /* fall-through */ + case DRM_MODE_SCALE_ASPECT: + if (asyh->view.oH < asyh->view.oW) { + u32 r = (asyh->view.iW << 19) / asyh->view.iH; + asyh->view.oW = ((asyh->view.oH * r) + (r / 2)) >> 19; + } else { + u32 r = (asyh->view.iH << 19) / asyh->view.iW; + asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19; + } + break; + default: + break; } - nv_connector = nouveau_crtc_connector_get(nv_crtc); - nv50_crtc_set_dither(nv_crtc, false); - nv50_crtc_set_scale(nv_crtc, false); + asyh->set.view = true; +} + +static void +nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh) +{ + struct drm_display_mode *mode = &asyh->state.adjusted_mode; + u32 ilace = (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 2 : 1; + u32 vscan = (mode->flags & DRM_MODE_FLAG_DBLSCAN) ? 2 : 1; + u32 hbackp = mode->htotal - mode->hsync_end; + u32 vbackp = (mode->vtotal - mode->vsync_end) * vscan / ilace; + u32 hfrontp = mode->hsync_start - mode->hdisplay; + u32 vfrontp = (mode->vsync_start - mode->vdisplay) * vscan / ilace; + struct nv50_head_mode *m = &asyh->mode; + + m->h.active = mode->htotal; + m->h.synce = mode->hsync_end - mode->hsync_start - 1; + m->h.blanke = m->h.synce + hbackp; + m->h.blanks = mode->htotal - hfrontp - 1; + + m->v.active = mode->vtotal * vscan / ilace; + m->v.synce = ((mode->vsync_end - mode->vsync_start) * vscan / ilace) - 1; + m->v.blanke = m->v.synce + vbackp; + m->v.blanks = m->v.active - vfrontp - 1; + + /*XXX: Safe underestimate, even "0" works */ + m->v.blankus = (m->v.active - mode->vdisplay - 2) * m->h.active; + m->v.blankus *= 1000; + m->v.blankus /= mode->clock; - /* G94 only accepts this after setting scale */ - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) - nv50_crtc_set_raster_vblank_dmi(nv_crtc, vblankus); + if (mode->flags & DRM_MODE_FLAG_INTERLACE) { + m->v.blank2e = m->v.active + m->v.synce + vbackp; + m->v.blank2s = m->v.blank2e + (mode->vdisplay * vscan / ilace); + m->v.active = (m->v.active * 2) + 1; + m->interlace = true; + } else { + m->v.blank2e = 0; + m->v.blank2s = 1; + m->interlace = false; + } + m->clock = mode->clock; - nv50_crtc_set_color_vibrance(nv_crtc, false); - nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, false); - return 0; + drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V); + asyh->set.mode = true; } static int -nv50_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y, - struct drm_framebuffer *old_fb) +nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state) { struct nouveau_drm *drm = nouveau_drm(crtc->dev); - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - int ret; + struct nv50_disp *disp = nv50_disp(crtc->dev); + struct nv50_head *head = nv50_head(crtc); + struct nv50_head_atom *armh = nv50_head_atom(crtc->state); + struct nv50_head_atom *asyh = nv50_head_atom(state); + struct nouveau_conn_atom *asyc = NULL; + struct drm_connector_state *conns; + struct drm_connector *conn; + int i; - if (!crtc->primary->fb) { - NV_DEBUG(drm, "No FB bound\n"); - return 0; + NV_ATOMIC(drm, "%s atomic_check %d\n", crtc->name, asyh->state.active); + if (asyh->state.active) { + for_each_connector_in_state(asyh->state.state, conn, conns, i) { + if (conns->crtc == crtc) { + asyc = nouveau_conn_atom(conns); + break; + } + } + + if (armh->state.active) { + if (asyc) { + if (asyh->state.mode_changed) + asyc->set.scaler = true; + if (armh->base.depth != asyh->base.depth) + asyc->set.dither = true; + } + } else { + asyc->set.mask = ~0; + asyh->set.mask = ~0; + } + + if (asyh->state.mode_changed) + nv50_head_atomic_check_mode(head, asyh); + + if (asyc) { + if (asyc->set.scaler) + nv50_head_atomic_check_view(armh, asyh, asyc); + if (asyc->set.dither) + nv50_head_atomic_check_dither(armh, asyh, asyc); + if (asyc->set.procamp) + nv50_head_atomic_check_procamp(armh, asyh, asyc); + } + + if ((asyh->core.visible = (asyh->base.cpp != 0))) { + asyh->core.x = asyh->base.x; + asyh->core.y = asyh->base.y; + asyh->core.w = asyh->base.w; + asyh->core.h = asyh->base.h; + } else + if ((asyh->core.visible = asyh->curs.visible)) { + /*XXX: We need to either find some way of having the + * primary base layer appear black, while still + * being able to display the other layers, or we + * need to allocate a dummy black surface here. + */ + asyh->core.x = 0; + asyh->core.y = 0; + asyh->core.w = asyh->state.mode.hdisplay; + asyh->core.h = asyh->state.mode.vdisplay; + } + asyh->core.handle = disp->mast.base.vram.handle; + asyh->core.offset = 0; + asyh->core.format = 0xcf; + asyh->core.kind = 0; + asyh->core.layout = 1; + asyh->core.block = 0; + asyh->core.pitch = ALIGN(asyh->core.w, 64) * 4; + asyh->lut.handle = disp->mast.base.vram.handle; + asyh->lut.offset = head->base.lut.nvbo->bo.offset; + asyh->set.base = armh->base.cpp != asyh->base.cpp; + asyh->set.ovly = armh->ovly.cpp != asyh->ovly.cpp; + } else { + asyh->core.visible = false; + asyh->curs.visible = false; + asyh->base.cpp = 0; + asyh->ovly.cpp = 0; } - ret = nv50_crtc_swap_fbs(crtc, old_fb); - if (ret) - return ret; + if (!drm_atomic_crtc_needs_modeset(&asyh->state)) { + if (asyh->core.visible) { + if (memcmp(&armh->core, &asyh->core, sizeof(asyh->core))) + asyh->set.core = true; + } else + if (armh->core.visible) { + asyh->clr.core = true; + } - nv50_display_flip_stop(crtc); - nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, true); - nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1); - return 0; -} + if (asyh->curs.visible) { + if (memcmp(&armh->curs, &asyh->curs, sizeof(asyh->curs))) + asyh->set.curs = true; + } else + if (armh->curs.visible) { + asyh->clr.curs = true; + } + } else { + asyh->clr.core = armh->core.visible; + asyh->clr.curs = armh->curs.visible; + asyh->set.core = asyh->core.visible; + asyh->set.curs = asyh->curs.visible; + } -static int -nv50_crtc_mode_set_base_atomic(struct drm_crtc *crtc, - struct drm_framebuffer *fb, int x, int y, - enum mode_set_atomic state) -{ - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - nv50_display_flip_stop(crtc); - nv50_crtc_set_image(nv_crtc, fb, x, y, true); + if (asyh->clr.mask || asyh->set.mask) + nv50_atom(asyh->state.state)->lock_core = true; return 0; } static void -nv50_crtc_lut_load(struct drm_crtc *crtc) +nv50_head_lut_load(struct drm_crtc *crtc) { struct nv50_disp *disp = nv50_disp(crtc->dev); struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); @@ -1292,64 +2200,95 @@ nv50_crtc_lut_load(struct drm_crtc *crtc) } } -static void -nv50_crtc_disable(struct drm_crtc *crtc) +static int +nv50_head_mode_set_base_atomic(struct drm_crtc *crtc, + struct drm_framebuffer *fb, int x, int y, + enum mode_set_atomic state) { - struct nv50_head *head = nv50_head(crtc); - evo_sync(crtc->dev); - if (head->image) - nouveau_bo_unpin(head->image); - nouveau_bo_ref(NULL, &head->image); + WARN_ON(1); + return 0; } +static const struct drm_crtc_helper_funcs +nv50_head_help = { + .mode_set_base_atomic = nv50_head_mode_set_base_atomic, + .load_lut = nv50_head_lut_load, + .atomic_check = nv50_head_atomic_check, +}; + +/* This is identical to the version in the atomic helpers, except that + * it supports non-vblanked ("async") page flips. + */ static int -nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv, - uint32_t handle, uint32_t width, uint32_t height) +nv50_head_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb, + struct drm_pending_vblank_event *event, u32 flags) { - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - struct drm_gem_object *gem = NULL; - struct nouveau_bo *nvbo = NULL; + struct drm_plane *plane = crtc->primary; + struct drm_atomic_state *state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; int ret = 0; - if (handle) { - if (width != 64 || height != 64) - return -EINVAL; + state = drm_atomic_state_alloc(plane->dev); + if (!state) + return -ENOMEM; - gem = drm_gem_object_lookup(file_priv, handle); - if (unlikely(!gem)) - return -ENOENT; - nvbo = nouveau_gem_object(gem); + state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc); +retry: + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + crtc_state->event = event; - ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, true); + plane_state = drm_atomic_get_plane_state(state, plane); + if (IS_ERR(plane_state)) { + ret = PTR_ERR(plane_state); + goto fail; } - if (ret == 0) { - if (nv_crtc->cursor.nvbo) - nouveau_bo_unpin(nv_crtc->cursor.nvbo); - nouveau_bo_ref(nvbo, &nv_crtc->cursor.nvbo); + ret = drm_atomic_set_crtc_for_plane(plane_state, crtc); + if (ret != 0) + goto fail; + drm_atomic_set_fb_for_plane(plane_state, fb); + + /* Make sure we don't accidentally do a full modeset. */ + state->allow_modeset = false; + if (!crtc_state->active) { + DRM_DEBUG_ATOMIC("[CRTC:%d] disabled, rejecting legacy flip\n", + crtc->base.id); + ret = -EINVAL; + goto fail; } - drm_gem_object_unreference_unlocked(gem); - nv50_crtc_cursor_show_hide(nv_crtc, true, true); + if (flags & DRM_MODE_PAGE_FLIP_ASYNC) + nv50_wndw_atom(plane_state)->interval = 0; + + ret = drm_atomic_nonblocking_commit(state); +fail: + if (ret == -EDEADLK) + goto backoff; + + drm_atomic_state_put(state); return ret; -} -static int -nv50_crtc_cursor_move(struct drm_crtc *crtc, int x, int y) -{ - struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); - struct nv50_curs *curs = nv50_curs(crtc); - struct nv50_chan *chan = nv50_chan(curs); - nvif_wr32(&chan->user, 0x0084, (y << 16) | (x & 0xffff)); - nvif_wr32(&chan->user, 0x0080, 0x00000000); +backoff: + drm_atomic_state_clear(state); + drm_atomic_legacy_backoff(state); - nv_crtc->cursor_saved_x = x; - nv_crtc->cursor_saved_y = y; - return 0; + /* + * Someone might have exchanged the framebuffer while we dropped locks + * in the backoff code. We need to fix up the fb refcount tracking the + * core does for us. + */ + plane->old_fb = plane->fb; + + goto retry; } static int -nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, +nv50_head_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, uint32_t size) { struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); @@ -1361,47 +2300,71 @@ nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, nv_crtc->lut.b[i] = b[i]; } - nv50_crtc_lut_load(crtc); - + nv50_head_lut_load(crtc); return 0; } static void -nv50_crtc_cursor_restore(struct nouveau_crtc *nv_crtc, int x, int y) +nv50_head_atomic_destroy_state(struct drm_crtc *crtc, + struct drm_crtc_state *state) { - nv50_crtc_cursor_move(&nv_crtc->base, x, y); + struct nv50_head_atom *asyh = nv50_head_atom(state); + __drm_atomic_helper_crtc_destroy_state(&asyh->state); + kfree(asyh); +} - nv50_crtc_cursor_show_hide(nv_crtc, true, true); +static struct drm_crtc_state * +nv50_head_atomic_duplicate_state(struct drm_crtc *crtc) +{ + struct nv50_head_atom *armh = nv50_head_atom(crtc->state); + struct nv50_head_atom *asyh; + if (!(asyh = kmalloc(sizeof(*asyh), GFP_KERNEL))) + return NULL; + __drm_atomic_helper_crtc_duplicate_state(crtc, &asyh->state); + asyh->view = armh->view; + asyh->mode = armh->mode; + asyh->lut = armh->lut; + asyh->core = armh->core; + asyh->curs = armh->curs; + asyh->base = armh->base; + asyh->ovly = armh->ovly; + asyh->dither = armh->dither; + asyh->procamp = armh->procamp; + asyh->clr.mask = 0; + asyh->set.mask = 0; + return &asyh->state; +} + +static void +__drm_atomic_helper_crtc_reset(struct drm_crtc *crtc, + struct drm_crtc_state *state) +{ + if (crtc->state) + crtc->funcs->atomic_destroy_state(crtc, crtc->state); + crtc->state = state; + crtc->state->crtc = crtc; } static void -nv50_crtc_destroy(struct drm_crtc *crtc) +nv50_head_reset(struct drm_crtc *crtc) +{ + struct nv50_head_atom *asyh; + + if (WARN_ON(!(asyh = kzalloc(sizeof(*asyh), GFP_KERNEL)))) + return; + + __drm_atomic_helper_crtc_reset(crtc, &asyh->state); +} + +static void +nv50_head_destroy(struct drm_crtc *crtc) { struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc); struct nv50_disp *disp = nv50_disp(crtc->dev); struct nv50_head *head = nv50_head(crtc); - struct nv50_fbdma *fbdma; - - list_for_each_entry(fbdma, &disp->fbdma, head) { - nvif_object_fini(&fbdma->base[nv_crtc->index]); - } nv50_dmac_destroy(&head->ovly.base, disp->disp); nv50_pioc_destroy(&head->oimm.base); - nv50_dmac_destroy(&head->sync.base, disp->disp); - nv50_pioc_destroy(&head->curs.base); - - /*XXX: this shouldn't be necessary, but the core doesn't call - * disconnect() during the cleanup paths - */ - if (head->image) - nouveau_bo_unpin(head->image); - nouveau_bo_ref(NULL, &head->image); - - /*XXX: ditto */ - if (nv_crtc->cursor.nvbo) - nouveau_bo_unpin(nv_crtc->cursor.nvbo); - nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo); nouveau_bo_unmap(nv_crtc->lut.nvbo); if (nv_crtc->lut.nvbo) @@ -1412,34 +2375,27 @@ nv50_crtc_destroy(struct drm_crtc *crtc) kfree(crtc); } -static const struct drm_crtc_helper_funcs nv50_crtc_hfunc = { - .dpms = nv50_crtc_dpms, - .prepare = nv50_crtc_prepare, - .commit = nv50_crtc_commit, - .mode_fixup = nv50_crtc_mode_fixup, - .mode_set = nv50_crtc_mode_set, - .mode_set_base = nv50_crtc_mode_set_base, - .mode_set_base_atomic = nv50_crtc_mode_set_base_atomic, - .load_lut = nv50_crtc_lut_load, - .disable = nv50_crtc_disable, -}; - -static const struct drm_crtc_funcs nv50_crtc_func = { - .cursor_set = nv50_crtc_cursor_set, - .cursor_move = nv50_crtc_cursor_move, - .gamma_set = nv50_crtc_gamma_set, - .set_config = nouveau_crtc_set_config, - .destroy = nv50_crtc_destroy, - .page_flip = nouveau_crtc_page_flip, +static const struct drm_crtc_funcs +nv50_head_func = { + .reset = nv50_head_reset, + .gamma_set = nv50_head_gamma_set, + .destroy = nv50_head_destroy, + .set_config = drm_atomic_helper_set_config, + .page_flip = nv50_head_page_flip, + .set_property = drm_atomic_helper_crtc_set_property, + .atomic_duplicate_state = nv50_head_atomic_duplicate_state, + .atomic_destroy_state = nv50_head_atomic_destroy_state, }; static int -nv50_crtc_create(struct drm_device *dev, int index) +nv50_head_create(struct drm_device *dev, int index) { struct nouveau_drm *drm = nouveau_drm(dev); struct nvif_device *device = &drm->device; struct nv50_disp *disp = nv50_disp(dev); struct nv50_head *head; + struct nv50_base *base; + struct nv50_curs *curs; struct drm_crtc *crtc; int ret, i; @@ -1448,21 +2404,25 @@ nv50_crtc_create(struct drm_device *dev, int index) return -ENOMEM; head->base.index = index; - head->base.set_dither = nv50_crtc_set_dither; - head->base.set_scale = nv50_crtc_set_scale; - head->base.set_color_vibrance = nv50_crtc_set_color_vibrance; - head->base.color_vibrance = 50; - head->base.vibrant_hue = 0; - head->base.cursor.set_pos = nv50_crtc_cursor_restore; for (i = 0; i < 256; i++) { head->base.lut.r[i] = i << 8; head->base.lut.g[i] = i << 8; head->base.lut.b[i] = i << 8; } + ret = nv50_base_new(drm, head, &base); + if (ret == 0) + ret = nv50_curs_new(drm, head, &curs); + if (ret) { + kfree(head); + return ret; + } + crtc = &head->base.base; - drm_crtc_init(dev, crtc, &nv50_crtc_func); - drm_crtc_helper_add(crtc, &nv50_crtc_hfunc); + drm_crtc_init_with_planes(dev, crtc, &base->wndw.plane, + &curs->wndw.plane, &nv50_head_func, + "head-%d", head->base.index); + drm_crtc_helper_add(crtc, &nv50_head_help); drm_mode_crtc_set_gamma_size(crtc, 256); ret = nouveau_bo_new(dev, 8192, 0x100, TTM_PL_FLAG_VRAM, @@ -1481,20 +2441,6 @@ nv50_crtc_create(struct drm_device *dev, int index) if (ret) goto out; - /* allocate cursor resources */ - ret = nv50_curs_create(device, disp->disp, index, &head->curs); - if (ret) - goto out; - - /* allocate page flip / sync resources */ - ret = nv50_base_create(device, disp->disp, index, disp->sync->bo.offset, - &head->sync); - if (ret) - goto out; - - head->sync.addr = EVO_FLIP_SEM0(index); - head->sync.data = 0x00000000; - /* allocate overlay resources */ ret = nv50_oimm_create(device, disp->disp, index, &head->oimm); if (ret) @@ -1507,43 +2453,64 @@ nv50_crtc_create(struct drm_device *dev, int index) out: if (ret) - nv50_crtc_destroy(crtc); + nv50_head_destroy(crtc); return ret; } /****************************************************************************** - * Encoder helpers + * Output path helpers *****************************************************************************/ -static bool -nv50_encoder_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +static int +nv50_outp_atomic_check_view(struct drm_encoder *encoder, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state, + struct drm_display_mode *native_mode) { - struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nouveau_connector *nv_connector; + struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode; + struct drm_display_mode *mode = &crtc_state->mode; + struct drm_connector *connector = conn_state->connector; + struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state); + struct nouveau_drm *drm = nouveau_drm(encoder->dev); + + NV_ATOMIC(drm, "%s atomic_check\n", encoder->name); + asyc->scaler.full = false; + if (!native_mode) + return 0; - nv_connector = nouveau_encoder_connector_get(nv_encoder); - if (nv_connector && nv_connector->native_mode) { - nv_connector->scaling_full = false; - if (nv_connector->scaling_mode == DRM_MODE_SCALE_NONE) { - switch (nv_connector->type) { - case DCB_CONNECTOR_LVDS: - case DCB_CONNECTOR_LVDS_SPWG: - case DCB_CONNECTOR_eDP: - /* force use of scaler for non-edid modes */ - if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER) - return true; - nv_connector->scaling_full = true; + if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) { + switch (connector->connector_type) { + case DRM_MODE_CONNECTOR_LVDS: + case DRM_MODE_CONNECTOR_eDP: + /* Force use of scaler for non-EDID modes. */ + if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER) break; - default: - return true; - } + mode = native_mode; + asyc->scaler.full = true; + break; + default: + break; } + } else { + mode = native_mode; + } - drm_mode_copy(adjusted_mode, nv_connector->native_mode); + if (!drm_mode_equal(adjusted_mode, mode)) { + drm_mode_copy(adjusted_mode, mode); + crtc_state->mode_changed = true; } - return true; + return 0; +} + +static int +nv50_outp_atomic_check(struct drm_encoder *encoder, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + struct nouveau_connector *nv_connector = + nouveau_connector(conn_state->connector); + return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state, + nv_connector->native_mode); } /****************************************************************************** @@ -1574,21 +2541,39 @@ nv50_dac_dpms(struct drm_encoder *encoder, int mode) } static void -nv50_dac_commit(struct drm_encoder *encoder) +nv50_dac_disable(struct drm_encoder *encoder) { + struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); + struct nv50_mast *mast = nv50_mast(encoder->dev); + const int or = nv_encoder->or; + u32 *push; + + if (nv_encoder->crtc) { + push = evo_wait(mast, 4); + if (push) { + if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0400 + (or * 0x080), 1); + evo_data(push, 0x00000000); + } else { + evo_mthd(push, 0x0180 + (or * 0x020), 1); + evo_data(push, 0x00000000); + } + evo_kick(push, mast); + } + } + + nv_encoder->crtc = NULL; } static void -nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +nv50_dac_enable(struct drm_encoder *encoder) { struct nv50_mast *mast = nv50_mast(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc); + struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode; u32 *push; - nv50_dac_dpms(encoder, DRM_MODE_DPMS_ON); - push = evo_wait(mast, 8); if (push) { if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { @@ -1627,33 +2612,6 @@ nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, nv_encoder->crtc = encoder->crtc; } -static void -nv50_dac_disconnect(struct drm_encoder *encoder) -{ - struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_mast *mast = nv50_mast(encoder->dev); - const int or = nv_encoder->or; - u32 *push; - - if (nv_encoder->crtc) { - nv50_crtc_prepare(nv_encoder->crtc); - - push = evo_wait(mast, 4); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0400 + (or * 0x080), 1); - evo_data(push, 0x00000000); - } else { - evo_mthd(push, 0x0180 + (or * 0x020), 1); - evo_data(push, 0x00000000); - } - evo_kick(push, mast); - } - } - - nv_encoder->crtc = NULL; -} - static enum drm_connector_status nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector) { @@ -1681,6 +2639,15 @@ nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector) return connector_status_connected; } +static const struct drm_encoder_helper_funcs +nv50_dac_help = { + .dpms = nv50_dac_dpms, + .atomic_check = nv50_outp_atomic_check, + .enable = nv50_dac_enable, + .disable = nv50_dac_disable, + .detect = nv50_dac_detect +}; + static void nv50_dac_destroy(struct drm_encoder *encoder) { @@ -1688,18 +2655,8 @@ nv50_dac_destroy(struct drm_encoder *encoder) kfree(encoder); } -static const struct drm_encoder_helper_funcs nv50_dac_hfunc = { - .dpms = nv50_dac_dpms, - .mode_fixup = nv50_encoder_mode_fixup, - .prepare = nv50_dac_disconnect, - .commit = nv50_dac_commit, - .mode_set = nv50_dac_mode_set, - .disable = nv50_dac_disconnect, - .get_crtc = nv50_display_crtc_get, - .detect = nv50_dac_detect -}; - -static const struct drm_encoder_funcs nv50_dac_func = { +static const struct drm_encoder_funcs +nv50_dac_func = { .destroy = nv50_dac_destroy, }; @@ -1726,8 +2683,9 @@ nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe) encoder = to_drm_encoder(nv_encoder); encoder->possible_crtcs = dcbe->heads; encoder->possible_clones = 0; - drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type, NULL); - drm_encoder_helper_add(encoder, &nv50_dac_hfunc); + drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type, + "dac-%04x-%04x", dcbe->hasht, dcbe->hashm); + drm_encoder_helper_add(encoder, &nv50_dac_help); drm_mode_connector_attach_encoder(connector, encoder); return 0; @@ -1737,7 +2695,26 @@ nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe) * Audio *****************************************************************************/ static void -nv50_audio_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode) +nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) +{ + struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); + struct nv50_disp *disp = nv50_disp(encoder->dev); + struct { + struct nv50_disp_mthd_v1 base; + struct nv50_disp_sor_hda_eld_v0 eld; + } args = { + .base.version = 1, + .base.method = NV50_DISP_MTHD_V1_SOR_HDA_ELD, + .base.hasht = nv_encoder->dcb->hasht, + .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | + (0x0100 << nv_crtc->index), + }; + + nvif_mthd(disp->disp, 0, &args, sizeof(args)); +} + +static void +nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc); @@ -1768,30 +2745,30 @@ nv50_audio_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode) sizeof(args.base) + drm_eld_size(args.data)); } +/****************************************************************************** + * HDMI + *****************************************************************************/ static void -nv50_audio_disconnect(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) +nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nv50_disp *disp = nv50_disp(encoder->dev); struct { struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_hda_eld_v0 eld; + struct nv50_disp_sor_hdmi_pwr_v0 pwr; } args = { .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_HDA_ELD, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | - (0x0100 << nv_crtc->index), + .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR, + .base.hasht = nv_encoder->dcb->hasht, + .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | + (0x0100 << nv_crtc->index), }; nvif_mthd(disp->disp, 0, &args, sizeof(args)); } -/****************************************************************************** - * HDMI - *****************************************************************************/ static void -nv50_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode) +nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc); @@ -1821,26 +2798,632 @@ nv50_hdmi_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode) args.pwr.max_ac_packet = max_ac_packet / 32; nvif_mthd(disp->disp, 0, &args, sizeof(args)); - nv50_audio_mode_set(encoder, mode); + nv50_audio_enable(encoder, mode); +} + +/****************************************************************************** + * MST + *****************************************************************************/ +#define nv50_mstm(p) container_of((p), struct nv50_mstm, mgr) +#define nv50_mstc(p) container_of((p), struct nv50_mstc, connector) +#define nv50_msto(p) container_of((p), struct nv50_msto, encoder) + +struct nv50_mstm { + struct nouveau_encoder *outp; + + struct drm_dp_mst_topology_mgr mgr; + struct nv50_msto *msto[4]; + + bool modified; +}; + +struct nv50_mstc { + struct nv50_mstm *mstm; + struct drm_dp_mst_port *port; + struct drm_connector connector; + + struct drm_display_mode *native; + struct edid *edid; + + int pbn; +}; + +struct nv50_msto { + struct drm_encoder encoder; + + struct nv50_head *head; + struct nv50_mstc *mstc; + bool disabled; +}; + +static struct drm_dp_payload * +nv50_msto_payload(struct nv50_msto *msto) +{ + struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev); + struct nv50_mstc *mstc = msto->mstc; + struct nv50_mstm *mstm = mstc->mstm; + int vcpi = mstc->port->vcpi.vcpi, i; + + NV_ATOMIC(drm, "%s: vcpi %d\n", msto->encoder.name, vcpi); + for (i = 0; i < mstm->mgr.max_payloads; i++) { + struct drm_dp_payload *payload = &mstm->mgr.payloads[i]; + NV_ATOMIC(drm, "%s: %d: vcpi %d start 0x%02x slots 0x%02x\n", + mstm->outp->base.base.name, i, payload->vcpi, + payload->start_slot, payload->num_slots); + } + + for (i = 0; i < mstm->mgr.max_payloads; i++) { + struct drm_dp_payload *payload = &mstm->mgr.payloads[i]; + if (payload->vcpi == vcpi) + return payload; + } + + return NULL; } static void -nv50_hdmi_disconnect(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) +nv50_msto_cleanup(struct nv50_msto *msto) { - struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_disp *disp = nv50_disp(encoder->dev); + struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev); + struct nv50_mstc *mstc = msto->mstc; + struct nv50_mstm *mstm = mstc->mstm; + + NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name); + if (mstc->port && mstc->port->vcpi.vcpi > 0 && !nv50_msto_payload(msto)) + drm_dp_mst_deallocate_vcpi(&mstm->mgr, mstc->port); + if (msto->disabled) { + msto->mstc = NULL; + msto->head = NULL; + msto->disabled = false; + } +} + +static void +nv50_msto_prepare(struct nv50_msto *msto) +{ + struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev); + struct nv50_mstc *mstc = msto->mstc; + struct nv50_mstm *mstm = mstc->mstm; struct { struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_hdmi_pwr_v0 pwr; + struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi; } args = { .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | - (0x0100 << nv_crtc->index), + .base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI, + .base.hasht = mstm->outp->dcb->hasht, + .base.hashm = (0xf0ff & mstm->outp->dcb->hashm) | + (0x0100 << msto->head->base.index), }; - nvif_mthd(disp->disp, 0, &args, sizeof(args)); + NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name); + if (mstc->port && mstc->port->vcpi.vcpi > 0) { + struct drm_dp_payload *payload = nv50_msto_payload(msto); + if (payload) { + args.vcpi.start_slot = payload->start_slot; + args.vcpi.num_slots = payload->num_slots; + args.vcpi.pbn = mstc->port->vcpi.pbn; + args.vcpi.aligned_pbn = mstc->port->vcpi.aligned_pbn; + } + } + + NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n", + msto->encoder.name, msto->head->base.base.name, + args.vcpi.start_slot, args.vcpi.num_slots, + args.vcpi.pbn, args.vcpi.aligned_pbn); + nvif_mthd(&drm->display->disp, 0, &args, sizeof(args)); +} + +static int +nv50_msto_atomic_check(struct drm_encoder *encoder, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + struct nv50_mstc *mstc = nv50_mstc(conn_state->connector); + struct nv50_mstm *mstm = mstc->mstm; + int bpp = conn_state->connector->display_info.bpc * 3; + int slots; + + mstc->pbn = drm_dp_calc_pbn_mode(crtc_state->adjusted_mode.clock, bpp); + + slots = drm_dp_find_vcpi_slots(&mstm->mgr, mstc->pbn); + if (slots < 0) + return slots; + + return nv50_outp_atomic_check_view(encoder, crtc_state, conn_state, + mstc->native); +} + +static void +nv50_msto_enable(struct drm_encoder *encoder) +{ + struct nv50_head *head = nv50_head(encoder->crtc); + struct nv50_msto *msto = nv50_msto(encoder); + struct nv50_mstc *mstc = NULL; + struct nv50_mstm *mstm = NULL; + struct drm_connector *connector; + u8 proto, depth; + int slots; + bool r; + + drm_for_each_connector(connector, encoder->dev) { + if (connector->state->best_encoder == &msto->encoder) { + mstc = nv50_mstc(connector); + mstm = mstc->mstm; + break; + } + } + + if (WARN_ON(!mstc)) + return; + + r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, mstc->pbn, &slots); + WARN_ON(!r); + + if (mstm->outp->dcb->sorconf.link & 1) + proto = 0x8; + else + proto = 0x9; + + switch (mstc->connector.display_info.bpc) { + case 6: depth = 0x2; break; + case 8: depth = 0x5; break; + case 10: + default: depth = 0x6; break; + } + + mstm->outp->update(mstm->outp, head->base.index, + &head->base.base.state->adjusted_mode, proto, depth); + + msto->head = head; + msto->mstc = mstc; + mstm->modified = true; +} + +static void +nv50_msto_disable(struct drm_encoder *encoder) +{ + struct nv50_msto *msto = nv50_msto(encoder); + struct nv50_mstc *mstc = msto->mstc; + struct nv50_mstm *mstm = mstc->mstm; + + if (mstc->port) + drm_dp_mst_reset_vcpi_slots(&mstm->mgr, mstc->port); + + mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0); + mstm->modified = true; + msto->disabled = true; +} + +static const struct drm_encoder_helper_funcs +nv50_msto_help = { + .disable = nv50_msto_disable, + .enable = nv50_msto_enable, + .atomic_check = nv50_msto_atomic_check, +}; + +static void +nv50_msto_destroy(struct drm_encoder *encoder) +{ + struct nv50_msto *msto = nv50_msto(encoder); + drm_encoder_cleanup(&msto->encoder); + kfree(msto); +} + +static const struct drm_encoder_funcs +nv50_msto = { + .destroy = nv50_msto_destroy, +}; + +static int +nv50_msto_new(struct drm_device *dev, u32 heads, const char *name, int id, + struct nv50_msto **pmsto) +{ + struct nv50_msto *msto; + int ret; + + if (!(msto = *pmsto = kzalloc(sizeof(*msto), GFP_KERNEL))) + return -ENOMEM; + + ret = drm_encoder_init(dev, &msto->encoder, &nv50_msto, + DRM_MODE_ENCODER_DPMST, "%s-mst-%d", name, id); + if (ret) { + kfree(*pmsto); + *pmsto = NULL; + return ret; + } + + drm_encoder_helper_add(&msto->encoder, &nv50_msto_help); + msto->encoder.possible_crtcs = heads; + return 0; +} + +static struct drm_encoder * +nv50_mstc_atomic_best_encoder(struct drm_connector *connector, + struct drm_connector_state *connector_state) +{ + struct nv50_head *head = nv50_head(connector_state->crtc); + struct nv50_mstc *mstc = nv50_mstc(connector); + if (mstc->port) { + struct nv50_mstm *mstm = mstc->mstm; + return &mstm->msto[head->base.index]->encoder; + } + return NULL; +} + +static struct drm_encoder * +nv50_mstc_best_encoder(struct drm_connector *connector) +{ + struct nv50_mstc *mstc = nv50_mstc(connector); + if (mstc->port) { + struct nv50_mstm *mstm = mstc->mstm; + return &mstm->msto[0]->encoder; + } + return NULL; +} + +static enum drm_mode_status +nv50_mstc_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) +{ + return MODE_OK; +} + +static int +nv50_mstc_get_modes(struct drm_connector *connector) +{ + struct nv50_mstc *mstc = nv50_mstc(connector); + int ret = 0; + + mstc->edid = drm_dp_mst_get_edid(&mstc->connector, mstc->port->mgr, mstc->port); + drm_mode_connector_update_edid_property(&mstc->connector, mstc->edid); + if (mstc->edid) { + ret = drm_add_edid_modes(&mstc->connector, mstc->edid); + drm_edid_to_eld(&mstc->connector, mstc->edid); + } + + if (!mstc->connector.display_info.bpc) + mstc->connector.display_info.bpc = 8; + + if (mstc->native) + drm_mode_destroy(mstc->connector.dev, mstc->native); + mstc->native = nouveau_conn_native_mode(&mstc->connector); + return ret; +} + +static const struct drm_connector_helper_funcs +nv50_mstc_help = { + .get_modes = nv50_mstc_get_modes, + .mode_valid = nv50_mstc_mode_valid, + .best_encoder = nv50_mstc_best_encoder, + .atomic_best_encoder = nv50_mstc_atomic_best_encoder, +}; + +static enum drm_connector_status +nv50_mstc_detect(struct drm_connector *connector, bool force) +{ + struct nv50_mstc *mstc = nv50_mstc(connector); + if (!mstc->port) + return connector_status_disconnected; + return drm_dp_mst_detect_port(connector, mstc->port->mgr, mstc->port); +} + +static void +nv50_mstc_destroy(struct drm_connector *connector) +{ + struct nv50_mstc *mstc = nv50_mstc(connector); + drm_connector_cleanup(&mstc->connector); + kfree(mstc); +} + +static const struct drm_connector_funcs +nv50_mstc = { + .dpms = drm_atomic_helper_connector_dpms, + .reset = nouveau_conn_reset, + .detect = nv50_mstc_detect, + .fill_modes = drm_helper_probe_single_connector_modes, + .set_property = drm_atomic_helper_connector_set_property, + .destroy = nv50_mstc_destroy, + .atomic_duplicate_state = nouveau_conn_atomic_duplicate_state, + .atomic_destroy_state = nouveau_conn_atomic_destroy_state, + .atomic_set_property = nouveau_conn_atomic_set_property, + .atomic_get_property = nouveau_conn_atomic_get_property, +}; + +static int +nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port, + const char *path, struct nv50_mstc **pmstc) +{ + struct drm_device *dev = mstm->outp->base.base.dev; + struct nv50_mstc *mstc; + int ret, i; + + if (!(mstc = *pmstc = kzalloc(sizeof(*mstc), GFP_KERNEL))) + return -ENOMEM; + mstc->mstm = mstm; + mstc->port = port; + + ret = drm_connector_init(dev, &mstc->connector, &nv50_mstc, + DRM_MODE_CONNECTOR_DisplayPort); + if (ret) { + kfree(*pmstc); + *pmstc = NULL; + return ret; + } + + drm_connector_helper_add(&mstc->connector, &nv50_mstc_help); + + mstc->connector.funcs->reset(&mstc->connector); + nouveau_conn_attach_properties(&mstc->connector); + + for (i = 0; i < ARRAY_SIZE(mstm->msto) && mstm->msto; i++) + drm_mode_connector_attach_encoder(&mstc->connector, &mstm->msto[i]->encoder); + + drm_object_attach_property(&mstc->connector.base, dev->mode_config.path_property, 0); + drm_object_attach_property(&mstc->connector.base, dev->mode_config.tile_property, 0); + drm_mode_connector_set_path_property(&mstc->connector, path); + return 0; +} + +static void +nv50_mstm_cleanup(struct nv50_mstm *mstm) +{ + struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev); + struct drm_encoder *encoder; + int ret; + + NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name); + ret = drm_dp_check_act_status(&mstm->mgr); + + ret = drm_dp_update_payload_part2(&mstm->mgr); + + drm_for_each_encoder(encoder, mstm->outp->base.base.dev) { + if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) { + struct nv50_msto *msto = nv50_msto(encoder); + struct nv50_mstc *mstc = msto->mstc; + if (mstc && mstc->mstm == mstm) + nv50_msto_cleanup(msto); + } + } + + mstm->modified = false; +} + +static void +nv50_mstm_prepare(struct nv50_mstm *mstm) +{ + struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev); + struct drm_encoder *encoder; + int ret; + + NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name); + ret = drm_dp_update_payload_part1(&mstm->mgr); + + drm_for_each_encoder(encoder, mstm->outp->base.base.dev) { + if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) { + struct nv50_msto *msto = nv50_msto(encoder); + struct nv50_mstc *mstc = msto->mstc; + if (mstc && mstc->mstm == mstm) + nv50_msto_prepare(msto); + } + } +} + +static void +nv50_mstm_hotplug(struct drm_dp_mst_topology_mgr *mgr) +{ + struct nv50_mstm *mstm = nv50_mstm(mgr); + drm_kms_helper_hotplug_event(mstm->outp->base.base.dev); +} + +static void +nv50_mstm_destroy_connector(struct drm_dp_mst_topology_mgr *mgr, + struct drm_connector *connector) +{ + struct nouveau_drm *drm = nouveau_drm(connector->dev); + struct nv50_mstc *mstc = nv50_mstc(connector); + + drm_connector_unregister(&mstc->connector); + + drm_modeset_lock_all(drm->dev); + drm_fb_helper_remove_one_connector(&drm->fbcon->helper, &mstc->connector); + mstc->port = NULL; + drm_modeset_unlock_all(drm->dev); + + drm_connector_unreference(&mstc->connector); +} + +static void +nv50_mstm_register_connector(struct drm_connector *connector) +{ + struct nouveau_drm *drm = nouveau_drm(connector->dev); + + drm_modeset_lock_all(drm->dev); + drm_fb_helper_add_one_connector(&drm->fbcon->helper, connector); + drm_modeset_unlock_all(drm->dev); + + drm_connector_register(connector); +} + +static struct drm_connector * +nv50_mstm_add_connector(struct drm_dp_mst_topology_mgr *mgr, + struct drm_dp_mst_port *port, const char *path) +{ + struct nv50_mstm *mstm = nv50_mstm(mgr); + struct nv50_mstc *mstc; + int ret; + + ret = nv50_mstc_new(mstm, port, path, &mstc); + if (ret) { + if (mstc) + mstc->connector.funcs->destroy(&mstc->connector); + return NULL; + } + + return &mstc->connector; +} + +static const struct drm_dp_mst_topology_cbs +nv50_mstm = { + .add_connector = nv50_mstm_add_connector, + .register_connector = nv50_mstm_register_connector, + .destroy_connector = nv50_mstm_destroy_connector, + .hotplug = nv50_mstm_hotplug, +}; + +void +nv50_mstm_service(struct nv50_mstm *mstm) +{ + struct drm_dp_aux *aux = mstm->mgr.aux; + bool handled = true; + int ret; + u8 esi[8] = {}; + + while (handled) { + ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT_ESI, esi, 8); + if (ret != 8) { + drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); + return; + } + + drm_dp_mst_hpd_irq(&mstm->mgr, esi, &handled); + if (!handled) + break; + + drm_dp_dpcd_write(aux, DP_SINK_COUNT_ESI + 1, &esi[1], 3); + } +} + +void +nv50_mstm_remove(struct nv50_mstm *mstm) +{ + if (mstm) + drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); +} + +static int +nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state) +{ + struct nouveau_encoder *outp = mstm->outp; + struct { + struct nv50_disp_mthd_v1 base; + struct nv50_disp_sor_dp_mst_link_v0 mst; + } args = { + .base.version = 1, + .base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK, + .base.hasht = outp->dcb->hasht, + .base.hashm = outp->dcb->hashm, + .mst.state = state, + }; + struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev); + struct nvif_object *disp = &drm->display->disp; + int ret; + + if (dpcd >= 0x12) { + ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CTRL, &dpcd); + if (ret < 0) + return ret; + + dpcd &= ~DP_MST_EN; + if (state) + dpcd |= DP_MST_EN; + + ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, dpcd); + if (ret < 0) + return ret; + } + + return nvif_mthd(disp, 0, &args, sizeof(args)); +} + +int +nv50_mstm_detect(struct nv50_mstm *mstm, u8 dpcd[8], int allow) +{ + int ret, state = 0; + + if (!mstm) + return 0; + + if (dpcd[0] >= 0x12 && allow) { + ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CAP, &dpcd[1]); + if (ret < 0) + return ret; + + state = dpcd[1] & DP_MST_CAP; + } + + ret = nv50_mstm_enable(mstm, dpcd[0], state); + if (ret) + return ret; + + ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, state); + if (ret) + return nv50_mstm_enable(mstm, dpcd[0], 0); + + return mstm->mgr.mst_state; +} + +static void +nv50_mstm_fini(struct nv50_mstm *mstm) +{ + if (mstm && mstm->mgr.mst_state) + drm_dp_mst_topology_mgr_suspend(&mstm->mgr); +} + +static void +nv50_mstm_init(struct nv50_mstm *mstm) +{ + if (mstm && mstm->mgr.mst_state) + drm_dp_mst_topology_mgr_resume(&mstm->mgr); +} + +static void +nv50_mstm_del(struct nv50_mstm **pmstm) +{ + struct nv50_mstm *mstm = *pmstm; + if (mstm) { + kfree(*pmstm); + *pmstm = NULL; + } +} + +static int +nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max, + int conn_base_id, struct nv50_mstm **pmstm) +{ + const int max_payloads = hweight8(outp->dcb->heads); + struct drm_device *dev = outp->base.base.dev; + struct nv50_mstm *mstm; + int ret, i; + u8 dpcd; + + /* This is a workaround for some monitors not functioning + * correctly in MST mode on initial module load. I think + * some bad interaction with the VBIOS may be responsible. + * + * A good ol' off and on again seems to work here ;) + */ + ret = drm_dp_dpcd_readb(aux, DP_DPCD_REV, &dpcd); + if (ret >= 0 && dpcd >= 0x12) + drm_dp_dpcd_writeb(aux, DP_MSTM_CTRL, 0); + + if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL))) + return -ENOMEM; + mstm->outp = outp; + mstm->mgr.cbs = &nv50_mstm; + + ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev->dev, aux, aux_max, + max_payloads, conn_base_id); + if (ret) + return ret; + + for (i = 0; i < max_payloads; i++) { + ret = nv50_msto_new(dev, outp->dcb->heads, outp->base.base.name, + i, &mstm->msto[i]); + if (ret) + return ret; + } + + return 0; } /****************************************************************************** @@ -1861,89 +3444,91 @@ nv50_sor_dpms(struct drm_encoder *encoder, int mode) .base.hashm = nv_encoder->dcb->hashm, .pwr.state = mode == DRM_MODE_DPMS_ON, }; - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_dp_pwr_v0 pwr; - } link = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_DP_PWR, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = nv_encoder->dcb->hashm, - .pwr.state = mode == DRM_MODE_DPMS_ON, - }; - struct drm_device *dev = encoder->dev; - struct drm_encoder *partner; - nv_encoder->last_dpms = mode; - - list_for_each_entry(partner, &dev->mode_config.encoder_list, head) { - struct nouveau_encoder *nv_partner = nouveau_encoder(partner); - - if (partner->encoder_type != DRM_MODE_ENCODER_TMDS) - continue; + nvif_mthd(disp->disp, 0, &args, sizeof(args)); +} - if (nv_partner != nv_encoder && - nv_partner->dcb->or == nv_encoder->dcb->or) { - if (nv_partner->last_dpms == DRM_MODE_DPMS_ON) - return; - break; - } - } +static void +nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head, + struct drm_display_mode *mode, u8 proto, u8 depth) +{ + struct nv50_dmac *core = &nv50_mast(nv_encoder->base.base.dev)->base; + u32 *push; - if (nv_encoder->dcb->type == DCB_OUTPUT_DP) { - args.pwr.state = 1; - nvif_mthd(disp->disp, 0, &args, sizeof(args)); - nvif_mthd(disp->disp, 0, &link, sizeof(link)); + if (!mode) { + nv_encoder->ctrl &= ~BIT(head); + if (!(nv_encoder->ctrl & 0x0000000f)) + nv_encoder->ctrl = 0; } else { - nvif_mthd(disp->disp, 0, &args, sizeof(args)); + nv_encoder->ctrl |= proto << 8; + nv_encoder->ctrl |= BIT(head); } -} -static void -nv50_sor_ctrl(struct nouveau_encoder *nv_encoder, u32 mask, u32 data) -{ - struct nv50_mast *mast = nv50_mast(nv_encoder->base.base.dev); - u32 temp = (nv_encoder->ctrl & ~mask) | (data & mask), *push; - if (temp != nv_encoder->ctrl && (push = evo_wait(mast, 2))) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { + if ((push = evo_wait(core, 6))) { + if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) { + if (mode) { + if (mode->flags & DRM_MODE_FLAG_NHSYNC) + nv_encoder->ctrl |= 0x00001000; + if (mode->flags & DRM_MODE_FLAG_NVSYNC) + nv_encoder->ctrl |= 0x00002000; + nv_encoder->ctrl |= depth << 16; + } evo_mthd(push, 0x0600 + (nv_encoder->or * 0x40), 1); - evo_data(push, (nv_encoder->ctrl = temp)); } else { + if (mode) { + u32 magic = 0x31ec6000 | (head << 25); + u32 syncs = 0x00000001; + if (mode->flags & DRM_MODE_FLAG_NHSYNC) + syncs |= 0x00000008; + if (mode->flags & DRM_MODE_FLAG_NVSYNC) + syncs |= 0x00000010; + if (mode->flags & DRM_MODE_FLAG_INTERLACE) + magic |= 0x00000001; + + evo_mthd(push, 0x0404 + (head * 0x300), 2); + evo_data(push, syncs | (depth << 6)); + evo_data(push, magic); + } evo_mthd(push, 0x0200 + (nv_encoder->or * 0x20), 1); - evo_data(push, (nv_encoder->ctrl = temp)); } - evo_kick(push, mast); + evo_data(push, nv_encoder->ctrl); + evo_kick(push, core); } } static void -nv50_sor_disconnect(struct drm_encoder *encoder) +nv50_sor_disable(struct drm_encoder *encoder) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc); - nv_encoder->last_dpms = DRM_MODE_DPMS_OFF; nv_encoder->crtc = NULL; if (nv_crtc) { - nv50_crtc_prepare(&nv_crtc->base); - nv50_sor_ctrl(nv_encoder, 1 << nv_crtc->index, 0); - nv50_audio_disconnect(encoder, nv_crtc); - nv50_hdmi_disconnect(&nv_encoder->base.base, nv_crtc); - } -} + struct nvkm_i2c_aux *aux = nv_encoder->aux; + u8 pwr; -static void -nv50_sor_commit(struct drm_encoder *encoder) -{ + if (aux) { + int ret = nvkm_rdaux(aux, DP_SET_POWER, &pwr, 1); + if (ret == 0) { + pwr &= ~DP_SET_POWER_MASK; + pwr |= DP_SET_POWER_D3; + nvkm_wraux(aux, DP_SET_POWER, &pwr, 1); + } + } + + nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0); + nv50_audio_disable(encoder, nv_crtc); + nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc); + } } static void -nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode, - struct drm_display_mode *mode) +nv50_sor_enable(struct drm_encoder *encoder) { struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc); + struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode; struct { struct nv50_disp_mthd_v1 base; struct nv50_disp_sor_lvds_script_v0 lvds; @@ -1954,13 +3539,10 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode, .base.hashm = nv_encoder->dcb->hashm, }; struct nv50_disp *disp = nv50_disp(encoder->dev); - struct nv50_mast *mast = nv50_mast(encoder->dev); struct drm_device *dev = encoder->dev; struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_connector *nv_connector; struct nvbios *bios = &drm->vbios; - u32 mask, ctrl; - u8 owner = 1 << nv_crtc->index; u8 proto = 0xf; u8 depth = 0x0; @@ -1985,7 +3567,7 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode, proto = 0x2; } - nv50_hdmi_mode_set(&nv_encoder->base.base, mode); + nv50_hdmi_enable(&nv_encoder->base.base, mode); break; case DCB_OUTPUT_LVDS: proto = 0x0; @@ -2019,94 +3601,60 @@ nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode, nvif_mthd(disp->disp, 0, &lvds, sizeof(lvds)); break; case DCB_OUTPUT_DP: - if (nv_connector->base.display_info.bpc == 6) { - nv_encoder->dp.datarate = mode->clock * 18 / 8; + if (nv_connector->base.display_info.bpc == 6) depth = 0x2; - } else - if (nv_connector->base.display_info.bpc == 8) { - nv_encoder->dp.datarate = mode->clock * 24 / 8; + else + if (nv_connector->base.display_info.bpc == 8) depth = 0x5; - } else { - nv_encoder->dp.datarate = mode->clock * 30 / 8; + else depth = 0x6; - } if (nv_encoder->dcb->sorconf.link & 1) proto = 0x8; else proto = 0x9; - nv50_audio_mode_set(encoder, mode); + + nv50_audio_enable(encoder, mode); break; default: BUG_ON(1); break; } - nv50_sor_dpms(&nv_encoder->base.base, DRM_MODE_DPMS_ON); - - if (nv50_vers(mast) >= GF110_DISP) { - u32 *push = evo_wait(mast, 3); - if (push) { - u32 magic = 0x31ec6000 | (nv_crtc->index << 25); - u32 syncs = 0x00000001; - - if (mode->flags & DRM_MODE_FLAG_NHSYNC) - syncs |= 0x00000008; - if (mode->flags & DRM_MODE_FLAG_NVSYNC) - syncs |= 0x00000010; - - if (mode->flags & DRM_MODE_FLAG_INTERLACE) - magic |= 0x00000001; - - evo_mthd(push, 0x0404 + (nv_crtc->index * 0x300), 2); - evo_data(push, syncs | (depth << 6)); - evo_data(push, magic); - evo_kick(push, mast); - } - - ctrl = proto << 8; - mask = 0x00000f00; - } else { - ctrl = (depth << 16) | (proto << 8); - if (mode->flags & DRM_MODE_FLAG_NHSYNC) - ctrl |= 0x00001000; - if (mode->flags & DRM_MODE_FLAG_NVSYNC) - ctrl |= 0x00002000; - mask = 0x000f3f00; - } - - nv50_sor_ctrl(nv_encoder, mask | owner, ctrl | owner); + nv_encoder->update(nv_encoder, nv_crtc->index, mode, proto, depth); } +static const struct drm_encoder_helper_funcs +nv50_sor_help = { + .dpms = nv50_sor_dpms, + .atomic_check = nv50_outp_atomic_check, + .enable = nv50_sor_enable, + .disable = nv50_sor_disable, +}; + static void nv50_sor_destroy(struct drm_encoder *encoder) { + struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); + nv50_mstm_del(&nv_encoder->dp.mstm); drm_encoder_cleanup(encoder); kfree(encoder); } -static const struct drm_encoder_helper_funcs nv50_sor_hfunc = { - .dpms = nv50_sor_dpms, - .mode_fixup = nv50_encoder_mode_fixup, - .prepare = nv50_sor_disconnect, - .commit = nv50_sor_commit, - .mode_set = nv50_sor_mode_set, - .disable = nv50_sor_disconnect, - .get_crtc = nv50_display_crtc_get, -}; - -static const struct drm_encoder_funcs nv50_sor_func = { +static const struct drm_encoder_funcs +nv50_sor_func = { .destroy = nv50_sor_destroy, }; static int nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe) { + struct nouveau_connector *nv_connector = nouveau_connector(connector); struct nouveau_drm *drm = nouveau_drm(connector->dev); struct nvkm_i2c *i2c = nvxx_i2c(&drm->device); struct nouveau_encoder *nv_encoder; struct drm_encoder *encoder; - int type; + int type, ret; switch (dcbe->type) { case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break; @@ -2122,7 +3670,16 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe) return -ENOMEM; nv_encoder->dcb = dcbe; nv_encoder->or = ffs(dcbe->or) - 1; - nv_encoder->last_dpms = DRM_MODE_DPMS_OFF; + nv_encoder->update = nv50_sor_update; + + encoder = to_drm_encoder(nv_encoder); + encoder->possible_crtcs = dcbe->heads; + encoder->possible_clones = 0; + drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type, + "sor-%04x-%04x", dcbe->hasht, dcbe->hashm); + drm_encoder_helper_add(encoder, &nv50_sor_help); + + drm_mode_connector_attach_encoder(connector, encoder); if (dcbe->type == DCB_OUTPUT_DP) { struct nvkm_i2c_aux *aux = @@ -2131,6 +3688,15 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe) nv_encoder->i2c = &aux->i2c; nv_encoder->aux = aux; } + + /*TODO: Use DP Info Table to check for support. */ + if (nv50_disp(encoder->dev)->disp->oclass >= GF110_DISP) { + ret = nv50_mstm_new(nv_encoder, &nv_connector->aux, 16, + nv_connector->base.base.id, + &nv_encoder->dp.mstm); + if (ret) + return ret; + } } else { struct nvkm_i2c_bus *bus = nvkm_i2c_bus_find(i2c, dcbe->i2c_index); @@ -2138,20 +3704,12 @@ nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe) nv_encoder->i2c = &bus->i2c; } - encoder = to_drm_encoder(nv_encoder); - encoder->possible_crtcs = dcbe->heads; - encoder->possible_clones = 0; - drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type, NULL); - drm_encoder_helper_add(encoder, &nv50_sor_hfunc); - - drm_mode_connector_attach_encoder(connector, encoder); return 0; } /****************************************************************************** * PIOR *****************************************************************************/ - static void nv50_pior_dpms(struct drm_encoder *encoder, int mode) { @@ -2172,30 +3730,48 @@ nv50_pior_dpms(struct drm_encoder *encoder, int mode) nvif_mthd(disp->disp, 0, &args, sizeof(args)); } -static bool -nv50_pior_mode_fixup(struct drm_encoder *encoder, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +static int +nv50_pior_atomic_check(struct drm_encoder *encoder, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) { - if (!nv50_encoder_mode_fixup(encoder, mode, adjusted_mode)) - return false; - adjusted_mode->clock *= 2; - return true; + int ret = nv50_outp_atomic_check(encoder, crtc_state, conn_state); + if (ret) + return ret; + crtc_state->adjusted_mode.clock *= 2; + return 0; } static void -nv50_pior_commit(struct drm_encoder *encoder) +nv50_pior_disable(struct drm_encoder *encoder) { + struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); + struct nv50_mast *mast = nv50_mast(encoder->dev); + const int or = nv_encoder->or; + u32 *push; + + if (nv_encoder->crtc) { + push = evo_wait(mast, 4); + if (push) { + if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { + evo_mthd(push, 0x0700 + (or * 0x040), 1); + evo_data(push, 0x00000000); + } + evo_kick(push, mast); + } + } + + nv_encoder->crtc = NULL; } static void -nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +nv50_pior_enable(struct drm_encoder *encoder) { struct nv50_mast *mast = nv50_mast(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc); struct nouveau_connector *nv_connector; + struct drm_display_mode *mode = &nv_crtc->base.state->adjusted_mode; u8 owner = 1 << nv_crtc->index; u8 proto, depth; u32 *push; @@ -2218,8 +3794,6 @@ nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, break; } - nv50_pior_dpms(encoder, DRM_MODE_DPMS_ON); - push = evo_wait(mast, 8); if (push) { if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { @@ -2238,29 +3812,13 @@ nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode, nv_encoder->crtc = encoder->crtc; } -static void -nv50_pior_disconnect(struct drm_encoder *encoder) -{ - struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_mast *mast = nv50_mast(encoder->dev); - const int or = nv_encoder->or; - u32 *push; - - if (nv_encoder->crtc) { - nv50_crtc_prepare(nv_encoder->crtc); - - push = evo_wait(mast, 4); - if (push) { - if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) { - evo_mthd(push, 0x0700 + (or * 0x040), 1); - evo_data(push, 0x00000000); - } - evo_kick(push, mast); - } - } - - nv_encoder->crtc = NULL; -} +static const struct drm_encoder_helper_funcs +nv50_pior_help = { + .dpms = nv50_pior_dpms, + .atomic_check = nv50_pior_atomic_check, + .enable = nv50_pior_enable, + .disable = nv50_pior_disable, +}; static void nv50_pior_destroy(struct drm_encoder *encoder) @@ -2269,17 +3827,8 @@ nv50_pior_destroy(struct drm_encoder *encoder) kfree(encoder); } -static const struct drm_encoder_helper_funcs nv50_pior_hfunc = { - .dpms = nv50_pior_dpms, - .mode_fixup = nv50_pior_mode_fixup, - .prepare = nv50_pior_disconnect, - .commit = nv50_pior_commit, - .mode_set = nv50_pior_mode_set, - .disable = nv50_pior_disconnect, - .get_crtc = nv50_display_crtc_get, -}; - -static const struct drm_encoder_funcs nv50_pior_func = { +static const struct drm_encoder_funcs +nv50_pior_func = { .destroy = nv50_pior_destroy, }; @@ -2321,149 +3870,462 @@ nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe) encoder = to_drm_encoder(nv_encoder); encoder->possible_crtcs = dcbe->heads; encoder->possible_clones = 0; - drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type, NULL); - drm_encoder_helper_add(encoder, &nv50_pior_hfunc); + drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type, + "pior-%04x-%04x", dcbe->hasht, dcbe->hashm); + drm_encoder_helper_add(encoder, &nv50_pior_help); drm_mode_connector_attach_encoder(connector, encoder); return 0; } /****************************************************************************** - * Framebuffer + * Atomic *****************************************************************************/ static void -nv50_fbdma_fini(struct nv50_fbdma *fbdma) +nv50_disp_atomic_commit_core(struct nouveau_drm *drm, u32 interlock) { - int i; - for (i = 0; i < ARRAY_SIZE(fbdma->base); i++) - nvif_object_fini(&fbdma->base[i]); - nvif_object_fini(&fbdma->core); - list_del(&fbdma->head); - kfree(fbdma); + struct nv50_disp *disp = nv50_disp(drm->dev); + struct nv50_dmac *core = &disp->mast.base; + struct nv50_mstm *mstm; + struct drm_encoder *encoder; + u32 *push; + + NV_ATOMIC(drm, "commit core %08x\n", interlock); + + drm_for_each_encoder(encoder, drm->dev) { + if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) { + mstm = nouveau_encoder(encoder)->dp.mstm; + if (mstm && mstm->modified) + nv50_mstm_prepare(mstm); + } + } + + if ((push = evo_wait(core, 5))) { + evo_mthd(push, 0x0084, 1); + evo_data(push, 0x80000000); + evo_mthd(push, 0x0080, 2); + evo_data(push, interlock); + evo_data(push, 0x00000000); + nouveau_bo_wr32(disp->sync, 0, 0x00000000); + evo_kick(push, core); + if (nvif_msec(&drm->device, 2000ULL, + if (nouveau_bo_rd32(disp->sync, 0)) + break; + usleep_range(1, 2); + ) < 0) + NV_ERROR(drm, "EVO timeout\n"); + } + + drm_for_each_encoder(encoder, drm->dev) { + if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) { + mstm = nouveau_encoder(encoder)->dp.mstm; + if (mstm && mstm->modified) + nv50_mstm_cleanup(mstm); + } + } } -static int -nv50_fbdma_init(struct drm_device *dev, u32 name, u64 offset, u64 length, u8 kind) +static void +nv50_disp_atomic_commit_tail(struct drm_atomic_state *state) { + struct drm_device *dev = state->dev; + struct drm_crtc_state *crtc_state; + struct drm_crtc *crtc; + struct drm_plane_state *plane_state; + struct drm_plane *plane; struct nouveau_drm *drm = nouveau_drm(dev); struct nv50_disp *disp = nv50_disp(dev); - struct nv50_mast *mast = nv50_mast(dev); - struct __attribute__ ((packed)) { - struct nv_dma_v0 base; - union { - struct nv50_dma_v0 nv50; - struct gf100_dma_v0 gf100; - struct gf119_dma_v0 gf119; - }; - } args = {}; - struct nv50_fbdma *fbdma; - struct drm_crtc *crtc; - u32 size = sizeof(args.base); - int ret; + struct nv50_atom *atom = nv50_atom(state); + struct nv50_outp_atom *outp, *outt; + u32 interlock_core = 0; + u32 interlock_chan = 0; + int i; + + NV_ATOMIC(drm, "commit %d %d\n", atom->lock_core, atom->flush_disable); + drm_atomic_helper_wait_for_fences(dev, state, false); + drm_atomic_helper_wait_for_dependencies(state); + drm_atomic_helper_update_legacy_modeset_state(dev, state); - list_for_each_entry(fbdma, &disp->fbdma, head) { - if (fbdma->core.handle == name) - return 0; + if (atom->lock_core) + mutex_lock(&disp->mutex); + + /* Disable head(s). */ + for_each_crtc_in_state(state, crtc, crtc_state, i) { + struct nv50_head_atom *asyh = nv50_head_atom(crtc->state); + struct nv50_head *head = nv50_head(crtc); + + NV_ATOMIC(drm, "%s: clr %04x (set %04x)\n", crtc->name, + asyh->clr.mask, asyh->set.mask); + + if (asyh->clr.mask) { + nv50_head_flush_clr(head, asyh, atom->flush_disable); + interlock_core |= 1; + } } - fbdma = kzalloc(sizeof(*fbdma), GFP_KERNEL); - if (!fbdma) - return -ENOMEM; - list_add(&fbdma->head, &disp->fbdma); + /* Disable plane(s). */ + for_each_plane_in_state(state, plane, plane_state, i) { + struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state); + struct nv50_wndw *wndw = nv50_wndw(plane); - args.base.target = NV_DMA_V0_TARGET_VRAM; - args.base.access = NV_DMA_V0_ACCESS_RDWR; - args.base.start = offset; - args.base.limit = offset + length - 1; + NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", plane->name, + asyw->clr.mask, asyw->set.mask); + if (!asyw->clr.mask) + continue; - if (drm->device.info.chipset < 0x80) { - args.nv50.part = NV50_DMA_V0_PART_256; - size += sizeof(args.nv50); - } else - if (drm->device.info.chipset < 0xc0) { - args.nv50.part = NV50_DMA_V0_PART_256; - args.nv50.kind = kind; - size += sizeof(args.nv50); - } else - if (drm->device.info.chipset < 0xd0) { - args.gf100.kind = kind; - size += sizeof(args.gf100); - } else { - args.gf119.page = GF119_DMA_V0_PAGE_LP; - args.gf119.kind = kind; - size += sizeof(args.gf119); + interlock_chan |= nv50_wndw_flush_clr(wndw, interlock_core, + atom->flush_disable, + asyw); + } + + /* Disable output path(s). */ + list_for_each_entry(outp, &atom->outp, head) { + const struct drm_encoder_helper_funcs *help; + struct drm_encoder *encoder; + + encoder = outp->encoder; + help = encoder->helper_private; + + NV_ATOMIC(drm, "%s: clr %02x (set %02x)\n", encoder->name, + outp->clr.mask, outp->set.mask); + + if (outp->clr.mask) { + help->disable(encoder); + interlock_core |= 1; + if (outp->flush_disable) { + nv50_disp_atomic_commit_core(drm, interlock_chan); + interlock_core = 0; + interlock_chan = 0; + } + } + } + + /* Flush disable. */ + if (interlock_core) { + if (atom->flush_disable) { + nv50_disp_atomic_commit_core(drm, interlock_chan); + interlock_core = 0; + interlock_chan = 0; + } } - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + /* Update output path(s). */ + list_for_each_entry_safe(outp, outt, &atom->outp, head) { + const struct drm_encoder_helper_funcs *help; + struct drm_encoder *encoder; + + encoder = outp->encoder; + help = encoder->helper_private; + + NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", encoder->name, + outp->set.mask, outp->clr.mask); + + if (outp->set.mask) { + help->enable(encoder); + interlock_core = 1; + } + + list_del(&outp->head); + kfree(outp); + } + + /* Update head(s). */ + for_each_crtc_in_state(state, crtc, crtc_state, i) { + struct nv50_head_atom *asyh = nv50_head_atom(crtc->state); struct nv50_head *head = nv50_head(crtc); - int ret = nvif_object_init(&head->sync.base.base.user, name, - NV_DMA_IN_MEMORY, &args, size, - &fbdma->base[head->base.index]); - if (ret) { - nv50_fbdma_fini(fbdma); - return ret; + + NV_ATOMIC(drm, "%s: set %04x (clr %04x)\n", crtc->name, + asyh->set.mask, asyh->clr.mask); + + if (asyh->set.mask) { + nv50_head_flush_set(head, asyh); + interlock_core = 1; } } - ret = nvif_object_init(&mast->base.base.user, name, NV_DMA_IN_MEMORY, - &args, size, &fbdma->core); - if (ret) { - nv50_fbdma_fini(fbdma); + /* Update plane(s). */ + for_each_plane_in_state(state, plane, plane_state, i) { + struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state); + struct nv50_wndw *wndw = nv50_wndw(plane); + + NV_ATOMIC(drm, "%s: set %02x (clr %02x)\n", plane->name, + asyw->set.mask, asyw->clr.mask); + if ( !asyw->set.mask && + (!asyw->clr.mask || atom->flush_disable)) + continue; + + interlock_chan |= nv50_wndw_flush_set(wndw, interlock_core, asyw); + } + + /* Flush update. */ + if (interlock_core) { + if (!interlock_chan && atom->state.legacy_cursor_update) { + u32 *push = evo_wait(&disp->mast, 2); + if (push) { + evo_mthd(push, 0x0080, 1); + evo_data(push, 0x00000000); + evo_kick(push, &disp->mast); + } + } else { + nv50_disp_atomic_commit_core(drm, interlock_chan); + } + } + + if (atom->lock_core) + mutex_unlock(&disp->mutex); + + /* Wait for HW to signal completion. */ + for_each_plane_in_state(state, plane, plane_state, i) { + struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane->state); + struct nv50_wndw *wndw = nv50_wndw(plane); + int ret = nv50_wndw_wait_armed(wndw, asyw); + if (ret) + NV_ERROR(drm, "%s: timeout\n", plane->name); + } + + for_each_crtc_in_state(state, crtc, crtc_state, i) { + if (crtc->state->event) { + unsigned long flags; + spin_lock_irqsave(&crtc->dev->event_lock, flags); + drm_crtc_send_vblank_event(crtc, crtc->state->event); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); + crtc->state->event = NULL; + } + } + + drm_atomic_helper_commit_hw_done(state); + drm_atomic_helper_cleanup_planes(dev, state); + drm_atomic_helper_commit_cleanup_done(state); + drm_atomic_state_put(state); +} + +static void +nv50_disp_atomic_commit_work(struct work_struct *work) +{ + struct drm_atomic_state *state = + container_of(work, typeof(*state), commit_work); + nv50_disp_atomic_commit_tail(state); +} + +static int +nv50_disp_atomic_commit(struct drm_device *dev, + struct drm_atomic_state *state, bool nonblock) +{ + struct nouveau_drm *drm = nouveau_drm(dev); + struct nv50_disp *disp = nv50_disp(dev); + struct drm_plane_state *plane_state; + struct drm_plane *plane; + struct drm_crtc *crtc; + bool active = false; + int ret, i; + + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0 && ret != -EACCES) return ret; + + ret = drm_atomic_helper_setup_commit(state, nonblock); + if (ret) + goto done; + + INIT_WORK(&state->commit_work, nv50_disp_atomic_commit_work); + + ret = drm_atomic_helper_prepare_planes(dev, state); + if (ret) + goto done; + + if (!nonblock) { + ret = drm_atomic_helper_wait_for_fences(dev, state, true); + if (ret) + goto done; + } + + for_each_plane_in_state(state, plane, plane_state, i) { + struct nv50_wndw_atom *asyw = nv50_wndw_atom(plane_state); + struct nv50_wndw *wndw = nv50_wndw(plane); + if (asyw->set.image) { + asyw->ntfy.handle = wndw->dmac->sync.handle; + asyw->ntfy.offset = wndw->ntfy; + asyw->ntfy.awaken = false; + asyw->set.ntfy = true; + nouveau_bo_wr32(disp->sync, wndw->ntfy / 4, 0x00000000); + wndw->ntfy ^= 0x10; + } + } + + drm_atomic_helper_swap_state(state, true); + drm_atomic_state_get(state); + + if (nonblock) + queue_work(system_unbound_wq, &state->commit_work); + else + nv50_disp_atomic_commit_tail(state); + + drm_for_each_crtc(crtc, dev) { + if (crtc->state->enable) { + if (!drm->have_disp_power_ref) { + drm->have_disp_power_ref = true; + return ret; + } + active = true; + break; + } + } + + if (!active && drm->have_disp_power_ref) { + pm_runtime_put_autosuspend(dev->dev); + drm->have_disp_power_ref = false; + } + +done: + pm_runtime_put_autosuspend(dev->dev); + return ret; +} + +static struct nv50_outp_atom * +nv50_disp_outp_atomic_add(struct nv50_atom *atom, struct drm_encoder *encoder) +{ + struct nv50_outp_atom *outp; + + list_for_each_entry(outp, &atom->outp, head) { + if (outp->encoder == encoder) + return outp; + } + + outp = kzalloc(sizeof(*outp), GFP_KERNEL); + if (!outp) + return ERR_PTR(-ENOMEM); + + list_add(&outp->head, &atom->outp); + outp->encoder = encoder; + return outp; +} + +static int +nv50_disp_outp_atomic_check_clr(struct nv50_atom *atom, + struct drm_connector *connector) +{ + struct drm_encoder *encoder = connector->state->best_encoder; + struct drm_crtc_state *crtc_state; + struct drm_crtc *crtc; + struct nv50_outp_atom *outp; + + if (!(crtc = connector->state->crtc)) + return 0; + + crtc_state = drm_atomic_get_existing_crtc_state(&atom->state, crtc); + if (crtc->state->active && drm_atomic_crtc_needs_modeset(crtc_state)) { + outp = nv50_disp_outp_atomic_add(atom, encoder); + if (IS_ERR(outp)) + return PTR_ERR(outp); + + if (outp->encoder->encoder_type == DRM_MODE_ENCODER_DPMST) { + outp->flush_disable = true; + atom->flush_disable = true; + } + outp->clr.ctrl = true; + atom->lock_core = true; } return 0; } -static void -nv50_fb_dtor(struct drm_framebuffer *fb) +static int +nv50_disp_outp_atomic_check_set(struct nv50_atom *atom, + struct drm_connector_state *connector_state) { + struct drm_encoder *encoder = connector_state->best_encoder; + struct drm_crtc_state *crtc_state; + struct drm_crtc *crtc; + struct nv50_outp_atom *outp; + + if (!(crtc = connector_state->crtc)) + return 0; + + crtc_state = drm_atomic_get_existing_crtc_state(&atom->state, crtc); + if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) { + outp = nv50_disp_outp_atomic_add(atom, encoder); + if (IS_ERR(outp)) + return PTR_ERR(outp); + + outp->set.ctrl = true; + atom->lock_core = true; + } + + return 0; } static int -nv50_fb_ctor(struct drm_framebuffer *fb) -{ - struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb); - struct nouveau_drm *drm = nouveau_drm(fb->dev); - struct nouveau_bo *nvbo = nv_fb->nvbo; - struct nv50_disp *disp = nv50_disp(fb->dev); - u8 kind = nouveau_bo_tile_layout(nvbo) >> 8; - u8 tile = nvbo->tile_mode; - - if (drm->device.info.chipset >= 0xc0) - tile >>= 4; /* yep.. */ - - switch (fb->depth) { - case 8: nv_fb->r_format = 0x1e00; break; - case 15: nv_fb->r_format = 0xe900; break; - case 16: nv_fb->r_format = 0xe800; break; - case 24: - case 32: nv_fb->r_format = 0xcf00; break; - case 30: nv_fb->r_format = 0xd100; break; - default: - NV_ERROR(drm, "unknown depth %d\n", fb->depth); - return -EINVAL; +nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state) +{ + struct nv50_atom *atom = nv50_atom(state); + struct drm_connector_state *connector_state; + struct drm_connector *connector; + int ret, i; + + ret = drm_atomic_helper_check(dev, state); + if (ret) + return ret; + + for_each_connector_in_state(state, connector, connector_state, i) { + ret = nv50_disp_outp_atomic_check_clr(atom, connector); + if (ret) + return ret; + + ret = nv50_disp_outp_atomic_check_set(atom, connector_state); + if (ret) + return ret; } - if (disp->disp->oclass < G82_DISP) { - nv_fb->r_pitch = kind ? (((fb->pitches[0] / 4) << 4) | tile) : - (fb->pitches[0] | 0x00100000); - nv_fb->r_format |= kind << 16; - } else - if (disp->disp->oclass < GF110_DISP) { - nv_fb->r_pitch = kind ? (((fb->pitches[0] / 4) << 4) | tile) : - (fb->pitches[0] | 0x00100000); - } else { - nv_fb->r_pitch = kind ? (((fb->pitches[0] / 4) << 4) | tile) : - (fb->pitches[0] | 0x01000000); + return 0; +} + +static void +nv50_disp_atomic_state_clear(struct drm_atomic_state *state) +{ + struct nv50_atom *atom = nv50_atom(state); + struct nv50_outp_atom *outp, *outt; + + list_for_each_entry_safe(outp, outt, &atom->outp, head) { + list_del(&outp->head); + kfree(outp); } - nv_fb->r_handle = 0xffff0000 | kind; - return nv50_fbdma_init(fb->dev, nv_fb->r_handle, 0, - drm->device.info.ram_user, kind); + drm_atomic_state_default_clear(state); +} + +static void +nv50_disp_atomic_state_free(struct drm_atomic_state *state) +{ + struct nv50_atom *atom = nv50_atom(state); + drm_atomic_state_default_release(&atom->state); + kfree(atom); } +static struct drm_atomic_state * +nv50_disp_atomic_state_alloc(struct drm_device *dev) +{ + struct nv50_atom *atom; + if (!(atom = kzalloc(sizeof(*atom), GFP_KERNEL)) || + drm_atomic_state_init(dev, &atom->state) < 0) { + kfree(atom); + return NULL; + } + INIT_LIST_HEAD(&atom->outp); + return &atom->state; +} + +static const struct drm_mode_config_funcs +nv50_disp_func = { + .fb_create = nouveau_user_framebuffer_create, + .output_poll_changed = nouveau_fbcon_output_poll_changed, + .atomic_check = nv50_disp_atomic_check, + .atomic_commit = nv50_disp_atomic_commit, + .atomic_state_alloc = nv50_disp_atomic_state_alloc, + .atomic_state_clear = nv50_disp_atomic_state_clear, + .atomic_state_free = nv50_disp_atomic_state_free, +}; + /****************************************************************************** * Init *****************************************************************************/ @@ -2471,12 +4333,30 @@ nv50_fb_ctor(struct drm_framebuffer *fb) void nv50_display_fini(struct drm_device *dev) { + struct nouveau_encoder *nv_encoder; + struct drm_encoder *encoder; + struct drm_plane *plane; + + drm_for_each_plane(plane, dev) { + struct nv50_wndw *wndw = nv50_wndw(plane); + if (plane->funcs != &nv50_wndw) + continue; + nv50_wndw_fini(wndw); + } + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) { + nv_encoder = nouveau_encoder(encoder); + nv50_mstm_fini(nv_encoder->dp.mstm); + } + } } int nv50_display_init(struct drm_device *dev) { - struct nv50_disp *disp = nv50_disp(dev); + struct drm_encoder *encoder; + struct drm_plane *plane; struct drm_crtc *crtc; u32 *push; @@ -2484,16 +4364,35 @@ nv50_display_init(struct drm_device *dev) if (!push) return -EBUSY; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - struct nv50_sync *sync = nv50_sync(crtc); - - nv50_crtc_lut_load(crtc); - nouveau_bo_wr32(disp->sync, sync->addr / 4, sync->data); - } - evo_mthd(push, 0x0088, 1); evo_data(push, nv50_mast(dev)->base.sync.handle); evo_kick(push, nv50_mast(dev)); + + list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) { + const struct drm_encoder_helper_funcs *help; + struct nouveau_encoder *nv_encoder; + + nv_encoder = nouveau_encoder(encoder); + help = encoder->helper_private; + if (help && help->dpms) + help->dpms(encoder, DRM_MODE_DPMS_ON); + + nv50_mstm_init(nv_encoder->dp.mstm); + } + } + + drm_for_each_crtc(crtc, dev) { + nv50_head_lut_load(crtc); + } + + drm_for_each_plane(plane, dev) { + struct nv50_wndw *wndw = nv50_wndw(plane); + if (plane->funcs != &nv50_wndw) + continue; + nv50_wndw_init(wndw); + } + return 0; } @@ -2501,11 +4400,6 @@ void nv50_display_destroy(struct drm_device *dev) { struct nv50_disp *disp = nv50_disp(dev); - struct nv50_fbdma *fbdma, *fbtmp; - - list_for_each_entry_safe(fbdma, fbtmp, &disp->fbdma, head) { - nv50_fbdma_fini(fbdma); - } nv50_dmac_destroy(&disp->mast.base, disp->disp); @@ -2518,6 +4412,10 @@ nv50_display_destroy(struct drm_device *dev) kfree(disp); } +MODULE_PARM_DESC(atomic, "Expose atomic ioctl (default: disabled)"); +static int nouveau_atomic = 0; +module_param_named(atomic, nouveau_atomic, int, 0400); + int nv50_display_create(struct drm_device *dev) { @@ -2532,15 +4430,17 @@ nv50_display_create(struct drm_device *dev) disp = kzalloc(sizeof(*disp), GFP_KERNEL); if (!disp) return -ENOMEM; - INIT_LIST_HEAD(&disp->fbdma); + + mutex_init(&disp->mutex); nouveau_display(dev)->priv = disp; nouveau_display(dev)->dtor = nv50_display_destroy; nouveau_display(dev)->init = nv50_display_init; nouveau_display(dev)->fini = nv50_display_fini; - nouveau_display(dev)->fb_ctor = nv50_fb_ctor; - nouveau_display(dev)->fb_dtor = nv50_fb_dtor; disp->disp = &nouveau_display(dev)->disp; + dev->mode_config.funcs = &nv50_disp_func; + if (nouveau_atomic) + dev->driver->driver_features |= DRIVER_ATOMIC; /* small shared memory area we use for notifiers and semaphores */ ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM, @@ -2572,7 +4472,7 @@ nv50_display_create(struct drm_device *dev) crtcs = 2; for (i = 0; i < crtcs; i++) { - ret = nv50_crtc_create(dev, i); + ret = nv50_head_create(dev, i); if (ret) goto out; } diff --git a/drivers/gpu/drm/nouveau/nv50_display.h b/drivers/gpu/drm/nouveau/nv50_display.h index 70da347aa8c5..918187cee84b 100644 --- a/drivers/gpu/drm/nouveau/nv50_display.h +++ b/drivers/gpu/drm/nouveau/nv50_display.h @@ -35,11 +35,4 @@ int nv50_display_create(struct drm_device *); void nv50_display_destroy(struct drm_device *); int nv50_display_init(struct drm_device *); void nv50_display_fini(struct drm_device *); - -void nv50_display_flip_stop(struct drm_crtc *); -int nv50_display_flip_next(struct drm_crtc *, struct drm_framebuffer *, - struct nouveau_channel *, u32 swap_interval); - -struct nouveau_bo *nv50_display_crtc_sema(struct drm_device *, int head); - #endif /* __NV50_DISPLAY_H__ */ diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c index af3d3c49411a..327dcd7901ed 100644 --- a/drivers/gpu/drm/nouveau/nv50_fbcon.c +++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c @@ -30,7 +30,7 @@ int nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -65,7 +65,7 @@ int nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -93,7 +93,7 @@ int nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; uint32_t dwords, *data = (uint32_t *)image->data; uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel)); @@ -148,8 +148,8 @@ int nv50_fbcon_accel_init(struct fb_info *info) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb; - struct drm_device *dev = nfbdev->dev; + struct nouveau_framebuffer *fb = nouveau_framebuffer(nfbdev->helper.fb); + struct drm_device *dev = nfbdev->helper.dev; struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_channel *chan = drm->channel; int ret, format; diff --git a/drivers/gpu/drm/nouveau/nv50_fence.c b/drivers/gpu/drm/nouveau/nv50_fence.c index 8c5295414578..f68c7054fd53 100644 --- a/drivers/gpu/drm/nouveau/nv50_fence.c +++ b/drivers/gpu/drm/nouveau/nv50_fence.c @@ -35,13 +35,12 @@ static int nv50_fence_context_new(struct nouveau_channel *chan) { - struct drm_device *dev = chan->drm->dev; struct nv10_fence_priv *priv = chan->drm->fence; struct nv10_fence_chan *fctx; struct ttm_mem_reg *mem = &priv->bo->bo.mem; u32 start = mem->start * PAGE_SIZE; u32 limit = start + mem->size - 1; - int ret, i; + int ret; fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL); if (!fctx) @@ -60,23 +59,6 @@ nv50_fence_context_new(struct nouveau_channel *chan) .limit = limit, }, sizeof(struct nv_dma_v0), &fctx->sema); - - /* dma objects for display sync channel semaphore blocks */ - for (i = 0; !ret && i < dev->mode_config.num_crtc; i++) { - struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i); - u32 start = bo->bo.mem.start * PAGE_SIZE; - u32 limit = start + bo->bo.mem.size - 1; - - ret = nvif_object_init(&chan->user, NvEvoSema0 + i, - NV_DMA_IN_MEMORY, &(struct nv_dma_v0) { - .target = NV_DMA_V0_TARGET_VRAM, - .access = NV_DMA_V0_ACCESS_RDWR, - .start = start, - .limit = limit, - }, sizeof(struct nv_dma_v0), - &fctx->head[i]); - } - if (ret) nv10_fence_context_del(chan); return ret; diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index 23ef04b4e0b2..52b87ae83e7b 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -28,13 +28,6 @@ #include "nv50_display.h" -u64 -nv84_fence_crtc(struct nouveau_channel *chan, int crtc) -{ - struct nv84_fence_chan *fctx = chan->fence; - return fctx->dispc_vma[crtc].offset; -} - static int nv84_fence_emit32(struct nouveau_channel *chan, u64 virtual, u32 sequence) { @@ -110,15 +103,8 @@ nv84_fence_read(struct nouveau_channel *chan) static void nv84_fence_context_del(struct nouveau_channel *chan) { - struct drm_device *dev = chan->drm->dev; struct nv84_fence_priv *priv = chan->drm->fence; struct nv84_fence_chan *fctx = chan->fence; - int i; - - for (i = 0; i < dev->mode_config.num_crtc; i++) { - struct nouveau_bo *bo = nv50_display_crtc_sema(dev, i); - nouveau_bo_vma_del(bo, &fctx->dispc_vma[i]); - } nouveau_bo_wr32(priv->bo, chan->chid * 16 / 4, fctx->base.sequence); nouveau_bo_vma_del(priv->bo, &fctx->vma_gart); @@ -134,7 +120,7 @@ nv84_fence_context_new(struct nouveau_channel *chan) struct nouveau_cli *cli = (void *)chan->user.client; struct nv84_fence_priv *priv = chan->drm->fence; struct nv84_fence_chan *fctx; - int ret, i; + int ret; fctx = chan->fence = kzalloc(sizeof(*fctx), GFP_KERNEL); if (!fctx) @@ -154,12 +140,6 @@ nv84_fence_context_new(struct nouveau_channel *chan) &fctx->vma_gart); } - /* map display semaphore buffers into channel's vm */ - for (i = 0; !ret && i < chan->drm->dev->mode_config.num_crtc; i++) { - struct nouveau_bo *bo = nv50_display_crtc_sema(chan->drm->dev, i); - ret = nouveau_bo_vma_add(bo, cli->vm, &fctx->dispc_vma[i]); - } - if (ret) nv84_fence_context_del(chan); return ret; diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c index 054b6a056d99..90f27bfa381f 100644 --- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c +++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c @@ -30,7 +30,7 @@ int nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -65,7 +65,7 @@ int nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; int ret; @@ -93,7 +93,7 @@ int nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) { struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->dev); + struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); struct nouveau_channel *chan = drm->channel; uint32_t dwords, *data = (uint32_t *)image->data; uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel)); @@ -148,8 +148,8 @@ int nvc0_fbcon_accel_init(struct fb_info *info) { struct nouveau_fbdev *nfbdev = info->par; - struct drm_device *dev = nfbdev->dev; - struct nouveau_framebuffer *fb = &nfbdev->nouveau_fb; + struct drm_device *dev = nfbdev->helper.dev; + struct nouveau_framebuffer *fb = nouveau_framebuffer(nfbdev->helper.fb); struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_channel *chan = drm->channel; int ret, format; diff --git a/drivers/gpu/drm/nouveau/nvif/client.c b/drivers/gpu/drm/nouveau/nvif/client.c index 1ee9294eca2e..29c20dfd894d 100644 --- a/drivers/gpu/drm/nouveau/nvif/client.c +++ b/drivers/gpu/drm/nouveau/nvif/client.c @@ -55,7 +55,7 @@ nvif_client_fini(struct nvif_client *client) } } -const struct nvif_driver * +static const struct nvif_driver * nvif_drivers[] = { #ifdef __KERNEL__ &nvif_driver_nvkm, diff --git a/drivers/gpu/drm/nouveau/nvif/notify.c b/drivers/gpu/drm/nouveau/nvif/notify.c index b0787ff833ef..278b3933dc96 100644 --- a/drivers/gpu/drm/nouveau/nvif/notify.c +++ b/drivers/gpu/drm/nouveau/nvif/notify.c @@ -155,10 +155,8 @@ nvif_notify_fini(struct nvif_notify *notify) int ret = nvif_notify_put(notify); if (ret >= 0 && object) { ret = nvif_object_ioctl(object, &args, sizeof(args), NULL); - if (ret == 0) { - notify->object = NULL; - kfree((void *)notify->data); - } + notify->object = NULL; + kfree((void *)notify->data); } return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h index 05bb65608dfe..d9ca9636a3e3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gf100.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gf100_ce_data[] = { +static uint32_t gf100_ce_data[] = { /* 0x0000: ctx_object */ 0x00000000, /* 0x0004: ctx_query_address_high */ @@ -171,7 +171,7 @@ uint32_t gf100_ce_data[] = { 0x00000800, }; -uint32_t gf100_ce_code[] = { +static uint32_t gf100_ce_code[] = { /* 0x0000: main */ 0x04fe04bd, 0x3517f000, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h index 972281d10f38..f0a1cf31c7ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/fuc/gt215.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gt215_ce_data[] = { +static uint32_t gt215_ce_data[] = { /* 0x0000: ctx_object */ 0x00000000, /* 0x0004: ctx_dma */ @@ -183,7 +183,7 @@ uint32_t gt215_ce_data[] = { 0x00000800, }; -uint32_t gt215_ce_code[] = { +static uint32_t gt215_ce_code[] = { /* 0x0000: main */ 0x04fe04bd, 0x3517f000, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 53d171729353..bd22526edb0b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1852,7 +1852,7 @@ nvf1_chipset = { .fb = gk104_fb_new, .fuse = gf100_fuse_new, .gpio = gk104_gpio_new, - .i2c = gf119_i2c_new, + .i2c = gk104_i2c_new, .ibus = gk104_ibus_new, .iccsense = gf100_iccsense_new, .imem = nv50_instmem_new, @@ -1966,7 +1966,7 @@ nv117_chipset = { .fb = gm107_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, - .i2c = gf119_i2c_new, + .i2c = gk104_i2c_new, .ibus = gk104_ibus_new, .iccsense = gf100_iccsense_new, .imem = nv50_instmem_new, @@ -2000,7 +2000,7 @@ nv118_chipset = { .fb = gm107_fb_new, .fuse = gm107_fuse_new, .gpio = gk104_gpio_new, - .i2c = gf119_i2c_new, + .i2c = gk104_i2c_new, .ibus = gk104_ibus_new, .iccsense = gf100_iccsense_new, .imem = nv50_instmem_new, @@ -2131,7 +2131,7 @@ nv12b_chipset = { .bar = gk20a_bar_new, .bus = gf100_bus_new, .clk = gm20b_clk_new, - .fb = gk20a_fb_new, + .fb = gm20b_fb_new, .fuse = gm107_fuse_new, .ibus = gk20a_ibus_new, .imem = gk20a_instmem_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c index 9b638bd905ff..f2bc0b7d9b93 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c @@ -102,7 +102,7 @@ nvkm_device_tegra_probe_iommu(struct nvkm_device_tegra *tdev) if (iommu_present(&platform_bus_type)) { tdev->iommu.domain = iommu_domain_alloc(&platform_bus_type); - if (IS_ERR(tdev->iommu.domain)) + if (!tdev->iommu.domain) goto error; /* diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index 79a8f71cf788..513ee6b79553 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -326,7 +326,7 @@ nvkm_udevice = { .sclass = nvkm_udevice_child_get, }; -int +static int nvkm_udevice_new(const struct nvkm_oclass *oclass, void *data, u32 size, struct nvkm_object **pobject) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c index dd2953bc9264..26990d44ae75 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.c @@ -153,7 +153,7 @@ nv50_disp_chan_uevent = { .fini = nv50_disp_chan_uevent_fini, }; -int +static int nv50_disp_chan_rd32(struct nvkm_object *object, u64 addr, u32 *data) { struct nv50_disp_chan *chan = nv50_disp_chan(object); @@ -163,7 +163,7 @@ nv50_disp_chan_rd32(struct nvkm_object *object, u64 addr, u32 *data) return 0; } -int +static int nv50_disp_chan_wr32(struct nvkm_object *object, u64 addr, u32 data) { struct nv50_disp_chan *chan = nv50_disp_chan(object); @@ -173,7 +173,7 @@ nv50_disp_chan_wr32(struct nvkm_object *object, u64 addr, u32 data) return 0; } -int +static int nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type, struct nvkm_event **pevent) { @@ -189,7 +189,7 @@ nv50_disp_chan_ntfy(struct nvkm_object *object, u32 type, return -EINVAL; } -int +static int nv50_disp_chan_map(struct nvkm_object *object, u64 *addr, u32 *size) { struct nv50_disp_chan *chan = nv50_disp_chan(object); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c index 019379a3a01c..c65c9f3ff69f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coreg94.c @@ -26,7 +26,7 @@ #include <nvif/class.h> -const struct nv50_disp_mthd_list +static const struct nv50_disp_mthd_list g94_disp_core_mthd_sor = { .mthd = 0x0040, .addr = 0x000008, @@ -43,8 +43,8 @@ g94_disp_core_chan_mthd = { .prev = 0x000004, .data = { { "Global", 1, &nv50_disp_core_mthd_base }, - { "DAC", 3, &g84_disp_core_mthd_dac }, - { "SOR", 4, &g94_disp_core_mthd_sor }, + { "DAC", 3, &g84_disp_core_mthd_dac }, + { "SOR", 4, &g94_disp_core_mthd_sor }, { "PIOR", 3, &nv50_disp_core_mthd_pior }, { "HEAD", 2, &g84_disp_core_mthd_head }, {} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c index 6922f4007b61..e356f87fbe60 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c @@ -59,7 +59,7 @@ gp104_disp_core_init(struct nv50_disp_dmac *chan) return 0; } -const struct nv50_disp_dmac_func +static const struct nv50_disp_dmac_func gp104_disp_core_func = { .init = gp104_disp_core_init, .fini = gf119_disp_core_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c index 9688970eca47..4a93ceb850ac 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.c @@ -319,9 +319,8 @@ static const struct dp_rates { }; void -nvkm_dp_train(struct work_struct *w) +nvkm_dp_train(struct nvkm_output_dp *outp) { - struct nvkm_output_dp *outp = container_of(w, typeof(*outp), lt.work); struct nv50_disp *disp = nv50_disp(outp->base.disp); const struct dp_rates *cfg = nvkm_dp_rates; struct dp_state _dp = { @@ -353,9 +352,6 @@ nvkm_dp_train(struct work_struct *w) } cfg--; - /* disable link interrupt handling during link training */ - nvkm_notify_put(&outp->irq); - /* ensure sink is not in a low-power state */ if (!nvkm_rdaux(outp->aux, DPCD_SC00, &pwr, 1)) { if ((pwr & DPCD_SC00_SET_POWER) != DPCD_SC00_SET_POWER_D0) { @@ -400,9 +396,6 @@ nvkm_dp_train(struct work_struct *w) dp_link_train_fini(dp); - /* signal completion and enable link interrupt handling */ OUTP_DBG(&outp->base, "training complete"); atomic_set(&outp->lt.done, 1); - wake_up(&outp->lt.wait); - nvkm_notify_get(&outp->irq); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h index 6e10c5e0ef11..baf1dd9ff975 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dport.h @@ -1,6 +1,6 @@ #ifndef __NVKM_DISP_DPORT_H__ #define __NVKM_DISP_DPORT_H__ -#include <core/os.h> +struct nvkm_output_dp; /* DPCD Receiver Capabilities */ #define DPCD_RC00_DPCD_REV 0x00000 @@ -77,5 +77,5 @@ #define DPCD_SC00_SET_POWER_D0 0x01 #define DPCD_SC00_SET_POWER_D3 0x03 -void nvkm_dp_train(struct work_struct *); +void nvkm_dp_train(struct nvkm_output_dp *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c index 29e84b241cca..7b346ccc38b7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c @@ -203,17 +203,20 @@ gf119_disp_intr_unk2_0(struct nv50_disp *disp, int head) /* see note in nv50_disp_intr_unk20_0() */ if (outp && outp->info.type == DCB_OUTPUT_DP) { struct nvkm_output_dp *outpdp = nvkm_output_dp(outp); - struct nvbios_init init = { - .subdev = subdev, - .bios = subdev->device->bios, - .outp = &outp->info, - .crtc = head, - .offset = outpdp->info.script[4], - .execute = 1, - }; + if (!outpdp->lt.mst) { + struct nvbios_init init = { + .subdev = subdev, + .bios = subdev->device->bios, + .outp = &outp->info, + .crtc = head, + .offset = outpdp->info.script[4], + .execute = 1, + }; - nvbios_exec(&init); - atomic_set(&outpdp->lt.done, 0); + nvkm_notify_put(&outpdp->irq); + nvbios_exec(&init); + atomic_set(&outpdp->lt.done, 0); + } } } @@ -314,7 +317,7 @@ gf119_disp_intr_unk2_2(struct nv50_disp *disp, int head) break; } - if (nvkm_output_dp_train(outp, pclk, true)) + if (nvkm_output_dp_train(outp, pclk)) OUTP_ERR(outp, "link not trained before attach"); } else { if (disp->func->sor.magic) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c index fbb8c7dc18fd..567466f93cd5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c @@ -590,6 +590,7 @@ nv50_disp_intr_unk20_0(struct nv50_disp *disp, int head) .execute = 1, }; + nvkm_notify_put(&outpdp->irq); nvbios_exec(&init); atomic_set(&outpdp->lt.done, 0); } @@ -779,7 +780,7 @@ nv50_disp_intr_unk20_2(struct nv50_disp *disp, int head) break; } - if (nvkm_output_dp_train(outp, datarate / soff, true)) + if (nvkm_output_dp_train(outp, datarate / soff)) OUTP_ERR(outp, "link not trained before attach"); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c index 3b7a9e7a1ea8..de36f73b14dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.c @@ -31,7 +31,7 @@ #include <nvif/event.h> int -nvkm_output_dp_train(struct nvkm_output *base, u32 datarate, bool wait) +nvkm_output_dp_train(struct nvkm_output *base, u32 datarate) { struct nvkm_output_dp *outp = nvkm_output_dp(base); bool retrain = true; @@ -39,6 +39,8 @@ nvkm_output_dp_train(struct nvkm_output *base, u32 datarate, bool wait) u32 linkrate; int ret, i; + mutex_lock(&outp->mutex); + /* check that the link is trained at a high enough rate */ ret = nvkm_rdaux(outp->aux, DPCD_LC00_LINK_BW_SET, link, 2); if (ret) { @@ -88,19 +90,10 @@ done: outp->dpcd[DPCD_RC02] = outp->base.info.dpconf.link_nr; } - atomic_set(&outp->lt.done, 0); - schedule_work(&outp->lt.work); - } else { - nvkm_notify_get(&outp->irq); - } - - if (wait) { - if (!wait_event_timeout(outp->lt.wait, - atomic_read(&outp->lt.done), - msecs_to_jiffies(2000))) - ret = -ETIMEDOUT; + nvkm_dp_train(outp); } + mutex_unlock(&outp->mutex); return ret; } @@ -118,7 +111,7 @@ nvkm_output_dp_enable(struct nvkm_output_dp *outp, bool enable) if (!nvkm_rdaux(aux, DPCD_RC00_DPCD_REV, outp->dpcd, sizeof(outp->dpcd))) { - nvkm_output_dp_train(&outp->base, 0, true); + nvkm_output_dp_train(&outp->base, 0); return; } } @@ -165,10 +158,10 @@ nvkm_output_dp_irq(struct nvkm_notify *notify) }; OUTP_DBG(&outp->base, "IRQ: %d", line->mask); - nvkm_output_dp_train(&outp->base, 0, true); + nvkm_output_dp_train(&outp->base, 0); nvkm_event_send(&disp->hpd, rep.mask, conn->index, &rep, sizeof(rep)); - return NVKM_NOTIFY_DROP; + return NVKM_NOTIFY_KEEP; } static void @@ -177,7 +170,6 @@ nvkm_output_dp_fini(struct nvkm_output *base) struct nvkm_output_dp *outp = nvkm_output_dp(base); nvkm_notify_put(&outp->hpd); nvkm_notify_put(&outp->irq); - flush_work(&outp->lt.work); nvkm_output_dp_enable(outp, false); } @@ -187,6 +179,7 @@ nvkm_output_dp_init(struct nvkm_output *base) struct nvkm_output_dp *outp = nvkm_output_dp(base); nvkm_notify_put(&outp->base.conn->hpd); nvkm_output_dp_enable(outp, true); + nvkm_notify_get(&outp->irq); nvkm_notify_get(&outp->hpd); } @@ -238,11 +231,6 @@ nvkm_output_dp_ctor(const struct nvkm_output_dp_func *func, OUTP_DBG(&outp->base, "bios dp %02x %02x %02x %02x", outp->version, hdr, cnt, len); - /* link training */ - INIT_WORK(&outp->lt.work, nvkm_dp_train); - init_waitqueue_head(&outp->lt.wait); - atomic_set(&outp->lt.done, 0); - /* link maintenance */ ret = nvkm_notify_init(NULL, &i2c->event, nvkm_output_dp_irq, true, &(struct nvkm_i2c_ntfy_req) { @@ -257,6 +245,9 @@ nvkm_output_dp_ctor(const struct nvkm_output_dp_func *func, return ret; } + mutex_init(&outp->mutex); + atomic_set(&outp->lt.done, 0); + /* hotplug detect, replaces gpio-based mechanism with aux events */ ret = nvkm_notify_init(NULL, &i2c->event, nvkm_output_dp_hpd, true, &(struct nvkm_i2c_ntfy_req) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h index 4e983f6d7032..3c83a561cd88 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outpdp.h @@ -29,10 +29,10 @@ struct nvkm_output_dp { bool present; u8 dpcd[16]; + struct mutex mutex; struct { - struct work_struct work; - wait_queue_head_t wait; atomic_t done; + bool mst; } lt; }; @@ -41,9 +41,11 @@ struct nvkm_output_dp_func { int (*lnk_pwr)(struct nvkm_output_dp *, int nr); int (*lnk_ctl)(struct nvkm_output_dp *, int nr, int bw, bool ef); int (*drv_ctl)(struct nvkm_output_dp *, int ln, int vs, int pe, int pc); + void (*vcpi)(struct nvkm_output_dp *, int head, u8 start_slot, + u8 num_slots, u16 pbn, u16 aligned_pbn); }; -int nvkm_output_dp_train(struct nvkm_output *, u32 rate, bool wait); +int nvkm_output_dp_train(struct nvkm_output *, u32 rate); int nvkm_output_dp_ctor(const struct nvkm_output_dp_func *, struct nvkm_disp *, int index, struct dcb_output *, struct nvkm_i2c_aux *, @@ -63,6 +65,7 @@ int gf119_sor_dp_new(struct nvkm_disp *, int, struct dcb_output *, struct nvkm_output **); int gf119_sor_dp_lnk_ctl(struct nvkm_output_dp *, int, int, bool); int gf119_sor_dp_drv_ctl(struct nvkm_output_dp *, int, int, int, int); +void gf119_sor_dp_vcpi(struct nvkm_output_dp *, int, u8, u8, u16, u16); int gm107_sor_dp_new(struct nvkm_disp *, int, struct dcb_output *, struct nvkm_output **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index 2f9cecd81d04..c1158b22a721 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -66,7 +66,7 @@ nv50_disp_root_scanoutpos(NV50_DISP_MTHD_V0) return 0; } -int +static int nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) { union { @@ -173,13 +173,56 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) return 0; } else if (args->v0.state != 0) { - nvkm_output_dp_train(&outpdp->base, 0, true); + nvkm_output_dp_train(&outpdp->base, 0); return 0; } } else return ret; } break; + case NV50_DISP_MTHD_V1_SOR_DP_MST_LINK: { + struct nvkm_output_dp *outpdp = nvkm_output_dp(outp); + union { + struct nv50_disp_sor_dp_mst_link_v0 v0; + } *args = data; + int ret = -ENOSYS; + nvif_ioctl(object, "disp sor dp mst link size %d\n", size); + if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { + nvif_ioctl(object, "disp sor dp mst link vers %d state %d\n", + args->v0.version, args->v0.state); + if (outpdp->lt.mst != !!args->v0.state) { + outpdp->lt.mst = !!args->v0.state; + atomic_set(&outpdp->lt.done, 0); + nvkm_output_dp_train(&outpdp->base, 0); + } + return 0; + } else + return ret; + } + break; + case NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI: { + struct nvkm_output_dp *outpdp = nvkm_output_dp(outp); + union { + struct nv50_disp_sor_dp_mst_vcpi_v0 v0; + } *args = data; + int ret = -ENOSYS; + nvif_ioctl(object, "disp sor dp mst vcpi size %d\n", size); + if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { + nvif_ioctl(object, "disp sor dp mst vcpi vers %d " + "slot %02x/%02x pbn %04x/%04x\n", + args->v0.version, args->v0.start_slot, + args->v0.num_slots, args->v0.pbn, + args->v0.aligned_pbn); + if (!outpdp->func->vcpi) + return -ENODEV; + outpdp->func->vcpi(outpdp, head, args->v0.start_slot, + args->v0.num_slots, args->v0.pbn, + args->v0.aligned_pbn); + return 0; + } else + return ret; + } + break; case NV50_DISP_MTHD_V1_PIOR_PWR: if (!func->pior.power) return -ENODEV; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c index 49bd5da194e1..6ffdaa65aa77 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgf119.c @@ -56,11 +56,13 @@ gf119_sor_dp_lnk_ctl(struct nvkm_output_dp *outp, int nr, int bw, bool ef) clksor |= bw << 18; dpctrl |= ((1 << nr) - 1) << 16; + if (outp->lt.mst) + dpctrl |= 0x40000000; if (ef) dpctrl |= 0x00004000; nvkm_mask(device, 0x612300 + soff, 0x007c0000, clksor); - nvkm_mask(device, 0x61c10c + loff, 0x001f4000, dpctrl); + nvkm_mask(device, 0x61c10c + loff, 0x401f4000, dpctrl); return 0; } @@ -101,12 +103,24 @@ gf119_sor_dp_drv_ctl(struct nvkm_output_dp *outp, return 0; } +void +gf119_sor_dp_vcpi(struct nvkm_output_dp *outp, int head, u8 slot, + u8 slot_nr, u16 pbn, u16 aligned) +{ + struct nvkm_device *device = outp->base.disp->engine.subdev.device; + const u32 hoff = head * 0x800; + + nvkm_mask(device, 0x616588 + hoff, 0x00003f3f, (slot_nr << 8) | slot); + nvkm_mask(device, 0x61658c + hoff, 0xffffffff, (aligned << 16) | pbn); +} + static const struct nvkm_output_dp_func gf119_sor_dp_func = { .pattern = gf119_sor_dp_pattern, .lnk_pwr = g94_sor_dp_lnk_pwr, .lnk_ctl = gf119_sor_dp_lnk_ctl, .drv_ctl = gf119_sor_dp_drv_ctl, + .vcpi = gf119_sor_dp_vcpi, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c index 37790b2617c5..4cf8ad4d18ab 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm107.c @@ -43,6 +43,7 @@ gm107_sor_dp_func = { .lnk_pwr = g94_sor_dp_lnk_pwr, .lnk_ctl = gf119_sor_dp_lnk_ctl, .drv_ctl = gf119_sor_dp_drv_ctl, + .vcpi = gf119_sor_dp_vcpi, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c index c44fa7ea672a..81b788fa61be 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/sorgm200.c @@ -120,6 +120,7 @@ gm200_sor_dp_func = { .lnk_pwr = gm200_sor_dp_lnk_pwr, .lnk_ctl = gf119_sor_dp_lnk_ctl, .drv_ctl = gm200_sor_dp_drv_ctl, + .vcpi = gf119_sor_dp_vcpi, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index aeb3387a3fb0..15a992b3580a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -129,7 +129,7 @@ g84_fifo_chan_engine_fini(struct nvkm_fifo_chan *base, } -int +static int g84_fifo_chan_engine_init(struct nvkm_fifo_chan *base, struct nvkm_engine *engine) { @@ -170,7 +170,7 @@ g84_fifo_chan_engine_ctor(struct nvkm_fifo_chan *base, return nvkm_object_bind(object, NULL, 0, &chan->engn[engn]); } -int +static int g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, struct nvkm_object *object) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index cbc67f262322..12d964260a29 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -60,6 +60,7 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, struct nvkm_gpuobj *inst = chan->base.inst; int ret = 0; + mutex_lock(&subdev->mutex); nvkm_wr32(device, 0x002634, chan->base.chid); if (nvkm_msec(device, 2000, if (nvkm_rd32(device, 0x002634) == chan->base.chid) @@ -67,10 +68,12 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, ) < 0) { nvkm_error(subdev, "channel %d [%s] kick timeout\n", chan->base.chid, chan->base.object.client->name); - ret = -EBUSY; - if (suspend) - return ret; + ret = -ETIMEDOUT; } + mutex_unlock(&subdev->mutex); + + if (ret && suspend) + return ret; if (offset) { nvkm_kmap(inst); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index ed4351032ed6..a2df4f3e7763 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -40,7 +40,9 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan) struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; struct nvkm_client *client = chan->base.object.client; + int ret = 0; + mutex_lock(&subdev->mutex); nvkm_wr32(device, 0x002634, chan->base.chid); if (nvkm_msec(device, 2000, if (!(nvkm_rd32(device, 0x002634) & 0x00100000)) @@ -48,10 +50,10 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan) ) < 0) { nvkm_error(subdev, "channel %d [%s] kick timeout\n", chan->base.chid, client->name); - return -EBUSY; + ret = -ETIMEDOUT; } - - return 0; + mutex_unlock(&subdev->mutex); + return ret; } static u32 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c index c925ade5880e..74a64e3fd59a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c @@ -218,7 +218,7 @@ gf117_grctx_generate_attrib(struct gf100_grctx *info) } } -void +static void gf117_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) { struct nvkm_device *device = gr->base.engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c index 6d3c5011e18c..4c4b5ab6e46d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c @@ -933,7 +933,7 @@ gm107_grctx_generate_attrib(struct gf100_grctx *info) } } -void +static void gm107_grctx_generate_tpcid(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c index 1e13278cf306..c8bb9191f9a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxnv50.c @@ -106,6 +106,7 @@ #define CP_SEEK_2 0x00c800ff #include "ctxnv40.h" +#include "nv50.h" #include <subdev/fb.h> diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h index 8cb240b65ec2..12a703fe355d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf100.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gf100_grgpc_data[] = { +static uint32_t gf100_grgpc_data[] = { /* 0x0000: gpc_mmio_list_head */ 0x00000064, /* 0x0004: gpc_mmio_list_tail */ @@ -36,7 +36,7 @@ uint32_t gf100_grgpc_data[] = { 0x00000000, }; -uint32_t gf100_grgpc_code[] = { +static uint32_t gf100_grgpc_code[] = { 0x03a10ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h index 550d6ba0933b..ffbfc51200f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgf117.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gf117_grgpc_data[] = { +static uint32_t gf117_grgpc_data[] = { /* 0x0000: gpc_mmio_list_head */ 0x0000006c, /* 0x0004: gpc_mmio_list_tail */ @@ -40,7 +40,7 @@ uint32_t gf117_grgpc_data[] = { 0x00000000, }; -uint32_t gf117_grgpc_code[] = { +static uint32_t gf117_grgpc_code[] = { 0x03a10ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h index 271b59d365e5..357f662de571 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk104.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gk104_grgpc_data[] = { +static uint32_t gk104_grgpc_data[] = { /* 0x0000: gpc_mmio_list_head */ 0x0000006c, /* 0x0004: gpc_mmio_list_tail */ @@ -40,7 +40,7 @@ uint32_t gk104_grgpc_data[] = { 0x00000000, }; -uint32_t gk104_grgpc_code[] = { +static uint32_t gk104_grgpc_code[] = { 0x03a10ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h index 73b4a32c5d29..4ffc8212a85c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk110.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gk110_grgpc_data[] = { +static uint32_t gk110_grgpc_data[] = { /* 0x0000: gpc_mmio_list_head */ 0x0000006c, /* 0x0004: gpc_mmio_list_tail */ @@ -40,7 +40,7 @@ uint32_t gk110_grgpc_data[] = { 0x00000000, }; -uint32_t gk110_grgpc_code[] = { +static uint32_t gk110_grgpc_code[] = { 0x03a10ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h index 018169818317..09196206c9bc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgk208.fuc5.h @@ -1,4 +1,4 @@ -uint32_t gk208_grgpc_data[] = { +static uint32_t gk208_grgpc_data[] = { /* 0x0000: gpc_mmio_list_head */ 0x0000006c, /* 0x0004: gpc_mmio_list_tail */ @@ -40,7 +40,7 @@ uint32_t gk208_grgpc_data[] = { 0x00000000, }; -uint32_t gk208_grgpc_code[] = { +static uint32_t gk208_grgpc_code[] = { 0x03140ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h index eca007f03fa9..6d7d004363d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/gpcgm107.fuc5.h @@ -1,4 +1,4 @@ -uint32_t gm107_grgpc_data[] = { +static uint32_t gm107_grgpc_data[] = { /* 0x0000: gpc_mmio_list_head */ 0x0000006c, /* 0x0004: gpc_mmio_list_tail */ @@ -40,7 +40,7 @@ uint32_t gm107_grgpc_data[] = { 0x00000000, }; -uint32_t gm107_grgpc_code[] = { +static uint32_t gm107_grgpc_code[] = { 0x03410ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h index 8015b40a61d6..7538404b8b13 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf100.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gf100_grhub_data[] = { +static uint32_t gf100_grhub_data[] = { /* 0x0000: hub_mmio_list_head */ 0x00000300, /* 0x0004: hub_mmio_list_tail */ @@ -205,7 +205,7 @@ uint32_t gf100_grhub_data[] = { 0x0417e91c, }; -uint32_t gf100_grhub_code[] = { +static uint32_t gf100_grhub_code[] = { 0x039b0ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h index 2af90ec6852a..ce000a47ec6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgf117.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gf117_grhub_data[] = { +static uint32_t gf117_grhub_data[] = { /* 0x0000: hub_mmio_list_head */ 0x00000300, /* 0x0004: hub_mmio_list_tail */ @@ -205,7 +205,7 @@ uint32_t gf117_grhub_data[] = { 0x0417e91c, }; -uint32_t gf117_grhub_code[] = { +static uint32_t gf117_grhub_code[] = { 0x039b0ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h index e8b8c1c94700..1f26cb6a233c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk104.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gk104_grhub_data[] = { +static uint32_t gk104_grhub_data[] = { /* 0x0000: hub_mmio_list_head */ 0x00000300, /* 0x0004: hub_mmio_list_tail */ @@ -205,7 +205,7 @@ uint32_t gk104_grhub_data[] = { 0x0417e91c, }; -uint32_t gk104_grhub_code[] = { +static uint32_t gk104_grhub_code[] = { 0x039b0ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h index f4ed2fb6f714..70436d93efe3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk110.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gk110_grhub_data[] = { +static uint32_t gk110_grhub_data[] = { /* 0x0000: hub_mmio_list_head */ 0x00000300, /* 0x0004: hub_mmio_list_tail */ @@ -205,7 +205,7 @@ uint32_t gk110_grhub_data[] = { 0x0417e91c, }; -uint32_t gk110_grhub_code[] = { +static uint32_t gk110_grhub_code[] = { 0x039b0ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h index ed488973c117..e0933a07426a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgk208.fuc5.h @@ -1,4 +1,4 @@ -uint32_t gk208_grhub_data[] = { +static uint32_t gk208_grhub_data[] = { /* 0x0000: hub_mmio_list_head */ 0x00000300, /* 0x0004: hub_mmio_list_tail */ @@ -205,7 +205,7 @@ uint32_t gk208_grhub_data[] = { 0x0417e91c, }; -uint32_t gk208_grhub_code[] = { +static uint32_t gk208_grhub_code[] = { 0x030e0ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h index 5c9051839557..9b432823bcbe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/fuc/hubgm107.fuc5.h @@ -1,4 +1,4 @@ -uint32_t gm107_grhub_data[] = { +static uint32_t gm107_grhub_data[] = { /* 0x0000: hub_mmio_list_head */ 0x00000300, /* 0x0004: hub_mmio_list_tail */ @@ -205,7 +205,7 @@ uint32_t gm107_grhub_data[] = { 0x0417e91c, }; -uint32_t gm107_grhub_code[] = { +static uint32_t gm107_grhub_code[] = { 0x030e0ef5, /* 0x0004: queue_put */ 0x9800d898, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 157919c788e6..60a1b5c8214b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1384,7 +1384,7 @@ gf100_gr_intr(struct nvkm_gr *base) nvkm_fifo_chan_put(device->fifo, flags, &chan); } -void +static void gf100_gr_init_fw(struct gf100_gr *gr, u32 fuc_base, struct gf100_gr_fuc *code, struct gf100_gr_fuc *data) { @@ -1701,7 +1701,7 @@ gf100_gr_oneinit(struct nvkm_gr *base) return 0; } -int +static int gf100_gr_init_(struct nvkm_gr *base) { struct gf100_gr *gr = gf100_gr(base); @@ -1756,6 +1756,50 @@ gf100_gr_ = { }; int +gf100_gr_ctor_fw_legacy(struct gf100_gr *gr, const char *fwname, + struct gf100_gr_fuc *fuc, int ret) +{ + struct nvkm_subdev *subdev = &gr->base.engine.subdev; + struct nvkm_device *device = subdev->device; + const struct firmware *fw; + char f[32]; + + /* see if this firmware has a legacy path */ + if (!strcmp(fwname, "fecs_inst")) + fwname = "fuc409c"; + else if (!strcmp(fwname, "fecs_data")) + fwname = "fuc409d"; + else if (!strcmp(fwname, "gpccs_inst")) + fwname = "fuc41ac"; + else if (!strcmp(fwname, "gpccs_data")) + fwname = "fuc41ad"; + else { + /* nope, let's just return the error we got */ + nvkm_error(subdev, "failed to load %s\n", fwname); + return ret; + } + + /* yes, try to load from the legacy path */ + nvkm_debug(subdev, "%s: falling back to legacy path\n", fwname); + + snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname); + ret = request_firmware(&fw, f, device->dev); + if (ret) { + snprintf(f, sizeof(f), "nouveau/%s", fwname); + ret = request_firmware(&fw, f, device->dev); + if (ret) { + nvkm_error(subdev, "failed to load %s\n", fwname); + return ret; + } + } + + fuc->size = fw->size; + fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL); + release_firmware(fw); + return (fuc->data != NULL) ? 0 : -ENOMEM; +} + +int gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname, struct gf100_gr_fuc *fuc) { @@ -1765,10 +1809,8 @@ gf100_gr_ctor_fw(struct gf100_gr *gr, const char *fwname, int ret; ret = nvkm_firmware_get(device, fwname, &fw); - if (ret) { - nvkm_error(subdev, "failed to load %s\n", fwname); - return ret; - } + if (ret) + return gf100_gr_ctor_fw_legacy(gr, fwname, fuc, ret); fuc->size = fw->size; fuc->data = kmemdup(fw->data, fuc->size, GFP_KERNEL); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c index 70335f65c51e..0124e468086e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c @@ -102,7 +102,7 @@ gf117_gr_pack_mmio[] = { #include "fuc/hubgf117.fuc3.h" -struct gf100_gr_ucode +static struct gf100_gr_ucode gf117_gr_fecs_ucode = { .code.data = gf117_grhub_code, .code.size = sizeof(gf117_grhub_code), @@ -112,7 +112,7 @@ gf117_gr_fecs_ucode = { #include "fuc/gpcgf117.fuc3.h" -struct gf100_gr_ucode +static struct gf100_gr_ucode gf117_gr_gpccs_ucode = { .code.data = gf117_grgpc_code, .code.size = sizeof(gf117_grgpc_code), diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c index 45f965f608a7..2c67fac576d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c @@ -308,7 +308,7 @@ gm107_gr_init_bios(struct gf100_gr *gr) } } -int +static int gm107_gr_init(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c index 8616636ad7b4..dde89a4a0f5b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/base.c @@ -71,7 +71,7 @@ nvkm_perfdom_find(struct nvkm_pm *pm, int di) return NULL; } -struct nvkm_perfsig * +static struct nvkm_perfsig * nvkm_perfsig_find(struct nvkm_pm *pm, u8 di, u8 si, struct nvkm_perfdom **pdom) { struct nvkm_perfdom *dom = *pdom; @@ -699,7 +699,7 @@ nvkm_pm_oclass_get(struct nvkm_oclass *oclass, int index, return 1; } -int +static int nvkm_perfsrc_new(struct nvkm_pm *pm, struct nvkm_perfsig *sig, const struct nvkm_specsrc *spec) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c index d2901e9a7808..fe2532ee4145 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/pm/gf100.c @@ -102,7 +102,7 @@ gf100_pm_gpc[] = { {} }; -const struct nvkm_specdom +static const struct nvkm_specdom gf100_pm_part[] = { { 0xe0, (const struct nvkm_specsig[]) { { 0x0f, "part00_pbfb_00", gf100_pbfb_sources }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h index eca62221f299..4b57f8814560 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec/fuc/g98.fuc0s.h @@ -1,4 +1,4 @@ -uint32_t g98_sec_data[] = { +static uint32_t g98_sec_data[] = { /* 0x0000: ctx_dma */ /* 0x0000: ctx_dma_query */ 0x00000000, @@ -150,7 +150,7 @@ uint32_t g98_sec_data[] = { 0x00000000, }; -uint32_t g98_sec_code[] = { +static uint32_t g98_sec_code[] = { 0x17f004bd, 0x0010fe35, 0xf10004fe, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c index 80fed7e78dcb..e2905815049b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/fan.c @@ -25,7 +25,7 @@ #include <subdev/bios/bit.h> #include <subdev/bios/fan.h> -u16 +static u16 nvbios_fan_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len) { struct bit_entry bit_P; @@ -52,7 +52,7 @@ nvbios_fan_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len) return 0x0000; } -u16 +static u16 nvbios_fan_entry(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr, u8 *cnt, u8 *len) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h index 212800ecdce9..7d1d3c6b4b72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/priv.h @@ -12,6 +12,7 @@ struct nvbios_source { bool rw; bool ignore_checksum; bool no_pcir; + bool require_checksum; }; int nvbios_extend(struct nvkm_bios *, u32 length); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c index b2557e87afdd..7deb81b6dbac 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadow.c @@ -86,9 +86,12 @@ shadow_image(struct nvkm_bios *bios, int idx, u32 offset, struct shadow *mthd) nvbios_checksum(&bios->data[image.base], image.size)) { nvkm_debug(subdev, "%08x: checksum failed\n", image.base); - if (mthd->func->rw) + if (!mthd->func->require_checksum) { + if (mthd->func->rw) + score += 1; score += 1; - score += 1; + } else + return 0; } else { score += 3; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c index 8fecb5ff22a0..06572f8ce914 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowacpi.c @@ -99,6 +99,7 @@ nvbios_acpi_fast = { .init = acpi_init, .read = acpi_read_fast, .rw = false, + .require_checksum = true, }; const struct nvbios_source diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c index 056702ef69aa..96e0941c8edd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gt215.c @@ -180,7 +180,7 @@ gt215_clk_read(struct nvkm_clk *base, enum nv_clk_src src) return 0; } -int +static int gt215_clk_info(struct nvkm_clk *base, int idx, u32 khz, struct gt215_clk_info *info) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild index edcc157e6ac8..ef47d57fcb87 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild @@ -24,6 +24,7 @@ nvkm-y += nvkm/subdev/fb/gk104.o nvkm-y += nvkm/subdev/fb/gk20a.o nvkm-y += nvkm/subdev/fb/gm107.o nvkm-y += nvkm/subdev/fb/gm200.o +nvkm-y += nvkm/subdev/fb/gm20b.o nvkm-y += nvkm/subdev/fb/gp100.o nvkm-y += nvkm/subdev/fb/gp104.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h index 449f431644b3..412eb89834e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h @@ -16,4 +16,8 @@ void gf100_fb_init(struct nvkm_fb *); void gf100_fb_intr(struct nvkm_fb *); void gp100_fb_init(struct nvkm_fb *); + +void gm200_fb_init_page(struct nvkm_fb *fb); +void gm200_fb_init(struct nvkm_fb *base); + #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c index f815fe2bbf08..5d34d6136616 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -20,27 +20,21 @@ * DEALINGS IN THE SOFTWARE. */ #include "priv.h" +#include "gf100.h" -#include <core/memory.h> - -static void -gk20a_fb_init(struct nvkm_fb *fb) -{ - struct nvkm_device *device = fb->subdev.device; - nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->mmu_wr) >> 8); - nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->mmu_rd) >> 8); -} - +/* GK20A's FB is similar to GF100's, but without the ability to allocate VRAM */ static const struct nvkm_fb_func gk20a_fb = { + .dtor = gf100_fb_dtor, .oneinit = gf100_fb_oneinit, - .init = gk20a_fb_init, + .init = gf100_fb_init, .init_page = gf100_fb_init_page, + .intr = gf100_fb_intr, .memtype_valid = gf100_fb_memtype_valid, }; int gk20a_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb) { - return nvkm_fb_new_(&gk20a_fb, device, index, pfb); + return gf100_fb_new_(&gk20a_fb, device, index, pfb); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c index 62f653240be3..fe5886013ac0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c @@ -44,7 +44,7 @@ gm200_fb_init_page(struct nvkm_fb *fb) } } -static void +void gm200_fb_init(struct nvkm_fb *base) { struct gf100_fb *fb = gf100_fb(base); diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c index 91315557e421..b87c233bcd6d 100644 --- a/drivers/gpu/drm/i915/i915_gem_dmabuf.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c @@ -1,5 +1,5 @@ /* - * Copyright 2016 Intel Corporation + * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -8,9 +8,8 @@ * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -19,27 +18,23 @@ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. - * */ +#include "priv.h" +#include "gf100.h" -#ifndef _I915_GEM_DMABUF_H_ -#define _I915_GEM_DMABUF_H_ - -#include <linux/dma-buf.h> +/* GM20B's FB is similar to GM200, but without the ability to allocate VRAM */ +static const struct nvkm_fb_func +gm20b_fb = { + .dtor = gf100_fb_dtor, + .oneinit = gf100_fb_oneinit, + .init = gm200_fb_init, + .init_page = gm200_fb_init_page, + .intr = gf100_fb_intr, + .memtype_valid = gf100_fb_memtype_valid, +}; -static inline struct reservation_object * -i915_gem_object_get_dmabuf_resv(struct drm_i915_gem_object *obj) +int +gm20b_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb) { - struct dma_buf *dma_buf; - - if (obj->base.dma_buf) - dma_buf = obj->base.dma_buf; - else if (obj->base.import_attach) - dma_buf = obj->base.import_attach->dmabuf; - else - return NULL; - - return dma_buf->resv; + return gf100_fb_new_(&gm20b_fb, device, index, pfb); } - -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c index 772425ca5a9e..093223d1df4f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgf100.c @@ -420,8 +420,6 @@ gf100_ram_tidy(struct nvkm_ram *base) ram_exec(&ram->fuc, false); } -extern const u8 gf100_pte_storage_type_map[256]; - void gf100_ram_put(struct nvkm_ram *ram, struct nvkm_mem **pmem) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c index d15ea886df27..f10664372161 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgt215.c @@ -95,7 +95,7 @@ struct gt215_ram { struct gt215_ltrain ltrain; }; -void +static void gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train) { int i, lo, hi; @@ -149,7 +149,7 @@ gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train) /* * Link training for (at least) DDR3 */ -int +static int gt215_link_train(struct gt215_ram *ram) { struct gt215_ltrain *train = &ram->ltrain; @@ -267,7 +267,7 @@ out: return ret; } -int +static int gt215_link_train_init(struct gt215_ram *ram) { static const u32 pattern[16] = { @@ -333,7 +333,7 @@ gt215_link_train_init(struct gt215_ram *ram) return 0; } -void +static void gt215_link_train_fini(struct gt215_ram *ram) { if (ram->ltrain.mem) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c index b9f1ffdfc602..4dcd8742f2da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr2.c @@ -23,6 +23,7 @@ * Ben Skeggs */ #include "priv.h" +#include "ram.h" struct ramxlat { int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c index 26900333b1d6..eca8a445eab3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/sddr3.c @@ -23,6 +23,7 @@ * Roy Spliet <rspliet@eclipso.eu> */ #include "priv.h" +#include "ram.h" struct ramxlat { int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c index 3f45afd17d5a..2ead515b8530 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/gk104.c @@ -37,7 +37,7 @@ gk104_gpio_intr_stat(struct nvkm_gpio *gpio, u32 *hi, u32 *lo) nvkm_wr32(device, 0x00dc80, intr1); } -void +static void gk104_gpio_intr_mask(struct nvkm_gpio *gpio, u32 type, u32 mask, u32 data) { struct nvkm_device *device = gpio->subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c index f0851d57df2f..01d5c5a56e2e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/aux.c @@ -74,7 +74,7 @@ nvkm_i2c_aux_i2c_func(struct i2c_adapter *adap) return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; } -const struct i2c_algorithm +static const struct i2c_algorithm nvkm_i2c_aux_i2c_algo = { .master_xfer = nvkm_i2c_aux_i2c_xfer, .functionality = nvkm_i2c_aux_i2c_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c index 954f5b76bfcf..b80236a4eeac 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxg94.c @@ -79,7 +79,7 @@ g94_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry, struct g94_i2c_aux *aux = g94_i2c_aux(obj); struct nvkm_device *device = aux->base.pad->i2c->subdev.device; const u32 base = aux->ch * 0x50; - u32 ctrl, stat, timeout, retries; + u32 ctrl, stat, timeout, retries = 0; u32 xbuf[4] = {}; int ret, i; @@ -111,7 +111,7 @@ g94_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry, nvkm_wr32(device, 0x00e4e0 + base, addr); /* (maybe) retry transaction a number of times on failure... */ - for (retries = 0; !ret && retries < 32; retries++) { + do { /* reset, and delay a while if this is a retry */ nvkm_wr32(device, 0x00e4e4 + base, 0x80000000 | ctrl); nvkm_wr32(device, 0x00e4e4 + base, 0x00000000 | ctrl); @@ -131,20 +131,20 @@ g94_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry, goto out; } } while (ctrl & 0x00010000); - ret = 1; + ret = 0; /* read status, and check if transaction completed ok */ stat = nvkm_mask(device, 0x00e4e8 + base, 0, 0); if ((stat & 0x000f0000) == 0x00080000 || (stat & 0x000f0000) == 0x00020000) - ret = retry ? 0 : 1; + ret = 1; if ((stat & 0x00000100)) ret = -ETIMEDOUT; if ((stat & 0x00000e00)) ret = -EIO; AUX_TRACE(&aux->base, "%02d %08x %08x", retries, ctrl, stat); - } + } while (ret && retry && retries++ < 32); if (type & 1) { for (i = 0; i < 16; i += 4) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c index 61d729b82c69..ed458c7f056b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/auxgm200.c @@ -79,7 +79,7 @@ gm200_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry, struct gm200_i2c_aux *aux = gm200_i2c_aux(obj); struct nvkm_device *device = aux->base.pad->i2c->subdev.device; const u32 base = aux->ch * 0x50; - u32 ctrl, stat, timeout, retries; + u32 ctrl, stat, timeout, retries = 0; u32 xbuf[4] = {}; int ret, i; @@ -111,7 +111,7 @@ gm200_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry, nvkm_wr32(device, 0x00d950 + base, addr); /* (maybe) retry transaction a number of times on failure... */ - for (retries = 0; !ret && retries < 32; retries++) { + do { /* reset, and delay a while if this is a retry */ nvkm_wr32(device, 0x00d954 + base, 0x80000000 | ctrl); nvkm_wr32(device, 0x00d954 + base, 0x00000000 | ctrl); @@ -131,20 +131,20 @@ gm200_i2c_aux_xfer(struct nvkm_i2c_aux *obj, bool retry, goto out; } } while (ctrl & 0x00010000); - ret = 1; + ret = 0; /* read status, and check if transaction completed ok */ stat = nvkm_mask(device, 0x00d958 + base, 0, 0); if ((stat & 0x000f0000) == 0x00080000 || (stat & 0x000f0000) == 0x00020000) - ret = retry ? 0 : 1; + ret = 1; if ((stat & 0x00000100)) ret = -ETIMEDOUT; if ((stat & 0x00000e00)) ret = -EIO; AUX_TRACE(&aux->base, "%02d %08x %08x", retries, ctrl, stat); - } + } while (ret && retry && retries++ < 32); if (type & 1) { for (i = 0; i < 16; i += 4) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c index 658355fc9354..f0af2a381eea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/iccsense/base.c @@ -288,7 +288,8 @@ nvkm_iccsense_init(struct nvkm_subdev *subdev) return 0; } -struct nvkm_subdev_func iccsense_func = { +static const struct nvkm_subdev_func +iccsense_func = { .oneinit = nvkm_iccsense_oneinit, .init = nvkm_iccsense_init, .dtor = nvkm_iccsense_dtor, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index 8ed8f65ff664..10c987a654ec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -104,7 +104,7 @@ nvkm_instobj_dtor(struct nvkm_memory *memory) return iobj; } -const struct nvkm_memory_func +static const struct nvkm_memory_func nvkm_instobj_func = { .dtor = nvkm_instobj_dtor, .target = nvkm_instobj_target, @@ -156,7 +156,7 @@ nvkm_instobj_wr32_slow(struct nvkm_memory *memory, u64 offset, u32 data) return nvkm_wo32(iobj->parent, offset, data); } -const struct nvkm_memory_func +static const struct nvkm_memory_func nvkm_instobj_func_slow = { .dtor = nvkm_instobj_dtor, .target = nvkm_instobj_target, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c index c3d66ef5dc12..430a61c3df44 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c @@ -34,7 +34,7 @@ g84_mc_reset[] = { {} }; -const struct nvkm_mc_map +static const struct nvkm_mc_map g84_mc_intr[] = { { 0x04000000, NVKM_ENGINE_DISP }, { 0x00020000, NVKM_ENGINE_VP }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c index 21b65ee254e4..e3e2f5e83815 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mxm/base.c @@ -250,6 +250,10 @@ nvkm_mxm_new_(struct nvkm_device *device, int index, struct nvkm_mxm **pmxm) } nvkm_info(&mxm->subdev, "BIOS version %d.%d\n", ver >> 4, ver & 0x0f); + nvkm_debug(&mxm->subdev, "module flags: %02x\n", + nvbios_rd08(bios, data + 0x01)); + nvkm_debug(&mxm->subdev, "config flags: %02x\n", + nvbios_rd08(bios, data + 0x02)); if (mxm_shadow(mxm, ver)) { nvkm_warn(&mxm->subdev, "failed to locate valid SIS\n"); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h index e2faccffee6f..0bcf0b307a61 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf100.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gf100_pmu_data[] = { +static uint32_t gf100_pmu_data[] = { /* 0x0000: proc_kern */ 0x52544e49, 0x00000000, @@ -916,7 +916,7 @@ uint32_t gf100_pmu_data[] = { 0x00000000, }; -uint32_t gf100_pmu_code[] = { +static uint32_t gf100_pmu_code[] = { 0x03920ef5, /* 0x0004: rd32 */ 0x07a007f1, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h index 2d5bdc539697..fe8905666c67 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gf119.fuc4.h @@ -1,4 +1,4 @@ -uint32_t gf119_pmu_data[] = { +static uint32_t gf119_pmu_data[] = { /* 0x0000: proc_kern */ 0x52544e49, 0x00000000, @@ -915,7 +915,7 @@ uint32_t gf119_pmu_data[] = { 0x00000000, }; -uint32_t gf119_pmu_code[] = { +static uint32_t gf119_pmu_code[] = { 0x03410ef5, /* 0x0004: rd32 */ 0x07a007f1, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h index 3c731ff12871..9cf4e6fc724e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gk208.fuc5.h @@ -1,4 +1,4 @@ -uint32_t gk208_pmu_data[] = { +static uint32_t gk208_pmu_data[] = { /* 0x0000: proc_kern */ 0x52544e49, 0x00000000, @@ -915,7 +915,7 @@ uint32_t gk208_pmu_data[] = { 0x00000000, }; -uint32_t gk208_pmu_code[] = { +static uint32_t gk208_pmu_code[] = { 0x02f90ef5, /* 0x0004: rd32 */ 0xf607a040, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h index e83341815ec6..5d692425b190 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/fuc/gt215.fuc3.h @@ -1,4 +1,4 @@ -uint32_t gt215_pmu_data[] = { +static uint32_t gt215_pmu_data[] = { /* 0x0000: proc_kern */ 0x52544e49, 0x00000000, @@ -916,7 +916,7 @@ uint32_t gt215_pmu_data[] = { 0x00000000, }; -uint32_t gt215_pmu_code[] = { +static uint32_t gt215_pmu_code[] = { 0x03920ef5, /* 0x0004: rd32 */ 0x07a007f1, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c index 8ba7fa4ca75b..dcf9eaf274aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c @@ -24,17 +24,8 @@ #include "priv.h" #include "fuc/gt215.fuc3.h" -static void -gt215_pmu_reset(struct nvkm_pmu *pmu) -{ - struct nvkm_device *device = pmu->subdev.device; - nvkm_mask(device, 0x022210, 0x00000001, 0x00000000); - nvkm_mask(device, 0x022210, 0x00000001, 0x00000001); -} - static const struct nvkm_pmu_func gt215_pmu = { - .reset = gt215_pmu_reset, .code.data = gt215_pmu_code, .code.size = sizeof(gt215_pmu_code), .data.data = gt215_pmu_data, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h index f38c88fae3d6..73b811ccc2d5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h @@ -8,8 +8,6 @@ int nvkm_pmu_new_(const struct nvkm_pmu_func *, struct nvkm_device *, int index, struct nvkm_pmu **); struct nvkm_pmu_func { - void (*reset)(struct nvkm_pmu *); - struct { u32 *data; u32 size; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c index f1e2dc914366..ec48e4ace37a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c @@ -1364,7 +1364,7 @@ gm200_secboot_init(struct nvkm_secboot *sb) return 0; } -int +static int gm200_secboot_fini(struct nvkm_secboot *sb, bool suspend) { struct gm200_secboot *gsb = gm200_secboot(sb); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c index b2c5d1166a13..1c744e029454 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk104.c @@ -35,7 +35,7 @@ struct gk104_volt { struct nvbios_volt bios; }; -int +static int gk104_volt_get(struct nvkm_volt *base) { struct nvbios_volt *bios = &gk104_volt(base)->bios; @@ -48,7 +48,7 @@ gk104_volt_get(struct nvkm_volt *base) return bios->base + bios->pwm_range * duty / div; } -int +static int gk104_volt_set(struct nvkm_volt *base, u32 uv) { struct nvbios_volt *bios = &gk104_volt(base)->bios; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c index 74db4d28930f..2925b9cae681 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c @@ -25,7 +25,7 @@ #include <core/tegra.h> -const struct cvb_coef gm20b_cvb_coef[] = { +static const struct cvb_coef gm20b_cvb_coef[] = { /* KHz, c0, c1, c2 */ /* 76800 */ { 1786666, -85625, 1632 }, /* 153600 */ { 1846729, -87525, 1632 }, @@ -58,7 +58,7 @@ static const struct cvb_coef gm20b_na_cvb_coef[] = { /* 998400 */ { 1316991, 8144, -940, 808, -21583, 226 }, }; -const u32 speedo_to_vmin[] = { +static const u32 speedo_to_vmin[] = { /* 0, 1, 2, 3, 4, */ 950000, 840000, 818750, 840000, 810000, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c index d2bac1d77819..443c031b966b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gpio.c @@ -25,6 +25,7 @@ #include <subdev/bios.h> #include <subdev/bios/gpio.h> #include <subdev/gpio.h> +#include "priv.h" static const u8 tags[] = { DCB_GPIO_VID0, DCB_GPIO_VID1, DCB_GPIO_VID2, DCB_GPIO_VID3, diff --git a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c index 3485d1ecd655..aaa8a58390f1 100644 --- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c +++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c @@ -24,23 +24,24 @@ struct panel_drv_data { struct device *dev; - struct omap_video_timings timings; + struct videomode vm; bool invert_polarity; }; -static const struct omap_video_timings tvc_pal_timings = { - .x_res = 720, - .y_res = 574, +static const struct videomode tvc_pal_vm = { + .hactive = 720, + .vactive = 574, .pixelclock = 13500000, - .hsw = 64, - .hfp = 12, - .hbp = 68, - .vsw = 5, - .vfp = 5, - .vbp = 41, - - .interlace = true, + .hsync_len = 64, + .hfront_porch = 12, + .hback_porch = 68, + .vsync_len = 5, + .vfront_porch = 5, + .vback_porch = 41, + + .flags = DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW | + DISPLAY_FLAGS_VSYNC_LOW, }; static const struct of_device_id tvc_of_match[]; @@ -92,7 +93,7 @@ static int tvc_enable(struct omap_dss_device *dssdev) if (omapdss_device_is_enabled(dssdev)) return 0; - in->ops.atv->set_timings(in, &ddata->timings); + in->ops.atv->set_timings(in, &ddata->vm); if (!ddata->dev->of_node) { in->ops.atv->set_type(in, OMAP_DSS_VENC_TYPE_COMPOSITE); @@ -126,32 +127,32 @@ static void tvc_disable(struct omap_dss_device *dssdev) } static void tvc_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->timings = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.atv->set_timings(in, timings); + in->ops.atv->set_timings(in, vm); } static void tvc_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->timings; + *vm = ddata->vm; } static int tvc_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.atv->check_timings(in, timings); + return in->ops.atv->check_timings(in, vm); } static u32 tvc_get_wss(struct omap_dss_device *dssdev) @@ -253,14 +254,14 @@ static int tvc_probe(struct platform_device *pdev) return -ENODEV; } - ddata->timings = tvc_pal_timings; + ddata->vm = tvc_pal_vm; dssdev = &ddata->dssdev; dssdev->driver = &tvc_driver; dssdev->dev = &pdev->dev; dssdev->type = OMAP_DISPLAY_TYPE_VENC; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = tvc_pal_timings; + dssdev->panel.vm = tvc_pal_vm; r = omapdss_register_display(dssdev); if (r) { diff --git a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c index 684b7aeda411..d6875d9fcefa 100644 --- a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c +++ b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c @@ -19,32 +19,30 @@ #include "../dss/omapdss.h" -static const struct omap_video_timings dvic_default_timings = { - .x_res = 640, - .y_res = 480, +static const struct videomode dvic_default_vm = { + .hactive = 640, + .vactive = 480, .pixelclock = 23500000, - .hfp = 48, - .hsw = 32, - .hbp = 80, + .hfront_porch = 48, + .hsync_len = 32, + .hback_porch = 80, - .vfp = 3, - .vsw = 4, - .vbp = 7, + .vfront_porch = 3, + .vsync_len = 4, + .vback_porch = 7, - .vsync_level = OMAPDSS_SIG_ACTIVE_HIGH, - .hsync_level = OMAPDSS_SIG_ACTIVE_HIGH, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, + .flags = DISPLAY_FLAGS_HSYNC_HIGH | DISPLAY_FLAGS_VSYNC_HIGH | + DISPLAY_FLAGS_SYNC_NEGEDGE | DISPLAY_FLAGS_DE_HIGH | + DISPLAY_FLAGS_PIXDATA_POSEDGE, }; struct panel_drv_data { struct omap_dss_device dssdev; struct omap_dss_device *in; - struct omap_video_timings timings; + struct videomode vm; struct i2c_adapter *i2c_adapter; }; @@ -90,7 +88,7 @@ static int dvic_enable(struct omap_dss_device *dssdev) if (omapdss_device_is_enabled(dssdev)) return 0; - in->ops.dvi->set_timings(in, &ddata->timings); + in->ops.dvi->set_timings(in, &ddata->vm); r = in->ops.dvi->enable(in); if (r) @@ -115,32 +113,32 @@ static void dvic_disable(struct omap_dss_device *dssdev) } static void dvic_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->timings = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dvi->set_timings(in, timings); + in->ops.dvi->set_timings(in, vm); } static void dvic_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->timings; + *vm = ddata->vm; } static int dvic_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dvi->check_timings(in, timings); + return in->ops.dvi->check_timings(in, vm); } static int dvic_ddc_read(struct i2c_adapter *adapter, @@ -287,14 +285,14 @@ static int dvic_probe(struct platform_device *pdev) if (r) return r; - ddata->timings = dvic_default_timings; + ddata->vm = dvic_default_vm; dssdev = &ddata->dssdev; dssdev->driver = &dvic_driver; dssdev->dev = &pdev->dev; dssdev->type = OMAP_DISPLAY_TYPE_DVI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = dvic_default_timings; + dssdev->panel.vm = dvic_default_vm; r = omapdss_register_display(dssdev); if (r) { diff --git a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c index 7bdf83af9797..1ef130641bae 100644 --- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c +++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c @@ -21,21 +21,18 @@ #include "../dss/omapdss.h" -static const struct omap_video_timings hdmic_default_timings = { - .x_res = 640, - .y_res = 480, +static const struct videomode hdmic_default_vm = { + .hactive = 640, + .vactive = 480, .pixelclock = 25175000, - .hsw = 96, - .hfp = 16, - .hbp = 48, - .vsw = 2, - .vfp = 11, - .vbp = 31, - - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - - .interlace = false, + .hsync_len = 96, + .hfront_porch = 16, + .hback_porch = 48, + .vsync_len = 2, + .vfront_porch = 11, + .vback_porch = 31, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW, }; struct panel_drv_data { @@ -44,7 +41,7 @@ struct panel_drv_data { struct device *dev; - struct omap_video_timings timings; + struct videomode vm; int hpd_gpio; }; @@ -96,7 +93,7 @@ static int hdmic_enable(struct omap_dss_device *dssdev) if (omapdss_device_is_enabled(dssdev)) return 0; - in->ops.hdmi->set_timings(in, &ddata->timings); + in->ops.hdmi->set_timings(in, &ddata->vm); r = in->ops.hdmi->enable(in); if (r) @@ -123,32 +120,32 @@ static void hdmic_disable(struct omap_dss_device *dssdev) } static void hdmic_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->timings = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.hdmi->set_timings(in, timings); + in->ops.hdmi->set_timings(in, vm); } static void hdmic_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->timings; + *vm = ddata->vm; } static int hdmic_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.hdmi->check_timings(in, timings); + return in->ops.hdmi->check_timings(in, vm); } static int hdmic_read_edid(struct omap_dss_device *dssdev, @@ -259,14 +256,14 @@ static int hdmic_probe(struct platform_device *pdev) goto err_reg; } - ddata->timings = hdmic_default_timings; + ddata->vm = hdmic_default_vm; dssdev = &ddata->dssdev; dssdev->driver = &hdmic_driver; dssdev->dev = &pdev->dev; dssdev->type = OMAP_DISPLAY_TYPE_HDMI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = hdmic_default_timings; + dssdev->panel.vm = hdmic_default_vm; r = omapdss_register_display(dssdev); if (r) { diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c index fe4e7ec3bab0..f7a5731492d0 100644 --- a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c +++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c @@ -27,7 +27,7 @@ struct panel_drv_data { struct gpio_desc *enable_gpio; - struct omap_video_timings timings; + struct videomode vm; }; #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev) @@ -90,7 +90,7 @@ static int opa362_enable(struct omap_dss_device *dssdev) if (omapdss_device_is_enabled(dssdev)) return 0; - in->ops.atv->set_timings(in, &ddata->timings); + in->ops.atv->set_timings(in, &ddata->vm); r = in->ops.atv->enable(in); if (r) @@ -123,38 +123,38 @@ static void opa362_disable(struct omap_dss_device *dssdev) } static void opa362_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; dev_dbg(dssdev->dev, "set_timings\n"); - ddata->timings = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.atv->set_timings(in, timings); + in->ops.atv->set_timings(in, vm); } static void opa362_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); dev_dbg(dssdev->dev, "get_timings\n"); - *timings = ddata->timings; + *vm = ddata->vm; } static int opa362_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; dev_dbg(dssdev->dev, "check_timings\n"); - return in->ops.atv->check_timings(in, timings); + return in->ops.atv->check_timings(in, vm); } static void opa362_set_type(struct omap_dss_device *dssdev, diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c index d768217cefe0..13e32d02c884 100644 --- a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c +++ b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c @@ -24,7 +24,7 @@ struct panel_drv_data { int pd_gpio; int data_lines; - struct omap_video_timings timings; + struct videomode vm; }; #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev) @@ -81,7 +81,7 @@ static int tfp410_enable(struct omap_dss_device *dssdev) if (omapdss_device_is_enabled(dssdev)) return 0; - in->ops.dpi->set_timings(in, &ddata->timings); + in->ops.dpi->set_timings(in, &ddata->vm); if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); @@ -113,44 +113,43 @@ static void tfp410_disable(struct omap_dss_device *dssdev) dssdev->state = OMAP_DSS_DISPLAY_DISABLED; } -static void tfp410_fix_timings(struct omap_video_timings *timings) +static void tfp410_fix_timings(struct videomode *vm) { - timings->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; - timings->sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; - timings->de_level = OMAPDSS_SIG_ACTIVE_HIGH; + vm->flags |= DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE | + DISPLAY_FLAGS_SYNC_POSEDGE; } static void tfp410_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - tfp410_fix_timings(timings); + tfp410_fix_timings(vm); - ddata->timings = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void tfp410_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->timings; + *vm = ddata->vm; } static int tfp410_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - tfp410_fix_timings(timings); + tfp410_fix_timings(vm); - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static const struct omapdss_dvi_ops tfp410_dvi_ops = { diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c index 46855c8f5cbf..6d8f79b29af6 100644 --- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c +++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c @@ -26,7 +26,7 @@ struct panel_drv_data { struct gpio_desc *ls_oe_gpio; struct gpio_desc *hpd_gpio; - struct omap_video_timings timings; + struct videomode vm; }; #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev) @@ -80,7 +80,7 @@ static int tpd_enable(struct omap_dss_device *dssdev) if (dssdev->state == OMAP_DSS_DISPLAY_ACTIVE) return 0; - in->ops.hdmi->set_timings(in, &ddata->timings); + in->ops.hdmi->set_timings(in, &ddata->vm); r = in->ops.hdmi->enable(in); if (r) @@ -105,33 +105,33 @@ static void tpd_disable(struct omap_dss_device *dssdev) } static void tpd_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->timings = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.hdmi->set_timings(in, timings); + in->ops.hdmi->set_timings(in, vm); } static void tpd_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->timings; + *vm = ddata->vm; } static int tpd_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; int r; - r = in->ops.hdmi->check_timings(in, timings); + r = in->ops.hdmi->check_timings(in, vm); return r; } diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c index 7f16f985ab22..38003208d9ca 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c @@ -28,7 +28,7 @@ struct panel_drv_data { int data_lines; - struct omap_video_timings videomode; + struct videomode vm; /* used for non-DT boot, to be removed */ int backlight_gpio; @@ -80,7 +80,7 @@ static int panel_dpi_enable(struct omap_dss_device *dssdev) if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); - in->ops.dpi->set_timings(in, &ddata->videomode); + in->ops.dpi->set_timings(in, &ddata->vm); r = in->ops.dpi->enable(in); if (r) @@ -122,32 +122,32 @@ static void panel_dpi_disable(struct omap_dss_device *dssdev) } static void panel_dpi_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void panel_dpi_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int panel_dpi_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static struct omap_dss_driver panel_dpi_ops = { @@ -169,7 +169,6 @@ static int panel_dpi_probe_pdata(struct platform_device *pdev) const struct panel_dpi_platform_data *pdata; struct panel_drv_data *ddata = platform_get_drvdata(pdev); struct omap_dss_device *dssdev, *in; - struct videomode vm; int r; pdata = dev_get_platdata(&pdev->dev); @@ -185,8 +184,7 @@ static int panel_dpi_probe_pdata(struct platform_device *pdev) ddata->data_lines = pdata->data_lines; - videomode_from_timing(pdata->display_timing, &vm); - videomode_to_omap_video_timings(&vm, &ddata->videomode); + videomode_from_timing(pdata->display_timing, &ddata->vm); dssdev = &ddata->dssdev; dssdev->name = pdata->name; @@ -214,7 +212,6 @@ static int panel_dpi_probe_of(struct platform_device *pdev) struct omap_dss_device *in; int r; struct display_timing timing; - struct videomode vm; struct gpio_desc *gpio; gpio = devm_gpiod_get_optional(&pdev->dev, "enable", GPIOD_OUT_LOW); @@ -245,8 +242,7 @@ static int panel_dpi_probe_of(struct platform_device *pdev) return r; } - videomode_from_timing(&timing, &vm); - videomode_to_omap_video_timings(&vm, &ddata->videomode); + videomode_from_timing(&timing, &ddata->vm); in = omapdss_of_find_source_for_first_ep(node); if (IS_ERR(in)) { @@ -295,7 +291,7 @@ static int panel_dpi_probe(struct platform_device *pdev) dssdev->driver = &panel_dpi_ops; dssdev->type = OMAP_DISPLAY_TYPE_DPI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; dssdev->phy.dpi.data_lines = ddata->data_lines; r = omapdss_register_display(dssdev); diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c index b1f3b818edf4..dc026a843712 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c @@ -42,7 +42,7 @@ struct panel_drv_data { struct omap_dss_device dssdev; struct omap_dss_device *in; - struct omap_video_timings timings; + struct videomode vm; struct platform_device *pdev; @@ -382,8 +382,8 @@ static const struct backlight_ops dsicm_bl_ops = { static void dsicm_get_resolution(struct omap_dss_device *dssdev, u16 *xres, u16 *yres) { - *xres = dssdev->panel.timings.x_res; - *yres = dssdev->panel.timings.y_res; + *xres = dssdev->panel.vm.hactive; + *yres = dssdev->panel.vm.vactive; } static ssize_t dsicm_num_errors_show(struct device *dev, @@ -589,7 +589,7 @@ static int dsicm_power_on(struct panel_drv_data *ddata) struct omap_dss_dsi_config dsi_config = { .mode = OMAP_DSS_DSI_CMD_MODE, .pixel_format = OMAP_DSS_DSI_FMT_RGB888, - .timings = &ddata->timings, + .vm = &ddata->vm, .hs_clk_min = 150000000, .hs_clk_max = 300000000, .lp_clk_min = 7000000, @@ -892,8 +892,8 @@ static int dsicm_update(struct omap_dss_device *dssdev, /* XXX no need to send this every frame, but dsi break if not done */ r = dsicm_set_update_window(ddata, 0, 0, - dssdev->panel.timings.x_res, - dssdev->panel.timings.y_res); + dssdev->panel.vm.hactive, + dssdev->panel.vm.vactive); if (r) goto err; @@ -1023,9 +1023,8 @@ static int dsicm_memory_read(struct omap_dss_device *dssdev, goto err1; } - size = min(w * h * 3, - dssdev->panel.timings.x_res * - dssdev->panel.timings.y_res * 3); + size = min((u32)w * h * 3, + dssdev->panel.vm.hactive * dssdev->panel.vm.vactive * 3); in->ops.dsi->bus_lock(in); @@ -1186,14 +1185,14 @@ static int dsicm_probe(struct platform_device *pdev) if (r) return r; - ddata->timings.x_res = 864; - ddata->timings.y_res = 480; - ddata->timings.pixelclock = 864 * 480 * 60; + ddata->vm.hactive = 864; + ddata->vm.vactive = 480; + ddata->vm.pixelclock = 864 * 480 * 60; dssdev = &ddata->dssdev; dssdev->dev = dev; dssdev->driver = &dsicm_ops; - dssdev->panel.timings = ddata->timings; + dssdev->panel.vm = ddata->vm; dssdev->type = OMAP_DISPLAY_TYPE_DSI; dssdev->owner = THIS_MODULE; diff --git a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c index 6dfb96cea293..43d21edb51f5 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c @@ -19,25 +19,28 @@ #include "../dss/omapdss.h" -static struct omap_video_timings lb035q02_timings = { - .x_res = 320, - .y_res = 240, +static struct videomode lb035q02_vm = { + .hactive = 320, + .vactive = 240, .pixelclock = 6500000, - .hsw = 2, - .hfp = 20, - .hbp = 68, - - .vsw = 2, - .vfp = 4, - .vbp = 18, - - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, + .hsync_len = 2, + .hfront_porch = 20, + .hback_porch = 68, + + .vsync_len = 2, + .vfront_porch = 4, + .vback_porch = 18, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE | + DISPLAY_FLAGS_PIXDATA_POSEDGE, + /* + * Note: According to the panel documentation: + * DE is active LOW + * DATA needs to be driven on the FALLING edge + */ }; struct panel_drv_data { @@ -48,7 +51,7 @@ struct panel_drv_data { int data_lines; - struct omap_video_timings videomode; + struct videomode vm; struct gpio_desc *enable_gpio; }; @@ -158,7 +161,7 @@ static int lb035q02_enable(struct omap_dss_device *dssdev) if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); - in->ops.dpi->set_timings(in, &ddata->videomode); + in->ops.dpi->set_timings(in, &ddata->vm); r = in->ops.dpi->enable(in); if (r) @@ -189,32 +192,32 @@ static void lb035q02_disable(struct omap_dss_device *dssdev) } static void lb035q02_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void lb035q02_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int lb035q02_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static struct omap_dss_driver lb035q02_ops = { @@ -278,14 +281,14 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi) if (r) return r; - ddata->videomode = lb035q02_timings; + ddata->vm = lb035q02_vm; dssdev = &ddata->dssdev; dssdev->dev = &spi->dev; dssdev->driver = &lb035q02_ops; dssdev->type = OMAP_DISPLAY_TYPE_DPI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; dssdev->phy.dpi.data_lines = ddata->data_lines; r = omapdss_register_display(dssdev); diff --git a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c index 9f3d6f48f3e1..2de27ba01552 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c @@ -23,7 +23,7 @@ struct panel_drv_data { struct omap_dss_device dssdev; struct omap_dss_device *in; - struct omap_video_timings videomode; + struct videomode vm; int data_lines; @@ -65,22 +65,20 @@ static const struct { { 156, 0x00 }, { 157, 0x00 }, { 2, 0x00 }, }; -static const struct omap_video_timings nec_8048_panel_timings = { - .x_res = LCD_XRES, - .y_res = LCD_YRES, +static const struct videomode nec_8048_panel_vm = { + .hactive = LCD_XRES, + .vactive = LCD_YRES, .pixelclock = LCD_PIXEL_CLOCK, - .hfp = 6, - .hsw = 1, - .hbp = 4, - .vfp = 3, - .vsw = 1, - .vbp = 4, - - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, + .hfront_porch = 6, + .hsync_len = 1, + .hback_porch = 4, + .vfront_porch = 3, + .vsync_len = 1, + .vback_porch = 4, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE | + DISPLAY_FLAGS_PIXDATA_POSEDGE, }; #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev) @@ -157,7 +155,7 @@ static int nec_8048_enable(struct omap_dss_device *dssdev) if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); - in->ops.dpi->set_timings(in, &ddata->videomode); + in->ops.dpi->set_timings(in, &ddata->vm); r = in->ops.dpi->enable(in); if (r) @@ -188,32 +186,32 @@ static void nec_8048_disable(struct omap_dss_device *dssdev) } static void nec_8048_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void nec_8048_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int nec_8048_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static struct omap_dss_driver nec_8048_ops = { @@ -306,14 +304,14 @@ static int nec_8048_probe(struct spi_device *spi) goto err_gpio; } - ddata->videomode = nec_8048_panel_timings; + ddata->vm = nec_8048_panel_vm; dssdev = &ddata->dssdev; dssdev->dev = &spi->dev; dssdev->driver = &nec_8048_ops; dssdev->type = OMAP_DISPLAY_TYPE_DPI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; r = omapdss_register_display(dssdev); if (r) { diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c index 3d3efc561ea9..04fe235b7cac 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c @@ -26,7 +26,7 @@ struct panel_drv_data { int data_lines; - struct omap_video_timings videomode; + struct videomode vm; struct gpio_desc *resb_gpio; /* low = reset active min 20 us */ struct gpio_desc *ini_gpio; /* high = power on */ @@ -35,25 +35,27 @@ struct panel_drv_data { struct gpio_desc *ud_gpio; /* high = conventional vertical scanning */ }; -static const struct omap_video_timings sharp_ls_timings = { - .x_res = 480, - .y_res = 640, +static const struct videomode sharp_ls_vm = { + .hactive = 480, + .vactive = 640, .pixelclock = 19200000, - .hsw = 2, - .hfp = 1, - .hbp = 28, - - .vsw = 1, - .vfp = 1, - .vbp = 1, - - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, + .hsync_len = 2, + .hfront_porch = 1, + .hback_porch = 28, + + .vsync_len = 1, + .vfront_porch = 1, + .vback_porch = 1, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE | + DISPLAY_FLAGS_PIXDATA_POSEDGE, + /* + * Note: According to the panel documentation: + * DATA needs to be driven on the FALLING edge + */ }; #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev) @@ -99,7 +101,7 @@ static int sharp_ls_enable(struct omap_dss_device *dssdev) if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); - in->ops.dpi->set_timings(in, &ddata->videomode); + in->ops.dpi->set_timings(in, &ddata->vm); if (ddata->vcc) { r = regulator_enable(ddata->vcc); @@ -154,32 +156,32 @@ static void sharp_ls_disable(struct omap_dss_device *dssdev) } static void sharp_ls_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void sharp_ls_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int sharp_ls_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static struct omap_dss_driver sharp_ls_ops = { @@ -279,14 +281,14 @@ static int sharp_ls_probe(struct platform_device *pdev) if (r) return r; - ddata->videomode = sharp_ls_timings; + ddata->vm = sharp_ls_vm; dssdev = &ddata->dssdev; dssdev->dev = &pdev->dev; dssdev->driver = &sharp_ls_ops; dssdev->type = OMAP_DISPLAY_TYPE_DPI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; dssdev->phy.dpi.data_lines = ddata->data_lines; r = omapdss_register_display(dssdev); diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c index 3557a4c7dd7b..746cb8d9cba1 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c @@ -71,7 +71,7 @@ struct panel_drv_data { int reset_gpio; int datapairs; - struct omap_video_timings videomode; + struct videomode vm; char *name; int enabled; @@ -92,23 +92,20 @@ struct panel_drv_data { struct backlight_device *bl_dev; }; -static const struct omap_video_timings acx565akm_panel_timings = { - .x_res = 800, - .y_res = 480, +static const struct videomode acx565akm_panel_vm = { + .hactive = 800, + .vactive = 480, .pixelclock = 24000000, - .hfp = 28, - .hsw = 4, - .hbp = 24, - .vfp = 3, - .vsw = 3, - .vbp = 4, - - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, + .hfront_porch = 28, + .hsync_len = 4, + .hback_porch = 24, + .vfront_porch = 3, + .vsync_len = 3, + .vback_porch = 4, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_NEGEDGE | + DISPLAY_FLAGS_PIXDATA_POSEDGE, }; #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev) @@ -548,7 +545,7 @@ static int acx565akm_panel_power_on(struct omap_dss_device *dssdev) dev_dbg(&ddata->spi->dev, "%s\n", __func__); - in->ops.sdi->set_timings(in, &ddata->videomode); + in->ops.sdi->set_timings(in, &ddata->vm); if (ddata->datapairs > 0) in->ops.sdi->set_datapairs(in, ddata->datapairs); @@ -662,32 +659,32 @@ static void acx565akm_disable(struct omap_dss_device *dssdev) } static void acx565akm_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.sdi->set_timings(in, timings); + in->ops.sdi->set_timings(in, vm); } static void acx565akm_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int acx565akm_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.sdi->check_timings(in, timings); + return in->ops.sdi->check_timings(in, vm); } static struct omap_dss_driver acx565akm_ops = { @@ -845,14 +842,14 @@ static int acx565akm_probe(struct spi_device *spi) acx565akm_bl_update_status(bldev); - ddata->videomode = acx565akm_panel_timings; + ddata->vm = acx565akm_panel_vm; dssdev = &ddata->dssdev; dssdev->dev = &spi->dev; dssdev->driver = &acx565akm_ops; dssdev->type = OMAP_DISPLAY_TYPE_SDI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; r = omapdss_register_display(dssdev); if (r) { diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c index e859b3f893f7..f313dbfcbacb 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c @@ -37,28 +37,29 @@ struct panel_drv_data { int data_lines; - struct omap_video_timings videomode; + struct videomode vm; struct spi_device *spi_dev; }; -static struct omap_video_timings td028ttec1_panel_timings = { - .x_res = 480, - .y_res = 640, +static struct videomode td028ttec1_panel_vm = { + .hactive = 480, + .vactive = 640, .pixelclock = 22153000, - .hfp = 24, - .hsw = 8, - .hbp = 8, - .vfp = 4, - .vsw = 2, - .vbp = 2, - - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - - .data_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, + .hfront_porch = 24, + .hsync_len = 8, + .hback_porch = 8, + .vfront_porch = 4, + .vsync_len = 2, + .vback_porch = 2, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE | + DISPLAY_FLAGS_PIXDATA_NEGEDGE, + /* + * Note: According to the panel documentation: + * SYNC needs to be driven on the FALLING edge + */ }; #define JBT_COMMAND 0x000 @@ -208,7 +209,7 @@ static int td028ttec1_panel_enable(struct omap_dss_device *dssdev) if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); - in->ops.dpi->set_timings(in, &ddata->videomode); + in->ops.dpi->set_timings(in, &ddata->vm); r = in->ops.dpi->enable(in); if (r) @@ -325,32 +326,32 @@ static void td028ttec1_panel_disable(struct omap_dss_device *dssdev) } static void td028ttec1_panel_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void td028ttec1_panel_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int td028ttec1_panel_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static struct omap_dss_driver td028ttec1_ops = { @@ -414,14 +415,14 @@ static int td028ttec1_panel_probe(struct spi_device *spi) if (r) return r; - ddata->videomode = td028ttec1_panel_timings; + ddata->vm = td028ttec1_panel_vm; dssdev = &ddata->dssdev; dssdev->dev = &spi->dev; dssdev->driver = &td028ttec1_ops; dssdev->type = OMAP_DISPLAY_TYPE_DPI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; dssdev->phy.dpi.data_lines = ddata->data_lines; r = omapdss_register_display(dssdev); diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c index 66c6bbe6472b..0787dba44faa 100644 --- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c +++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c @@ -56,7 +56,7 @@ struct panel_drv_data { struct omap_dss_device dssdev; struct omap_dss_device *in; - struct omap_video_timings videomode; + struct videomode vm; int data_lines; @@ -72,25 +72,27 @@ struct panel_drv_data { u32 power_on_resume:1; }; -static const struct omap_video_timings tpo_td043_timings = { - .x_res = 800, - .y_res = 480, +static const struct videomode tpo_td043_vm = { + .hactive = 800, + .vactive = 480, .pixelclock = 36000000, - .hsw = 1, - .hfp = 68, - .hbp = 214, + .hsync_len = 1, + .hfront_porch = 68, + .hback_porch = 214, - .vsw = 1, - .vfp = 39, - .vbp = 34, + .vsync_len = 1, + .vfront_porch = 39, + .vback_porch = 34, - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE | + DISPLAY_FLAGS_PIXDATA_NEGEDGE, + /* + * Note: According to the panel documentation: + * SYNC needs to be driven on the FALLING edge + */ }; #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev) @@ -378,7 +380,7 @@ static int tpo_td043_enable(struct omap_dss_device *dssdev) if (ddata->data_lines) in->ops.dpi->set_data_lines(in, ddata->data_lines); - in->ops.dpi->set_timings(in, &ddata->videomode); + in->ops.dpi->set_timings(in, &ddata->vm); r = in->ops.dpi->enable(in); if (r) @@ -418,32 +420,32 @@ static void tpo_td043_disable(struct omap_dss_device *dssdev) } static void tpo_td043_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - ddata->videomode = *timings; - dssdev->panel.timings = *timings; + ddata->vm = *vm; + dssdev->panel.vm = *vm; - in->ops.dpi->set_timings(in, timings); + in->ops.dpi->set_timings(in, vm); } static void tpo_td043_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); - *timings = ddata->videomode; + *vm = ddata->vm; } static int tpo_td043_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct panel_drv_data *ddata = to_panel_data(dssdev); struct omap_dss_device *in = ddata->in; - return in->ops.dpi->check_timings(in, timings); + return in->ops.dpi->check_timings(in, vm); } static struct omap_dss_driver tpo_td043_ops = { @@ -546,14 +548,14 @@ static int tpo_td043_probe(struct spi_device *spi) goto err_sysfs; } - ddata->videomode = tpo_td043_timings; + ddata->vm = tpo_td043_vm; dssdev = &ddata->dssdev; dssdev->dev = &spi->dev; dssdev->driver = &tpo_td043_ops; dssdev->type = OMAP_DISPLAY_TYPE_DPI; dssdev->owner = THIS_MODULE; - dssdev->panel.timings = ddata->videomode; + dssdev->panel.vm = ddata->vm; r = omapdss_register_display(dssdev); if (r) { diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c index 535240fba671..c839f6456db2 100644 --- a/drivers/gpu/drm/omapdrm/dss/dispc.c +++ b/drivers/gpu/drm/omapdrm/dss/dispc.c @@ -75,7 +75,7 @@ struct dispc_features { unsigned long max_lcd_pclk; unsigned long max_tv_pclk; int (*calc_scaling) (unsigned long pclk, unsigned long lclk, - const struct omap_video_timings *mgr_timings, + const struct videomode *vm, u16 width, u16 height, u16 out_width, u16 out_height, enum omap_color_mode color_mode, bool *five_taps, int *x_predecim, int *y_predecim, int *decim_x, int *decim_y, @@ -1679,7 +1679,7 @@ static void dispc_ovl_set_scaling_uv(enum omap_plane plane, { int scale_x = out_width != orig_width; int scale_y = out_height != orig_height; - bool chroma_upscale = plane != OMAP_DSS_WB ? true : false; + bool chroma_upscale = plane != OMAP_DSS_WB; if (!dss_has_feature(FEAT_HANDLE_UV_SEPARATE)) return; @@ -2179,7 +2179,7 @@ static void calc_tiler_rotation_offset(u16 screen_width, u16 width, * undocumented horizontal position and timing related limitations. */ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk, - const struct omap_video_timings *t, u16 pos_x, + const struct videomode *vm, u16 pos_x, u16 width, u16 height, u16 out_width, u16 out_height, bool five_taps) { @@ -2189,14 +2189,16 @@ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk, u64 val, blank; int i; - nonactive = t->x_res + t->hfp + t->hsw + t->hbp - out_width; + nonactive = vm->hactive + vm->hfront_porch + vm->hsync_len + + vm->hback_porch - out_width; i = 0; if (out_height < height) i++; if (out_width < width) i++; - blank = div_u64((u64)(t->hbp + t->hsw + t->hfp) * lclk, pclk); + blank = div_u64((u64)(vm->hback_porch + vm->hsync_len + vm->hfront_porch) * + lclk, pclk); DSSDBG("blanking period + ppl = %llu (limit = %u)\n", blank, limits[i]); if (blank <= limits[i]) return -EINVAL; @@ -2231,7 +2233,7 @@ static int check_horiz_timing_omap3(unsigned long pclk, unsigned long lclk, } static unsigned long calc_core_clk_five_taps(unsigned long pclk, - const struct omap_video_timings *mgr_timings, u16 width, + const struct videomode *vm, u16 width, u16 height, u16 out_width, u16 out_height, enum omap_color_mode color_mode) { @@ -2242,7 +2244,7 @@ static unsigned long calc_core_clk_five_taps(unsigned long pclk, return (unsigned long) pclk; if (height > out_height) { - unsigned int ppl = mgr_timings->x_res; + unsigned int ppl = vm->hactive; tmp = (u64)pclk * height * out_width; do_div(tmp, 2 * out_height * ppl); @@ -2324,7 +2326,7 @@ static unsigned long calc_core_clk_44xx(unsigned long pclk, u16 width, } static int dispc_ovl_calc_scaling_24xx(unsigned long pclk, unsigned long lclk, - const struct omap_video_timings *mgr_timings, + const struct videomode *vm, u16 width, u16 height, u16 out_width, u16 out_height, enum omap_color_mode color_mode, bool *five_taps, int *x_predecim, int *y_predecim, int *decim_x, int *decim_y, @@ -2370,7 +2372,7 @@ static int dispc_ovl_calc_scaling_24xx(unsigned long pclk, unsigned long lclk, } static int dispc_ovl_calc_scaling_34xx(unsigned long pclk, unsigned long lclk, - const struct omap_video_timings *mgr_timings, + const struct videomode *vm, u16 width, u16 height, u16 out_width, u16 out_height, enum omap_color_mode color_mode, bool *five_taps, int *x_predecim, int *y_predecim, int *decim_x, int *decim_y, @@ -2392,7 +2394,7 @@ static int dispc_ovl_calc_scaling_34xx(unsigned long pclk, unsigned long lclk, *five_taps = false; again: if (*five_taps) - *core_clk = calc_core_clk_five_taps(pclk, mgr_timings, + *core_clk = calc_core_clk_five_taps(pclk, vm, in_width, in_height, out_width, out_height, color_mode); else @@ -2400,7 +2402,7 @@ again: in_height, out_width, out_height, mem_to_mem); - error = check_horiz_timing_omap3(pclk, lclk, mgr_timings, + error = check_horiz_timing_omap3(pclk, lclk, vm, pos_x, in_width, in_height, out_width, out_height, *five_taps); if (error && *five_taps) { @@ -2435,7 +2437,7 @@ again: return -EINVAL; } - if (check_horiz_timing_omap3(pclk, lclk, mgr_timings, pos_x, in_width, + if (check_horiz_timing_omap3(pclk, lclk, vm, pos_x, in_width, in_height, out_width, out_height, *five_taps)) { DSSERR("horizontal timing too tight\n"); return -EINVAL; @@ -2455,7 +2457,7 @@ again: } static int dispc_ovl_calc_scaling_44xx(unsigned long pclk, unsigned long lclk, - const struct omap_video_timings *mgr_timings, + const struct videomode *vm, u16 width, u16 height, u16 out_width, u16 out_height, enum omap_color_mode color_mode, bool *five_taps, int *x_predecim, int *y_predecim, int *decim_x, int *decim_y, @@ -2501,7 +2503,7 @@ static int dispc_ovl_calc_scaling_44xx(unsigned long pclk, unsigned long lclk, static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk, enum omap_overlay_caps caps, - const struct omap_video_timings *mgr_timings, + const struct videomode *vm, u16 width, u16 height, u16 out_width, u16 out_height, enum omap_color_mode color_mode, bool *five_taps, int *x_predecim, int *y_predecim, u16 pos_x, @@ -2515,7 +2517,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk, if (width == out_width && height == out_height) return 0; - if (!mem_to_mem && (pclk == 0 || mgr_timings->pixelclock == 0)) { + if (!mem_to_mem && (pclk == 0 || vm->pixelclock == 0)) { DSSERR("cannot calculate scaling settings: pclk is zero\n"); return -EINVAL; } @@ -2551,7 +2553,7 @@ static int dispc_ovl_calc_scaling(unsigned long pclk, unsigned long lclk, if (decim_y > *y_predecim || out_height > height * 8) return -EINVAL; - ret = dispc.feat->calc_scaling(pclk, lclk, mgr_timings, width, height, + ret = dispc.feat->calc_scaling(pclk, lclk, vm, width, height, out_width, out_height, color_mode, five_taps, x_predecim, y_predecim, &decim_x, &decim_y, pos_x, &core_clk, mem_to_mem); @@ -2591,7 +2593,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane, u16 out_width, u16 out_height, enum omap_color_mode color_mode, u8 rotation, bool mirror, u8 zorder, u8 pre_mult_alpha, u8 global_alpha, enum omap_dss_rotation_type rotation_type, - bool replication, const struct omap_video_timings *mgr_timings, + bool replication, const struct videomode *vm, bool mem_to_mem) { bool five_taps = true; @@ -2605,7 +2607,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane, u16 in_height = height; u16 in_width = width; int x_predecim = 1, y_predecim = 1; - bool ilace = mgr_timings->interlace; + bool ilace = !!(vm->flags & DISPLAY_FLAGS_INTERLACED); unsigned long pclk = dispc_plane_pclk_rate(plane); unsigned long lclk = dispc_plane_lclk_rate(plane); @@ -2647,7 +2649,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane, if (!dss_feat_color_mode_supported(plane, color_mode)) return -EINVAL; - r = dispc_ovl_calc_scaling(pclk, lclk, caps, mgr_timings, in_width, + r = dispc_ovl_calc_scaling(pclk, lclk, caps, vm, in_width, in_height, out_width, out_height, color_mode, &five_taps, &x_predecim, &y_predecim, pos_x, rotation_type, mem_to_mem); @@ -2784,7 +2786,7 @@ static int dispc_ovl_setup_common(enum omap_plane plane, } int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi, - bool replication, const struct omap_video_timings *mgr_timings, + bool replication, const struct videomode *vm, bool mem_to_mem) { int r; @@ -2803,14 +2805,14 @@ int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi, oi->screen_width, oi->pos_x, oi->pos_y, oi->width, oi->height, oi->out_width, oi->out_height, oi->color_mode, oi->rotation, oi->mirror, oi->zorder, oi->pre_mult_alpha, oi->global_alpha, - oi->rotation_type, replication, mgr_timings, mem_to_mem); + oi->rotation_type, replication, vm, mem_to_mem); return r; } EXPORT_SYMBOL(dispc_ovl_setup); int dispc_wb_setup(const struct omap_dss_writeback_info *wi, - bool mem_to_mem, const struct omap_video_timings *mgr_timings) + bool mem_to_mem, const struct videomode *vm) { int r; u32 l; @@ -2819,8 +2821,8 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi, const u8 zorder = 0, global_alpha = 0; const bool replication = false; bool truncation; - int in_width = mgr_timings->x_res; - int in_height = mgr_timings->y_res; + int in_width = vm->hactive; + int in_height = vm->vactive; enum omap_overlay_caps caps = OMAP_DSS_OVL_CAP_SCALE | OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA; @@ -2833,7 +2835,7 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi, wi->buf_width, pos_x, pos_y, in_width, in_height, wi->width, wi->height, wi->color_mode, wi->rotation, wi->mirror, zorder, wi->pre_mult_alpha, global_alpha, wi->rotation_type, - replication, mgr_timings, mem_to_mem); + replication, vm, mem_to_mem); switch (wi->color_mode) { case OMAP_DSS_COLOR_RGB16: @@ -2867,8 +2869,8 @@ int dispc_wb_setup(const struct omap_dss_writeback_info *wi, } else { int wbdelay; - wbdelay = min(mgr_timings->vfp + mgr_timings->vsw + - mgr_timings->vbp, 255); + wbdelay = min(vm->vfront_porch + + vm->vsync_len + vm->vback_porch, (u32)255); /* WBDELAYCOUNT */ REG_FLD_MOD(DISPC_OVL_ATTRIBUTES2(plane), wbdelay, 7, 0); @@ -3093,10 +3095,10 @@ static bool _dispc_mgr_size_ok(u16 width, u16 height) height <= dispc.feat->mgr_height_max; } -static bool _dispc_lcd_timings_ok(int hsw, int hfp, int hbp, +static bool _dispc_lcd_timings_ok(int hsync_len, int hfp, int hbp, int vsw, int vfp, int vbp) { - if (hsw < 1 || hsw > dispc.feat->sw_max || + if (hsync_len < 1 || hsync_len > dispc.feat->sw_max || hfp < 1 || hfp > dispc.feat->hp_max || hbp < 1 || hbp > dispc.feat->hp_max || vsw < 1 || vsw > dispc.feat->sw_max || @@ -3110,113 +3112,77 @@ static bool _dispc_mgr_pclk_ok(enum omap_channel channel, unsigned long pclk) { if (dss_mgr_is_lcd(channel)) - return pclk <= dispc.feat->max_lcd_pclk ? true : false; + return pclk <= dispc.feat->max_lcd_pclk; else - return pclk <= dispc.feat->max_tv_pclk ? true : false; + return pclk <= dispc.feat->max_tv_pclk; } -bool dispc_mgr_timings_ok(enum omap_channel channel, - const struct omap_video_timings *timings) +bool dispc_mgr_timings_ok(enum omap_channel channel, const struct videomode *vm) { - if (!_dispc_mgr_size_ok(timings->x_res, timings->y_res)) + if (!_dispc_mgr_size_ok(vm->hactive, vm->vactive)) return false; - if (!_dispc_mgr_pclk_ok(channel, timings->pixelclock)) + if (!_dispc_mgr_pclk_ok(channel, vm->pixelclock)) return false; if (dss_mgr_is_lcd(channel)) { /* TODO: OMAP4+ supports interlace for LCD outputs */ - if (timings->interlace) + if (vm->flags & DISPLAY_FLAGS_INTERLACED) return false; - if (!_dispc_lcd_timings_ok(timings->hsw, timings->hfp, - timings->hbp, timings->vsw, timings->vfp, - timings->vbp)) + if (!_dispc_lcd_timings_ok(vm->hsync_len, + vm->hfront_porch, vm->hback_porch, + vm->vsync_len, vm->vfront_porch, + vm->vback_porch)) return false; } return true; } -static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, int hsw, - int hfp, int hbp, int vsw, int vfp, int vbp, - enum omap_dss_signal_level vsync_level, - enum omap_dss_signal_level hsync_level, - enum omap_dss_signal_edge data_pclk_edge, - enum omap_dss_signal_level de_level, - enum omap_dss_signal_edge sync_pclk_edge) - +static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, + const struct videomode *vm) { u32 timing_h, timing_v, l; bool onoff, rf, ipc, vs, hs, de; - timing_h = FLD_VAL(hsw-1, dispc.feat->sw_start, 0) | - FLD_VAL(hfp-1, dispc.feat->fp_start, 8) | - FLD_VAL(hbp-1, dispc.feat->bp_start, 20); - timing_v = FLD_VAL(vsw-1, dispc.feat->sw_start, 0) | - FLD_VAL(vfp, dispc.feat->fp_start, 8) | - FLD_VAL(vbp, dispc.feat->bp_start, 20); + timing_h = FLD_VAL(vm->hsync_len - 1, dispc.feat->sw_start, 0) | + FLD_VAL(vm->hfront_porch - 1, dispc.feat->fp_start, 8) | + FLD_VAL(vm->hback_porch - 1, dispc.feat->bp_start, 20); + timing_v = FLD_VAL(vm->vsync_len - 1, dispc.feat->sw_start, 0) | + FLD_VAL(vm->vfront_porch, dispc.feat->fp_start, 8) | + FLD_VAL(vm->vback_porch, dispc.feat->bp_start, 20); dispc_write_reg(DISPC_TIMING_H(channel), timing_h); dispc_write_reg(DISPC_TIMING_V(channel), timing_v); - switch (vsync_level) { - case OMAPDSS_SIG_ACTIVE_LOW: - vs = true; - break; - case OMAPDSS_SIG_ACTIVE_HIGH: + if (vm->flags & DISPLAY_FLAGS_VSYNC_HIGH) vs = false; - break; - default: - BUG(); - } + else + vs = true; - switch (hsync_level) { - case OMAPDSS_SIG_ACTIVE_LOW: - hs = true; - break; - case OMAPDSS_SIG_ACTIVE_HIGH: + if (vm->flags & DISPLAY_FLAGS_HSYNC_HIGH) hs = false; - break; - default: - BUG(); - } + else + hs = true; - switch (de_level) { - case OMAPDSS_SIG_ACTIVE_LOW: - de = true; - break; - case OMAPDSS_SIG_ACTIVE_HIGH: + if (vm->flags & DISPLAY_FLAGS_DE_HIGH) de = false; - break; - default: - BUG(); - } + else + de = true; - switch (data_pclk_edge) { - case OMAPDSS_DRIVE_SIG_RISING_EDGE: + if (vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE) ipc = false; - break; - case OMAPDSS_DRIVE_SIG_FALLING_EDGE: + else ipc = true; - break; - default: - BUG(); - } /* always use the 'rf' setting */ onoff = true; - switch (sync_pclk_edge) { - case OMAPDSS_DRIVE_SIG_FALLING_EDGE: - rf = false; - break; - case OMAPDSS_DRIVE_SIG_RISING_EDGE: + if (vm->flags & DISPLAY_FLAGS_SYNC_POSEDGE) rf = true; - break; - default: - BUG(); - } + else + rf = false; l = FLD_VAL(onoff, 17, 17) | FLD_VAL(rf, 16, 16) | @@ -3253,13 +3219,13 @@ static void _dispc_mgr_set_lcd_timings(enum omap_channel channel, int hsw, /* change name to mode? */ void dispc_mgr_set_timings(enum omap_channel channel, - const struct omap_video_timings *timings) + const struct videomode *vm) { unsigned xtot, ytot; unsigned long ht, vt; - struct omap_video_timings t = *timings; + struct videomode t = *vm; - DSSDBG("channel %d xres %u yres %u\n", channel, t.x_res, t.y_res); + DSSDBG("channel %d xres %u yres %u\n", channel, t.hactive, t.vactive); if (!dispc_mgr_timings_ok(channel, &t)) { BUG(); @@ -3267,34 +3233,37 @@ void dispc_mgr_set_timings(enum omap_channel channel, } if (dss_mgr_is_lcd(channel)) { - _dispc_mgr_set_lcd_timings(channel, t.hsw, t.hfp, t.hbp, t.vsw, - t.vfp, t.vbp, t.vsync_level, t.hsync_level, - t.data_pclk_edge, t.de_level, t.sync_pclk_edge); + _dispc_mgr_set_lcd_timings(channel, &t); - xtot = t.x_res + t.hfp + t.hsw + t.hbp; - ytot = t.y_res + t.vfp + t.vsw + t.vbp; + xtot = t.hactive + t.hfront_porch + t.hsync_len + t.hback_porch; + ytot = t.vactive + t.vfront_porch + t.vsync_len + t.vback_porch; - ht = timings->pixelclock / xtot; - vt = timings->pixelclock / xtot / ytot; + ht = vm->pixelclock / xtot; + vt = vm->pixelclock / xtot / ytot; - DSSDBG("pck %u\n", timings->pixelclock); - DSSDBG("hsw %d hfp %d hbp %d vsw %d vfp %d vbp %d\n", - t.hsw, t.hfp, t.hbp, t.vsw, t.vfp, t.vbp); + DSSDBG("pck %lu\n", vm->pixelclock); + DSSDBG("hsync_len %d hfp %d hbp %d vsw %d vfp %d vbp %d\n", + t.hsync_len, t.hfront_porch, t.hback_porch, + t.vsync_len, t.vfront_porch, t.vback_porch); DSSDBG("vsync_level %d hsync_level %d data_pclk_edge %d de_level %d sync_pclk_edge %d\n", - t.vsync_level, t.hsync_level, t.data_pclk_edge, - t.de_level, t.sync_pclk_edge); + !!(t.flags & DISPLAY_FLAGS_VSYNC_HIGH), + !!(t.flags & DISPLAY_FLAGS_HSYNC_HIGH), + !!(t.flags & DISPLAY_FLAGS_PIXDATA_POSEDGE), + !!(t.flags & DISPLAY_FLAGS_DE_HIGH), + !!(t.flags & DISPLAY_FLAGS_SYNC_POSEDGE)); DSSDBG("hsync %luHz, vsync %luHz\n", ht, vt); } else { - if (t.interlace) - t.y_res /= 2; + if (t.flags & DISPLAY_FLAGS_INTERLACED) + t.vactive /= 2; if (dispc.feat->supports_double_pixel) - REG_FLD_MOD(DISPC_CONTROL, t.double_pixel ? 1 : 0, - 19, 17); + REG_FLD_MOD(DISPC_CONTROL, + !!(t.flags & DISPLAY_FLAGS_DOUBLECLK), + 19, 17); } - dispc_mgr_set_size(channel, t.x_res, t.y_res); + dispc_mgr_set_size(channel, t.hactive, t.vactive); } EXPORT_SYMBOL(dispc_mgr_set_timings); @@ -4214,23 +4183,20 @@ EXPORT_SYMBOL(dispc_free_irq); */ static const struct dispc_errata_i734_data { - struct omap_video_timings timings; + struct videomode vm; struct omap_overlay_info ovli; struct omap_overlay_manager_info mgri; struct dss_lcd_mgr_config lcd_conf; } i734 = { - .timings = { - .x_res = 8, .y_res = 1, + .vm = { + .hactive = 8, .vactive = 1, .pixelclock = 16000000, - .hsw = 8, .hfp = 4, .hbp = 4, - .vsw = 1, .vfp = 1, .vbp = 1, - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .interlace = false, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .double_pixel = false, + .hsync_len = 8, .hfront_porch = 4, .hback_porch = 4, + .vsync_len = 1, .vfront_porch = 1, .vback_porch = 1, + + .flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW | + DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_SYNC_POSEDGE | + DISPLAY_FLAGS_PIXDATA_POSEDGE, }, .ovli = { .screen_width = 1, @@ -4320,7 +4286,7 @@ static void dispc_errata_i734_wa(void) /* Setup and enable GFX plane */ dispc_ovl_set_channel_out(OMAP_DSS_GFX, OMAP_DSS_CHANNEL_LCD); - dispc_ovl_setup(OMAP_DSS_GFX, &ovli, false, &i734.timings, false); + dispc_ovl_setup(OMAP_DSS_GFX, &ovli, false, &i734.vm, false); dispc_ovl_enable(OMAP_DSS_GFX, true); /* Set up and enable display manager for LCD1 */ @@ -4328,7 +4294,7 @@ static void dispc_errata_i734_wa(void) dispc_calc_clock_rates(dss_get_dispc_clk_rate(), &lcd_conf.clock_info); dispc_mgr_set_lcd_config(OMAP_DSS_CHANNEL_LCD, &lcd_conf); - dispc_mgr_set_timings(OMAP_DSS_CHANNEL_LCD, &i734.timings); + dispc_mgr_set_timings(OMAP_DSS_CHANNEL_LCD, &i734.vm); dispc_clear_irqstatus(framedone_irq); diff --git a/drivers/gpu/drm/omapdrm/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c index 8dcdd7cf9937..425a5a8dff8b 100644 --- a/drivers/gpu/drm/omapdrm/dss/display.c +++ b/drivers/gpu/drm/omapdrm/dss/display.c @@ -35,8 +35,8 @@ void omapdss_default_get_resolution(struct omap_dss_device *dssdev, u16 *xres, u16 *yres) { - *xres = dssdev->panel.timings.x_res; - *yres = dssdev->panel.timings.y_res; + *xres = dssdev->panel.vm.hactive; + *yres = dssdev->panel.vm.vactive; } EXPORT_SYMBOL(omapdss_default_get_resolution); @@ -72,9 +72,9 @@ int omapdss_default_get_recommended_bpp(struct omap_dss_device *dssdev) EXPORT_SYMBOL(omapdss_default_get_recommended_bpp); void omapdss_default_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - *timings = dssdev->panel.timings; + *vm = dssdev->panel.vm; } EXPORT_SYMBOL(omapdss_default_get_timings); @@ -217,73 +217,3 @@ struct omap_dss_device *omap_dss_find_device(void *data, return NULL; } EXPORT_SYMBOL(omap_dss_find_device); - -void videomode_to_omap_video_timings(const struct videomode *vm, - struct omap_video_timings *ovt) -{ - memset(ovt, 0, sizeof(*ovt)); - - ovt->pixelclock = vm->pixelclock; - ovt->x_res = vm->hactive; - ovt->hbp = vm->hback_porch; - ovt->hfp = vm->hfront_porch; - ovt->hsw = vm->hsync_len; - ovt->y_res = vm->vactive; - ovt->vbp = vm->vback_porch; - ovt->vfp = vm->vfront_porch; - ovt->vsw = vm->vsync_len; - - ovt->vsync_level = vm->flags & DISPLAY_FLAGS_VSYNC_HIGH ? - OMAPDSS_SIG_ACTIVE_HIGH : - OMAPDSS_SIG_ACTIVE_LOW; - ovt->hsync_level = vm->flags & DISPLAY_FLAGS_HSYNC_HIGH ? - OMAPDSS_SIG_ACTIVE_HIGH : - OMAPDSS_SIG_ACTIVE_LOW; - ovt->de_level = vm->flags & DISPLAY_FLAGS_DE_HIGH ? - OMAPDSS_SIG_ACTIVE_HIGH : - OMAPDSS_SIG_ACTIVE_LOW; - ovt->data_pclk_edge = vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE ? - OMAPDSS_DRIVE_SIG_RISING_EDGE : - OMAPDSS_DRIVE_SIG_FALLING_EDGE; - - ovt->sync_pclk_edge = ovt->data_pclk_edge; -} -EXPORT_SYMBOL(videomode_to_omap_video_timings); - -void omap_video_timings_to_videomode(const struct omap_video_timings *ovt, - struct videomode *vm) -{ - memset(vm, 0, sizeof(*vm)); - - vm->pixelclock = ovt->pixelclock; - - vm->hactive = ovt->x_res; - vm->hback_porch = ovt->hbp; - vm->hfront_porch = ovt->hfp; - vm->hsync_len = ovt->hsw; - vm->vactive = ovt->y_res; - vm->vback_porch = ovt->vbp; - vm->vfront_porch = ovt->vfp; - vm->vsync_len = ovt->vsw; - - if (ovt->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH) - vm->flags |= DISPLAY_FLAGS_HSYNC_HIGH; - else - vm->flags |= DISPLAY_FLAGS_HSYNC_LOW; - - if (ovt->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH) - vm->flags |= DISPLAY_FLAGS_VSYNC_HIGH; - else - vm->flags |= DISPLAY_FLAGS_VSYNC_LOW; - - if (ovt->de_level == OMAPDSS_SIG_ACTIVE_HIGH) - vm->flags |= DISPLAY_FLAGS_DE_HIGH; - else - vm->flags |= DISPLAY_FLAGS_DE_LOW; - - if (ovt->data_pclk_edge == OMAPDSS_DRIVE_SIG_RISING_EDGE) - vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE; - else - vm->flags |= DISPLAY_FLAGS_PIXDATA_NEGEDGE; -} -EXPORT_SYMBOL(omap_video_timings_to_videomode); diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c index b268295b76cf..e75162d26ac0 100644 --- a/drivers/gpu/drm/omapdrm/dss/dpi.c +++ b/drivers/gpu/drm/omapdrm/dss/dpi.c @@ -47,7 +47,7 @@ struct dpi_data { struct mutex lock; - struct omap_video_timings timings; + struct videomode vm; struct dss_lcd_mgr_config mgr_config; int data_lines; @@ -333,31 +333,31 @@ static int dpi_set_mode(struct dpi_data *dpi) { struct omap_dss_device *out = &dpi->output; enum omap_channel channel = out->dispc_channel; - struct omap_video_timings *t = &dpi->timings; + struct videomode *vm = &dpi->vm; int lck_div = 0, pck_div = 0; unsigned long fck = 0; unsigned long pck; int r = 0; if (dpi->pll) - r = dpi_set_pll_clk(dpi, channel, t->pixelclock, &fck, + r = dpi_set_pll_clk(dpi, channel, vm->pixelclock, &fck, &lck_div, &pck_div); else - r = dpi_set_dispc_clk(dpi, t->pixelclock, &fck, + r = dpi_set_dispc_clk(dpi, vm->pixelclock, &fck, &lck_div, &pck_div); if (r) return r; pck = fck / lck_div / pck_div; - if (pck != t->pixelclock) { - DSSWARN("Could not find exact pixel clock. Requested %d Hz, got %lu Hz\n", - t->pixelclock, pck); + if (pck != vm->pixelclock) { + DSSWARN("Could not find exact pixel clock. Requested %lu Hz, got %lu Hz\n", + vm->pixelclock, pck); - t->pixelclock = pck; + vm->pixelclock = pck; } - dss_mgr_set_timings(channel, t); + dss_mgr_set_timings(channel, vm); return 0; } @@ -476,7 +476,7 @@ static void dpi_display_disable(struct omap_dss_device *dssdev) } static void dpi_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev); @@ -484,25 +484,25 @@ static void dpi_set_timings(struct omap_dss_device *dssdev, mutex_lock(&dpi->lock); - dpi->timings = *timings; + dpi->vm = *vm; mutex_unlock(&dpi->lock); } static void dpi_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev); mutex_lock(&dpi->lock); - *timings = dpi->timings; + *vm = dpi->vm; mutex_unlock(&dpi->lock); } static int dpi_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { struct dpi_data *dpi = dpi_get_data_from_dssdev(dssdev); enum omap_channel channel = dpi->output.dispc_channel; @@ -512,23 +512,23 @@ static int dpi_check_timings(struct omap_dss_device *dssdev, struct dpi_clk_calc_ctx ctx; bool ok; - if (timings->x_res % 8 != 0) + if (vm->hactive % 8 != 0) return -EINVAL; - if (!dispc_mgr_timings_ok(channel, timings)) + if (!dispc_mgr_timings_ok(channel, vm)) return -EINVAL; - if (timings->pixelclock == 0) + if (vm->pixelclock == 0) return -EINVAL; if (dpi->pll) { - ok = dpi_pll_clk_calc(dpi, timings->pixelclock, &ctx); + ok = dpi_pll_clk_calc(dpi, vm->pixelclock, &ctx); if (!ok) return -EINVAL; fck = ctx.pll_cinfo.clkout[ctx.clkout_idx]; } else { - ok = dpi_dss_clk_calc(timings->pixelclock, &ctx); + ok = dpi_dss_clk_calc(vm->pixelclock, &ctx); if (!ok) return -EINVAL; @@ -540,7 +540,7 @@ static int dpi_check_timings(struct omap_dss_device *dssdev, pck = fck / lck_div / pck_div; - timings->pixelclock = pck; + vm->pixelclock = pck; return 0; } diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c index e1be5e795cd8..f060bda31235 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.c +++ b/drivers/gpu/drm/omapdrm/dss/dsi.c @@ -289,7 +289,7 @@ struct dsi_clk_calc_ctx { struct dss_pll_clock_info dsi_cinfo; struct dispc_clock_info dispc_cinfo; - struct omap_video_timings dispc_vm; + struct videomode vm; struct omap_dss_dsi_videomode_timings dsi_vm; }; @@ -383,7 +383,7 @@ struct dsi_data { unsigned scp_clk_refcount; struct dss_lcd_mgr_config mgr_config; - struct omap_video_timings timings; + struct videomode vm; enum omap_dss_dsi_pixel_format pix_fmt; enum omap_dss_dsi_mode mode; struct omap_dss_dsi_videomode_timings vm_timings; @@ -3321,12 +3321,12 @@ static void dsi_config_vp_num_line_buffers(struct platform_device *dsidev) if (dsi->mode == OMAP_DSS_DSI_VIDEO_MODE) { int bpp = dsi_get_pixel_size(dsi->pix_fmt); - struct omap_video_timings *timings = &dsi->timings; + struct videomode *vm = &dsi->vm; /* * Don't use line buffers if width is greater than the video * port's line buffer size */ - if (dsi->line_buffer_size <= timings->x_res * bpp / 8) + if (dsi->line_buffer_size <= vm->hactive * bpp / 8) num_line_buffers = 0; else num_line_buffers = 2; @@ -3453,7 +3453,7 @@ static void dsi_config_cmd_mode_interleaving(struct platform_device *dsidev) int ddr_clk_pre, ddr_clk_post, enter_hs_mode_lat, exit_hs_mode_lat; int tclk_trail, ths_exit, exiths_clk; bool ddr_alwon; - struct omap_video_timings *timings = &dsi->timings; + struct videomode *vm = &dsi->vm; int bpp = dsi_get_pixel_size(dsi->pix_fmt); int ndl = dsi->num_lanes_used - 1; int dsi_fclk_hsdiv = dsi->user_dsi_cinfo.mX[HSDIV_DSI] + 1; @@ -3494,7 +3494,7 @@ static void dsi_config_cmd_mode_interleaving(struct platform_device *dsidev) exiths_clk = ths_exit + tclk_trail; - width_bytes = DIV_ROUND_UP(timings->x_res * bpp, 8); + width_bytes = DIV_ROUND_UP(vm->hactive * bpp, 8); bllp = hbp + hfp + hsa + DIV_ROUND_UP(width_bytes + 6, ndl); if (!hsa_blanking_mode) { @@ -3705,7 +3705,7 @@ static void dsi_proto_timings(struct platform_device *dsidev) int vbp = dsi->vm_timings.vbp; int window_sync = dsi->vm_timings.window_sync; bool hsync_end; - struct omap_video_timings *timings = &dsi->timings; + struct videomode *vm = &dsi->vm; int bpp = dsi_get_pixel_size(dsi->pix_fmt); int tl, t_he, width_bytes; @@ -3713,7 +3713,7 @@ static void dsi_proto_timings(struct platform_device *dsidev) t_he = hsync_end ? ((hsa == 0 && ndl == 3) ? 1 : DIV_ROUND_UP(4, ndl)) : 0; - width_bytes = DIV_ROUND_UP(timings->x_res * bpp, 8); + width_bytes = DIV_ROUND_UP(vm->hactive * bpp, 8); /* TL = t_HS + HSA + t_HE + HFP + ceil((WC + 6) / NDL) + HBP */ tl = DIV_ROUND_UP(4, ndl) + (hsync_end ? hsa : 0) + t_he + hfp + @@ -3722,7 +3722,7 @@ static void dsi_proto_timings(struct platform_device *dsidev) DSSDBG("HBP: %d, HFP: %d, HSA: %d, TL: %d TXBYTECLKHS\n", hbp, hfp, hsync_end ? hsa : 0, tl); DSSDBG("VBP: %d, VFP: %d, VSA: %d, VACT: %d lines\n", vbp, vfp, - vsa, timings->y_res); + vsa, vm->vactive); r = dsi_read_reg(dsidev, DSI_VM_TIMING1); r = FLD_MOD(r, hbp, 11, 0); /* HBP */ @@ -3738,7 +3738,7 @@ static void dsi_proto_timings(struct platform_device *dsidev) dsi_write_reg(dsidev, DSI_VM_TIMING2, r); r = dsi_read_reg(dsidev, DSI_VM_TIMING3); - r = FLD_MOD(r, timings->y_res, 14, 0); /* VACT */ + r = FLD_MOD(r, vm->vactive, 14, 0); /* VACT */ r = FLD_MOD(r, tl, 31, 16); /* TL */ dsi_write_reg(dsidev, DSI_VM_TIMING3, r); } @@ -3856,7 +3856,7 @@ static int dsi_enable_video_output(struct omap_dss_device *dssdev, int channel) /* MODE, 1 = video mode */ REG_FLD_MOD(dsidev, DSI_VC_CTRL(channel), 1, 4, 4); - word_count = DIV_ROUND_UP(dsi->timings.x_res * bpp, 8); + word_count = DIV_ROUND_UP(dsi->vm.hactive * bpp, 8); dsi_vc_write_long_header(dsidev, channel, data_type, word_count, 0); @@ -3918,8 +3918,8 @@ static void dsi_update_screen_dispc(struct platform_device *dsidev) int r; const unsigned channel = dsi->update_channel; const unsigned line_buf_size = dsi->line_buffer_size; - u16 w = dsi->timings.x_res; - u16 h = dsi->timings.y_res; + u16 w = dsi->vm.hactive; + u16 h = dsi->vm.vactive; DSSDBG("dsi_update_screen_dispc(%dx%d)\n", w, h); @@ -3969,7 +3969,7 @@ static void dsi_update_screen_dispc(struct platform_device *dsidev) msecs_to_jiffies(250)); BUG_ON(r == 0); - dss_mgr_set_timings(dispc_channel, &dsi->timings); + dss_mgr_set_timings(dispc_channel, &dsi->vm); dss_mgr_start_update(dispc_channel); @@ -4056,8 +4056,8 @@ static int dsi_update(struct omap_dss_device *dssdev, int channel, dsi->framedone_callback = callback; dsi->framedone_data = data; - dw = dsi->timings.x_res; - dh = dsi->timings.y_res; + dw = dsi->vm.hactive; + dh = dsi->vm.vactive; #ifdef DSI_PERF_MEASURE dsi->update_bytes = dw * dh * @@ -4120,16 +4120,21 @@ static int dsi_display_init_dispc(struct platform_device *dsidev, /* * override interlace, logic level and edge related parameters in - * omap_video_timings with default values + * videomode with default values */ - dsi->timings.interlace = false; - dsi->timings.hsync_level = OMAPDSS_SIG_ACTIVE_HIGH; - dsi->timings.vsync_level = OMAPDSS_SIG_ACTIVE_HIGH; - dsi->timings.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; - dsi->timings.de_level = OMAPDSS_SIG_ACTIVE_HIGH; - dsi->timings.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE; - - dss_mgr_set_timings(channel, &dsi->timings); + dsi->vm.flags &= ~DISPLAY_FLAGS_INTERLACED; + dsi->vm.flags &= ~DISPLAY_FLAGS_HSYNC_LOW; + dsi->vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH; + dsi->vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW; + dsi->vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH; + dsi->vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE; + dsi->vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE; + dsi->vm.flags &= ~DISPLAY_FLAGS_DE_LOW; + dsi->vm.flags |= DISPLAY_FLAGS_DE_HIGH; + dsi->vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE; + dsi->vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE; + + dss_mgr_set_timings(channel, &dsi->vm); r = dsi_configure_dispc_clocks(dsidev); if (r) @@ -4331,7 +4336,7 @@ static void print_dsi_vm(const char *str, wc = DIV_ROUND_UP(t->hact * t->bitspp, 8); pps = DIV_ROUND_UP(wc + 6, t->ndl); /* pixel packet size */ - bl = t->hss + t->hsa + t->hse + t->hbp + t->hfp; + bl = t->hss + t->hsa + t->hse + t->hbp + t->hfront_porch; tot = bl + pps; #define TO_DSI_T(x) ((u32)div64_u64((u64)x * 1000000000llu, byteclk)) @@ -4340,14 +4345,14 @@ static void print_dsi_vm(const char *str, "%u/%u/%u/%u/%u/%u = %u + %u = %u\n", str, byteclk, - t->hss, t->hsa, t->hse, t->hbp, pps, t->hfp, + t->hss, t->hsa, t->hse, t->hbp, pps, t->hfront_porch, bl, pps, tot, TO_DSI_T(t->hss), TO_DSI_T(t->hsa), TO_DSI_T(t->hse), TO_DSI_T(t->hbp), TO_DSI_T(pps), - TO_DSI_T(t->hfp), + TO_DSI_T(t->hfront_porch), TO_DSI_T(bl), TO_DSI_T(pps), @@ -4356,13 +4361,13 @@ static void print_dsi_vm(const char *str, #undef TO_DSI_T } -static void print_dispc_vm(const char *str, const struct omap_video_timings *t) +static void print_dispc_vm(const char *str, const struct videomode *vm) { - unsigned long pck = t->pixelclock; + unsigned long pck = vm->pixelclock; int hact, bl, tot; - hact = t->x_res; - bl = t->hsw + t->hbp + t->hfp; + hact = vm->hactive; + bl = vm->hsync_len + vm->hbp + vm->hfront_porch; tot = hact + bl; #define TO_DISPC_T(x) ((u32)div64_u64((u64)x * 1000000000llu, pck)) @@ -4371,12 +4376,12 @@ static void print_dispc_vm(const char *str, const struct omap_video_timings *t) "%u/%u/%u/%u = %u + %u = %u\n", str, pck, - t->hsw, t->hbp, hact, t->hfp, + vm->hsync_len, vm->hbp, hact, vm->hfront_porch, bl, hact, tot, - TO_DISPC_T(t->hsw), - TO_DISPC_T(t->hbp), + TO_DISPC_T(vm->hsync_len), + TO_DISPC_T(vm->hbp), TO_DISPC_T(hact), - TO_DISPC_T(t->hfp), + TO_DISPC_T(vm->hfront_porch), TO_DISPC_T(bl), TO_DISPC_T(hact), TO_DISPC_T(tot)); @@ -4387,7 +4392,7 @@ static void print_dispc_vm(const char *str, const struct omap_video_timings *t) static void print_dsi_dispc_vm(const char *str, const struct omap_dss_dsi_videomode_timings *t) { - struct omap_video_timings vm = { 0 }; + struct videomode vm = { 0 }; unsigned long byteclk = t->hsclk / 4; unsigned long pck; u64 dsi_tput; @@ -4396,13 +4401,13 @@ static void print_dsi_dispc_vm(const char *str, dsi_tput = (u64)byteclk * t->ndl * 8; pck = (u32)div64_u64(dsi_tput, t->bitspp); dsi_hact = DIV_ROUND_UP(DIV_ROUND_UP(t->hact * t->bitspp, 8) + 6, t->ndl); - dsi_htot = t->hss + t->hsa + t->hse + t->hbp + dsi_hact + t->hfp; + dsi_htot = t->hss + t->hsa + t->hse + t->hbp + dsi_hact + t->hfront_porch; vm.pixelclock = pck; - vm.hsw = div64_u64((u64)(t->hsa + t->hse) * pck, byteclk); + vm.hsync_len = div64_u64((u64)(t->hsa + t->hse) * pck, byteclk); vm.hbp = div64_u64((u64)t->hbp * pck, byteclk); - vm.hfp = div64_u64((u64)t->hfp * pck, byteclk); - vm.x_res = t->hact; + vm.hfront_porch = div64_u64((u64)t->hfront_porch * pck, byteclk); + vm.hactive = t->hact; print_dispc_vm(str, &vm); } @@ -4412,19 +4417,19 @@ static bool dsi_cm_calc_dispc_cb(int lckd, int pckd, unsigned long lck, unsigned long pck, void *data) { struct dsi_clk_calc_ctx *ctx = data; - struct omap_video_timings *t = &ctx->dispc_vm; + struct videomode *vm = &ctx->vm; ctx->dispc_cinfo.lck_div = lckd; ctx->dispc_cinfo.pck_div = pckd; ctx->dispc_cinfo.lck = lck; ctx->dispc_cinfo.pck = pck; - *t = *ctx->config->timings; - t->pixelclock = pck; - t->x_res = ctx->config->timings->x_res; - t->y_res = ctx->config->timings->y_res; - t->hsw = t->hfp = t->hbp = t->vsw = 1; - t->vfp = t->vbp = 0; + *vm = *ctx->config->vm; + vm->pixelclock = pck; + vm->hactive = ctx->config->vm->hactive; + vm->vactive = ctx->config->vm->vactive; + vm->hsync_len = vm->hfront_porch = vm->hback_porch = vm->vsync_len = 1; + vm->vfront_porch = vm->vback_porch = 0; return true; } @@ -4475,7 +4480,7 @@ static bool dsi_cm_calc(struct dsi_data *dsi, * especially as we go to LP between each pixel packet due to HW * "feature". So let's just estimate very roughly and multiply by 1.5. */ - pck = cfg->timings->pixelclock; + pck = cfg->vm->pixelclock; pck = pck * 3 / 2; txbyteclk = pck * bitspp / 8 / ndl; @@ -4510,14 +4515,14 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) int dispc_htot, dispc_hbl; /* pixels */ int dsi_htot, dsi_hact, dsi_hbl, hss, hse; /* byteclks */ int hfp, hsa, hbp; - const struct omap_video_timings *req_vm; - struct omap_video_timings *dispc_vm; + const struct videomode *req_vm; + struct videomode *dispc_vm; struct omap_dss_dsi_videomode_timings *dsi_vm; u64 dsi_tput, dispc_tput; dsi_tput = (u64)byteclk * ndl * 8; - req_vm = cfg->timings; + req_vm = cfg->vm; req_pck_min = ctx->req_pck_min; req_pck_max = ctx->req_pck_max; req_pck_nom = ctx->req_pck_nom; @@ -4525,9 +4530,10 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) dispc_pck = ctx->dispc_cinfo.pck; dispc_tput = (u64)dispc_pck * bitspp; - xres = req_vm->x_res; + xres = req_vm->hactive; - panel_hbl = req_vm->hfp + req_vm->hbp + req_vm->hsw; + panel_hbl = req_vm->hfront_porch + req_vm->hback_porch + + req_vm->hsync_len; panel_htot = xres + panel_hbl; dsi_hact = DIV_ROUND_UP(DIV_ROUND_UP(xres * bitspp, 8) + 6, ndl); @@ -4557,7 +4563,7 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) hss = DIV_ROUND_UP(4, ndl); if (cfg->trans_mode == OMAP_DSS_DSI_PULSE_MODE) { - if (ndl == 3 && req_vm->hsw == 0) + if (ndl == 3 && req_vm->hsync_len == 0) hse = 1; else hse = DIV_ROUND_UP(4, ndl); @@ -4596,14 +4602,14 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) if (cfg->trans_mode != OMAP_DSS_DSI_PULSE_MODE) { hsa = 0; - } else if (ndl == 3 && req_vm->hsw == 0) { + } else if (ndl == 3 && req_vm->hsync_len == 0) { hsa = 0; } else { - hsa = div64_u64((u64)req_vm->hsw * byteclk, req_pck_nom); + hsa = div64_u64((u64)req_vm->hsync_len * byteclk, req_pck_nom); hsa = max(hsa - hse, 1); } - hbp = div64_u64((u64)req_vm->hbp * byteclk, req_pck_nom); + hbp = div64_u64((u64)req_vm->hback_porch * byteclk, req_pck_nom); hbp = max(hbp, 1); hfp = dsi_hbl - (hss + hsa + hse + hbp); @@ -4633,10 +4639,10 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) dsi_vm->hact = xres; dsi_vm->hfp = hfp; - dsi_vm->vsa = req_vm->vsw; - dsi_vm->vbp = req_vm->vbp; - dsi_vm->vact = req_vm->y_res; - dsi_vm->vfp = req_vm->vfp; + dsi_vm->vsa = req_vm->vsync_len; + dsi_vm->vbp = req_vm->vback_porch; + dsi_vm->vact = req_vm->vactive; + dsi_vm->vfp = req_vm->vfront_porch; dsi_vm->trans_mode = cfg->trans_mode; @@ -4650,19 +4656,19 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) /* setup DISPC videomode */ - dispc_vm = &ctx->dispc_vm; + dispc_vm = &ctx->vm; *dispc_vm = *req_vm; dispc_vm->pixelclock = dispc_pck; if (cfg->trans_mode == OMAP_DSS_DSI_PULSE_MODE) { - hsa = div64_u64((u64)req_vm->hsw * dispc_pck, + hsa = div64_u64((u64)req_vm->hsync_len * dispc_pck, req_pck_nom); hsa = max(hsa, 1); } else { hsa = 1; } - hbp = div64_u64((u64)req_vm->hbp * dispc_pck, req_pck_nom); + hbp = div64_u64((u64)req_vm->hback_porch * dispc_pck, req_pck_nom); hbp = max(hbp, 1); hfp = dispc_hbl - hsa - hbp; @@ -4685,9 +4691,9 @@ static bool dsi_vm_calc_blanking(struct dsi_clk_calc_ctx *ctx) if (hfp < 1) return false; - dispc_vm->hfp = hfp; - dispc_vm->hsw = hsa; - dispc_vm->hbp = hbp; + dispc_vm->hfront_porch = hfp; + dispc_vm->hsync_len = hsa; + dispc_vm->hback_porch = hbp; return true; } @@ -4707,9 +4713,9 @@ static bool dsi_vm_calc_dispc_cb(int lckd, int pckd, unsigned long lck, return false; #ifdef PRINT_VERBOSE_VM_TIMINGS - print_dispc_vm("dispc", &ctx->dispc_vm); + print_dispc_vm("dispc", &ctx->vm); print_dsi_vm("dsi ", &ctx->dsi_vm); - print_dispc_vm("req ", ctx->config->timings); + print_dispc_vm("req ", ctx->config->vm); print_dsi_dispc_vm("act ", &ctx->dsi_vm); #endif @@ -4758,7 +4764,7 @@ static bool dsi_vm_calc(struct dsi_data *dsi, const struct omap_dss_dsi_config *cfg, struct dsi_clk_calc_ctx *ctx) { - const struct omap_video_timings *t = cfg->timings; + const struct videomode *vm = cfg->vm; unsigned long clkin; unsigned long pll_min; unsigned long pll_max; @@ -4774,9 +4780,9 @@ static bool dsi_vm_calc(struct dsi_data *dsi, ctx->config = cfg; /* these limits should come from the panel driver */ - ctx->req_pck_min = t->pixelclock - 1000; - ctx->req_pck_nom = t->pixelclock; - ctx->req_pck_max = t->pixelclock + 1000; + ctx->req_pck_min = vm->pixelclock - 1000; + ctx->req_pck_nom = vm->pixelclock; + ctx->req_pck_max = vm->pixelclock + 1000; byteclk_min = div64_u64((u64)ctx->req_pck_min * bitspp, ndl * 8); pll_min = max(cfg->hs_clk_min * 4, byteclk_min * 4 * 4); @@ -4833,7 +4839,7 @@ static int dsi_set_config(struct omap_dss_device *dssdev, dsi->user_dsi_cinfo = ctx.dsi_cinfo; dsi->user_dispc_cinfo = ctx.dispc_cinfo; - dsi->timings = ctx.dispc_vm; + dsi->vm = ctx.vm; dsi->vm_timings = ctx.dsi_vm; mutex_unlock(&dsi->lock); @@ -5342,7 +5348,7 @@ static int dsi_bind(struct device *dev, struct device *master, void *data) dsi->phy_base = devm_ioremap(&dsidev->dev, res->start, resource_size(res)); - if (!dsi->proto_base) { + if (!dsi->phy_base) { DSSERR("can't ioremap DSI PHY\n"); return -ENOMEM; } @@ -5362,7 +5368,7 @@ static int dsi_bind(struct device *dev, struct device *master, void *data) dsi->pll_base = devm_ioremap(&dsidev->dev, res->start, resource_size(res)); - if (!dsi->proto_base) { + if (!dsi->pll_base) { DSSERR("can't ioremap DSI PLL\n"); return -ENOMEM; } diff --git a/drivers/gpu/drm/omapdrm/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h index 4fd06dc41cb3..56493b290731 100644 --- a/drivers/gpu/drm/omapdrm/dss/dss.h +++ b/drivers/gpu/drm/omapdrm/dss/dss.h @@ -366,8 +366,7 @@ bool dispc_div_calc(unsigned long dispc, unsigned long pck_min, unsigned long pck_max, dispc_div_calc_func func, void *data); -bool dispc_mgr_timings_ok(enum omap_channel channel, - const struct omap_video_timings *timings); +bool dispc_mgr_timings_ok(enum omap_channel channel, const struct videomode *vm); int dispc_calc_clock_rates(unsigned long dispc_fclk_rate, struct dispc_clock_info *cinfo); @@ -390,7 +389,7 @@ void dispc_wb_enable(bool enable); bool dispc_wb_is_enabled(void); void dispc_wb_set_channel_in(enum dss_writeback_channel channel); int dispc_wb_setup(const struct omap_dss_writeback_info *wi, - bool mem_to_mem, const struct omap_video_timings *timings); + bool mem_to_mem, const struct videomode *vm); /* VENC */ int venc_init_platform_driver(void) __init; diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h index 63e711545865..fb6cccd02374 100644 --- a/drivers/gpu/drm/omapdrm/dss/hdmi.h +++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h @@ -181,7 +181,7 @@ struct hdmi_video_format { }; struct hdmi_config { - struct omap_video_timings timings; + struct videomode vm; struct hdmi_avi_infoframe infoframe; enum hdmi_core_hdmi_dvi hdmi_dvi_mode; }; @@ -298,11 +298,11 @@ int hdmi_wp_set_pll_pwr(struct hdmi_wp_data *wp, enum hdmi_pll_pwr val); void hdmi_wp_video_config_format(struct hdmi_wp_data *wp, struct hdmi_video_format *video_fmt); void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp, - struct omap_video_timings *timings); + struct videomode *vm); void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp, - struct omap_video_timings *timings); + struct videomode *vm); void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt, - struct omap_video_timings *timings, struct hdmi_config *param); + struct videomode *vm, struct hdmi_config *param); int hdmi_wp_init(struct platform_device *pdev, struct hdmi_wp_data *wp); phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp); diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c index cbd28dfdb86a..e7162c16de2e 100644 --- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c +++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c @@ -155,7 +155,7 @@ static void hdmi_power_off_core(struct omap_dss_device *dssdev) static int hdmi_power_on_full(struct omap_dss_device *dssdev) { int r; - struct omap_video_timings *p; + struct videomode *vm; enum omap_channel channel = dssdev->dispc_channel; struct hdmi_wp_data *wp = &hdmi.wp; struct dss_pll_clock_info hdmi_cinfo = { 0 }; @@ -169,12 +169,13 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev) hdmi_wp_clear_irqenable(wp, 0xffffffff); hdmi_wp_set_irqstatus(wp, 0xffffffff); - p = &hdmi.cfg.timings; + vm = &hdmi.cfg.vm; - DSSDBG("hdmi_power_on x_res= %d y_res = %d\n", p->x_res, p->y_res); + DSSDBG("hdmi_power_on hactive= %d vactive = %d\n", vm->hactive, + vm->vactive); - pc = p->pixelclock; - if (p->double_pixel) + pc = vm->pixelclock; + if (vm->flags & DISPLAY_FLAGS_DOUBLECLK) pc *= 2; /* DSS_HDMI_TCLK is bitclk / 10 */ @@ -209,7 +210,7 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev) hdmi4_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg); /* tv size */ - dss_mgr_set_timings(channel, p); + dss_mgr_set_timings(channel, vm); r = dss_mgr_enable(channel); if (r) @@ -255,30 +256,30 @@ static void hdmi_power_off_full(struct omap_dss_device *dssdev) } static int hdmi_display_check_timing(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - if (!dispc_mgr_timings_ok(dssdev->dispc_channel, timings)) + if (!dispc_mgr_timings_ok(dssdev->dispc_channel, vm)) return -EINVAL; return 0; } static void hdmi_display_set_timing(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { mutex_lock(&hdmi.lock); - hdmi.cfg.timings = *timings; + hdmi.cfg.vm = *vm; - dispc_set_tv_pclk(timings->pixelclock); + dispc_set_tv_pclk(vm->pixelclock); mutex_unlock(&hdmi.lock); } static void hdmi_display_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - *timings = hdmi.cfg.timings; + *vm = hdmi.cfg.vm; } static void hdmi_dump_regs(struct seq_file *s) @@ -352,7 +353,7 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev) if (hdmi.audio_configured) { r = hdmi4_audio_config(&hdmi.core, &hdmi.wp, &hdmi.audio_config, - hdmi.cfg.timings.pixelclock); + hdmi.cfg.vm.pixelclock); if (r) { DSSERR("Error restoring audio configuration: %d", r); hdmi.audio_abort_cb(&hdmi.pdev->dev); @@ -643,7 +644,7 @@ static int hdmi_audio_config(struct device *dev, } ret = hdmi4_audio_config(&hd->core, &hd->wp, dss_audio, - hd->cfg.timings.pixelclock); + hd->cfg.vm.pixelclock); if (!ret) { hd->audio_configured = true; hd->audio_config = *dss_audio; diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c index ef3afe99e487..e05b7ac4f7dd 100644 --- a/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c +++ b/drivers/gpu/drm/omapdrm/dss/hdmi4_core.c @@ -310,7 +310,7 @@ void hdmi4_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp, struct hdmi_config *cfg) { /* HDMI */ - struct omap_video_timings video_timing; + struct videomode vm; struct hdmi_video_format video_format; /* HDMI core */ struct hdmi_core_video_config v_core_cfg; @@ -318,16 +318,16 @@ void hdmi4_configure(struct hdmi_core_data *core, hdmi_core_init(&v_core_cfg); - hdmi_wp_init_vid_fmt_timings(&video_format, &video_timing, cfg); + hdmi_wp_init_vid_fmt_timings(&video_format, &vm, cfg); - hdmi_wp_video_config_timing(wp, &video_timing); + hdmi_wp_video_config_timing(wp, &vm); /* video config */ video_format.packing_mode = HDMI_PACK_24b_RGB_YUV444_YUV422; hdmi_wp_video_config_format(wp, &video_format); - hdmi_wp_video_config_interface(wp, &video_timing); + hdmi_wp_video_config_interface(wp, &vm); /* * configure core video part diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c index 0c0a5139a301..678dfb02764a 100644 --- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c +++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c @@ -172,7 +172,7 @@ static void hdmi_power_off_core(struct omap_dss_device *dssdev) static int hdmi_power_on_full(struct omap_dss_device *dssdev) { int r; - struct omap_video_timings *p; + struct videomode *vm; enum omap_channel channel = dssdev->dispc_channel; struct dss_pll_clock_info hdmi_cinfo = { 0 }; unsigned pc; @@ -181,12 +181,13 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev) if (r) return r; - p = &hdmi.cfg.timings; + vm = &hdmi.cfg.vm; - DSSDBG("hdmi_power_on x_res= %d y_res = %d\n", p->x_res, p->y_res); + DSSDBG("hdmi_power_on hactive= %d vactive = %d\n", vm->hactive, + vm->vactive); - pc = p->pixelclock; - if (p->double_pixel) + pc = vm->pixelclock; + if (vm->flags & DISPLAY_FLAGS_DOUBLECLK) pc *= 2; /* DSS_HDMI_TCLK is bitclk / 10 */ @@ -226,7 +227,7 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev) hdmi5_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg); /* tv size */ - dss_mgr_set_timings(channel, p); + dss_mgr_set_timings(channel, vm); r = dss_mgr_enable(channel); if (r) @@ -272,30 +273,30 @@ static void hdmi_power_off_full(struct omap_dss_device *dssdev) } static int hdmi_display_check_timing(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - if (!dispc_mgr_timings_ok(dssdev->dispc_channel, timings)) + if (!dispc_mgr_timings_ok(dssdev->dispc_channel, vm)) return -EINVAL; return 0; } static void hdmi_display_set_timing(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { mutex_lock(&hdmi.lock); - hdmi.cfg.timings = *timings; + hdmi.cfg.vm = *vm; - dispc_set_tv_pclk(timings->pixelclock); + dispc_set_tv_pclk(vm->pixelclock); mutex_unlock(&hdmi.lock); } static void hdmi_display_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - *timings = hdmi.cfg.timings; + *vm = hdmi.cfg.vm; } static void hdmi_dump_regs(struct seq_file *s) @@ -378,7 +379,7 @@ static int hdmi_display_enable(struct omap_dss_device *dssdev) if (hdmi.audio_configured) { r = hdmi5_audio_config(&hdmi.core, &hdmi.wp, &hdmi.audio_config, - hdmi.cfg.timings.pixelclock); + hdmi.cfg.vm.pixelclock); if (r) { DSSERR("Error restoring audio configuration: %d", r); hdmi.audio_abort_cb(&hdmi.pdev->dev); @@ -669,7 +670,7 @@ static int hdmi_audio_config(struct device *dev, } ret = hdmi5_audio_config(&hd->core, &hd->wp, dss_audio, - hd->cfg.timings.pixelclock); + hd->cfg.vm.pixelclock); if (!ret) { hd->audio_configured = true; diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c index 8ab2093daa12..8de1d7b2ae55 100644 --- a/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c +++ b/drivers/gpu/drm/omapdrm/dss/hdmi5_core.c @@ -292,35 +292,35 @@ static void hdmi_core_init(struct hdmi_core_vid_config *video_cfg, { DSSDBG("hdmi_core_init\n"); - video_cfg->v_fc_config.timings = cfg->timings; + video_cfg->v_fc_config.vm = cfg->vm; /* video core */ video_cfg->data_enable_pol = 1; /* It is always 1*/ - video_cfg->hblank = cfg->timings.hfp + - cfg->timings.hbp + cfg->timings.hsw; + video_cfg->hblank = cfg->vm.hfront_porch + + cfg->vm.hback_porch + cfg->vm.hsync_len; video_cfg->vblank_osc = 0; - video_cfg->vblank = cfg->timings.vsw + - cfg->timings.vfp + cfg->timings.vbp; + video_cfg->vblank = cfg->vm.vsync_len + cfg->vm.vfront_porch + + cfg->vm.vback_porch; video_cfg->v_fc_config.hdmi_dvi_mode = cfg->hdmi_dvi_mode; - if (cfg->timings.interlace) { + if (cfg->vm.flags & DISPLAY_FLAGS_INTERLACED) { /* set vblank_osc if vblank is fractional */ if (video_cfg->vblank % 2 != 0) video_cfg->vblank_osc = 1; - video_cfg->v_fc_config.timings.y_res /= 2; + video_cfg->v_fc_config.vm.vactive /= 2; video_cfg->vblank /= 2; - video_cfg->v_fc_config.timings.vfp /= 2; - video_cfg->v_fc_config.timings.vsw /= 2; - video_cfg->v_fc_config.timings.vbp /= 2; + video_cfg->v_fc_config.vm.vfront_porch /= 2; + video_cfg->v_fc_config.vm.vsync_len /= 2; + video_cfg->v_fc_config.vm.vback_porch /= 2; } - if (cfg->timings.double_pixel) { - video_cfg->v_fc_config.timings.x_res *= 2; + if (cfg->vm.flags & DISPLAY_FLAGS_DOUBLECLK) { + video_cfg->v_fc_config.vm.hactive *= 2; video_cfg->hblank *= 2; - video_cfg->v_fc_config.timings.hfp *= 2; - video_cfg->v_fc_config.timings.hsw *= 2; - video_cfg->v_fc_config.timings.hbp *= 2; + video_cfg->v_fc_config.vm.hfront_porch *= 2; + video_cfg->v_fc_config.vm.hsync_len *= 2; + video_cfg->v_fc_config.vm.hback_porch *= 2; } } @@ -329,13 +329,12 @@ static void hdmi_core_video_config(struct hdmi_core_data *core, struct hdmi_core_vid_config *cfg) { void __iomem *base = core->base; + struct videomode *vm = &cfg->v_fc_config.vm; unsigned char r = 0; bool vsync_pol, hsync_pol; - vsync_pol = - cfg->v_fc_config.timings.vsync_level == OMAPDSS_SIG_ACTIVE_HIGH; - hsync_pol = - cfg->v_fc_config.timings.hsync_level == OMAPDSS_SIG_ACTIVE_HIGH; + vsync_pol = !!(vm->flags & DISPLAY_FLAGS_VSYNC_HIGH); + hsync_pol = !!(vm->flags & DISPLAY_FLAGS_HSYNC_HIGH); /* Set hsync, vsync and data-enable polarity */ r = hdmi_read_reg(base, HDMI_CORE_FC_INVIDCONF); @@ -343,20 +342,16 @@ static void hdmi_core_video_config(struct hdmi_core_data *core, r = FLD_MOD(r, hsync_pol, 5, 5); r = FLD_MOD(r, cfg->data_enable_pol, 4, 4); r = FLD_MOD(r, cfg->vblank_osc, 1, 1); - r = FLD_MOD(r, cfg->v_fc_config.timings.interlace, 0, 0); + r = FLD_MOD(r, !!(vm->flags & DISPLAY_FLAGS_INTERLACED), 0, 0); hdmi_write_reg(base, HDMI_CORE_FC_INVIDCONF, r); /* set x resolution */ - REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV1, - cfg->v_fc_config.timings.x_res >> 8, 4, 0); - REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV0, - cfg->v_fc_config.timings.x_res & 0xFF, 7, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV1, vm->hactive >> 8, 4, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_INHACTIV0, vm->hactive & 0xFF, 7, 0); /* set y resolution */ - REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV1, - cfg->v_fc_config.timings.y_res >> 8, 4, 0); - REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV0, - cfg->v_fc_config.timings.y_res & 0xFF, 7, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV1, vm->vactive >> 8, 4, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_INVACTIV0, vm->vactive & 0xFF, 7, 0); /* set horizontal blanking pixels */ REG_FLD_MOD(base, HDMI_CORE_FC_INHBLANK1, cfg->hblank >> 8, 4, 0); @@ -366,30 +361,28 @@ static void hdmi_core_video_config(struct hdmi_core_data *core, REG_FLD_MOD(base, HDMI_CORE_FC_INVBLANK, cfg->vblank, 7, 0); /* set horizontal sync offset */ - REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY1, - cfg->v_fc_config.timings.hfp >> 8, 4, 0); - REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY0, - cfg->v_fc_config.timings.hfp & 0xFF, 7, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY1, vm->hfront_porch >> 8, + 4, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINDELAY0, vm->hfront_porch & 0xFF, + 7, 0); /* set vertical sync offset */ - REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINDELAY, - cfg->v_fc_config.timings.vfp, 7, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINDELAY, vm->vfront_porch, 7, 0); /* set horizontal sync pulse width */ - REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH1, - (cfg->v_fc_config.timings.hsw >> 8), 1, 0); - REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH0, - cfg->v_fc_config.timings.hsw & 0xFF, 7, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH1, (vm->hsync_len >> 8), + 1, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_HSYNCINWIDTH0, vm->hsync_len & 0xFF, + 7, 0); /* set vertical sync pulse width */ - REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINWIDTH, - cfg->v_fc_config.timings.vsw, 5, 0); + REG_FLD_MOD(base, HDMI_CORE_FC_VSYNCINWIDTH, vm->vsync_len, 5, 0); /* select DVI mode */ REG_FLD_MOD(base, HDMI_CORE_FC_INVIDCONF, - cfg->v_fc_config.hdmi_dvi_mode, 3, 3); + cfg->v_fc_config.hdmi_dvi_mode, 3, 3); - if (cfg->v_fc_config.timings.double_pixel) + if (vm->flags & DISPLAY_FLAGS_DOUBLECLK) REG_FLD_MOD(base, HDMI_CORE_FC_PRCONF, 2, 7, 4); else REG_FLD_MOD(base, HDMI_CORE_FC_PRCONF, 1, 7, 4); @@ -616,7 +609,7 @@ int hdmi5_core_handle_irqs(struct hdmi_core_data *core) void hdmi5_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp, struct hdmi_config *cfg) { - struct omap_video_timings video_timing; + struct videomode vm; struct hdmi_video_format video_format; struct hdmi_core_vid_config v_core_cfg; @@ -624,16 +617,16 @@ void hdmi5_configure(struct hdmi_core_data *core, struct hdmi_wp_data *wp, hdmi_core_init(&v_core_cfg, cfg); - hdmi_wp_init_vid_fmt_timings(&video_format, &video_timing, cfg); + hdmi_wp_init_vid_fmt_timings(&video_format, &vm, cfg); - hdmi_wp_video_config_timing(wp, &video_timing); + hdmi_wp_video_config_timing(wp, &vm); /* video config */ video_format.packing_mode = HDMI_PACK_24b_RGB_YUV444_YUV422; hdmi_wp_video_config_format(wp, &video_format); - hdmi_wp_video_config_interface(wp, &video_timing); + hdmi_wp_video_config_interface(wp, &vm); /* support limited range with 24 bit color depth for now */ hdmi_core_configure_range(core); diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c index 203694a52d18..b783d5a0750e 100644 --- a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c +++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c @@ -144,87 +144,84 @@ void hdmi_wp_video_config_format(struct hdmi_wp_data *wp, } void hdmi_wp_video_config_interface(struct hdmi_wp_data *wp, - struct omap_video_timings *timings) + struct videomode *vm) { u32 r; bool vsync_pol, hsync_pol; DSSDBG("Enter hdmi_wp_video_config_interface\n"); - vsync_pol = timings->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH; - hsync_pol = timings->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH; + vsync_pol = !!(vm->flags & DISPLAY_FLAGS_VSYNC_HIGH); + hsync_pol = !!(vm->flags & DISPLAY_FLAGS_HSYNC_HIGH); r = hdmi_read_reg(wp->base, HDMI_WP_VIDEO_CFG); r = FLD_MOD(r, vsync_pol, 7, 7); r = FLD_MOD(r, hsync_pol, 6, 6); - r = FLD_MOD(r, timings->interlace, 3, 3); + r = FLD_MOD(r, !!(vm->flags & DISPLAY_FLAGS_INTERLACED), 3, 3); r = FLD_MOD(r, 1, 1, 0); /* HDMI_TIMING_MASTER_24BIT */ hdmi_write_reg(wp->base, HDMI_WP_VIDEO_CFG, r); } void hdmi_wp_video_config_timing(struct hdmi_wp_data *wp, - struct omap_video_timings *timings) + struct videomode *vm) { u32 timing_h = 0; u32 timing_v = 0; - unsigned hsw_offset = 1; + unsigned hsync_len_offset = 1; DSSDBG("Enter hdmi_wp_video_config_timing\n"); /* * On OMAP4 and OMAP5 ES1 the HSW field is programmed as is. On OMAP5 - * ES2+ (including DRA7/AM5 SoCs) HSW field is programmed to hsw-1. + * ES2+ (including DRA7/AM5 SoCs) HSW field is programmed to hsync_len-1. * However, we don't support OMAP5 ES1 at all, so we can just check for * OMAP4 here. */ if (omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES1 || omapdss_get_version() == OMAPDSS_VER_OMAP4430_ES2 || omapdss_get_version() == OMAPDSS_VER_OMAP4) - hsw_offset = 0; + hsync_len_offset = 0; - timing_h |= FLD_VAL(timings->hbp, 31, 20); - timing_h |= FLD_VAL(timings->hfp, 19, 8); - timing_h |= FLD_VAL(timings->hsw - hsw_offset, 7, 0); + timing_h |= FLD_VAL(vm->hback_porch, 31, 20); + timing_h |= FLD_VAL(vm->hfront_porch, 19, 8); + timing_h |= FLD_VAL(vm->hsync_len - hsync_len_offset, 7, 0); hdmi_write_reg(wp->base, HDMI_WP_VIDEO_TIMING_H, timing_h); - timing_v |= FLD_VAL(timings->vbp, 31, 20); - timing_v |= FLD_VAL(timings->vfp, 19, 8); - timing_v |= FLD_VAL(timings->vsw, 7, 0); + timing_v |= FLD_VAL(vm->vback_porch, 31, 20); + timing_v |= FLD_VAL(vm->vfront_porch, 19, 8); + timing_v |= FLD_VAL(vm->vsync_len, 7, 0); hdmi_write_reg(wp->base, HDMI_WP_VIDEO_TIMING_V, timing_v); } void hdmi_wp_init_vid_fmt_timings(struct hdmi_video_format *video_fmt, - struct omap_video_timings *timings, struct hdmi_config *param) + struct videomode *vm, struct hdmi_config *param) { DSSDBG("Enter hdmi_wp_video_init_format\n"); video_fmt->packing_mode = HDMI_PACK_10b_RGB_YUV444; - video_fmt->y_res = param->timings.y_res; - video_fmt->x_res = param->timings.x_res; - - timings->hbp = param->timings.hbp; - timings->hfp = param->timings.hfp; - timings->hsw = param->timings.hsw; - timings->vbp = param->timings.vbp; - timings->vfp = param->timings.vfp; - timings->vsw = param->timings.vsw; - - timings->vsync_level = param->timings.vsync_level; - timings->hsync_level = param->timings.hsync_level; - timings->interlace = param->timings.interlace; - timings->double_pixel = param->timings.double_pixel; - - if (param->timings.interlace) { + video_fmt->y_res = param->vm.vactive; + video_fmt->x_res = param->vm.hactive; + + vm->hback_porch = param->vm.hback_porch; + vm->hfront_porch = param->vm.hfront_porch; + vm->hsync_len = param->vm.hsync_len; + vm->vback_porch = param->vm.vback_porch; + vm->vfront_porch = param->vm.vfront_porch; + vm->vsync_len = param->vm.vsync_len; + + vm->flags = param->vm.flags; + + if (param->vm.flags & DISPLAY_FLAGS_INTERLACED) { video_fmt->y_res /= 2; - timings->vbp /= 2; - timings->vfp /= 2; - timings->vsw /= 2; + vm->vback_porch /= 2; + vm->vfront_porch /= 2; + vm->vsync_len /= 2; } - if (param->timings.double_pixel) { + if (param->vm.flags & DISPLAY_FLAGS_DOUBLECLK) { video_fmt->x_res *= 2; - timings->hfp *= 2; - timings->hsw *= 2; - timings->hbp *= 2; + vm->hfront_porch *= 2; + vm->hsync_len *= 2; + vm->hback_porch *= 2; } } diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h index 6eaf1adbd606..b420dde8c0fb 100644 --- a/drivers/gpu/drm/omapdrm/dss/omapdss.h +++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h @@ -290,7 +290,7 @@ struct omap_dss_dsi_videomode_timings { struct omap_dss_dsi_config { enum omap_dss_dsi_mode mode; enum omap_dss_dsi_pixel_format pixel_format; - const struct omap_video_timings *timings; + const struct videomode *vm; unsigned long hs_clk_min, hs_clk_max; unsigned long lp_clk_min, lp_clk_max; @@ -299,48 +299,12 @@ struct omap_dss_dsi_config { enum omap_dss_dsi_trans_mode trans_mode; }; -struct omap_video_timings { - /* Unit: pixels */ - u16 x_res; - /* Unit: pixels */ - u16 y_res; - /* Unit: Hz */ - u32 pixelclock; - /* Unit: pixel clocks */ - u16 hsw; /* Horizontal synchronization pulse width */ - /* Unit: pixel clocks */ - u16 hfp; /* Horizontal front porch */ - /* Unit: pixel clocks */ - u16 hbp; /* Horizontal back porch */ - /* Unit: line clocks */ - u16 vsw; /* Vertical synchronization pulse width */ - /* Unit: line clocks */ - u16 vfp; /* Vertical front porch */ - /* Unit: line clocks */ - u16 vbp; /* Vertical back porch */ - - /* Vsync logic level */ - enum omap_dss_signal_level vsync_level; - /* Hsync logic level */ - enum omap_dss_signal_level hsync_level; - /* Interlaced or Progressive timings */ - bool interlace; - /* Pixel clock edge to drive LCD data */ - enum omap_dss_signal_edge data_pclk_edge; - /* Data enable logic level */ - enum omap_dss_signal_level de_level; - /* Pixel clock edges to drive HSYNC and VSYNC signals */ - enum omap_dss_signal_edge sync_pclk_edge; - - bool double_pixel; -}; - -/* Hardcoded timings for tv modes. Venc only uses these to +/* Hardcoded videomodes for tv. Venc only uses these to * identify the mode, and does not actually use the configs * itself. However, the configs should be something that * a normal monitor can also show */ -extern const struct omap_video_timings omap_dss_pal_timings; -extern const struct omap_video_timings omap_dss_ntsc_timings; +extern const struct videomode omap_dss_pal_vm; +extern const struct videomode omap_dss_ntsc_vm; struct omap_dss_cpr_coefs { s16 rr, rg, rb; @@ -502,11 +466,11 @@ struct omapdss_dpi_ops { void (*disable)(struct omap_dss_device *dssdev); int (*check_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*get_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_data_lines)(struct omap_dss_device *dssdev, int data_lines); }; @@ -521,11 +485,11 @@ struct omapdss_sdi_ops { void (*disable)(struct omap_dss_device *dssdev); int (*check_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*get_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_datapairs)(struct omap_dss_device *dssdev, int datapairs); }; @@ -540,11 +504,11 @@ struct omapdss_dvi_ops { void (*disable)(struct omap_dss_device *dssdev); int (*check_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*get_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); }; struct omapdss_atv_ops { @@ -557,11 +521,11 @@ struct omapdss_atv_ops { void (*disable)(struct omap_dss_device *dssdev); int (*check_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*get_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_type)(struct omap_dss_device *dssdev, enum omap_dss_venc_type type); @@ -582,11 +546,11 @@ struct omapdss_hdmi_ops { void (*disable)(struct omap_dss_device *dssdev); int (*check_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*get_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len); bool (*detect)(struct omap_dss_device *dssdev); @@ -692,7 +656,7 @@ struct omap_dss_device { } phy; struct { - struct omap_video_timings timings; + struct videomode vm; enum omap_dss_dsi_pixel_format dsi_pix_fmt; enum omap_dss_dsi_mode dsi_mode; @@ -785,11 +749,11 @@ struct omap_dss_driver { int (*get_recommended_bpp)(struct omap_dss_device *dssdev); int (*check_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*set_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); void (*get_timings)(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); int (*set_wss)(struct omap_dss_device *dssdev, u32 wss); u32 (*get_wss)(struct omap_dss_device *dssdev); @@ -819,11 +783,6 @@ struct omap_dss_device *omap_dss_find_device(void *data, int (*match)(struct omap_dss_device *dssdev, void *data)); const char *omapdss_get_default_display_name(void); -void videomode_to_omap_video_timings(const struct videomode *vm, - struct omap_video_timings *ovt); -void omap_video_timings_to_videomode(const struct omap_video_timings *ovt, - struct videomode *vm); - int dss_feat_get_num_mgrs(void); int dss_feat_get_num_ovls(void); enum omap_color_mode dss_feat_get_supported_color_modes(enum omap_plane plane); @@ -852,7 +811,7 @@ void omapdss_default_get_resolution(struct omap_dss_device *dssdev, u16 *xres, u16 *yres); int omapdss_default_get_recommended_bpp(struct omap_dss_device *dssdev); void omapdss_default_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings); + struct videomode *vm); typedef void (*omap_dispc_isr_t) (void *arg, u32 mask); int omap_dispc_register_isr(omap_dispc_isr_t isr, void *arg, u32 mask); @@ -906,7 +865,7 @@ void dispc_mgr_go(enum omap_channel channel); void dispc_mgr_set_lcd_config(enum omap_channel channel, const struct dss_lcd_mgr_config *config); void dispc_mgr_set_timings(enum omap_channel channel, - const struct omap_video_timings *timings); + const struct videomode *vm); void dispc_mgr_setup(enum omap_channel channel, const struct omap_overlay_manager_info *info); u32 dispc_mgr_gamma_size(enum omap_channel channel); @@ -919,8 +878,7 @@ bool dispc_ovl_enabled(enum omap_plane plane); void dispc_ovl_set_channel_out(enum omap_plane plane, enum omap_channel channel); int dispc_ovl_setup(enum omap_plane plane, const struct omap_overlay_info *oi, - bool replication, const struct omap_video_timings *mgr_timings, - bool mem_to_mem); + bool replication, const struct videomode *vm, bool mem_to_mem); enum omap_dss_output_id dispc_mgr_get_supported_outputs(enum omap_channel channel); @@ -934,7 +892,7 @@ struct dss_mgr_ops { int (*enable)(enum omap_channel channel); void (*disable)(enum omap_channel channel); void (*set_timings)(enum omap_channel channel, - const struct omap_video_timings *timings); + const struct videomode *vm); void (*set_lcd_config)(enum omap_channel channel, const struct dss_lcd_mgr_config *config); int (*register_framedone_handler)(enum omap_channel channel, @@ -951,7 +909,7 @@ int dss_mgr_connect(enum omap_channel channel, void dss_mgr_disconnect(enum omap_channel channel, struct omap_dss_device *dst); void dss_mgr_set_timings(enum omap_channel channel, - const struct omap_video_timings *timings); + const struct videomode *vm); void dss_mgr_set_lcd_config(enum omap_channel channel, const struct dss_lcd_mgr_config *config); int dss_mgr_enable(enum omap_channel channel); diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c index 24f859488201..a901af5a9bc3 100644 --- a/drivers/gpu/drm/omapdrm/dss/output.c +++ b/drivers/gpu/drm/omapdrm/dss/output.c @@ -201,10 +201,9 @@ void dss_mgr_disconnect(enum omap_channel channel, } EXPORT_SYMBOL(dss_mgr_disconnect); -void dss_mgr_set_timings(enum omap_channel channel, - const struct omap_video_timings *timings) +void dss_mgr_set_timings(enum omap_channel channel, const struct videomode *vm) { - dss_mgr_ops->set_timings(channel, timings); + dss_mgr_ops->set_timings(channel, vm); } EXPORT_SYMBOL(dss_mgr_set_timings); diff --git a/drivers/gpu/drm/omapdrm/dss/rfbi.c b/drivers/gpu/drm/omapdrm/dss/rfbi.c index cd53566d75eb..09724757366a 100644 --- a/drivers/gpu/drm/omapdrm/dss/rfbi.c +++ b/drivers/gpu/drm/omapdrm/dss/rfbi.c @@ -113,7 +113,7 @@ static struct { struct semaphore bus_lock; - struct omap_video_timings timings; + struct videomode vm; int pixel_size; int data_lines; struct rfbi_timings intf_timings; @@ -308,15 +308,15 @@ static int rfbi_transfer_area(struct omap_dss_device *dssdev, u32 l; int r; struct omap_overlay_manager *mgr = rfbi.output.manager; - u16 width = rfbi.timings.x_res; - u16 height = rfbi.timings.y_res; + u16 width = rfbi.vm.hactive; + u16 height = rfbi.vm.vactive; /*BUG_ON(callback == 0);*/ BUG_ON(rfbi.framedone_callback != NULL); DSSDBG("rfbi_transfer_area %dx%d\n", width, height); - dss_mgr_set_timings(mgr, &rfbi.timings); + dss_mgr_set_timings(mgr, &rfbi.vm); r = dss_mgr_enable(mgr); if (r) @@ -777,8 +777,8 @@ static int rfbi_update(struct omap_dss_device *dssdev, void (*callback)(void *), static void rfbi_set_size(struct omap_dss_device *dssdev, u16 w, u16 h) { - rfbi.timings.x_res = w; - rfbi.timings.y_res = h; + rfbi.vm.hactive = w; + rfbi.vm.vactive = h; } static void rfbi_set_pixel_size(struct omap_dss_device *dssdev, int pixel_size) @@ -854,25 +854,30 @@ static void rfbi_config_lcd_manager(struct omap_dss_device *dssdev) dss_mgr_set_lcd_config(mgr, &mgr_config); /* - * Set rfbi.timings with default values, the x_res and y_res fields + * Set rfbi.timings with default values, the hactive and vactive fields * are expected to be already configured by the panel driver via * omapdss_rfbi_set_size() */ - rfbi.timings.hsw = 1; - rfbi.timings.hfp = 1; - rfbi.timings.hbp = 1; - rfbi.timings.vsw = 1; - rfbi.timings.vfp = 0; - rfbi.timings.vbp = 0; - - rfbi.timings.interlace = false; - rfbi.timings.hsync_level = OMAPDSS_SIG_ACTIVE_HIGH; - rfbi.timings.vsync_level = OMAPDSS_SIG_ACTIVE_HIGH; - rfbi.timings.data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; - rfbi.timings.de_level = OMAPDSS_SIG_ACTIVE_HIGH; - rfbi.timings.sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE; - - dss_mgr_set_timings(mgr, &rfbi.timings); + rfbi.vm.hsync_len = 1; + rfbi.vm.hfront_porch = 1; + rfbi.vm.hback_porch = 1; + rfbi.vm.vsync_len = 1; + rfbi.vm.vfront_porch = 0; + rfbi.vm.vback_porch = 0; + + rfbi.vm.flags &= ~DISPLAY_FLAGS_INTERLACED; + rfbi.vm.flags &= ~DISPLAY_FLAGS_HSYNC_LOW; + rfbi.vm.flags |= DISPLAY_FLAGS_HSYNC_HIGH; + rfbi.vm.flags &= ~DISPLAY_FLAGS_VSYNC_LOW; + rfbi.vm.flags |= DISPLAY_FLAGS_VSYNC_HIGH; + rfbi.vm.flags &= ~DISPLAY_FLAGS_PIXDATA_NEGEDGE; + rfbi.vm.flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE; + rfbi.vm.flags &= ~DISPLAY_FLAGS_DE_LOW; + rfbi.vm.flags |= DISPLAY_FLAGS_DE_HIGH; + rfbi.vm.flags &= ~DISPLAY_FLAGS_SYNC_POSEDGE; + rfbi.vm.flags |= DISPLAY_FLAGS_SYNC_NEGEDGE; + + dss_mgr_set_timings(mgr, &rfbi.vm); } static int rfbi_display_enable(struct omap_dss_device *dssdev) diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c index 0a96c321ce62..b3bda2d3c08d 100644 --- a/drivers/gpu/drm/omapdrm/dss/sdi.c +++ b/drivers/gpu/drm/omapdrm/dss/sdi.c @@ -39,7 +39,7 @@ static struct { struct regulator *vdds_sdi_reg; struct dss_lcd_mgr_config mgr_config; - struct omap_video_timings timings; + struct videomode vm; int datapairs; struct omap_dss_device output; @@ -131,7 +131,7 @@ static int sdi_display_enable(struct omap_dss_device *dssdev) { struct omap_dss_device *out = &sdi.output; enum omap_channel channel = dssdev->dispc_channel; - struct omap_video_timings *t = &sdi.timings; + struct videomode *vm = &sdi.vm; unsigned long fck; struct dispc_clock_info dispc_cinfo; unsigned long pck; @@ -151,10 +151,9 @@ static int sdi_display_enable(struct omap_dss_device *dssdev) goto err_get_dispc; /* 15.5.9.1.2 */ - t->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; - t->sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; + vm->flags |= DISPLAY_FLAGS_PIXDATA_POSEDGE | DISPLAY_FLAGS_SYNC_POSEDGE; - r = sdi_calc_clock_div(t->pixelclock, &fck, &dispc_cinfo); + r = sdi_calc_clock_div(vm->pixelclock, &fck, &dispc_cinfo); if (r) goto err_calc_clock_div; @@ -162,15 +161,15 @@ static int sdi_display_enable(struct omap_dss_device *dssdev) pck = fck / dispc_cinfo.lck_div / dispc_cinfo.pck_div; - if (pck != t->pixelclock) { - DSSWARN("Could not find exact pixel clock. Requested %d Hz, got %lu Hz\n", - t->pixelclock, pck); + if (pck != vm->pixelclock) { + DSSWARN("Could not find exact pixel clock. Requested %lu Hz, got %lu Hz\n", + vm->pixelclock, pck); - t->pixelclock = pck; + vm->pixelclock = pck; } - dss_mgr_set_timings(channel, t); + dss_mgr_set_timings(channel, vm); r = dss_set_fck_rate(fck); if (r) @@ -229,26 +228,26 @@ static void sdi_display_disable(struct omap_dss_device *dssdev) } static void sdi_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - sdi.timings = *timings; + sdi.vm = *vm; } static void sdi_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { - *timings = sdi.timings; + *vm = sdi.vm; } static int sdi_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { enum omap_channel channel = dssdev->dispc_channel; - if (!dispc_mgr_timings_ok(channel, timings)) + if (!dispc_mgr_timings_ok(channel, vm)) return -EINVAL; - if (timings->pixelclock == 0) + if (vm->pixelclock == 0) return -EINVAL; return 0; diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c index 6eedf2118708..d74f7fcc2e46 100644 --- a/drivers/gpu/drm/omapdrm/dss/venc.c +++ b/drivers/gpu/drm/omapdrm/dss/venc.c @@ -262,47 +262,41 @@ static const struct venc_config venc_config_pal_bdghi = { .fid_ext_start_y__fid_ext_offset_y = 0x01380005, }; -const struct omap_video_timings omap_dss_pal_timings = { - .x_res = 720, - .y_res = 574, +const struct videomode omap_dss_pal_vm = { + .hactive = 720, + .vactive = 574, .pixelclock = 13500000, - .hsw = 64, - .hfp = 12, - .hbp = 68, - .vsw = 5, - .vfp = 5, - .vbp = 41, - - .interlace = true, - - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, + .hsync_len = 64, + .hfront_porch = 12, + .hback_porch = 68, + .vsync_len = 5, + .vfront_porch = 5, + .vback_porch = 41, + + .flags = DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW | + DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH | + DISPLAY_FLAGS_PIXDATA_POSEDGE | + DISPLAY_FLAGS_SYNC_NEGEDGE, }; -EXPORT_SYMBOL(omap_dss_pal_timings); +EXPORT_SYMBOL(omap_dss_pal_vm); -const struct omap_video_timings omap_dss_ntsc_timings = { - .x_res = 720, - .y_res = 482, +const struct videomode omap_dss_ntsc_vm = { + .hactive = 720, + .vactive = 482, .pixelclock = 13500000, - .hsw = 64, - .hfp = 16, - .hbp = 58, - .vsw = 6, - .vfp = 6, - .vbp = 31, - - .interlace = true, - - .hsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .vsync_level = OMAPDSS_SIG_ACTIVE_LOW, - .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE, - .de_level = OMAPDSS_SIG_ACTIVE_HIGH, - .sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE, + .hsync_len = 64, + .hfront_porch = 16, + .hback_porch = 58, + .vsync_len = 6, + .vfront_porch = 6, + .vback_porch = 31, + + .flags = DISPLAY_FLAGS_INTERLACED | DISPLAY_FLAGS_HSYNC_LOW | + DISPLAY_FLAGS_VSYNC_LOW | DISPLAY_FLAGS_DE_HIGH | + DISPLAY_FLAGS_PIXDATA_POSEDGE | + DISPLAY_FLAGS_SYNC_NEGEDGE, }; -EXPORT_SYMBOL(omap_dss_ntsc_timings); +EXPORT_SYMBOL(omap_dss_ntsc_vm); static struct { struct platform_device *pdev; @@ -313,7 +307,7 @@ static struct { struct clk *tv_dac_clk; - struct omap_video_timings timings; + struct videomode vm; enum omap_dss_venc_type type; bool invert_polarity; @@ -427,13 +421,12 @@ static void venc_runtime_put(void) WARN_ON(r < 0 && r != -ENOSYS); } -static const struct venc_config *venc_timings_to_config( - struct omap_video_timings *timings) +static const struct venc_config *venc_timings_to_config(struct videomode *vm) { - if (memcmp(&omap_dss_pal_timings, timings, sizeof(*timings)) == 0) + if (memcmp(&omap_dss_pal_vm, vm, sizeof(*vm)) == 0) return &venc_config_pal_trm; - if (memcmp(&omap_dss_ntsc_timings, timings, sizeof(*timings)) == 0) + if (memcmp(&omap_dss_ntsc_vm, vm, sizeof(*vm)) == 0) return &venc_config_ntsc_trm; BUG(); @@ -451,7 +444,7 @@ static int venc_power_on(struct omap_dss_device *dssdev) goto err0; venc_reset(); - venc_write_config(venc_timings_to_config(&venc.timings)); + venc_write_config(venc_timings_to_config(&venc.vm)); dss_set_venc_output(venc.type); dss_set_dac_pwrdn_bgz(1); @@ -468,7 +461,7 @@ static int venc_power_on(struct omap_dss_device *dssdev) venc_write_reg(VENC_OUTPUT_CONTROL, l); - dss_mgr_set_timings(channel, &venc.timings); + dss_mgr_set_timings(channel, &venc.vm); r = regulator_enable(venc.vdda_dac_reg); if (r) @@ -546,17 +539,17 @@ static void venc_display_disable(struct omap_dss_device *dssdev) } static void venc_set_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { DSSDBG("venc_set_timings\n"); mutex_lock(&venc.venc_lock); /* Reset WSS data when the TV standard changes. */ - if (memcmp(&venc.timings, timings, sizeof(*timings))) + if (memcmp(&venc.vm, vm, sizeof(*vm))) venc.wss_data = 0; - venc.timings = *timings; + venc.vm = *vm; dispc_set_tv_pclk(13500000); @@ -564,25 +557,25 @@ static void venc_set_timings(struct omap_dss_device *dssdev, } static int venc_check_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { DSSDBG("venc_check_timings\n"); - if (memcmp(&omap_dss_pal_timings, timings, sizeof(*timings)) == 0) + if (memcmp(&omap_dss_pal_vm, vm, sizeof(*vm)) == 0) return 0; - if (memcmp(&omap_dss_ntsc_timings, timings, sizeof(*timings)) == 0) + if (memcmp(&omap_dss_ntsc_vm, vm, sizeof(*vm)) == 0) return 0; return -EINVAL; } static void venc_get_timings(struct omap_dss_device *dssdev, - struct omap_video_timings *timings) + struct videomode *vm) { mutex_lock(&venc.venc_lock); - *timings = venc.timings; + *vm = venc.vm; mutex_unlock(&venc.venc_lock); } @@ -602,7 +595,7 @@ static int venc_set_wss(struct omap_dss_device *dssdev, u32 wss) mutex_lock(&venc.venc_lock); - config = venc_timings_to_config(&venc.timings); + config = venc_timings_to_config(&venc.vm); /* Invert due to VENC_L21_WC_CTL:INV=1 */ venc.wss_data = (wss ^ 0xfffff) << 8; diff --git a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c index 137fe690a0da..2580e8673908 100644 --- a/drivers/gpu/drm/omapdrm/omap_connector.c +++ b/drivers/gpu/drm/omapdrm/omap_connector.c @@ -42,73 +42,6 @@ bool omap_connector_get_hdmi_mode(struct drm_connector *connector) return omap_connector->hdmi_mode; } -void copy_timings_omap_to_drm(struct drm_display_mode *mode, - struct omap_video_timings *timings) -{ - mode->clock = timings->pixelclock / 1000; - - mode->hdisplay = timings->x_res; - mode->hsync_start = mode->hdisplay + timings->hfp; - mode->hsync_end = mode->hsync_start + timings->hsw; - mode->htotal = mode->hsync_end + timings->hbp; - - mode->vdisplay = timings->y_res; - mode->vsync_start = mode->vdisplay + timings->vfp; - mode->vsync_end = mode->vsync_start + timings->vsw; - mode->vtotal = mode->vsync_end + timings->vbp; - - mode->flags = 0; - - if (timings->interlace) - mode->flags |= DRM_MODE_FLAG_INTERLACE; - - if (timings->double_pixel) - mode->flags |= DRM_MODE_FLAG_DBLCLK; - - if (timings->hsync_level == OMAPDSS_SIG_ACTIVE_HIGH) - mode->flags |= DRM_MODE_FLAG_PHSYNC; - else - mode->flags |= DRM_MODE_FLAG_NHSYNC; - - if (timings->vsync_level == OMAPDSS_SIG_ACTIVE_HIGH) - mode->flags |= DRM_MODE_FLAG_PVSYNC; - else - mode->flags |= DRM_MODE_FLAG_NVSYNC; -} - -void copy_timings_drm_to_omap(struct omap_video_timings *timings, - struct drm_display_mode *mode) -{ - timings->pixelclock = mode->clock * 1000; - - timings->x_res = mode->hdisplay; - timings->hfp = mode->hsync_start - mode->hdisplay; - timings->hsw = mode->hsync_end - mode->hsync_start; - timings->hbp = mode->htotal - mode->hsync_end; - - timings->y_res = mode->vdisplay; - timings->vfp = mode->vsync_start - mode->vdisplay; - timings->vsw = mode->vsync_end - mode->vsync_start; - timings->vbp = mode->vtotal - mode->vsync_end; - - timings->interlace = !!(mode->flags & DRM_MODE_FLAG_INTERLACE); - timings->double_pixel = !!(mode->flags & DRM_MODE_FLAG_DBLCLK); - - if (mode->flags & DRM_MODE_FLAG_PHSYNC) - timings->hsync_level = OMAPDSS_SIG_ACTIVE_HIGH; - else - timings->hsync_level = OMAPDSS_SIG_ACTIVE_LOW; - - if (mode->flags & DRM_MODE_FLAG_PVSYNC) - timings->vsync_level = OMAPDSS_SIG_ACTIVE_HIGH; - else - timings->vsync_level = OMAPDSS_SIG_ACTIVE_LOW; - - timings->data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE; - timings->de_level = OMAPDSS_SIG_ACTIVE_HIGH; - timings->sync_pclk_edge = OMAPDSS_DRIVE_SIG_FALLING_EDGE; -} - static enum drm_connector_status omap_connector_detect( struct drm_connector *connector, bool force) { @@ -185,11 +118,11 @@ static int omap_connector_get_modes(struct drm_connector *connector) kfree(edid); } else { struct drm_display_mode *mode = drm_mode_create(dev); - struct omap_video_timings timings = {0}; + struct videomode vm = {0}; - dssdrv->get_timings(dssdev, &timings); + dssdrv->get_timings(dssdev, &vm); - copy_timings_omap_to_drm(mode, &timings); + drm_display_mode_from_videomode(&vm, mode); mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED; drm_mode_set_name(mode); @@ -207,12 +140,14 @@ static int omap_connector_mode_valid(struct drm_connector *connector, struct omap_connector *omap_connector = to_omap_connector(connector); struct omap_dss_device *dssdev = omap_connector->dssdev; struct omap_dss_driver *dssdrv = dssdev->driver; - struct omap_video_timings timings = {0}; + struct videomode vm = {0}; struct drm_device *dev = connector->dev; struct drm_display_mode *new_mode; int r, ret = MODE_BAD; - copy_timings_drm_to_omap(&timings, mode); + drm_display_mode_to_videomode(mode, &vm); + vm.flags |= DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE | + DISPLAY_FLAGS_SYNC_NEGEDGE; mode->vrefresh = drm_mode_vrefresh(mode); /* @@ -221,13 +156,13 @@ static int omap_connector_mode_valid(struct drm_connector *connector, * panel's timings */ if (dssdrv->check_timings) { - r = dssdrv->check_timings(dssdev, &timings); + r = dssdrv->check_timings(dssdev, &vm); } else { - struct omap_video_timings t = {0}; + struct videomode t = {0}; dssdrv->get_timings(dssdev, &t); - if (memcmp(&timings, &t, sizeof(struct omap_video_timings))) + if (memcmp(&vm, &t, sizeof(struct videomode))) r = -EINVAL; else r = 0; @@ -236,7 +171,7 @@ static int omap_connector_mode_valid(struct drm_connector *connector, if (!r) { /* check if vrefresh is still valid */ new_mode = drm_mode_duplicate(dev, mode); - new_mode->clock = timings.pixelclock / 1000; + new_mode->clock = vm.pixelclock / 1000; new_mode->vrefresh = 0; if (mode->vrefresh == drm_mode_vrefresh(new_mode)) ret = MODE_OK; diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c index 16c691dbc372..8dea89030e66 100644 --- a/drivers/gpu/drm/omapdrm/omap_crtc.c +++ b/drivers/gpu/drm/omapdrm/omap_crtc.c @@ -34,7 +34,7 @@ struct omap_crtc { const char *name; enum omap_channel channel; - struct omap_video_timings timings; + struct videomode vm; struct omap_drm_irq vblank_irq; struct omap_drm_irq error_irq; @@ -56,10 +56,10 @@ uint32_t pipe2vbl(struct drm_crtc *crtc) return dispc_mgr_get_vsync_irq(omap_crtc->channel); } -struct omap_video_timings *omap_crtc_timings(struct drm_crtc *crtc) +struct videomode *omap_crtc_timings(struct drm_crtc *crtc) { struct omap_crtc *omap_crtc = to_omap_crtc(crtc); - return &omap_crtc->timings; + return &omap_crtc->vm; } enum omap_channel omap_crtc_channel(struct drm_crtc *crtc) @@ -201,7 +201,7 @@ static int omap_crtc_dss_enable(enum omap_channel channel) dispc_mgr_setup(omap_crtc->channel, &info); dispc_mgr_set_timings(omap_crtc->channel, - &omap_crtc->timings); + &omap_crtc->vm); omap_crtc_set_enabled(&omap_crtc->base, true); return 0; @@ -215,11 +215,11 @@ static void omap_crtc_dss_disable(enum omap_channel channel) } static void omap_crtc_dss_set_timings(enum omap_channel channel, - const struct omap_video_timings *timings) + const struct videomode *vm) { struct omap_crtc *omap_crtc = omap_crtcs[channel]; DBG("%s", omap_crtc->name); - omap_crtc->timings = *timings; + omap_crtc->vm = *vm; } static void omap_crtc_dss_set_lcd_config(enum omap_channel channel, @@ -369,7 +369,10 @@ static void omap_crtc_mode_set_nofb(struct drm_crtc *crtc) mode->vdisplay, mode->vsync_start, mode->vsync_end, mode->vtotal, mode->type, mode->flags); - copy_timings_drm_to_omap(&omap_crtc->timings, mode); + drm_display_mode_to_videomode(mode, &omap_crtc->vm); + omap_crtc->vm.flags |= DISPLAY_FLAGS_DE_HIGH | + DISPLAY_FLAGS_PIXDATA_POSEDGE | + DISPLAY_FLAGS_SYNC_NEGEDGE; } static int omap_crtc_atomic_check(struct drm_crtc *crtc, @@ -411,19 +414,6 @@ static void omap_crtc_atomic_flush(struct drm_crtc *crtc, dispc_mgr_set_gamma(omap_crtc->channel, lut, length); } - if (crtc->state->color_mgmt_changed) { - struct drm_color_lut *lut = NULL; - uint length = 0; - - if (crtc->state->gamma_lut) { - lut = (struct drm_color_lut *) - crtc->state->gamma_lut->data; - length = crtc->state->gamma_lut->length / - sizeof(*lut); - } - dispc_mgr_set_gamma(omap_crtc->channel, lut, length); - } - if (dispc_mgr_is_enabled(omap_crtc->channel)) { DBG("%s: GO", omap_crtc->name); diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h index dcc30a98b9d4..4c51135eb9a6 100644 --- a/drivers/gpu/drm/omapdrm/omap_drv.h +++ b/drivers/gpu/drm/omapdrm/omap_drv.h @@ -148,7 +148,7 @@ static inline void omap_fbdev_free(struct drm_device *dev) } #endif -struct omap_video_timings *omap_crtc_timings(struct drm_crtc *crtc); +struct videomode *omap_crtc_timings(struct drm_crtc *crtc); enum omap_channel omap_crtc_channel(struct drm_crtc *crtc); void omap_crtc_pre_init(void); void omap_crtc_pre_uninit(void); @@ -171,11 +171,6 @@ struct drm_encoder *omap_connector_attached_encoder( struct drm_connector *connector); bool omap_connector_get_hdmi_mode(struct drm_connector *connector); -void copy_timings_omap_to_drm(struct drm_display_mode *mode, - struct omap_video_timings *timings); -void copy_timings_drm_to_omap(struct omap_video_timings *timings, - struct drm_display_mode *mode); - uint32_t omap_framebuffer_get_formats(uint32_t *pixel_formats, uint32_t max_formats, enum omap_color_mode supported_modes); struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev, diff --git a/drivers/gpu/drm/omapdrm/omap_encoder.c b/drivers/gpu/drm/omapdrm/omap_encoder.c index 0bbb9c59622e..a20f30039aee 100644 --- a/drivers/gpu/drm/omapdrm/omap_encoder.c +++ b/drivers/gpu/drm/omapdrm/omap_encoder.c @@ -102,7 +102,7 @@ static void omap_encoder_disable(struct drm_encoder *encoder) static int omap_encoder_update(struct drm_encoder *encoder, enum omap_channel channel, - struct omap_video_timings *timings) + struct videomode *vm) { struct drm_device *dev = encoder->dev; struct omap_encoder *omap_encoder = to_omap_encoder(encoder); @@ -111,13 +111,13 @@ static int omap_encoder_update(struct drm_encoder *encoder, int ret; if (dssdrv->check_timings) { - ret = dssdrv->check_timings(dssdev, timings); + ret = dssdrv->check_timings(dssdev, vm); } else { - struct omap_video_timings t = {0}; + struct videomode t = {0}; dssdrv->get_timings(dssdev, &t); - if (memcmp(timings, &t, sizeof(struct omap_video_timings))) + if (memcmp(vm, &t, sizeof(struct videomode))) ret = -EINVAL; else ret = 0; @@ -129,7 +129,7 @@ static int omap_encoder_update(struct drm_encoder *encoder, } if (dssdrv->set_timings) - dssdrv->set_timings(dssdev, timings); + dssdrv->set_timings(dssdev, vm); return 0; } diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c index 505dee0db973..d4e1e11466f8 100644 --- a/drivers/gpu/drm/omapdrm/omap_gem.c +++ b/drivers/gpu/drm/omapdrm/omap_gem.c @@ -336,8 +336,10 @@ static void omap_gem_detach_pages(struct drm_gem_object *obj) if (omap_obj->flags & (OMAP_BO_WC|OMAP_BO_UNCACHED)) { int i, npages = obj->size >> PAGE_SHIFT; for (i = 0; i < npages; i++) { - dma_unmap_page(obj->dev->dev, omap_obj->addrs[i], - PAGE_SIZE, DMA_BIDIRECTIONAL); + if (omap_obj->addrs[i]) + dma_unmap_page(obj->dev->dev, + omap_obj->addrs[i], + PAGE_SIZE, DMA_BIDIRECTIONAL); } } diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c index 0ffd5b930ec0..9c43cb481e62 100644 --- a/drivers/gpu/drm/omapdrm/omap_plane.c +++ b/drivers/gpu/drm/omapdrm/omap_plane.c @@ -131,7 +131,9 @@ static void omap_plane_atomic_update(struct drm_plane *plane, /* and finally, update omapdss: */ ret = dispc_ovl_setup(omap_plane->id, &info, false, omap_crtc_timings(state->crtc), false); - if (WARN_ON(ret)) { + if (ret) { + dev_err(plane->dev->dev, "Failed to setup plane %s\n", + omap_plane->name); dispc_ovl_enable(omap_plane->id, false); return; } @@ -157,12 +159,20 @@ static int omap_plane_atomic_check(struct drm_plane *plane, { struct drm_crtc_state *crtc_state; - if (!state->crtc) + if (!state->fb) return 0; - crtc_state = drm_atomic_get_crtc_state(state->state, state->crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); + /* crtc should only be NULL when disabling (i.e., !state->fb) */ + if (WARN_ON(!state->crtc)) + return 0; + + crtc_state = drm_atomic_get_existing_crtc_state(state->state, state->crtc); + /* we should have a crtc state if the plane is attached to a crtc */ + if (WARN_ON(!crtc_state)) + return 0; + + if (!crtc_state->enable) + return 0; if (state->crtc_x < 0 || state->crtc_y < 0) return -EINVAL; @@ -173,11 +183,9 @@ static int omap_plane_atomic_check(struct drm_plane *plane, if (state->crtc_y + state->crtc_h > crtc_state->adjusted_mode.vdisplay) return -EINVAL; - if (state->fb) { - if (state->rotation != DRM_ROTATE_0 && - !omap_framebuffer_supports_rotation(state->fb)) - return -EINVAL; - } + if (state->rotation != DRM_ROTATE_0 && + !omap_framebuffer_supports_rotation(state->fb)) + return -EINVAL; return 0; } diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c index 103fc8650197..a0d4a0522fdc 100644 --- a/drivers/gpu/drm/radeon/ni.c +++ b/drivers/gpu/drm/radeon/ni.c @@ -1396,9 +1396,7 @@ static void cayman_pcie_gart_fini(struct radeon_device *rdev) void cayman_cp_int_cntl_setup(struct radeon_device *rdev, int ring, u32 cp_int_cntl) { - u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3; - - WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3)); + WREG32(SRBM_GFX_CNTL, RINGID(ring)); WREG32(CP_INT_CNTL, cp_int_cntl); } diff --git a/drivers/gpu/drm/radeon/radeon_dp_auxch.c b/drivers/gpu/drm/radeon/radeon_dp_auxch.c index 2d465648856a..474a8a1886f7 100644 --- a/drivers/gpu/drm/radeon/radeon_dp_auxch.c +++ b/drivers/gpu/drm/radeon/radeon_dp_auxch.c @@ -105,7 +105,7 @@ radeon_dp_aux_transfer_native(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg tmp &= AUX_HPD_SEL(0x7); tmp |= AUX_HPD_SEL(chan->rec.hpd); - tmp |= AUX_EN | AUX_LS_READ_EN | AUX_HPD_DISCON(0x1); + tmp |= AUX_EN | AUX_LS_READ_EN; WREG32(AUX_CONTROL + aux_offset[instance], tmp); diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c index be30861afae9..41b72ce6613f 100644 --- a/drivers/gpu/drm/radeon/radeon_object.c +++ b/drivers/gpu/drm/radeon/radeon_object.c @@ -446,6 +446,10 @@ void radeon_bo_force_delete(struct radeon_device *rdev) int radeon_bo_init(struct radeon_device *rdev) { + /* reserve PAT memory space to WC for VRAM */ + arch_io_reserve_memtype_wc(rdev->mc.aper_base, + rdev->mc.aper_size); + /* Add an MTRR for the VRAM */ if (!rdev->fastfb_working) { rdev->mc.vram_mtrr = arch_phys_wc_add(rdev->mc.aper_base, @@ -463,6 +467,7 @@ void radeon_bo_fini(struct radeon_device *rdev) { radeon_ttm_fini(rdev); arch_phys_wc_del(rdev->mc.vram_mtrr); + arch_io_free_memtype_wc(rdev->mc.aper_base, rdev->mc.aper_size); } /* Returns how many bytes TTM can move per IB. diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c index 89bdf20344ae..c49934527a87 100644 --- a/drivers/gpu/drm/radeon/si_dpm.c +++ b/drivers/gpu/drm/radeon/si_dpm.c @@ -2999,6 +2999,49 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, int i; struct si_dpm_quirk *p = si_dpm_quirk_list; + /* limit all SI kickers */ + if (rdev->family == CHIP_PITCAIRN) { + if ((rdev->pdev->revision == 0x81) || + (rdev->pdev->device == 0x6810) || + (rdev->pdev->device == 0x6811) || + (rdev->pdev->device == 0x6816) || + (rdev->pdev->device == 0x6817) || + (rdev->pdev->device == 0x6806)) + max_mclk = 120000; + } else if (rdev->family == CHIP_VERDE) { + if ((rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0x87) || + (rdev->pdev->device == 0x6820) || + (rdev->pdev->device == 0x6821) || + (rdev->pdev->device == 0x6822) || + (rdev->pdev->device == 0x6823) || + (rdev->pdev->device == 0x682A) || + (rdev->pdev->device == 0x682B)) { + max_sclk = 75000; + max_mclk = 80000; + } + } else if (rdev->family == CHIP_OLAND) { + if ((rdev->pdev->revision == 0xC7) || + (rdev->pdev->revision == 0x80) || + (rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83) || + (rdev->pdev->device == 0x6604) || + (rdev->pdev->device == 0x6605)) { + max_sclk = 75000; + max_mclk = 80000; + } + } else if (rdev->family == CHIP_HAINAN) { + if ((rdev->pdev->revision == 0x81) || + (rdev->pdev->revision == 0x83) || + (rdev->pdev->revision == 0xC3) || + (rdev->pdev->device == 0x6664) || + (rdev->pdev->device == 0x6665) || + (rdev->pdev->device == 0x6667)) { + max_sclk = 75000; + max_mclk = 80000; + } + } /* Apply dpm quirks */ while (p && p->chip_device != 0) { if (rdev->pdev->vendor == p->chip_vendor && @@ -3011,16 +3054,6 @@ static void si_apply_state_adjust_rules(struct radeon_device *rdev, } ++p; } - /* limit mclk on all R7 370 parts for stability */ - if (rdev->pdev->device == 0x6811 && - rdev->pdev->revision == 0x81) - max_mclk = 120000; - /* limit sclk/mclk on Jet parts for stability */ - if (rdev->pdev->device == 0x6665 && - rdev->pdev->revision == 0xc3) { - max_sclk = 75000; - max_mclk = 80000; - } if (rps->vce_active) { rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk; diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c index c76ed9ee6a01..4220d621f55f 100644 --- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c +++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c @@ -231,8 +231,16 @@ static int rcar_du_atomic_check(struct drm_device *dev, struct rcar_du_device *rcdu = dev->dev_private; int ret; - ret = drm_atomic_helper_check(dev, state); - if (ret < 0) + ret = drm_atomic_helper_check_modeset(dev, state); + if (ret) + return ret; + + ret = drm_atomic_normalize_zpos(dev, state); + if (ret) + return ret; + + ret = drm_atomic_helper_check_planes(dev, state); + if (ret) return ret; if (rcar_du_has(rcdu, RCAR_DU_FEATURE_VSP1_SOURCE)) diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c index 80ecee9d52b8..ff71e25ab5bf 100644 --- a/drivers/gpu/drm/sti/sti_drv.c +++ b/drivers/gpu/drm/sti/sti_drv.c @@ -196,6 +196,26 @@ static void sti_atomic_work(struct work_struct *work) sti_atomic_complete(private, private->commit.state); } +static int sti_atomic_check(struct drm_device *dev, + struct drm_atomic_state *state) +{ + int ret; + + ret = drm_atomic_helper_check_modeset(dev, state); + if (ret) + return ret; + + ret = drm_atomic_normalize_zpos(dev, state); + if (ret) + return ret; + + ret = drm_atomic_helper_check_planes(dev, state); + if (ret) + return ret; + + return ret; +} + static int sti_atomic_commit(struct drm_device *drm, struct drm_atomic_state *state, bool nonblock) { @@ -250,7 +270,7 @@ static void sti_output_poll_changed(struct drm_device *ddev) static const struct drm_mode_config_funcs sti_mode_config_funcs = { .fb_create = drm_fb_cma_create, .output_poll_changed = sti_output_poll_changed, - .atomic_check = drm_atomic_helper_check, + .atomic_check = sti_atomic_check, .atomic_commit = sti_atomic_commit, }; diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c index 7cf3678623c3..58048709c34e 100644 --- a/drivers/gpu/drm/virtio/virtgpu_display.c +++ b/drivers/gpu/drm/virtio/virtgpu_display.c @@ -338,8 +338,7 @@ static void vgdev_atomic_commit_tail(struct drm_atomic_state *state) drm_atomic_helper_commit_modeset_disables(dev, state); drm_atomic_helper_commit_modeset_enables(dev, state); - drm_atomic_helper_commit_planes(dev, state, - DRM_PLANE_COMMIT_ACTIVE_ONLY); + drm_atomic_helper_commit_planes(dev, state, 0); drm_atomic_helper_commit_hw_done(state); diff --git a/drivers/gpu/ipu-v3/ipu-image-convert.c b/drivers/gpu/ipu-v3/ipu-image-convert.c index 2ba7d437a2af..805b6fa7b5f4 100644 --- a/drivers/gpu/ipu-v3/ipu-image-convert.c +++ b/drivers/gpu/ipu-v3/ipu-image-convert.c @@ -1617,7 +1617,7 @@ ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task, ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode, complete, complete_context); if (IS_ERR(ctx)) - return ERR_PTR(PTR_ERR(ctx)); + return ERR_CAST(ctx); run = kzalloc(sizeof(*run), GFP_KERNEL); if (!run) { diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c index 4aa3cb63fd41..bcd06306f3e8 100644 --- a/drivers/hv/hv_util.c +++ b/drivers/hv/hv_util.c @@ -314,10 +314,14 @@ static void heartbeat_onchannelcallback(void *context) u8 *hbeat_txf_buf = util_heartbeat.recv_buffer; struct icmsg_negotiate *negop = NULL; - vmbus_recvpacket(channel, hbeat_txf_buf, - PAGE_SIZE, &recvlen, &requestid); + while (1) { + + vmbus_recvpacket(channel, hbeat_txf_buf, + PAGE_SIZE, &recvlen, &requestid); + + if (!recvlen) + break; - if (recvlen > 0) { icmsghdrp = (struct icmsg_hdr *)&hbeat_txf_buf[ sizeof(struct vmbuspipe_hdr)]; diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 6d94e2ec5b4f..d252276feadf 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -79,12 +79,12 @@ config I2C_AMD8111 config I2C_HIX5HD2 tristate "Hix5hd2 high-speed I2C driver" - depends on ARCH_HIX5HD2 || COMPILE_TEST + depends on ARCH_HISI || ARCH_HIX5HD2 || COMPILE_TEST help - Say Y here to include support for high-speed I2C controller in the - Hisilicon based hix5hd2 SoCs. + Say Y here to include support for the high-speed I2C controller + used in HiSilicon hix5hd2 SoCs. - This driver can also be built as a module. If so, the module + This driver can also be built as a module. If so, the module will be called i2c-hix5hd2. config I2C_I801 @@ -589,10 +589,10 @@ config I2C_IMG config I2C_IMX tristate "IMX I2C interface" - depends on ARCH_MXC || ARCH_LAYERSCAPE + depends on ARCH_MXC || ARCH_LAYERSCAPE || COLDFIRE help Say Y here if you want to use the IIC bus controller on - the Freescale i.MX/MXC or Layerscape processors. + the Freescale i.MX/MXC, Layerscape or ColdFire processors. This driver can also be built as a module. If so, the module will be called i2c-imx. diff --git a/drivers/i2c/busses/i2c-designware-core.c b/drivers/i2c/busses/i2c-designware-core.c index 1fe93c43215c..11e866d05368 100644 --- a/drivers/i2c/busses/i2c-designware-core.c +++ b/drivers/i2c/busses/i2c-designware-core.c @@ -95,6 +95,9 @@ #define DW_IC_STATUS_TFE BIT(2) #define DW_IC_STATUS_MST_ACTIVITY BIT(5) +#define DW_IC_SDA_HOLD_RX_SHIFT 16 +#define DW_IC_SDA_HOLD_RX_MASK GENMASK(23, DW_IC_SDA_HOLD_RX_SHIFT) + #define DW_IC_ERR_TX_ABRT 0x1 #define DW_IC_TAR_10BITADDR_MASTER BIT(12) @@ -420,12 +423,20 @@ int i2c_dw_init(struct dw_i2c_dev *dev) /* Configure SDA Hold Time if required */ reg = dw_readl(dev, DW_IC_COMP_VERSION); if (reg >= DW_IC_SDA_HOLD_MIN_VERS) { - if (dev->sda_hold_time) { - dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD); - } else { + if (!dev->sda_hold_time) { /* Keep previous hold time setting if no one set it */ dev->sda_hold_time = dw_readl(dev, DW_IC_SDA_HOLD); } + /* + * Workaround for avoiding TX arbitration lost in case I2C + * slave pulls SDA down "too quickly" after falling egde of + * SCL by enabling non-zero SDA RX hold. Specification says it + * extends incoming SDA low to high transition while SCL is + * high but it apprears to help also above issue. + */ + if (!(dev->sda_hold_time & DW_IC_SDA_HOLD_RX_MASK)) + dev->sda_hold_time |= 1 << DW_IC_SDA_HOLD_RX_SHIFT; + dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD); } else { dev_warn(dev->dev, "Hardware too old to adjust SDA hold time.\n"); diff --git a/drivers/i2c/busses/i2c-digicolor.c b/drivers/i2c/busses/i2c-digicolor.c index 9604024e0eb0..49f2084f7bb5 100644 --- a/drivers/i2c/busses/i2c-digicolor.c +++ b/drivers/i2c/busses/i2c-digicolor.c @@ -368,6 +368,7 @@ static const struct of_device_id dc_i2c_match[] = { { .compatible = "cnxt,cx92755-i2c" }, { }, }; +MODULE_DEVICE_TABLE(of, dc_i2c_match); static struct platform_driver dc_i2c_driver = { .probe = dc_i2c_probe, diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 08847e8b8998..eb3627f35d12 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -146,6 +146,7 @@ #define SMBHSTCFG_HST_EN 1 #define SMBHSTCFG_SMB_SMI_EN 2 #define SMBHSTCFG_I2C_EN 4 +#define SMBHSTCFG_SPD_WD 0x10 /* TCO configuration bits for TCOCTL */ #define TCOCTL_EN 0x0100 @@ -865,9 +866,16 @@ static s32 i801_access(struct i2c_adapter *adap, u16 addr, block = 1; break; case I2C_SMBUS_I2C_BLOCK_DATA: - /* NB: page 240 of ICH5 datasheet shows that the R/#W - * bit should be cleared here, even when reading */ - outb_p((addr & 0x7f) << 1, SMBHSTADD(priv)); + /* + * NB: page 240 of ICH5 datasheet shows that the R/#W + * bit should be cleared here, even when reading. + * However if SPD Write Disable is set (Lynx Point and later), + * the read will fail if we don't set the R/#W bit. + */ + outb_p(((addr & 0x7f) << 1) | + ((priv->original_hstcfg & SMBHSTCFG_SPD_WD) ? + (read_write & 0x01) : 0), + SMBHSTADD(priv)); if (read_write == I2C_SMBUS_READ) { /* NB: page 240 of ICH5 datasheet also shows * that DATA1 is the cmd field when reading */ @@ -1573,6 +1581,8 @@ static int i801_probe(struct pci_dev *dev, const struct pci_device_id *id) /* Disable SMBus interrupt feature if SMBus using SMI# */ priv->features &= ~FEATURE_IRQ; } + if (temp & SMBHSTCFG_SPD_WD) + dev_info(&dev->dev, "SPD Write Disable is set\n"); /* Clear special mode bits */ if (priv->features & (FEATURE_SMBUS_PEC | FEATURE_BLOCK_BUFFER)) diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index 592a8f26a708..47fc1f1acff7 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -1009,10 +1009,13 @@ static int i2c_imx_init_recovery_info(struct imx_i2c_struct *i2c_imx, rinfo->sda_gpio = of_get_named_gpio(pdev->dev.of_node, "sda-gpios", 0); rinfo->scl_gpio = of_get_named_gpio(pdev->dev.of_node, "scl-gpios", 0); - if (!gpio_is_valid(rinfo->sda_gpio) || - !gpio_is_valid(rinfo->scl_gpio) || - IS_ERR(i2c_imx->pinctrl_pins_default) || - IS_ERR(i2c_imx->pinctrl_pins_gpio)) { + if (rinfo->sda_gpio == -EPROBE_DEFER || + rinfo->scl_gpio == -EPROBE_DEFER) { + return -EPROBE_DEFER; + } else if (!gpio_is_valid(rinfo->sda_gpio) || + !gpio_is_valid(rinfo->scl_gpio) || + IS_ERR(i2c_imx->pinctrl_pins_default) || + IS_ERR(i2c_imx->pinctrl_pins_gpio)) { dev_dbg(&pdev->dev, "recovery information incomplete\n"); return 0; } diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index b8ea62105f42..30132c3957cd 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -729,6 +729,7 @@ static const struct of_device_id jz4780_i2c_of_matches[] = { { .compatible = "ingenic,jz4780-i2c", }, { /* sentinel */ } }; +MODULE_DEVICE_TABLE(of, jz4780_i2c_of_matches); static int jz4780_i2c_probe(struct platform_device *pdev) { diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 50702c7bb244..df220666d627 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -694,6 +694,8 @@ static int rk3x_i2c_v0_calc_timings(unsigned long clk_rate, t_calc->div_low--; t_calc->div_high--; + /* Give the tuning value 0, that would not update con register */ + t_calc->tuning = 0; /* Maximum divider supported by hw is 0xffff */ if (t_calc->div_low > 0xffff) { t_calc->div_low = 0xffff; diff --git a/drivers/i2c/busses/i2c-xgene-slimpro.c b/drivers/i2c/busses/i2c-xgene-slimpro.c index 263685c7a512..05cf192ef1ac 100644 --- a/drivers/i2c/busses/i2c-xgene-slimpro.c +++ b/drivers/i2c/busses/i2c-xgene-slimpro.c @@ -105,7 +105,7 @@ struct slimpro_i2c_dev { struct mbox_chan *mbox_chan; struct mbox_client mbox_client; struct completion rd_complete; - u8 dma_buffer[I2C_SMBUS_BLOCK_MAX]; + u8 dma_buffer[I2C_SMBUS_BLOCK_MAX + 1]; /* dma_buffer[0] is used for length */ u32 *resp_msg; }; diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c index 2a972ed7aa0d..e29ff37a43bd 100644 --- a/drivers/i2c/busses/i2c-xlp9xx.c +++ b/drivers/i2c/busses/i2c-xlp9xx.c @@ -426,6 +426,7 @@ static const struct of_device_id xlp9xx_i2c_of_match[] = { { .compatible = "netlogic,xlp980-i2c", }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, xlp9xx_i2c_of_match); #ifdef CONFIG_ACPI static const struct acpi_device_id xlp9xx_i2c_acpi_ids[] = { diff --git a/drivers/i2c/busses/i2c-xlr.c b/drivers/i2c/busses/i2c-xlr.c index 0968f59b6df5..ad17d88d8573 100644 --- a/drivers/i2c/busses/i2c-xlr.c +++ b/drivers/i2c/busses/i2c-xlr.c @@ -358,6 +358,7 @@ static const struct of_device_id xlr_i2c_dt_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, xlr_i2c_dt_ids); static int xlr_i2c_probe(struct platform_device *pdev) { diff --git a/drivers/i2c/i2c-core.c b/drivers/i2c/i2c-core.c index 5ab67219f71e..b432b64e307a 100644 --- a/drivers/i2c/i2c-core.c +++ b/drivers/i2c/i2c-core.c @@ -1681,6 +1681,7 @@ static struct i2c_client *of_i2c_register_device(struct i2c_adapter *adap, static void of_i2c_register_devices(struct i2c_adapter *adap) { struct device_node *bus, *node; + struct i2c_client *client; /* Only register child devices if the adapter has a node pointer set */ if (!adap->dev.of_node) @@ -1695,7 +1696,14 @@ static void of_i2c_register_devices(struct i2c_adapter *adap) for_each_available_child_of_node(bus, node) { if (of_node_test_and_set_flag(node, OF_POPULATED)) continue; - of_i2c_register_device(adap, node); + + client = of_i2c_register_device(adap, node); + if (IS_ERR(client)) { + dev_warn(&adap->dev, + "Failed to create I2C device for %s\n", + node->full_name); + of_node_clear_flag(node, OF_POPULATED); + } } of_node_put(bus); @@ -2171,6 +2179,7 @@ int i2c_register_driver(struct module *owner, struct i2c_driver *driver) /* add the driver to the list of i2c drivers in the driver core */ driver->driver.owner = owner; driver->driver.bus = &i2c_bus_type; + INIT_LIST_HEAD(&driver->clients); /* When registration returns, the driver core * will have called probe() for all matching-but-unbound devices. @@ -2181,7 +2190,6 @@ int i2c_register_driver(struct module *owner, struct i2c_driver *driver) pr_debug("driver [%s] registered\n", driver->driver.name); - INIT_LIST_HEAD(&driver->clients); /* Walk the adapters that are already present */ i2c_for_each_dev(driver, __process_new_driver); @@ -2299,6 +2307,7 @@ static int of_i2c_notify(struct notifier_block *nb, unsigned long action, if (IS_ERR(client)) { dev_err(&adap->dev, "failed to create client for '%s'\n", rd->dn->full_name); + of_node_clear_flag(rd->dn, OF_POPULATED); return notifier_from_errno(PTR_ERR(client)); } break; diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index 7edcf3238620..99c051490eff 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -437,6 +437,8 @@ config STX104 config TI_ADC081C tristate "Texas Instruments ADC081C/ADC101C/ADC121C family" depends on I2C + select IIO_BUFFER + select IIO_TRIGGERED_BUFFER help If you say yes here you get support for Texas Instruments ADC081C, ADC101C and ADC121C ADC chips. diff --git a/drivers/iio/chemical/atlas-ph-sensor.c b/drivers/iio/chemical/atlas-ph-sensor.c index bd321b305a0a..ef761a508630 100644 --- a/drivers/iio/chemical/atlas-ph-sensor.c +++ b/drivers/iio/chemical/atlas-ph-sensor.c @@ -213,13 +213,14 @@ static int atlas_check_ec_calibration(struct atlas_data *data) struct device *dev = &data->client->dev; int ret; unsigned int val; + __be16 rval; - ret = regmap_bulk_read(data->regmap, ATLAS_REG_EC_PROBE, &val, 2); + ret = regmap_bulk_read(data->regmap, ATLAS_REG_EC_PROBE, &rval, 2); if (ret) return ret; - dev_info(dev, "probe set to K = %d.%.2d", be16_to_cpu(val) / 100, - be16_to_cpu(val) % 100); + val = be16_to_cpu(rval); + dev_info(dev, "probe set to K = %d.%.2d", val / 100, val % 100); ret = regmap_read(data->regmap, ATLAS_REG_EC_CALIB_STATUS, &val); if (ret) diff --git a/drivers/iio/temperature/maxim_thermocouple.c b/drivers/iio/temperature/maxim_thermocouple.c index 39dd2026ccc9..066161a4bccd 100644 --- a/drivers/iio/temperature/maxim_thermocouple.c +++ b/drivers/iio/temperature/maxim_thermocouple.c @@ -123,22 +123,24 @@ static int maxim_thermocouple_read(struct maxim_thermocouple_data *data, { unsigned int storage_bytes = data->chip->read_size; unsigned int shift = chan->scan_type.shift + (chan->address * 8); - unsigned int buf; + __be16 buf16; + __be32 buf32; int ret; - ret = spi_read(data->spi, (void *) &buf, storage_bytes); - if (ret) - return ret; - switch (storage_bytes) { case 2: - *val = be16_to_cpu(buf); + ret = spi_read(data->spi, (void *)&buf16, storage_bytes); + *val = be16_to_cpu(buf16); break; case 4: - *val = be32_to_cpu(buf); + ret = spi_read(data->spi, (void *)&buf32, storage_bytes); + *val = be32_to_cpu(buf32); break; } + if (ret) + return ret; + /* check to be sure this is a valid reading */ if (*val & data->chip->status_bit) return -EINVAL; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 22174774dbb8..63036c731626 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1019,7 +1019,7 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev, resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp); if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf)) resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size); - resp.cache_line_size = L1_CACHE_BYTES; + resp.cache_line_size = cache_line_size(); resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq); resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq); resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz); diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 41f4c2afbcdd..7ce97daf26c6 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -52,7 +52,6 @@ enum { enum { MLX5_IB_SQ_STRIDE = 6, - MLX5_IB_CACHE_LINE_SIZE = 64, }; static const u32 mlx5_ib_opcode[] = { diff --git a/drivers/infiniband/hw/qedr/Kconfig b/drivers/infiniband/hw/qedr/Kconfig index 7c06d85568d4..6c9f3923e838 100644 --- a/drivers/infiniband/hw/qedr/Kconfig +++ b/drivers/infiniband/hw/qedr/Kconfig @@ -2,6 +2,7 @@ config INFINIBAND_QEDR tristate "QLogic RoCE driver" depends on 64BIT && QEDE select QED_LL2 + select QED_RDMA ---help--- This driver provides low-level InfiniBand over Ethernet support for QLogic QED host channel adapters (HCAs). diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h index 7b8d2d9e2263..da12717a3eb7 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib.h +++ b/drivers/infiniband/ulp/ipoib/ipoib.h @@ -63,6 +63,8 @@ enum ipoib_flush_level { enum { IPOIB_ENCAP_LEN = 4, + IPOIB_PSEUDO_LEN = 20, + IPOIB_HARD_LEN = IPOIB_ENCAP_LEN + IPOIB_PSEUDO_LEN, IPOIB_UD_HEAD_SIZE = IB_GRH_BYTES + IPOIB_ENCAP_LEN, IPOIB_UD_RX_SG = 2, /* max buffer needed for 4K mtu */ @@ -134,15 +136,21 @@ struct ipoib_header { u16 reserved; }; -struct ipoib_cb { - struct qdisc_skb_cb qdisc_cb; - u8 hwaddr[INFINIBAND_ALEN]; +struct ipoib_pseudo_header { + u8 hwaddr[INFINIBAND_ALEN]; }; -static inline struct ipoib_cb *ipoib_skb_cb(const struct sk_buff *skb) +static inline void skb_add_pseudo_hdr(struct sk_buff *skb) { - BUILD_BUG_ON(sizeof(skb->cb) < sizeof(struct ipoib_cb)); - return (struct ipoib_cb *)skb->cb; + char *data = skb_push(skb, IPOIB_PSEUDO_LEN); + + /* + * only the ipoib header is present now, make room for a dummy + * pseudo header and set skb field accordingly + */ + memset(data, 0, IPOIB_PSEUDO_LEN); + skb_reset_mac_header(skb); + skb_pull(skb, IPOIB_HARD_LEN); } /* Used for all multicast joins (broadcast, IPv4 mcast and IPv6 mcast) */ diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c index 4ad297d3de89..339a1eecdfe3 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c @@ -63,6 +63,8 @@ MODULE_PARM_DESC(cm_data_debug_level, #define IPOIB_CM_RX_DELAY (3 * 256 * HZ) #define IPOIB_CM_RX_UPDATE_MASK (0x3) +#define IPOIB_CM_RX_RESERVE (ALIGN(IPOIB_HARD_LEN, 16) - IPOIB_ENCAP_LEN) + static struct ib_qp_attr ipoib_cm_err_attr = { .qp_state = IB_QPS_ERR }; @@ -146,15 +148,15 @@ static struct sk_buff *ipoib_cm_alloc_rx_skb(struct net_device *dev, struct sk_buff *skb; int i; - skb = dev_alloc_skb(IPOIB_CM_HEAD_SIZE + 12); + skb = dev_alloc_skb(ALIGN(IPOIB_CM_HEAD_SIZE + IPOIB_PSEUDO_LEN, 16)); if (unlikely(!skb)) return NULL; /* - * IPoIB adds a 4 byte header. So we need 12 more bytes to align the + * IPoIB adds a IPOIB_ENCAP_LEN byte header, this will align the * IP header to a multiple of 16. */ - skb_reserve(skb, 12); + skb_reserve(skb, IPOIB_CM_RX_RESERVE); mapping[0] = ib_dma_map_single(priv->ca, skb->data, IPOIB_CM_HEAD_SIZE, DMA_FROM_DEVICE); @@ -624,9 +626,9 @@ void ipoib_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc) if (wc->byte_len < IPOIB_CM_COPYBREAK) { int dlen = wc->byte_len; - small_skb = dev_alloc_skb(dlen + 12); + small_skb = dev_alloc_skb(dlen + IPOIB_CM_RX_RESERVE); if (small_skb) { - skb_reserve(small_skb, 12); + skb_reserve(small_skb, IPOIB_CM_RX_RESERVE); ib_dma_sync_single_for_cpu(priv->ca, rx_ring[wr_id].mapping[0], dlen, DMA_FROM_DEVICE); skb_copy_from_linear_data(skb, small_skb->data, dlen); @@ -663,8 +665,7 @@ void ipoib_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc) copied: skb->protocol = ((struct ipoib_header *) skb->data)->proto; - skb_reset_mac_header(skb); - skb_pull(skb, IPOIB_ENCAP_LEN); + skb_add_pseudo_hdr(skb); ++dev->stats.rx_packets; dev->stats.rx_bytes += skb->len; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c index be11d5d5b8c1..830fecb6934c 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c @@ -128,16 +128,15 @@ static struct sk_buff *ipoib_alloc_rx_skb(struct net_device *dev, int id) buf_size = IPOIB_UD_BUF_SIZE(priv->max_ib_mtu); - skb = dev_alloc_skb(buf_size + IPOIB_ENCAP_LEN); + skb = dev_alloc_skb(buf_size + IPOIB_HARD_LEN); if (unlikely(!skb)) return NULL; /* - * IB will leave a 40 byte gap for a GRH and IPoIB adds a 4 byte - * header. So we need 4 more bytes to get to 48 and align the - * IP header to a multiple of 16. + * the IP header will be at IPOIP_HARD_LEN + IB_GRH_BYTES, that is + * 64 bytes aligned */ - skb_reserve(skb, 4); + skb_reserve(skb, sizeof(struct ipoib_pseudo_header)); mapping = priv->rx_ring[id].mapping; mapping[0] = ib_dma_map_single(priv->ca, skb->data, buf_size, @@ -253,8 +252,7 @@ static void ipoib_ib_handle_rx_wc(struct net_device *dev, struct ib_wc *wc) skb_pull(skb, IB_GRH_BYTES); skb->protocol = ((struct ipoib_header *) skb->data)->proto; - skb_reset_mac_header(skb); - skb_pull(skb, IPOIB_ENCAP_LEN); + skb_add_pseudo_hdr(skb); ++dev->stats.rx_packets; dev->stats.rx_bytes += skb->len; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 5636fc3da6b8..b58d9dca5c93 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -925,9 +925,12 @@ static void neigh_add_path(struct sk_buff *skb, u8 *daddr, ipoib_neigh_free(neigh); goto err_drop; } - if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) + if (skb_queue_len(&neigh->queue) < + IPOIB_MAX_PATH_REC_QUEUE) { + /* put pseudoheader back on for next time */ + skb_push(skb, IPOIB_PSEUDO_LEN); __skb_queue_tail(&neigh->queue, skb); - else { + } else { ipoib_warn(priv, "queue length limit %d. Packet drop.\n", skb_queue_len(&neigh->queue)); goto err_drop; @@ -964,7 +967,7 @@ err_drop: } static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, - struct ipoib_cb *cb) + struct ipoib_pseudo_header *phdr) { struct ipoib_dev_priv *priv = netdev_priv(dev); struct ipoib_path *path; @@ -972,16 +975,18 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, spin_lock_irqsave(&priv->lock, flags); - path = __path_find(dev, cb->hwaddr + 4); + path = __path_find(dev, phdr->hwaddr + 4); if (!path || !path->valid) { int new_path = 0; if (!path) { - path = path_rec_create(dev, cb->hwaddr + 4); + path = path_rec_create(dev, phdr->hwaddr + 4); new_path = 1; } if (path) { if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) { + /* put pseudoheader back on for next time */ + skb_push(skb, IPOIB_PSEUDO_LEN); __skb_queue_tail(&path->queue, skb); } else { ++dev->stats.tx_dropped; @@ -1009,10 +1014,12 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev, be16_to_cpu(path->pathrec.dlid)); spin_unlock_irqrestore(&priv->lock, flags); - ipoib_send(dev, skb, path->ah, IPOIB_QPN(cb->hwaddr)); + ipoib_send(dev, skb, path->ah, IPOIB_QPN(phdr->hwaddr)); return; } else if ((path->query || !path_rec_start(dev, path)) && skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) { + /* put pseudoheader back on for next time */ + skb_push(skb, IPOIB_PSEUDO_LEN); __skb_queue_tail(&path->queue, skb); } else { ++dev->stats.tx_dropped; @@ -1026,13 +1033,15 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct ipoib_dev_priv *priv = netdev_priv(dev); struct ipoib_neigh *neigh; - struct ipoib_cb *cb = ipoib_skb_cb(skb); + struct ipoib_pseudo_header *phdr; struct ipoib_header *header; unsigned long flags; + phdr = (struct ipoib_pseudo_header *) skb->data; + skb_pull(skb, sizeof(*phdr)); header = (struct ipoib_header *) skb->data; - if (unlikely(cb->hwaddr[4] == 0xff)) { + if (unlikely(phdr->hwaddr[4] == 0xff)) { /* multicast, arrange "if" according to probability */ if ((header->proto != htons(ETH_P_IP)) && (header->proto != htons(ETH_P_IPV6)) && @@ -1045,13 +1054,13 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; } /* Add in the P_Key for multicast*/ - cb->hwaddr[8] = (priv->pkey >> 8) & 0xff; - cb->hwaddr[9] = priv->pkey & 0xff; + phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff; + phdr->hwaddr[9] = priv->pkey & 0xff; - neigh = ipoib_neigh_get(dev, cb->hwaddr); + neigh = ipoib_neigh_get(dev, phdr->hwaddr); if (likely(neigh)) goto send_using_neigh; - ipoib_mcast_send(dev, cb->hwaddr, skb); + ipoib_mcast_send(dev, phdr->hwaddr, skb); return NETDEV_TX_OK; } @@ -1060,16 +1069,16 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev) case htons(ETH_P_IP): case htons(ETH_P_IPV6): case htons(ETH_P_TIPC): - neigh = ipoib_neigh_get(dev, cb->hwaddr); + neigh = ipoib_neigh_get(dev, phdr->hwaddr); if (unlikely(!neigh)) { - neigh_add_path(skb, cb->hwaddr, dev); + neigh_add_path(skb, phdr->hwaddr, dev); return NETDEV_TX_OK; } break; case htons(ETH_P_ARP): case htons(ETH_P_RARP): /* for unicast ARP and RARP should always perform path find */ - unicast_arp_send(skb, dev, cb); + unicast_arp_send(skb, dev, phdr); return NETDEV_TX_OK; default: /* ethertype not supported by IPoIB */ @@ -1086,11 +1095,13 @@ send_using_neigh: goto unref; } } else if (neigh->ah) { - ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(cb->hwaddr)); + ipoib_send(dev, skb, neigh->ah, IPOIB_QPN(phdr->hwaddr)); goto unref; } if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) { + /* put pseudoheader back on for next time */ + skb_push(skb, sizeof(*phdr)); spin_lock_irqsave(&priv->lock, flags); __skb_queue_tail(&neigh->queue, skb); spin_unlock_irqrestore(&priv->lock, flags); @@ -1122,8 +1133,8 @@ static int ipoib_hard_header(struct sk_buff *skb, unsigned short type, const void *daddr, const void *saddr, unsigned len) { + struct ipoib_pseudo_header *phdr; struct ipoib_header *header; - struct ipoib_cb *cb = ipoib_skb_cb(skb); header = (struct ipoib_header *) skb_push(skb, sizeof *header); @@ -1132,12 +1143,13 @@ static int ipoib_hard_header(struct sk_buff *skb, /* * we don't rely on dst_entry structure, always stuff the - * destination address into skb->cb so we can figure out where + * destination address into skb hard header so we can figure out where * to send the packet later. */ - memcpy(cb->hwaddr, daddr, INFINIBAND_ALEN); + phdr = (struct ipoib_pseudo_header *) skb_push(skb, sizeof(*phdr)); + memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN); - return sizeof *header; + return IPOIB_HARD_LEN; } static void ipoib_set_mcast_list(struct net_device *dev) @@ -1759,7 +1771,7 @@ void ipoib_setup(struct net_device *dev) dev->flags |= IFF_BROADCAST | IFF_MULTICAST; - dev->hard_header_len = IPOIB_ENCAP_LEN; + dev->hard_header_len = IPOIB_HARD_LEN; dev->addr_len = INFINIBAND_ALEN; dev->type = ARPHRD_INFINIBAND; dev->tx_queue_len = ipoib_sendq_size * 2; diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c index d3394b6add24..1909dd252c94 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c @@ -796,9 +796,11 @@ void ipoib_mcast_send(struct net_device *dev, u8 *daddr, struct sk_buff *skb) __ipoib_mcast_add(dev, mcast); list_add_tail(&mcast->list, &priv->multicast_list); } - if (skb_queue_len(&mcast->pkt_queue) < IPOIB_MAX_MCAST_QUEUE) + if (skb_queue_len(&mcast->pkt_queue) < IPOIB_MAX_MCAST_QUEUE) { + /* put pseudoheader back on for next time */ + skb_push(skb, sizeof(struct ipoib_pseudo_header)); skb_queue_tail(&mcast->pkt_queue, skb); - else { + } else { ++dev->stats.tx_dropped; dev_kfree_skb_any(skb); } diff --git a/drivers/input/mouse/focaltech.c b/drivers/input/mouse/focaltech.c index 54eceb30ede5..a7d39689bbfb 100644 --- a/drivers/input/mouse/focaltech.c +++ b/drivers/input/mouse/focaltech.c @@ -43,7 +43,7 @@ int focaltech_detect(struct psmouse *psmouse, bool set_properties) if (set_properties) { psmouse->vendor = "FocalTech"; - psmouse->name = "FocalTech Touchpad"; + psmouse->name = "Touchpad"; } return 0; @@ -146,8 +146,8 @@ static void focaltech_report_state(struct psmouse *psmouse) } input_mt_report_pointer_emulation(dev, true); - input_report_key(psmouse->dev, BTN_LEFT, state->pressed); - input_sync(psmouse->dev); + input_report_key(dev, BTN_LEFT, state->pressed); + input_sync(dev); } static void focaltech_process_touch_packet(struct psmouse *psmouse, diff --git a/drivers/input/serio/i8042-x86ia64io.h b/drivers/input/serio/i8042-x86ia64io.h index f4bfb4b2d50a..073246c7d163 100644 --- a/drivers/input/serio/i8042-x86ia64io.h +++ b/drivers/input/serio/i8042-x86ia64io.h @@ -877,6 +877,13 @@ static const struct dmi_system_id __initconst i8042_dmi_kbdreset_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "P34"), }, }, + { + /* Schenker XMG C504 - Elantech touchpad */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "XMG"), + DMI_MATCH(DMI_PRODUCT_NAME, "C504"), + }, + }, { } }; diff --git a/drivers/ipack/ipack.c b/drivers/ipack/ipack.c index c0e7b624ce54..12102448fddd 100644 --- a/drivers/ipack/ipack.c +++ b/drivers/ipack/ipack.c @@ -178,7 +178,7 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, idev->id_vendor, idev->id_device); } -ipack_device_attr(id_format, "0x%hhu\n"); +ipack_device_attr(id_format, "0x%hhx\n"); static DEVICE_ATTR_RO(id); static DEVICE_ATTR_RO(id_device); diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 8abde6b8cedc..6d53810963f7 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -266,7 +266,7 @@ static struct raid_type { {"raid10_offset", "raid10 offset (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_OFFSET}, {"raid10_near", "raid10 near (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_NEAR}, {"raid10", "raid10 (striped mirrors)", 0, 2, 10, ALGORITHM_RAID10_DEFAULT}, - {"raid4", "raid4 (dedicated last parity disk)", 1, 2, 4, ALGORITHM_PARITY_N}, /* raid4 layout = raid5_n */ + {"raid4", "raid4 (dedicated first parity disk)", 1, 2, 5, ALGORITHM_PARITY_0}, /* raid4 layout = raid5_0 */ {"raid5_n", "raid5 (dedicated last parity disk)", 1, 2, 5, ALGORITHM_PARITY_N}, {"raid5_ls", "raid5 (left symmetric)", 1, 2, 5, ALGORITHM_LEFT_SYMMETRIC}, {"raid5_rs", "raid5 (right symmetric)", 1, 2, 5, ALGORITHM_RIGHT_SYMMETRIC}, @@ -2087,11 +2087,11 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) /* * No takeover/reshaping, because we don't have the extended v1.9.0 metadata */ - if (le32_to_cpu(sb->level) != mddev->level) { + if (le32_to_cpu(sb->level) != mddev->new_level) { DMERR("Reshaping/takeover raid sets not yet supported. (raid level/stripes/size change)"); return -EINVAL; } - if (le32_to_cpu(sb->layout) != mddev->layout) { + if (le32_to_cpu(sb->layout) != mddev->new_layout) { DMERR("Reshaping raid sets not yet supported. (raid layout change)"); DMERR(" 0x%X vs 0x%X", le32_to_cpu(sb->layout), mddev->layout); DMERR(" Old layout: %s w/ %d copies", @@ -2102,7 +2102,7 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) raid10_md_layout_to_copies(mddev->layout)); return -EINVAL; } - if (le32_to_cpu(sb->stripe_sectors) != mddev->chunk_sectors) { + if (le32_to_cpu(sb->stripe_sectors) != mddev->new_chunk_sectors) { DMERR("Reshaping raid sets not yet supported. (stripe sectors change)"); return -EINVAL; } @@ -2115,6 +2115,8 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) return -EINVAL; } + DMINFO("Discovered old metadata format; upgrading to extended metadata format"); + /* Table line is checked vs. authoritative superblock */ rs_set_new(rs); } @@ -2258,7 +2260,8 @@ static int super_validate(struct raid_set *rs, struct md_rdev *rdev) if (!mddev->events && super_init_validation(rs, rdev)) return -EINVAL; - if (le32_to_cpu(sb->compat_features) != FEATURE_FLAG_SUPPORTS_V190) { + if (le32_to_cpu(sb->compat_features) && + le32_to_cpu(sb->compat_features) != FEATURE_FLAG_SUPPORTS_V190) { rs->ti->error = "Unable to assemble array: Unknown flag(s) in compatible feature flags"; return -EINVAL; } @@ -3646,7 +3649,7 @@ static void raid_resume(struct dm_target *ti) static struct target_type raid_target = { .name = "raid", - .version = {1, 9, 0}, + .version = {1, 9, 1}, .module = THIS_MODULE, .ctr = raid_ctr, .dtr = raid_dtr, diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index bdf1606f67bc..9a8b71067c6e 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -145,7 +145,6 @@ static void dispatch_bios(void *context, struct bio_list *bio_list) struct dm_raid1_bio_record { struct mirror *m; - /* if details->bi_bdev == NULL, details were not saved */ struct dm_bio_details details; region_t write_region; }; @@ -1200,8 +1199,6 @@ static int mirror_map(struct dm_target *ti, struct bio *bio) struct dm_raid1_bio_record *bio_record = dm_per_bio_data(bio, sizeof(struct dm_raid1_bio_record)); - bio_record->details.bi_bdev = NULL; - if (rw == WRITE) { /* Save region for mirror_end_io() handler */ bio_record->write_region = dm_rh_bio_to_region(ms->rh, bio); @@ -1260,22 +1257,12 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) } if (error == -EOPNOTSUPP) - goto out; + return error; if ((error == -EWOULDBLOCK) && (bio->bi_opf & REQ_RAHEAD)) - goto out; + return error; if (unlikely(error)) { - if (!bio_record->details.bi_bdev) { - /* - * There wasn't enough memory to record necessary - * information for a retry or there was no other - * mirror in-sync. - */ - DMERR_LIMIT("Mirror read failed."); - return -EIO; - } - m = bio_record->m; DMERR("Mirror read failed from %s. Trying alternative device.", @@ -1291,7 +1278,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) bd = &bio_record->details; dm_bio_restore(bd, bio); - bio_record->details.bi_bdev = NULL; + bio->bi_error = 0; queue_bio(ms, bio, rw); return DM_ENDIO_INCOMPLETE; @@ -1299,9 +1286,6 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) DMERR("All replicated volumes dead, failing I/O"); } -out: - bio_record->details.bi_bdev = NULL; - return error; } diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index dc75bea0d541..1d0d2adc050a 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -856,8 +856,11 @@ int dm_old_init_request_queue(struct mapped_device *md) kthread_init_worker(&md->kworker); md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, "kdmwork-%s", dm_device_name(md)); - if (IS_ERR(md->kworker_task)) - return PTR_ERR(md->kworker_task); + if (IS_ERR(md->kworker_task)) { + int error = PTR_ERR(md->kworker_task); + md->kworker_task = NULL; + return error; + } elv_register_queue(md->queue); diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c index 3e407a9cde1f..c4b53b332607 100644 --- a/drivers/md/dm-table.c +++ b/drivers/md/dm-table.c @@ -695,37 +695,32 @@ int dm_table_add_target(struct dm_table *t, const char *type, tgt->type = dm_get_target_type(type); if (!tgt->type) { - DMERR("%s: %s: unknown target type", dm_device_name(t->md), - type); + DMERR("%s: %s: unknown target type", dm_device_name(t->md), type); return -EINVAL; } if (dm_target_needs_singleton(tgt->type)) { if (t->num_targets) { - DMERR("%s: target type %s must appear alone in table", - dm_device_name(t->md), type); - return -EINVAL; + tgt->error = "singleton target type must appear alone in table"; + goto bad; } t->singleton = true; } if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) { - DMERR("%s: target type %s may not be included in read-only tables", - dm_device_name(t->md), type); - return -EINVAL; + tgt->error = "target type may not be included in a read-only table"; + goto bad; } if (t->immutable_target_type) { if (t->immutable_target_type != tgt->type) { - DMERR("%s: immutable target type %s cannot be mixed with other target types", - dm_device_name(t->md), t->immutable_target_type->name); - return -EINVAL; + tgt->error = "immutable target type cannot be mixed with other target types"; + goto bad; } } else if (dm_target_is_immutable(tgt->type)) { if (t->num_targets) { - DMERR("%s: immutable target type %s cannot be mixed with other target types", - dm_device_name(t->md), tgt->type->name); - return -EINVAL; + tgt->error = "immutable target type cannot be mixed with other target types"; + goto bad; } t->immutable_target_type = tgt->type; } @@ -740,7 +735,6 @@ int dm_table_add_target(struct dm_table *t, const char *type, */ if (!adjoin(t, tgt)) { tgt->error = "Gap in table"; - r = -EINVAL; goto bad; } diff --git a/drivers/md/dm.c b/drivers/md/dm.c index 147af9536d0c..ef7bf1dd6900 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -1423,8 +1423,6 @@ static void cleanup_mapped_device(struct mapped_device *md) if (md->bs) bioset_free(md->bs); - cleanup_srcu_struct(&md->io_barrier); - if (md->disk) { spin_lock(&_minor_lock); md->disk->private_data = NULL; @@ -1436,6 +1434,8 @@ static void cleanup_mapped_device(struct mapped_device *md) if (md->queue) blk_cleanup_queue(md->queue); + cleanup_srcu_struct(&md->io_barrier); + if (md->bdev) { bdput(md->bdev); md->bdev = NULL; diff --git a/drivers/md/md.c b/drivers/md/md.c index eac84d8ff724..2089d46b0eb8 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -3887,10 +3887,10 @@ array_state_show(struct mddev *mddev, char *page) st = read_auto; break; case 0: - if (mddev->in_sync) - st = clean; - else if (test_bit(MD_CHANGE_PENDING, &mddev->flags)) + if (test_bit(MD_CHANGE_PENDING, &mddev->flags)) st = write_pending; + else if (mddev->in_sync) + st = clean; else if (mddev->safemode) st = active_idle; else @@ -8144,14 +8144,14 @@ void md_do_sync(struct md_thread *thread) if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && !test_bit(MD_RECOVERY_INTR, &mddev->recovery) && - mddev->curr_resync > 2) { + mddev->curr_resync > 3) { mddev->curr_resync_completed = mddev->curr_resync; sysfs_notify(&mddev->kobj, NULL, "sync_completed"); } mddev->pers->sync_request(mddev, max_sectors, &skipped); if (!test_bit(MD_RECOVERY_CHECK, &mddev->recovery) && - mddev->curr_resync > 2) { + mddev->curr_resync > 3) { if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { if (mddev->curr_resync >= mddev->recovery_cp) { diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 1961d827dbd1..29e2df5cd77b 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -403,11 +403,14 @@ static void raid1_end_write_request(struct bio *bio) struct bio *to_put = NULL; int mirror = find_bio_disk(r1_bio, bio); struct md_rdev *rdev = conf->mirrors[mirror].rdev; + bool discard_error; + + discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD; /* * 'one mirror IO has finished' event handler: */ - if (bio->bi_error) { + if (bio->bi_error && !discard_error) { set_bit(WriteErrorSeen, &rdev->flags); if (!test_and_set_bit(WantReplacement, &rdev->flags)) set_bit(MD_RECOVERY_NEEDED, & @@ -444,7 +447,7 @@ static void raid1_end_write_request(struct bio *bio) /* Maybe we can clear some bad blocks. */ if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors, - &first_bad, &bad_sectors)) { + &first_bad, &bad_sectors) && !discard_error) { r1_bio->bios[mirror] = IO_MADE_GOOD; set_bit(R1BIO_MadeGood, &r1_bio->state); } @@ -2294,17 +2297,23 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) * This is all done synchronously while the array is * frozen */ + + bio = r1_bio->bios[r1_bio->read_disk]; + bdevname(bio->bi_bdev, b); + bio_put(bio); + r1_bio->bios[r1_bio->read_disk] = NULL; + if (mddev->ro == 0) { freeze_array(conf, 1); fix_read_error(conf, r1_bio->read_disk, r1_bio->sector, r1_bio->sectors); unfreeze_array(conf); - } else - md_error(mddev, conf->mirrors[r1_bio->read_disk].rdev); + } else { + r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; + } + rdev_dec_pending(conf->mirrors[r1_bio->read_disk].rdev, conf->mddev); - bio = r1_bio->bios[r1_bio->read_disk]; - bdevname(bio->bi_bdev, b); read_more: disk = read_balance(conf, r1_bio, &max_sectors); if (disk == -1) { @@ -2315,11 +2324,6 @@ read_more: } else { const unsigned long do_sync = r1_bio->master_bio->bi_opf & REQ_SYNC; - if (bio) { - r1_bio->bios[r1_bio->read_disk] = - mddev->ro ? IO_BLOCKED : NULL; - bio_put(bio); - } r1_bio->read_disk = disk; bio = bio_clone_mddev(r1_bio->master_bio, GFP_NOIO, mddev); bio_trim(bio, r1_bio->sector - bio->bi_iter.bi_sector, diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index be1a9fca3b2d..39fddda2fef2 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -447,6 +447,9 @@ static void raid10_end_write_request(struct bio *bio) struct r10conf *conf = r10_bio->mddev->private; int slot, repl; struct md_rdev *rdev = NULL; + bool discard_error; + + discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD; dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); @@ -460,7 +463,7 @@ static void raid10_end_write_request(struct bio *bio) /* * this branch is our 'one mirror IO has finished' event handler: */ - if (bio->bi_error) { + if (bio->bi_error && !discard_error) { if (repl) /* Never record new bad blocks to replacement, * just fail it. @@ -503,7 +506,7 @@ static void raid10_end_write_request(struct bio *bio) if (is_badblock(rdev, r10_bio->devs[slot].addr, r10_bio->sectors, - &first_bad, &bad_sectors)) { + &first_bad, &bad_sectors) && !discard_error) { bio_put(bio); if (repl) r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c index 1b1ab4a1d132..a227a9f3ee65 100644 --- a/drivers/md/raid5-cache.c +++ b/drivers/md/raid5-cache.c @@ -1087,7 +1087,7 @@ static int r5l_recovery_log(struct r5l_log *log) * 1's seq + 10 and let superblock points to meta2. The same recovery will * not think meta 3 is a valid meta, because its seq doesn't match */ - if (ctx.seq > log->last_cp_seq + 1) { + if (ctx.seq > log->last_cp_seq) { int ret; ret = r5l_log_write_empty_meta_block(log, ctx.pos, ctx.seq + 10); @@ -1096,6 +1096,8 @@ static int r5l_recovery_log(struct r5l_log *log) log->seq = ctx.seq + 11; log->log_start = r5l_ring_add(log, ctx.pos, BLOCK_SECTORS); r5l_write_super(log, ctx.pos); + log->last_checkpoint = ctx.pos; + log->next_checkpoint = ctx.pos; } else { log->log_start = ctx.pos; log->seq = ctx.seq; @@ -1154,6 +1156,7 @@ create: if (create_super) { log->last_cp_seq = prandom_u32(); cp = 0; + r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq); /* * Make sure super points to correct address. Log might have * data very soon. If super hasn't correct log tail address, @@ -1168,6 +1171,7 @@ create: if (log->max_free_space > RECLAIM_MAX_FREE_SPACE) log->max_free_space = RECLAIM_MAX_FREE_SPACE; log->last_checkpoint = cp; + log->next_checkpoint = cp; __free_page(page); diff --git a/drivers/media/usb/b2c2/flexcop-usb.c b/drivers/media/usb/b2c2/flexcop-usb.c index d4bdba60b0f7..52bc42da8a4c 100644 --- a/drivers/media/usb/b2c2/flexcop-usb.c +++ b/drivers/media/usb/b2c2/flexcop-usb.c @@ -73,23 +73,34 @@ static int flexcop_usb_readwrite_dw(struct flexcop_device *fc, u16 wRegOffsPCI, u8 request_type = (read ? USB_DIR_IN : USB_DIR_OUT) | USB_TYPE_VENDOR; u8 wAddress = B2C2_FLEX_PCIOFFSET_TO_INTERNALADDR(wRegOffsPCI) | (read ? 0x80 : 0); + int ret; + + mutex_lock(&fc_usb->data_mutex); + if (!read) + memcpy(fc_usb->data, val, sizeof(*val)); - int len = usb_control_msg(fc_usb->udev, + ret = usb_control_msg(fc_usb->udev, read ? B2C2_USB_CTRL_PIPE_IN : B2C2_USB_CTRL_PIPE_OUT, request, request_type, /* 0xc0 read or 0x40 write */ wAddress, 0, - val, + fc_usb->data, sizeof(u32), B2C2_WAIT_FOR_OPERATION_RDW * HZ); - if (len != sizeof(u32)) { + if (ret != sizeof(u32)) { err("error while %s dword from %d (%d).", read ? "reading" : "writing", wAddress, wRegOffsPCI); - return -EIO; + if (ret >= 0) + ret = -EIO; } - return 0; + + if (read && ret >= 0) + memcpy(val, fc_usb->data, sizeof(*val)); + mutex_unlock(&fc_usb->data_mutex); + + return ret; } /* * DKT 010817 - add support for V8 memory read/write and flash update @@ -100,9 +111,14 @@ static int flexcop_usb_v8_memory_req(struct flexcop_usb *fc_usb, { u8 request_type = USB_TYPE_VENDOR; u16 wIndex; - int nWaitTime, pipe, len; + int nWaitTime, pipe, ret; wIndex = page << 8; + if (buflen > sizeof(fc_usb->data)) { + err("Buffer size bigger than max URB control message\n"); + return -EIO; + } + switch (req) { case B2C2_USB_READ_V8_MEM: nWaitTime = B2C2_WAIT_FOR_OPERATION_V8READ; @@ -127,17 +143,32 @@ static int flexcop_usb_v8_memory_req(struct flexcop_usb *fc_usb, deb_v8("v8mem: %02x %02x %04x %04x, len: %d\n", request_type, req, wAddress, wIndex, buflen); - len = usb_control_msg(fc_usb->udev, pipe, + mutex_lock(&fc_usb->data_mutex); + + if ((request_type & USB_ENDPOINT_DIR_MASK) == USB_DIR_OUT) + memcpy(fc_usb->data, pbBuffer, buflen); + + ret = usb_control_msg(fc_usb->udev, pipe, req, request_type, wAddress, wIndex, - pbBuffer, + fc_usb->data, buflen, nWaitTime * HZ); + if (ret != buflen) + ret = -EIO; + + if (ret >= 0) { + ret = 0; + if ((request_type & USB_ENDPOINT_DIR_MASK) == USB_DIR_IN) + memcpy(pbBuffer, fc_usb->data, buflen); + } - debug_dump(pbBuffer, len, deb_v8); - return len == buflen ? 0 : -EIO; + mutex_unlock(&fc_usb->data_mutex); + + debug_dump(pbBuffer, ret, deb_v8); + return ret; } #define bytes_left_to_read_on_page(paddr,buflen) \ @@ -196,29 +227,6 @@ static int flexcop_usb_get_mac_addr(struct flexcop_device *fc, int extended) fc->dvb_adapter.proposed_mac, 6); } -#if 0 -static int flexcop_usb_utility_req(struct flexcop_usb *fc_usb, int set, - flexcop_usb_utility_function_t func, u8 extra, u16 wIndex, - u16 buflen, u8 *pvBuffer) -{ - u16 wValue; - u8 request_type = (set ? USB_DIR_OUT : USB_DIR_IN) | USB_TYPE_VENDOR; - int nWaitTime = 2, - pipe = set ? B2C2_USB_CTRL_PIPE_OUT : B2C2_USB_CTRL_PIPE_IN, len; - wValue = (func << 8) | extra; - - len = usb_control_msg(fc_usb->udev,pipe, - B2C2_USB_UTILITY, - request_type, - wValue, - wIndex, - pvBuffer, - buflen, - nWaitTime * HZ); - return len == buflen ? 0 : -EIO; -} -#endif - /* usb i2c stuff */ static int flexcop_usb_i2c_req(struct flexcop_i2c_adapter *i2c, flexcop_usb_request_t req, flexcop_usb_i2c_function_t func, @@ -226,9 +234,14 @@ static int flexcop_usb_i2c_req(struct flexcop_i2c_adapter *i2c, { struct flexcop_usb *fc_usb = i2c->fc->bus_specific; u16 wValue, wIndex; - int nWaitTime,pipe,len; + int nWaitTime, pipe, ret; u8 request_type = USB_TYPE_VENDOR; + if (buflen > sizeof(fc_usb->data)) { + err("Buffer size bigger than max URB control message\n"); + return -EIO; + } + switch (func) { case USB_FUNC_I2C_WRITE: case USB_FUNC_I2C_MULTIWRITE: @@ -257,15 +270,32 @@ static int flexcop_usb_i2c_req(struct flexcop_i2c_adapter *i2c, wValue & 0xff, wValue >> 8, wIndex & 0xff, wIndex >> 8); - len = usb_control_msg(fc_usb->udev,pipe, + mutex_lock(&fc_usb->data_mutex); + + if ((request_type & USB_ENDPOINT_DIR_MASK) == USB_DIR_OUT) + memcpy(fc_usb->data, buf, buflen); + + ret = usb_control_msg(fc_usb->udev, pipe, req, request_type, wValue, wIndex, - buf, + fc_usb->data, buflen, nWaitTime * HZ); - return len == buflen ? 0 : -EREMOTEIO; + + if (ret != buflen) + ret = -EIO; + + if (ret >= 0) { + ret = 0; + if ((request_type & USB_ENDPOINT_DIR_MASK) == USB_DIR_IN) + memcpy(buf, fc_usb->data, buflen); + } + + mutex_unlock(&fc_usb->data_mutex); + + return 0; } /* actual bus specific access functions, @@ -516,6 +546,7 @@ static int flexcop_usb_probe(struct usb_interface *intf, /* general flexcop init */ fc_usb = fc->bus_specific; fc_usb->fc_dev = fc; + mutex_init(&fc_usb->data_mutex); fc->read_ibi_reg = flexcop_usb_read_ibi_reg; fc->write_ibi_reg = flexcop_usb_write_ibi_reg; diff --git a/drivers/media/usb/b2c2/flexcop-usb.h b/drivers/media/usb/b2c2/flexcop-usb.h index 92529a9c4475..25ad43166e78 100644 --- a/drivers/media/usb/b2c2/flexcop-usb.h +++ b/drivers/media/usb/b2c2/flexcop-usb.h @@ -29,6 +29,10 @@ struct flexcop_usb { u8 tmp_buffer[1023+190]; int tmp_buffer_length; + + /* for URB control messages */ + u8 data[80]; + struct mutex data_mutex; }; #if 0 diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/media/usb/cpia2/cpia2_usb.c index 13620cdf0599..e9100a235831 100644 --- a/drivers/media/usb/cpia2/cpia2_usb.c +++ b/drivers/media/usb/cpia2/cpia2_usb.c @@ -545,18 +545,30 @@ static void free_sbufs(struct camera_data *cam) static int write_packet(struct usb_device *udev, u8 request, u8 * registers, u16 start, size_t size) { + unsigned char *buf; + int ret; + if (!registers || size <= 0) return -EINVAL; - return usb_control_msg(udev, + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + memcpy(buf, registers, size); + + ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), request, USB_TYPE_VENDOR | USB_RECIP_DEVICE, start, /* value */ 0, /* index */ - registers, /* buffer */ + buf, /* buffer */ size, HZ); + + kfree(buf); + return ret; } /**************************************************************************** @@ -567,18 +579,32 @@ static int write_packet(struct usb_device *udev, static int read_packet(struct usb_device *udev, u8 request, u8 * registers, u16 start, size_t size) { + unsigned char *buf; + int ret; + if (!registers || size <= 0) return -EINVAL; - return usb_control_msg(udev, + buf = kmalloc(size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), request, USB_DIR_IN|USB_TYPE_VENDOR|USB_RECIP_DEVICE, start, /* value */ 0, /* index */ - registers, /* buffer */ + buf, /* buffer */ size, HZ); + + if (ret >= 0) + memcpy(registers, buf, size); + + kfree(buf); + + return ret; } /****************************************************************************** diff --git a/drivers/media/usb/dvb-usb/af9005.c b/drivers/media/usb/dvb-usb/af9005.c index efa782ed6e2d..b257780fb380 100644 --- a/drivers/media/usb/dvb-usb/af9005.c +++ b/drivers/media/usb/dvb-usb/af9005.c @@ -52,17 +52,16 @@ u8 regmask[8] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff }; struct af9005_device_state { u8 sequence; int led_state; + unsigned char data[256]; + struct mutex data_mutex; }; static int af9005_generic_read_write(struct dvb_usb_device *d, u16 reg, int readwrite, int type, u8 * values, int len) { struct af9005_device_state *st = d->priv; - u8 obuf[16] = { 0 }; - u8 ibuf[17] = { 0 }; - u8 command; - int i; - int ret; + u8 command, seq; + int i, ret; if (len < 1) { err("generic read/write, less than 1 byte. Makes no sense."); @@ -73,16 +72,17 @@ static int af9005_generic_read_write(struct dvb_usb_device *d, u16 reg, return -EINVAL; } - obuf[0] = 14; /* rest of buffer length low */ - obuf[1] = 0; /* rest of buffer length high */ + mutex_lock(&st->data_mutex); + st->data[0] = 14; /* rest of buffer length low */ + st->data[1] = 0; /* rest of buffer length high */ - obuf[2] = AF9005_REGISTER_RW; /* register operation */ - obuf[3] = 12; /* rest of buffer length */ + st->data[2] = AF9005_REGISTER_RW; /* register operation */ + st->data[3] = 12; /* rest of buffer length */ - obuf[4] = st->sequence++; /* sequence number */ + st->data[4] = seq = st->sequence++; /* sequence number */ - obuf[5] = (u8) (reg >> 8); /* register address */ - obuf[6] = (u8) (reg & 0xff); + st->data[5] = (u8) (reg >> 8); /* register address */ + st->data[6] = (u8) (reg & 0xff); if (type == AF9005_OFDM_REG) { command = AF9005_CMD_OFDM_REG; @@ -96,51 +96,52 @@ static int af9005_generic_read_write(struct dvb_usb_device *d, u16 reg, command |= readwrite; if (readwrite == AF9005_CMD_WRITE) for (i = 0; i < len; i++) - obuf[8 + i] = values[i]; + st->data[8 + i] = values[i]; else if (type == AF9005_TUNER_REG) /* read command for tuner, the first byte contains the i2c address */ - obuf[8] = values[0]; - obuf[7] = command; + st->data[8] = values[0]; + st->data[7] = command; - ret = dvb_usb_generic_rw(d, obuf, 16, ibuf, 17, 0); + ret = dvb_usb_generic_rw(d, st->data, 16, st->data, 17, 0); if (ret) - return ret; + goto ret; /* sanity check */ - if (ibuf[2] != AF9005_REGISTER_RW_ACK) { + if (st->data[2] != AF9005_REGISTER_RW_ACK) { err("generic read/write, wrong reply code."); - return -EIO; + ret = -EIO; + goto ret; } - if (ibuf[3] != 0x0d) { + if (st->data[3] != 0x0d) { err("generic read/write, wrong length in reply."); - return -EIO; + ret = -EIO; + goto ret; } - if (ibuf[4] != obuf[4]) { + if (st->data[4] != seq) { err("generic read/write, wrong sequence in reply."); - return -EIO; + ret = -EIO; + goto ret; } /* - Windows driver doesn't check these fields, in fact sometimes - the register in the reply is different that what has been sent - - if (ibuf[5] != obuf[5] || ibuf[6] != obuf[6]) { - err("generic read/write, wrong register in reply."); - return -EIO; - } - if (ibuf[7] != command) { - err("generic read/write wrong command in reply."); - return -EIO; - } + * In thesis, both input and output buffers should have + * identical values for st->data[5] to st->data[8]. + * However, windows driver doesn't check these fields, in fact + * sometimes the register in the reply is different that what + * has been sent */ - if (ibuf[16] != 0x01) { + if (st->data[16] != 0x01) { err("generic read/write wrong status code in reply."); - return -EIO; + ret = -EIO; + goto ret; } + if (readwrite == AF9005_CMD_READ) for (i = 0; i < len; i++) - values[i] = ibuf[8 + i]; + values[i] = st->data[8 + i]; - return 0; +ret: + mutex_unlock(&st->data_mutex); + return ret; } @@ -464,8 +465,7 @@ int af9005_send_command(struct dvb_usb_device *d, u8 command, u8 * wbuf, struct af9005_device_state *st = d->priv; int ret, i, packet_len; - u8 buf[64]; - u8 ibuf[64]; + u8 seq; if (wlen < 0) { err("send command, wlen less than 0 bytes. Makes no sense."); @@ -480,94 +480,97 @@ int af9005_send_command(struct dvb_usb_device *d, u8 command, u8 * wbuf, return -EINVAL; } packet_len = wlen + 5; - buf[0] = (u8) (packet_len & 0xff); - buf[1] = (u8) ((packet_len & 0xff00) >> 8); - - buf[2] = 0x26; /* packet type */ - buf[3] = wlen + 3; - buf[4] = st->sequence++; - buf[5] = command; - buf[6] = wlen; + + mutex_lock(&st->data_mutex); + + st->data[0] = (u8) (packet_len & 0xff); + st->data[1] = (u8) ((packet_len & 0xff00) >> 8); + + st->data[2] = 0x26; /* packet type */ + st->data[3] = wlen + 3; + st->data[4] = seq = st->sequence++; + st->data[5] = command; + st->data[6] = wlen; for (i = 0; i < wlen; i++) - buf[7 + i] = wbuf[i]; - ret = dvb_usb_generic_rw(d, buf, wlen + 7, ibuf, rlen + 7, 0); - if (ret) - return ret; - if (ibuf[2] != 0x27) { + st->data[7 + i] = wbuf[i]; + ret = dvb_usb_generic_rw(d, st->data, wlen + 7, st->data, rlen + 7, 0); + if (st->data[2] != 0x27) { err("send command, wrong reply code."); - return -EIO; - } - if (ibuf[4] != buf[4]) { + ret = -EIO; + } else if (st->data[4] != seq) { err("send command, wrong sequence in reply."); - return -EIO; - } - if (ibuf[5] != 0x01) { + ret = -EIO; + } else if (st->data[5] != 0x01) { err("send command, wrong status code in reply."); - return -EIO; - } - if (ibuf[6] != rlen) { + ret = -EIO; + } else if (st->data[6] != rlen) { err("send command, invalid data length in reply."); - return -EIO; + ret = -EIO; } - for (i = 0; i < rlen; i++) - rbuf[i] = ibuf[i + 7]; - return 0; + if (!ret) { + for (i = 0; i < rlen; i++) + rbuf[i] = st->data[i + 7]; + } + + mutex_unlock(&st->data_mutex); + return ret; } int af9005_read_eeprom(struct dvb_usb_device *d, u8 address, u8 * values, int len) { struct af9005_device_state *st = d->priv; - u8 obuf[16], ibuf[14]; + u8 seq; int ret, i; - memset(obuf, 0, sizeof(obuf)); - memset(ibuf, 0, sizeof(ibuf)); + mutex_lock(&st->data_mutex); - obuf[0] = 14; /* length of rest of packet low */ - obuf[1] = 0; /* length of rest of packer high */ + memset(st->data, 0, sizeof(st->data)); - obuf[2] = 0x2a; /* read/write eeprom */ + st->data[0] = 14; /* length of rest of packet low */ + st->data[1] = 0; /* length of rest of packer high */ - obuf[3] = 12; /* size */ + st->data[2] = 0x2a; /* read/write eeprom */ - obuf[4] = st->sequence++; + st->data[3] = 12; /* size */ - obuf[5] = 0; /* read */ + st->data[4] = seq = st->sequence++; - obuf[6] = len; - obuf[7] = address; - ret = dvb_usb_generic_rw(d, obuf, 16, ibuf, 14, 0); - if (ret) - return ret; - if (ibuf[2] != 0x2b) { + st->data[5] = 0; /* read */ + + st->data[6] = len; + st->data[7] = address; + ret = dvb_usb_generic_rw(d, st->data, 16, st->data, 14, 0); + if (st->data[2] != 0x2b) { err("Read eeprom, invalid reply code"); - return -EIO; - } - if (ibuf[3] != 10) { + ret = -EIO; + } else if (st->data[3] != 10) { err("Read eeprom, invalid reply length"); - return -EIO; - } - if (ibuf[4] != obuf[4]) { + ret = -EIO; + } else if (st->data[4] != seq) { err("Read eeprom, wrong sequence in reply "); - return -EIO; - } - if (ibuf[5] != 1) { + ret = -EIO; + } else if (st->data[5] != 1) { err("Read eeprom, wrong status in reply "); - return -EIO; + ret = -EIO; } - for (i = 0; i < len; i++) { - values[i] = ibuf[6 + i]; + + if (!ret) { + for (i = 0; i < len; i++) + values[i] = st->data[6 + i]; } - return 0; + mutex_unlock(&st->data_mutex); + + return ret; } -static int af9005_boot_packet(struct usb_device *udev, int type, u8 * reply) +static int af9005_boot_packet(struct usb_device *udev, int type, u8 *reply, + u8 *buf, int size) { - u8 buf[FW_BULKOUT_SIZE + 2]; u16 checksum; int act_len, i, ret; - memset(buf, 0, sizeof(buf)); + + memset(buf, 0, size); buf[0] = (u8) (FW_BULKOUT_SIZE & 0xff); buf[1] = (u8) ((FW_BULKOUT_SIZE >> 8) & 0xff); switch (type) { @@ -720,15 +723,21 @@ static int af9005_download_firmware(struct usb_device *udev, const struct firmwa { int i, packets, ret, act_len; - u8 buf[FW_BULKOUT_SIZE + 2]; + u8 *buf; u8 reply; - ret = af9005_boot_packet(udev, FW_CONFIG, &reply); + buf = kmalloc(FW_BULKOUT_SIZE + 2, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = af9005_boot_packet(udev, FW_CONFIG, &reply, buf, + FW_BULKOUT_SIZE + 2); if (ret) - return ret; + goto err; if (reply != 0x01) { err("before downloading firmware, FW_CONFIG expected 0x01, received 0x%x", reply); - return -EIO; + ret = -EIO; + goto err; } packets = fw->size / FW_BULKOUT_SIZE; buf[0] = (u8) (FW_BULKOUT_SIZE & 0xff); @@ -743,28 +752,35 @@ static int af9005_download_firmware(struct usb_device *udev, const struct firmwa buf, FW_BULKOUT_SIZE + 2, &act_len, 1000); if (ret) { err("firmware download failed at packet %d with code %d", i, ret); - return ret; + goto err; } } - ret = af9005_boot_packet(udev, FW_CONFIRM, &reply); + ret = af9005_boot_packet(udev, FW_CONFIRM, &reply, + buf, FW_BULKOUT_SIZE + 2); if (ret) - return ret; + goto err; if (reply != (u8) (packets & 0xff)) { err("after downloading firmware, FW_CONFIRM expected 0x%x, received 0x%x", packets & 0xff, reply); - return -EIO; + ret = -EIO; + goto err; } - ret = af9005_boot_packet(udev, FW_BOOT, &reply); + ret = af9005_boot_packet(udev, FW_BOOT, &reply, buf, + FW_BULKOUT_SIZE + 2); if (ret) - return ret; - ret = af9005_boot_packet(udev, FW_CONFIG, &reply); + goto err; + ret = af9005_boot_packet(udev, FW_CONFIG, &reply, buf, + FW_BULKOUT_SIZE + 2); if (ret) - return ret; + goto err; if (reply != 0x02) { err("after downloading firmware, FW_CONFIG expected 0x02, received 0x%x", reply); - return -EIO; + ret = -EIO; + goto err; } - return 0; +err: + kfree(buf); + return ret; } @@ -823,53 +839,59 @@ static int af9005_rc_query(struct dvb_usb_device *d, u32 * event, int *state) { struct af9005_device_state *st = d->priv; int ret, len; - - u8 obuf[5]; - u8 ibuf[256]; + u8 seq; *state = REMOTE_NO_KEY_PRESSED; if (rc_decode == NULL) { /* it shouldn't never come here */ return 0; } + + mutex_lock(&st->data_mutex); + /* deb_info("rc_query\n"); */ - obuf[0] = 3; /* rest of packet length low */ - obuf[1] = 0; /* rest of packet lentgh high */ - obuf[2] = 0x40; /* read remote */ - obuf[3] = 1; /* rest of packet length */ - obuf[4] = st->sequence++; /* sequence number */ - ret = dvb_usb_generic_rw(d, obuf, 5, ibuf, 256, 0); + st->data[0] = 3; /* rest of packet length low */ + st->data[1] = 0; /* rest of packet lentgh high */ + st->data[2] = 0x40; /* read remote */ + st->data[3] = 1; /* rest of packet length */ + st->data[4] = seq = st->sequence++; /* sequence number */ + ret = dvb_usb_generic_rw(d, st->data, 5, st->data, 256, 0); if (ret) { err("rc query failed"); - return ret; + goto ret; } - if (ibuf[2] != 0x41) { + if (st->data[2] != 0x41) { err("rc query bad header."); - return -EIO; - } - if (ibuf[4] != obuf[4]) { + ret = -EIO; + goto ret; + } else if (st->data[4] != seq) { err("rc query bad sequence."); - return -EIO; + ret = -EIO; + goto ret; } - len = ibuf[5]; + len = st->data[5]; if (len > 246) { err("rc query invalid length"); - return -EIO; + ret = -EIO; + goto ret; } if (len > 0) { deb_rc("rc data (%d) ", len); - debug_dump((ibuf + 6), len, deb_rc); - ret = rc_decode(d, &ibuf[6], len, event, state); + debug_dump((st->data + 6), len, deb_rc); + ret = rc_decode(d, &st->data[6], len, event, state); if (ret) { err("rc_decode failed"); - return ret; + goto ret; } else { deb_rc("rc_decode state %x event %x\n", *state, *event); if (*state == REMOTE_KEY_REPEAT) *event = d->last_event; } } - return 0; + +ret: + mutex_unlock(&st->data_mutex); + return ret; } static int af9005_power_ctrl(struct dvb_usb_device *d, int onoff) @@ -953,10 +975,16 @@ static int af9005_identify_state(struct usb_device *udev, int *cold) { int ret; - u8 reply; - ret = af9005_boot_packet(udev, FW_CONFIG, &reply); + u8 reply, *buf; + + buf = kmalloc(FW_BULKOUT_SIZE + 2, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + ret = af9005_boot_packet(udev, FW_CONFIG, &reply, + buf, FW_BULKOUT_SIZE + 2); if (ret) - return ret; + goto err; deb_info("result of FW_CONFIG in identify state %d\n", reply); if (reply == 0x01) *cold = 1; @@ -965,7 +993,10 @@ static int af9005_identify_state(struct usb_device *udev, else return -EIO; deb_info("Identify state cold = %d\n", *cold); - return 0; + +err: + kfree(buf); + return ret; } static struct dvb_usb_device_properties af9005_properties; @@ -973,8 +1004,20 @@ static struct dvb_usb_device_properties af9005_properties; static int af9005_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) { - return dvb_usb_device_init(intf, &af9005_properties, - THIS_MODULE, NULL, adapter_nr); + struct dvb_usb_device *d; + struct af9005_device_state *st; + int ret; + + ret = dvb_usb_device_init(intf, &af9005_properties, + THIS_MODULE, &d, adapter_nr); + + if (ret < 0) + return ret; + + st = d->priv; + mutex_init(&st->data_mutex); + + return 0; } enum af9005_usb_table_entry { diff --git a/drivers/media/usb/dvb-usb/cinergyT2-core.c b/drivers/media/usb/dvb-usb/cinergyT2-core.c index 9fd1527494eb..8ac825413d5a 100644 --- a/drivers/media/usb/dvb-usb/cinergyT2-core.c +++ b/drivers/media/usb/dvb-usb/cinergyT2-core.c @@ -41,6 +41,8 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); struct cinergyt2_state { u8 rc_counter; + unsigned char data[64]; + struct mutex data_mutex; }; /* We are missing a release hook with usb_device data */ @@ -50,38 +52,57 @@ static struct dvb_usb_device_properties cinergyt2_properties; static int cinergyt2_streaming_ctrl(struct dvb_usb_adapter *adap, int enable) { - char buf[] = { CINERGYT2_EP1_CONTROL_STREAM_TRANSFER, enable ? 1 : 0 }; - char result[64]; - return dvb_usb_generic_rw(adap->dev, buf, sizeof(buf), result, - sizeof(result), 0); + struct dvb_usb_device *d = adap->dev; + struct cinergyt2_state *st = d->priv; + int ret; + + mutex_lock(&st->data_mutex); + st->data[0] = CINERGYT2_EP1_CONTROL_STREAM_TRANSFER; + st->data[1] = enable ? 1 : 0; + + ret = dvb_usb_generic_rw(d, st->data, 2, st->data, 64, 0); + mutex_unlock(&st->data_mutex); + + return ret; } static int cinergyt2_power_ctrl(struct dvb_usb_device *d, int enable) { - char buf[] = { CINERGYT2_EP1_SLEEP_MODE, enable ? 0 : 1 }; - char state[3]; - return dvb_usb_generic_rw(d, buf, sizeof(buf), state, sizeof(state), 0); + struct cinergyt2_state *st = d->priv; + int ret; + + mutex_lock(&st->data_mutex); + st->data[0] = CINERGYT2_EP1_SLEEP_MODE; + st->data[1] = enable ? 0 : 1; + + ret = dvb_usb_generic_rw(d, st->data, 2, st->data, 3, 0); + mutex_unlock(&st->data_mutex); + + return ret; } static int cinergyt2_frontend_attach(struct dvb_usb_adapter *adap) { - char query[] = { CINERGYT2_EP1_GET_FIRMWARE_VERSION }; - char state[3]; + struct dvb_usb_device *d = adap->dev; + struct cinergyt2_state *st = d->priv; int ret; adap->fe_adap[0].fe = cinergyt2_fe_attach(adap->dev); - ret = dvb_usb_generic_rw(adap->dev, query, sizeof(query), state, - sizeof(state), 0); + mutex_lock(&st->data_mutex); + st->data[0] = CINERGYT2_EP1_GET_FIRMWARE_VERSION; + + ret = dvb_usb_generic_rw(d, st->data, 1, st->data, 3, 0); if (ret < 0) { deb_rc("cinergyt2_power_ctrl() Failed to retrieve sleep " "state info\n"); } + mutex_unlock(&st->data_mutex); /* Copy this pointer as we are gonna need it in the release phase */ cinergyt2_usb_device = adap->dev; - return 0; + return ret; } static struct rc_map_table rc_map_cinergyt2_table[] = { @@ -141,13 +162,18 @@ static int repeatable_keys[] = { static int cinergyt2_rc_query(struct dvb_usb_device *d, u32 *event, int *state) { struct cinergyt2_state *st = d->priv; - u8 key[5] = {0, 0, 0, 0, 0}, cmd = CINERGYT2_EP1_GET_RC_EVENTS; - int i; + int i, ret; *state = REMOTE_NO_KEY_PRESSED; - dvb_usb_generic_rw(d, &cmd, 1, key, sizeof(key), 0); - if (key[4] == 0xff) { + mutex_lock(&st->data_mutex); + st->data[0] = CINERGYT2_EP1_GET_RC_EVENTS; + + ret = dvb_usb_generic_rw(d, st->data, 1, st->data, 5, 0); + if (ret < 0) + goto ret; + + if (st->data[4] == 0xff) { /* key repeat */ st->rc_counter++; if (st->rc_counter > RC_REPEAT_DELAY) { @@ -157,31 +183,45 @@ static int cinergyt2_rc_query(struct dvb_usb_device *d, u32 *event, int *state) *event = d->last_event; deb_rc("repeat key, event %x\n", *event); - return 0; + goto ret; } } deb_rc("repeated key (non repeatable)\n"); } - return 0; + goto ret; } /* hack to pass checksum on the custom field */ - key[2] = ~key[1]; - dvb_usb_nec_rc_key_to_event(d, key, event, state); - if (key[0] != 0) { + st->data[2] = ~st->data[1]; + dvb_usb_nec_rc_key_to_event(d, st->data, event, state); + if (st->data[0] != 0) { if (*event != d->last_event) st->rc_counter = 0; - deb_rc("key: %*ph\n", 5, key); + deb_rc("key: %*ph\n", 5, st->data); } - return 0; + +ret: + mutex_unlock(&st->data_mutex); + return ret; } static int cinergyt2_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) { - return dvb_usb_device_init(intf, &cinergyt2_properties, - THIS_MODULE, NULL, adapter_nr); + struct dvb_usb_device *d; + struct cinergyt2_state *st; + int ret; + + ret = dvb_usb_device_init(intf, &cinergyt2_properties, + THIS_MODULE, &d, adapter_nr); + if (ret < 0) + return ret; + + st = d->priv; + mutex_init(&st->data_mutex); + + return 0; } diff --git a/drivers/media/usb/dvb-usb/cinergyT2-fe.c b/drivers/media/usb/dvb-usb/cinergyT2-fe.c index b3ec743a7a2e..2d29b4174dba 100644 --- a/drivers/media/usb/dvb-usb/cinergyT2-fe.c +++ b/drivers/media/usb/dvb-usb/cinergyT2-fe.c @@ -139,32 +139,42 @@ static uint16_t compute_tps(struct dtv_frontend_properties *op) struct cinergyt2_fe_state { struct dvb_frontend fe; struct dvb_usb_device *d; + + unsigned char data[64]; + struct mutex data_mutex; + + struct dvbt_get_status_msg status; }; static int cinergyt2_fe_read_status(struct dvb_frontend *fe, enum fe_status *status) { struct cinergyt2_fe_state *state = fe->demodulator_priv; - struct dvbt_get_status_msg result; - u8 cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS }; int ret; - ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (u8 *)&result, - sizeof(result), 0); + mutex_lock(&state->data_mutex); + state->data[0] = CINERGYT2_EP1_GET_TUNER_STATUS; + + ret = dvb_usb_generic_rw(state->d, state->data, 1, + state->data, sizeof(state->status), 0); + if (!ret) + memcpy(&state->status, state->data, sizeof(state->status)); + mutex_unlock(&state->data_mutex); + if (ret < 0) return ret; *status = 0; - if (0xffff - le16_to_cpu(result.gain) > 30) + if (0xffff - le16_to_cpu(state->status.gain) > 30) *status |= FE_HAS_SIGNAL; - if (result.lock_bits & (1 << 6)) + if (state->status.lock_bits & (1 << 6)) *status |= FE_HAS_LOCK; - if (result.lock_bits & (1 << 5)) + if (state->status.lock_bits & (1 << 5)) *status |= FE_HAS_SYNC; - if (result.lock_bits & (1 << 4)) + if (state->status.lock_bits & (1 << 4)) *status |= FE_HAS_CARRIER; - if (result.lock_bits & (1 << 1)) + if (state->status.lock_bits & (1 << 1)) *status |= FE_HAS_VITERBI; if ((*status & (FE_HAS_CARRIER | FE_HAS_VITERBI | FE_HAS_SYNC)) != @@ -177,34 +187,16 @@ static int cinergyt2_fe_read_status(struct dvb_frontend *fe, static int cinergyt2_fe_read_ber(struct dvb_frontend *fe, u32 *ber) { struct cinergyt2_fe_state *state = fe->demodulator_priv; - struct dvbt_get_status_msg status; - char cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS }; - int ret; - - ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (char *)&status, - sizeof(status), 0); - if (ret < 0) - return ret; - *ber = le32_to_cpu(status.viterbi_error_rate); + *ber = le32_to_cpu(state->status.viterbi_error_rate); return 0; } static int cinergyt2_fe_read_unc_blocks(struct dvb_frontend *fe, u32 *unc) { struct cinergyt2_fe_state *state = fe->demodulator_priv; - struct dvbt_get_status_msg status; - u8 cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS }; - int ret; - ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (u8 *)&status, - sizeof(status), 0); - if (ret < 0) { - err("cinergyt2_fe_read_unc_blocks() Failed! (Error=%d)\n", - ret); - return ret; - } - *unc = le32_to_cpu(status.uncorrected_block_count); + *unc = le32_to_cpu(state->status.uncorrected_block_count); return 0; } @@ -212,35 +204,16 @@ static int cinergyt2_fe_read_signal_strength(struct dvb_frontend *fe, u16 *strength) { struct cinergyt2_fe_state *state = fe->demodulator_priv; - struct dvbt_get_status_msg status; - char cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS }; - int ret; - ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (char *)&status, - sizeof(status), 0); - if (ret < 0) { - err("cinergyt2_fe_read_signal_strength() Failed!" - " (Error=%d)\n", ret); - return ret; - } - *strength = (0xffff - le16_to_cpu(status.gain)); + *strength = (0xffff - le16_to_cpu(state->status.gain)); return 0; } static int cinergyt2_fe_read_snr(struct dvb_frontend *fe, u16 *snr) { struct cinergyt2_fe_state *state = fe->demodulator_priv; - struct dvbt_get_status_msg status; - char cmd[] = { CINERGYT2_EP1_GET_TUNER_STATUS }; - int ret; - ret = dvb_usb_generic_rw(state->d, cmd, sizeof(cmd), (char *)&status, - sizeof(status), 0); - if (ret < 0) { - err("cinergyt2_fe_read_snr() Failed! (Error=%d)\n", ret); - return ret; - } - *snr = (status.snr << 8) | status.snr; + *snr = (state->status.snr << 8) | state->status.snr; return 0; } @@ -266,34 +239,36 @@ static int cinergyt2_fe_set_frontend(struct dvb_frontend *fe) { struct dtv_frontend_properties *fep = &fe->dtv_property_cache; struct cinergyt2_fe_state *state = fe->demodulator_priv; - struct dvbt_set_parameters_msg param; - char result[2]; + struct dvbt_set_parameters_msg *param; int err; - param.cmd = CINERGYT2_EP1_SET_TUNER_PARAMETERS; - param.tps = cpu_to_le16(compute_tps(fep)); - param.freq = cpu_to_le32(fep->frequency / 1000); - param.flags = 0; + mutex_lock(&state->data_mutex); + + param = (void *)state->data; + param->cmd = CINERGYT2_EP1_SET_TUNER_PARAMETERS; + param->tps = cpu_to_le16(compute_tps(fep)); + param->freq = cpu_to_le32(fep->frequency / 1000); + param->flags = 0; switch (fep->bandwidth_hz) { default: case 8000000: - param.bandwidth = 8; + param->bandwidth = 8; break; case 7000000: - param.bandwidth = 7; + param->bandwidth = 7; break; case 6000000: - param.bandwidth = 6; + param->bandwidth = 6; break; } - err = dvb_usb_generic_rw(state->d, - (char *)¶m, sizeof(param), - result, sizeof(result), 0); + err = dvb_usb_generic_rw(state->d, state->data, sizeof(*param), + state->data, 2, 0); if (err < 0) err("cinergyt2_fe_set_frontend() Failed! err=%d\n", err); + mutex_unlock(&state->data_mutex); return (err < 0) ? err : 0; } @@ -315,6 +290,7 @@ struct dvb_frontend *cinergyt2_fe_attach(struct dvb_usb_device *d) s->d = d; memcpy(&s->fe.ops, &cinergyt2_fe_ops, sizeof(struct dvb_frontend_ops)); s->fe.demodulator_priv = s; + mutex_init(&s->data_mutex); return &s->fe; } diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c index 907ac01ae297..39772812269d 100644 --- a/drivers/media/usb/dvb-usb/cxusb.c +++ b/drivers/media/usb/dvb-usb/cxusb.c @@ -45,9 +45,6 @@ #include "si2168.h" #include "si2157.h" -/* Max transfer size done by I2C transfer functions */ -#define MAX_XFER_SIZE 80 - /* debug */ static int dvb_usb_cxusb_debug; module_param_named(debug, dvb_usb_cxusb_debug, int, 0644); @@ -61,23 +58,27 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); static int cxusb_ctrl_msg(struct dvb_usb_device *d, u8 cmd, u8 *wbuf, int wlen, u8 *rbuf, int rlen) { - int wo = (rbuf == NULL || rlen == 0); /* write-only */ - u8 sndbuf[MAX_XFER_SIZE]; + struct cxusb_state *st = d->priv; + int ret, wo; - if (1 + wlen > sizeof(sndbuf)) { - warn("i2c wr: len=%d is too big!\n", - wlen); + if (1 + wlen > MAX_XFER_SIZE) { + warn("i2c wr: len=%d is too big!\n", wlen); return -EOPNOTSUPP; } - memset(sndbuf, 0, 1+wlen); + wo = (rbuf == NULL || rlen == 0); /* write-only */ - sndbuf[0] = cmd; - memcpy(&sndbuf[1], wbuf, wlen); + mutex_lock(&st->data_mutex); + st->data[0] = cmd; + memcpy(&st->data[1], wbuf, wlen); if (wo) - return dvb_usb_generic_write(d, sndbuf, 1+wlen); + ret = dvb_usb_generic_write(d, st->data, 1 + wlen); else - return dvb_usb_generic_rw(d, sndbuf, 1+wlen, rbuf, rlen, 0); + ret = dvb_usb_generic_rw(d, st->data, 1 + wlen, + rbuf, rlen, 0); + + mutex_unlock(&st->data_mutex); + return ret; } /* GPIO */ @@ -1460,36 +1461,43 @@ static struct dvb_usb_device_properties cxusb_mygica_t230_properties; static int cxusb_probe(struct usb_interface *intf, const struct usb_device_id *id) { + struct dvb_usb_device *d; + struct cxusb_state *st; + if (0 == dvb_usb_device_init(intf, &cxusb_medion_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_lgh064f_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_dee1601_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_lgz201_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_dtt7579_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_dualdig4_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_nano2_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_nano2_needsfirmware_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_aver_a868r_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_bluebird_dualdig4_rev2_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_d680_dmb_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_mygica_d689_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &cxusb_mygica_t230_properties, - THIS_MODULE, NULL, adapter_nr) || - 0) + THIS_MODULE, &d, adapter_nr) || + 0) { + st = d->priv; + mutex_init(&st->data_mutex); + return 0; + } return -EINVAL; } diff --git a/drivers/media/usb/dvb-usb/cxusb.h b/drivers/media/usb/dvb-usb/cxusb.h index 527ff7905e15..9f3ee0e47d5c 100644 --- a/drivers/media/usb/dvb-usb/cxusb.h +++ b/drivers/media/usb/dvb-usb/cxusb.h @@ -28,10 +28,16 @@ #define CMD_ANALOG 0x50 #define CMD_DIGITAL 0x51 +/* Max transfer size done by I2C transfer functions */ +#define MAX_XFER_SIZE 80 + struct cxusb_state { u8 gpio_write_state[3]; struct i2c_client *i2c_client_demod; struct i2c_client *i2c_client_tuner; + + unsigned char data[MAX_XFER_SIZE]; + struct mutex data_mutex; }; #endif diff --git a/drivers/media/usb/dvb-usb/dib0700_core.c b/drivers/media/usb/dvb-usb/dib0700_core.c index f3196658fb70..92d5408684ac 100644 --- a/drivers/media/usb/dvb-usb/dib0700_core.c +++ b/drivers/media/usb/dvb-usb/dib0700_core.c @@ -213,7 +213,7 @@ static int dib0700_i2c_xfer_new(struct i2c_adapter *adap, struct i2c_msg *msg, usb_rcvctrlpipe(d->udev, 0), REQUEST_NEW_I2C_READ, USB_TYPE_VENDOR | USB_DIR_IN, - value, index, msg[i].buf, + value, index, st->buf, msg[i].len, USB_CTRL_GET_TIMEOUT); if (result < 0) { @@ -221,6 +221,14 @@ static int dib0700_i2c_xfer_new(struct i2c_adapter *adap, struct i2c_msg *msg, break; } + if (msg[i].len > sizeof(st->buf)) { + deb_info("buffer too small to fit %d bytes\n", + msg[i].len); + return -EIO; + } + + memcpy(msg[i].buf, st->buf, msg[i].len); + deb_data("<<< "); debug_dump(msg[i].buf, msg[i].len, deb_data); @@ -238,6 +246,13 @@ static int dib0700_i2c_xfer_new(struct i2c_adapter *adap, struct i2c_msg *msg, /* I2C ctrl + FE bus; */ st->buf[3] = ((gen_mode << 6) & 0xC0) | ((bus_mode << 4) & 0x30); + + if (msg[i].len > sizeof(st->buf) - 4) { + deb_info("i2c message to big: %d\n", + msg[i].len); + return -EIO; + } + /* The Actual i2c payload */ memcpy(&st->buf[4], msg[i].buf, msg[i].len); @@ -283,6 +298,11 @@ static int dib0700_i2c_xfer_legacy(struct i2c_adapter *adap, /* fill in the address */ st->buf[1] = msg[i].addr << 1; /* fill the buffer */ + if (msg[i].len > sizeof(st->buf) - 2) { + deb_info("i2c xfer to big: %d\n", + msg[i].len); + return -EIO; + } memcpy(&st->buf[2], msg[i].buf, msg[i].len); /* write/read request */ @@ -292,13 +312,20 @@ static int dib0700_i2c_xfer_legacy(struct i2c_adapter *adap, /* special thing in the current firmware: when length is zero the read-failed */ len = dib0700_ctrl_rd(d, st->buf, msg[i].len + 2, - msg[i+1].buf, msg[i+1].len); + st->buf, msg[i + 1].len); if (len <= 0) { deb_info("I2C read failed on address 0x%02x\n", msg[i].addr); break; } + if (msg[i + 1].len > sizeof(st->buf)) { + deb_info("i2c xfer buffer to small for %d\n", + msg[i].len); + return -EIO; + } + memcpy(msg[i + 1].buf, st->buf, msg[i + 1].len); + msg[i+1].len = len; i++; diff --git a/drivers/media/usb/dvb-usb/dib0700_devices.c b/drivers/media/usb/dvb-usb/dib0700_devices.c index 0857b56e652c..ef1b8ee75c57 100644 --- a/drivers/media/usb/dvb-usb/dib0700_devices.c +++ b/drivers/media/usb/dvb-usb/dib0700_devices.c @@ -508,8 +508,6 @@ static int stk7700ph_tuner_attach(struct dvb_usb_adapter *adap) #define DEFAULT_RC_INTERVAL 50 -static u8 rc_request[] = { REQUEST_POLL_RC, 0 }; - /* * This function is used only when firmware is < 1.20 version. Newer * firmwares use bulk mode, with functions implemented at dib0700_core, @@ -517,7 +515,6 @@ static u8 rc_request[] = { REQUEST_POLL_RC, 0 }; */ static int dib0700_rc_query_old_firmware(struct dvb_usb_device *d) { - u8 key[4]; enum rc_type protocol; u32 scancode; u8 toggle; @@ -532,39 +529,43 @@ static int dib0700_rc_query_old_firmware(struct dvb_usb_device *d) return 0; } - i = dib0700_ctrl_rd(d, rc_request, 2, key, 4); + st->buf[0] = REQUEST_POLL_RC; + st->buf[1] = 0; + + i = dib0700_ctrl_rd(d, st->buf, 2, st->buf, 4); if (i <= 0) { err("RC Query Failed"); - return -1; + return -EIO; } /* losing half of KEY_0 events from Philipps rc5 remotes.. */ - if (key[0] == 0 && key[1] == 0 && key[2] == 0 && key[3] == 0) + if (st->buf[0] == 0 && st->buf[1] == 0 + && st->buf[2] == 0 && st->buf[3] == 0) return 0; - /* info("%d: %2X %2X %2X %2X",dvb_usb_dib0700_ir_proto,(int)key[3-2],(int)key[3-3],(int)key[3-1],(int)key[3]); */ + /* info("%d: %2X %2X %2X %2X",dvb_usb_dib0700_ir_proto,(int)st->buf[3 - 2],(int)st->buf[3 - 3],(int)st->buf[3 - 1],(int)st->buf[3]); */ dib0700_rc_setup(d, NULL); /* reset ir sensor data to prevent false events */ switch (d->props.rc.core.protocol) { case RC_BIT_NEC: /* NEC protocol sends repeat code as 0 0 0 FF */ - if ((key[3-2] == 0x00) && (key[3-3] == 0x00) && - (key[3] == 0xff)) { + if ((st->buf[3 - 2] == 0x00) && (st->buf[3 - 3] == 0x00) && + (st->buf[3] == 0xff)) { rc_repeat(d->rc_dev); return 0; } protocol = RC_TYPE_NEC; - scancode = RC_SCANCODE_NEC(key[3-2], key[3-3]); + scancode = RC_SCANCODE_NEC(st->buf[3 - 2], st->buf[3 - 3]); toggle = 0; break; default: /* RC-5 protocol changes toggle bit on new keypress */ protocol = RC_TYPE_RC5; - scancode = RC_SCANCODE_RC5(key[3-2], key[3-3]); - toggle = key[3-1]; + scancode = RC_SCANCODE_RC5(st->buf[3 - 2], st->buf[3 - 3]); + toggle = st->buf[3 - 1]; break; } diff --git a/drivers/media/usb/dvb-usb/dibusb-common.c b/drivers/media/usb/dvb-usb/dibusb-common.c index 18ed3bfbb5e2..de3ee2547479 100644 --- a/drivers/media/usb/dvb-usb/dibusb-common.c +++ b/drivers/media/usb/dvb-usb/dibusb-common.c @@ -62,72 +62,117 @@ EXPORT_SYMBOL(dibusb_pid_filter_ctrl); int dibusb_power_ctrl(struct dvb_usb_device *d, int onoff) { - u8 b[3]; + u8 *b; int ret; + + b = kmalloc(3, GFP_KERNEL); + if (!b) + return -ENOMEM; + b[0] = DIBUSB_REQ_SET_IOCTL; b[1] = DIBUSB_IOCTL_CMD_POWER_MODE; b[2] = onoff ? DIBUSB_IOCTL_POWER_WAKEUP : DIBUSB_IOCTL_POWER_SLEEP; - ret = dvb_usb_generic_write(d,b,3); + + ret = dvb_usb_generic_write(d, b, 3); + + kfree(b); + msleep(10); + return ret; } EXPORT_SYMBOL(dibusb_power_ctrl); int dibusb2_0_streaming_ctrl(struct dvb_usb_adapter *adap, int onoff) { - u8 b[3] = { 0 }; int ret; + u8 *b; + + b = kmalloc(3, GFP_KERNEL); + if (!b) + return -ENOMEM; if ((ret = dibusb_streaming_ctrl(adap,onoff)) < 0) - return ret; + goto ret; if (onoff) { b[0] = DIBUSB_REQ_SET_STREAMING_MODE; b[1] = 0x00; - if ((ret = dvb_usb_generic_write(adap->dev,b,2)) < 0) - return ret; + ret = dvb_usb_generic_write(adap->dev, b, 2); + if (ret < 0) + goto ret; } b[0] = DIBUSB_REQ_SET_IOCTL; b[1] = onoff ? DIBUSB_IOCTL_CMD_ENABLE_STREAM : DIBUSB_IOCTL_CMD_DISABLE_STREAM; - return dvb_usb_generic_write(adap->dev,b,3); + ret = dvb_usb_generic_write(adap->dev, b, 3); + +ret: + kfree(b); + return ret; } EXPORT_SYMBOL(dibusb2_0_streaming_ctrl); int dibusb2_0_power_ctrl(struct dvb_usb_device *d, int onoff) { - if (onoff) { - u8 b[3] = { DIBUSB_REQ_SET_IOCTL, DIBUSB_IOCTL_CMD_POWER_MODE, DIBUSB_IOCTL_POWER_WAKEUP }; - return dvb_usb_generic_write(d,b,3); - } else + u8 *b; + int ret; + + if (!onoff) return 0; + + b = kmalloc(3, GFP_KERNEL); + if (!b) + return -ENOMEM; + + b[0] = DIBUSB_REQ_SET_IOCTL; + b[1] = DIBUSB_IOCTL_CMD_POWER_MODE; + b[2] = DIBUSB_IOCTL_POWER_WAKEUP; + + ret = dvb_usb_generic_write(d, b, 3); + + kfree(b); + + return ret; } EXPORT_SYMBOL(dibusb2_0_power_ctrl); static int dibusb_i2c_msg(struct dvb_usb_device *d, u8 addr, u8 *wbuf, u16 wlen, u8 *rbuf, u16 rlen) { - u8 sndbuf[MAX_XFER_SIZE]; /* lead(1) devaddr,direction(1) addr(2) data(wlen) (len(2) (when reading)) */ + u8 *sndbuf; + int ret, wo, len; + /* write only ? */ - int wo = (rbuf == NULL || rlen == 0), - len = 2 + wlen + (wo ? 0 : 2); + wo = (rbuf == NULL || rlen == 0); + + len = 2 + wlen + (wo ? 0 : 2); + + sndbuf = kmalloc(MAX_XFER_SIZE, GFP_KERNEL); + if (!sndbuf) + return -ENOMEM; - if (4 + wlen > sizeof(sndbuf)) { + if (4 + wlen > MAX_XFER_SIZE) { warn("i2c wr: len=%d is too big!\n", wlen); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; + goto ret; } sndbuf[0] = wo ? DIBUSB_REQ_I2C_WRITE : DIBUSB_REQ_I2C_READ; sndbuf[1] = (addr << 1) | (wo ? 0 : 1); - memcpy(&sndbuf[2],wbuf,wlen); + memcpy(&sndbuf[2], wbuf, wlen); if (!wo) { - sndbuf[wlen+2] = (rlen >> 8) & 0xff; - sndbuf[wlen+3] = rlen & 0xff; + sndbuf[wlen + 2] = (rlen >> 8) & 0xff; + sndbuf[wlen + 3] = rlen & 0xff; } - return dvb_usb_generic_rw(d,sndbuf,len,rbuf,rlen,0); + ret = dvb_usb_generic_rw(d, sndbuf, len, rbuf, rlen, 0); + +ret: + kfree(sndbuf); + return ret; } /* @@ -319,11 +364,27 @@ EXPORT_SYMBOL(rc_map_dibusb_table); int dibusb_rc_query(struct dvb_usb_device *d, u32 *event, int *state) { - u8 key[5],cmd = DIBUSB_REQ_POLL_REMOTE; - dvb_usb_generic_rw(d,&cmd,1,key,5,0); - dvb_usb_nec_rc_key_to_event(d,key,event,state); - if (key[0] != 0) - deb_info("key: %*ph\n", 5, key); - return 0; + u8 *buf; + int ret; + + buf = kmalloc(5, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf[0] = DIBUSB_REQ_POLL_REMOTE; + + ret = dvb_usb_generic_rw(d, buf, 1, buf, 5, 0); + if (ret < 0) + goto ret; + + dvb_usb_nec_rc_key_to_event(d, buf, event, state); + + if (buf[0] != 0) + deb_info("key: %*ph\n", 5, buf); + + kfree(buf); + +ret: + return ret; } EXPORT_SYMBOL(dibusb_rc_query); diff --git a/drivers/media/usb/dvb-usb/dibusb.h b/drivers/media/usb/dvb-usb/dibusb.h index 3f82163d8ab8..697be2a17ade 100644 --- a/drivers/media/usb/dvb-usb/dibusb.h +++ b/drivers/media/usb/dvb-usb/dibusb.h @@ -96,6 +96,9 @@ #define DIBUSB_IOCTL_CMD_ENABLE_STREAM 0x01 #define DIBUSB_IOCTL_CMD_DISABLE_STREAM 0x02 +/* Max transfer size done by I2C transfer functions */ +#define MAX_XFER_SIZE 64 + struct dibusb_state { struct dib_fe_xfer_ops ops; int mt2060_present; diff --git a/drivers/media/usb/dvb-usb/digitv.c b/drivers/media/usb/dvb-usb/digitv.c index 63134335c994..4284f6984dc1 100644 --- a/drivers/media/usb/dvb-usb/digitv.c +++ b/drivers/media/usb/dvb-usb/digitv.c @@ -28,22 +28,26 @@ DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); static int digitv_ctrl_msg(struct dvb_usb_device *d, u8 cmd, u8 vv, u8 *wbuf, int wlen, u8 *rbuf, int rlen) { - int wo = (rbuf == NULL || rlen == 0); /* write-only */ - u8 sndbuf[7],rcvbuf[7]; - memset(sndbuf,0,7); memset(rcvbuf,0,7); + struct digitv_state *st = d->priv; + int ret, wo; - sndbuf[0] = cmd; - sndbuf[1] = vv; - sndbuf[2] = wo ? wlen : rlen; + wo = (rbuf == NULL || rlen == 0); /* write-only */ + + memset(st->sndbuf, 0, 7); + memset(st->rcvbuf, 0, 7); + + st->sndbuf[0] = cmd; + st->sndbuf[1] = vv; + st->sndbuf[2] = wo ? wlen : rlen; if (wo) { - memcpy(&sndbuf[3],wbuf,wlen); - dvb_usb_generic_write(d,sndbuf,7); + memcpy(&st->sndbuf[3], wbuf, wlen); + ret = dvb_usb_generic_write(d, st->sndbuf, 7); } else { - dvb_usb_generic_rw(d,sndbuf,7,rcvbuf,7,10); - memcpy(rbuf,&rcvbuf[3],rlen); + ret = dvb_usb_generic_rw(d, st->sndbuf, 7, st->rcvbuf, 7, 10); + memcpy(rbuf, &st->rcvbuf[3], rlen); } - return 0; + return ret; } /* I2C */ diff --git a/drivers/media/usb/dvb-usb/digitv.h b/drivers/media/usb/dvb-usb/digitv.h index 908c09f4966b..581e09c25491 100644 --- a/drivers/media/usb/dvb-usb/digitv.h +++ b/drivers/media/usb/dvb-usb/digitv.h @@ -5,7 +5,10 @@ #include "dvb-usb.h" struct digitv_state { - int is_nxt6000; + int is_nxt6000; + + unsigned char sndbuf[7]; + unsigned char rcvbuf[7]; }; /* protocol (from usblogging and the SDK: diff --git a/drivers/media/usb/dvb-usb/dtt200u-fe.c b/drivers/media/usb/dvb-usb/dtt200u-fe.c index c09332bd99cb..f5c042baa254 100644 --- a/drivers/media/usb/dvb-usb/dtt200u-fe.c +++ b/drivers/media/usb/dvb-usb/dtt200u-fe.c @@ -18,17 +18,28 @@ struct dtt200u_fe_state { struct dtv_frontend_properties fep; struct dvb_frontend frontend; + + unsigned char data[80]; + struct mutex data_mutex; }; static int dtt200u_fe_read_status(struct dvb_frontend *fe, enum fe_status *stat) { struct dtt200u_fe_state *state = fe->demodulator_priv; - u8 st = GET_TUNE_STATUS, b[3]; + int ret; + + mutex_lock(&state->data_mutex); + state->data[0] = GET_TUNE_STATUS; - dvb_usb_generic_rw(state->d,&st,1,b,3,0); + ret = dvb_usb_generic_rw(state->d, state->data, 1, state->data, 3, 0); + if (ret < 0) { + *stat = 0; + mutex_unlock(&state->data_mutex); + return ret; + } - switch (b[0]) { + switch (state->data[0]) { case 0x01: *stat = FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK; @@ -41,51 +52,86 @@ static int dtt200u_fe_read_status(struct dvb_frontend *fe, *stat = 0; break; } + mutex_unlock(&state->data_mutex); return 0; } static int dtt200u_fe_read_ber(struct dvb_frontend* fe, u32 *ber) { struct dtt200u_fe_state *state = fe->demodulator_priv; - u8 bw = GET_VIT_ERR_CNT,b[3]; - dvb_usb_generic_rw(state->d,&bw,1,b,3,0); - *ber = (b[0] << 16) | (b[1] << 8) | b[2]; - return 0; + int ret; + + mutex_lock(&state->data_mutex); + state->data[0] = GET_VIT_ERR_CNT; + + ret = dvb_usb_generic_rw(state->d, state->data, 1, state->data, 3, 0); + if (ret >= 0) + *ber = (state->data[0] << 16) | (state->data[1] << 8) | state->data[2]; + + mutex_unlock(&state->data_mutex); + return ret; } static int dtt200u_fe_read_unc_blocks(struct dvb_frontend* fe, u32 *unc) { struct dtt200u_fe_state *state = fe->demodulator_priv; - u8 bw = GET_RS_UNCOR_BLK_CNT,b[2]; + int ret; - dvb_usb_generic_rw(state->d,&bw,1,b,2,0); - *unc = (b[0] << 8) | b[1]; - return 0; + mutex_lock(&state->data_mutex); + state->data[0] = GET_RS_UNCOR_BLK_CNT; + + ret = dvb_usb_generic_rw(state->d, state->data, 1, state->data, 2, 0); + if (ret >= 0) + *unc = (state->data[0] << 8) | state->data[1]; + + mutex_unlock(&state->data_mutex); + return ret; } static int dtt200u_fe_read_signal_strength(struct dvb_frontend* fe, u16 *strength) { struct dtt200u_fe_state *state = fe->demodulator_priv; - u8 bw = GET_AGC, b; - dvb_usb_generic_rw(state->d,&bw,1,&b,1,0); - *strength = (b << 8) | b; - return 0; + int ret; + + mutex_lock(&state->data_mutex); + state->data[0] = GET_AGC; + + ret = dvb_usb_generic_rw(state->d, state->data, 1, state->data, 1, 0); + if (ret >= 0) + *strength = (state->data[0] << 8) | state->data[0]; + + mutex_unlock(&state->data_mutex); + return ret; } static int dtt200u_fe_read_snr(struct dvb_frontend* fe, u16 *snr) { struct dtt200u_fe_state *state = fe->demodulator_priv; - u8 bw = GET_SNR,br; - dvb_usb_generic_rw(state->d,&bw,1,&br,1,0); - *snr = ~((br << 8) | br); - return 0; + int ret; + + mutex_lock(&state->data_mutex); + state->data[0] = GET_SNR; + + ret = dvb_usb_generic_rw(state->d, state->data, 1, state->data, 1, 0); + if (ret >= 0) + *snr = ~((state->data[0] << 8) | state->data[0]); + + mutex_unlock(&state->data_mutex); + return ret; } static int dtt200u_fe_init(struct dvb_frontend* fe) { struct dtt200u_fe_state *state = fe->demodulator_priv; - u8 b = SET_INIT; - return dvb_usb_generic_write(state->d,&b,1); + int ret; + + mutex_lock(&state->data_mutex); + state->data[0] = SET_INIT; + + ret = dvb_usb_generic_write(state->d, state->data, 1); + mutex_unlock(&state->data_mutex); + + return ret; } static int dtt200u_fe_sleep(struct dvb_frontend* fe) @@ -105,39 +151,40 @@ static int dtt200u_fe_set_frontend(struct dvb_frontend *fe) { struct dtv_frontend_properties *fep = &fe->dtv_property_cache; struct dtt200u_fe_state *state = fe->demodulator_priv; - int i; - enum fe_status st; + int ret; u16 freq = fep->frequency / 250000; - u8 bwbuf[2] = { SET_BANDWIDTH, 0 },freqbuf[3] = { SET_RF_FREQ, 0, 0 }; + mutex_lock(&state->data_mutex); + state->data[0] = SET_BANDWIDTH; switch (fep->bandwidth_hz) { case 8000000: - bwbuf[1] = 8; + state->data[1] = 8; break; case 7000000: - bwbuf[1] = 7; + state->data[1] = 7; break; case 6000000: - bwbuf[1] = 6; + state->data[1] = 6; break; default: - return -EINVAL; + ret = -EINVAL; + goto ret; } - dvb_usb_generic_write(state->d,bwbuf,2); + ret = dvb_usb_generic_write(state->d, state->data, 2); + if (ret < 0) + goto ret; - freqbuf[1] = freq & 0xff; - freqbuf[2] = (freq >> 8) & 0xff; - dvb_usb_generic_write(state->d,freqbuf,3); + state->data[0] = SET_RF_FREQ; + state->data[1] = freq & 0xff; + state->data[2] = (freq >> 8) & 0xff; + ret = dvb_usb_generic_write(state->d, state->data, 3); + if (ret < 0) + goto ret; - for (i = 0; i < 30; i++) { - msleep(20); - dtt200u_fe_read_status(fe, &st); - if (st & FE_TIMEDOUT) - continue; - } - - return 0; +ret: + mutex_unlock(&state->data_mutex); + return ret; } static int dtt200u_fe_get_frontend(struct dvb_frontend* fe, @@ -169,6 +216,7 @@ struct dvb_frontend* dtt200u_fe_attach(struct dvb_usb_device *d) deb_info("attaching frontend dtt200u\n"); state->d = d; + mutex_init(&state->data_mutex); memcpy(&state->frontend.ops,&dtt200u_fe_ops,sizeof(struct dvb_frontend_ops)); state->frontend.demodulator_priv = state; diff --git a/drivers/media/usb/dvb-usb/dtt200u.c b/drivers/media/usb/dvb-usb/dtt200u.c index d2a01b50af0d..f88572c7ae7c 100644 --- a/drivers/media/usb/dvb-usb/dtt200u.c +++ b/drivers/media/usb/dvb-usb/dtt200u.c @@ -20,75 +20,114 @@ MODULE_PARM_DESC(debug, "set debugging level (1=info,xfer=2 (or-able))." DVB_USB DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); +struct dtt200u_state { + unsigned char data[80]; + struct mutex data_mutex; +}; + static int dtt200u_power_ctrl(struct dvb_usb_device *d, int onoff) { - u8 b = SET_INIT; + struct dtt200u_state *st = d->priv; + int ret = 0; + + mutex_lock(&st->data_mutex); + + st->data[0] = SET_INIT; if (onoff) - dvb_usb_generic_write(d,&b,2); + ret = dvb_usb_generic_write(d, st->data, 2); - return 0; + mutex_unlock(&st->data_mutex); + return ret; } static int dtt200u_streaming_ctrl(struct dvb_usb_adapter *adap, int onoff) { - u8 b_streaming[2] = { SET_STREAMING, onoff }; - u8 b_rst_pid = RESET_PID_FILTER; + struct dtt200u_state *st = adap->dev->priv; + int ret; - dvb_usb_generic_write(adap->dev, b_streaming, 2); + mutex_lock(&st->data_mutex); + st->data[0] = SET_STREAMING; + st->data[1] = onoff; - if (onoff == 0) - dvb_usb_generic_write(adap->dev, &b_rst_pid, 1); - return 0; + ret = dvb_usb_generic_write(adap->dev, st->data, 2); + if (ret < 0) + goto ret; + + if (onoff) + goto ret; + + st->data[0] = RESET_PID_FILTER; + ret = dvb_usb_generic_write(adap->dev, st->data, 1); + +ret: + mutex_unlock(&st->data_mutex); + + return ret; } static int dtt200u_pid_filter(struct dvb_usb_adapter *adap, int index, u16 pid, int onoff) { - u8 b_pid[4]; + struct dtt200u_state *st = adap->dev->priv; + int ret; + pid = onoff ? pid : 0; - b_pid[0] = SET_PID_FILTER; - b_pid[1] = index; - b_pid[2] = pid & 0xff; - b_pid[3] = (pid >> 8) & 0x1f; + mutex_lock(&st->data_mutex); + st->data[0] = SET_PID_FILTER; + st->data[1] = index; + st->data[2] = pid & 0xff; + st->data[3] = (pid >> 8) & 0x1f; - return dvb_usb_generic_write(adap->dev, b_pid, 4); + ret = dvb_usb_generic_write(adap->dev, st->data, 4); + mutex_unlock(&st->data_mutex); + + return ret; } static int dtt200u_rc_query(struct dvb_usb_device *d) { - u8 key[5],cmd = GET_RC_CODE; + struct dtt200u_state *st = d->priv; u32 scancode; + int ret; + + mutex_lock(&st->data_mutex); + st->data[0] = GET_RC_CODE; + + ret = dvb_usb_generic_rw(d, st->data, 1, st->data, 5, 0); + if (ret < 0) + goto ret; - dvb_usb_generic_rw(d,&cmd,1,key,5,0); - if (key[0] == 1) { + if (st->data[0] == 1) { enum rc_type proto = RC_TYPE_NEC; - scancode = key[1]; - if ((u8) ~key[1] != key[2]) { + scancode = st->data[1]; + if ((u8) ~st->data[1] != st->data[2]) { /* Extended NEC */ scancode = scancode << 8; - scancode |= key[2]; + scancode |= st->data[2]; proto = RC_TYPE_NECX; } scancode = scancode << 8; - scancode |= key[3]; + scancode |= st->data[3]; /* Check command checksum is ok */ - if ((u8) ~key[3] == key[4]) + if ((u8) ~st->data[3] == st->data[4]) rc_keydown(d->rc_dev, proto, scancode, 0); else rc_keyup(d->rc_dev); - } else if (key[0] == 2) { + } else if (st->data[0] == 2) { rc_repeat(d->rc_dev); } else { rc_keyup(d->rc_dev); } - if (key[0] != 0) - deb_info("key: %*ph\n", 5, key); + if (st->data[0] != 0) + deb_info("st->data: %*ph\n", 5, st->data); - return 0; +ret: + mutex_unlock(&st->data_mutex); + return ret; } static int dtt200u_frontend_attach(struct dvb_usb_adapter *adap) @@ -106,17 +145,24 @@ static struct dvb_usb_device_properties wt220u_miglia_properties; static int dtt200u_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) { + struct dvb_usb_device *d; + struct dtt200u_state *st; + if (0 == dvb_usb_device_init(intf, &dtt200u_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &wt220u_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &wt220u_fc_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &wt220u_zl0353_properties, - THIS_MODULE, NULL, adapter_nr) || + THIS_MODULE, &d, adapter_nr) || 0 == dvb_usb_device_init(intf, &wt220u_miglia_properties, - THIS_MODULE, NULL, adapter_nr)) + THIS_MODULE, &d, adapter_nr)) { + st = d->priv; + mutex_init(&st->data_mutex); + return 0; + } return -ENODEV; } @@ -140,6 +186,8 @@ static struct dvb_usb_device_properties dtt200u_properties = { .usb_ctrl = CYPRESS_FX2, .firmware = "dvb-usb-dtt200u-01.fw", + .size_of_priv = sizeof(struct dtt200u_state), + .num_adapters = 1, .adapter = { { @@ -190,6 +238,8 @@ static struct dvb_usb_device_properties wt220u_properties = { .usb_ctrl = CYPRESS_FX2, .firmware = "dvb-usb-wt220u-02.fw", + .size_of_priv = sizeof(struct dtt200u_state), + .num_adapters = 1, .adapter = { { @@ -240,6 +290,8 @@ static struct dvb_usb_device_properties wt220u_fc_properties = { .usb_ctrl = CYPRESS_FX2, .firmware = "dvb-usb-wt220u-fc03.fw", + .size_of_priv = sizeof(struct dtt200u_state), + .num_adapters = 1, .adapter = { { @@ -290,6 +342,8 @@ static struct dvb_usb_device_properties wt220u_zl0353_properties = { .usb_ctrl = CYPRESS_FX2, .firmware = "dvb-usb-wt220u-zl0353-01.fw", + .size_of_priv = sizeof(struct dtt200u_state), + .num_adapters = 1, .adapter = { { @@ -340,6 +394,8 @@ static struct dvb_usb_device_properties wt220u_miglia_properties = { .usb_ctrl = CYPRESS_FX2, .firmware = "dvb-usb-wt220u-miglia-01.fw", + .size_of_priv = sizeof(struct dtt200u_state), + .num_adapters = 1, .generic_bulk_ctrl_endpoint = 0x01, diff --git a/drivers/media/usb/dvb-usb/dtv5100.c b/drivers/media/usb/dvb-usb/dtv5100.c index 3d11df41cac0..c60fb54f445f 100644 --- a/drivers/media/usb/dvb-usb/dtv5100.c +++ b/drivers/media/usb/dvb-usb/dtv5100.c @@ -31,9 +31,14 @@ module_param_named(debug, dvb_usb_dtv5100_debug, int, 0644); MODULE_PARM_DESC(debug, "set debugging level" DVB_USB_DEBUG_STATUS); DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); +struct dtv5100_state { + unsigned char data[80]; +}; + static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, u8 *wbuf, u16 wlen, u8 *rbuf, u16 rlen) { + struct dtv5100_state *st = d->priv; u8 request; u8 type; u16 value; @@ -60,9 +65,10 @@ static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, } index = (addr << 8) + wbuf[0]; + memcpy(st->data, rbuf, rlen); msleep(1); /* avoid I2C errors */ return usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev, 0), request, - type, value, index, rbuf, rlen, + type, value, index, st->data, rlen, DTV5100_USB_TIMEOUT); } @@ -176,7 +182,7 @@ static struct dvb_usb_device_properties dtv5100_properties = { .caps = DVB_USB_IS_AN_I2C_ADAPTER, .usb_ctrl = DEVICE_SPECIFIC, - .size_of_priv = 0, + .size_of_priv = sizeof(struct dtv5100_state), .num_adapters = 1, .adapter = {{ diff --git a/drivers/media/usb/dvb-usb/dw2102.c b/drivers/media/usb/dvb-usb/dw2102.c index 5fb0c650926e..2c720cb2fb00 100644 --- a/drivers/media/usb/dvb-usb/dw2102.c +++ b/drivers/media/usb/dvb-usb/dw2102.c @@ -852,7 +852,7 @@ static int su3000_power_ctrl(struct dvb_usb_device *d, int i) if (i && !state->initialized) { state->initialized = 1; /* reset board */ - dvb_usb_generic_rw(d, obuf, 2, NULL, 0, 0); + return dvb_usb_generic_rw(d, obuf, 2, NULL, 0, 0); } return 0; diff --git a/drivers/media/usb/dvb-usb/gp8psk.c b/drivers/media/usb/dvb-usb/gp8psk.c index 5d0384dd45b5..adfd76491451 100644 --- a/drivers/media/usb/dvb-usb/gp8psk.c +++ b/drivers/media/usb/dvb-usb/gp8psk.c @@ -24,6 +24,10 @@ MODULE_PARM_DESC(debug, "set debugging level (1=info,xfer=2,rc=4 (or-able))." DV DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); +struct gp8psk_state { + unsigned char data[80]; +}; + static int gp8psk_get_fw_version(struct dvb_usb_device *d, u8 *fw_vers) { return (gp8psk_usb_in_op(d, GET_FW_VERS, 0, 0, fw_vers, 6)); @@ -53,17 +57,22 @@ static void gp8psk_info(struct dvb_usb_device *d) int gp8psk_usb_in_op(struct dvb_usb_device *d, u8 req, u16 value, u16 index, u8 *b, int blen) { + struct gp8psk_state *st = d->priv; int ret = 0,try = 0; + if (blen > sizeof(st->data)) + return -EIO; + if ((ret = mutex_lock_interruptible(&d->usb_mutex))) return ret; while (ret >= 0 && ret != blen && try < 3) { + memcpy(st->data, b, blen); ret = usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev,0), req, USB_TYPE_VENDOR | USB_DIR_IN, - value,index,b,blen, + value, index, st->data, blen, 2000); deb_info("reading number %d (ret: %d)\n",try,ret); try++; @@ -86,19 +95,24 @@ int gp8psk_usb_in_op(struct dvb_usb_device *d, u8 req, u16 value, u16 index, u8 int gp8psk_usb_out_op(struct dvb_usb_device *d, u8 req, u16 value, u16 index, u8 *b, int blen) { + struct gp8psk_state *st = d->priv; int ret; deb_xfer("out: req. %x, val: %x, ind: %x, buffer: ",req,value,index); debug_dump(b,blen,deb_xfer); + if (blen > sizeof(st->data)) + return -EIO; + if ((ret = mutex_lock_interruptible(&d->usb_mutex))) return ret; + memcpy(st->data, b, blen); if (usb_control_msg(d->udev, usb_sndctrlpipe(d->udev,0), req, USB_TYPE_VENDOR | USB_DIR_OUT, - value,index,b,blen, + value, index, st->data, blen, 2000) != blen) { warn("usb out operation failed."); ret = -EIO; @@ -143,6 +157,11 @@ static int gp8psk_load_bcm4500fw(struct dvb_usb_device *d) err("failed to load bcm4500 firmware."); goto out_free; } + if (buflen > 64) { + err("firmare chunk size bigger than 64 bytes."); + goto out_free; + } + memcpy(buf, ptr, buflen); if (dvb_usb_generic_write(d, buf, buflen)) { err("failed to load bcm4500 firmware."); @@ -265,6 +284,8 @@ static struct dvb_usb_device_properties gp8psk_properties = { .usb_ctrl = CYPRESS_FX2, .firmware = "dvb-usb-gp8psk-01.fw", + .size_of_priv = sizeof(struct gp8psk_state), + .num_adapters = 1, .adapter = { { diff --git a/drivers/media/usb/dvb-usb/nova-t-usb2.c b/drivers/media/usb/dvb-usb/nova-t-usb2.c index fc7569e2728d..1babd3341910 100644 --- a/drivers/media/usb/dvb-usb/nova-t-usb2.c +++ b/drivers/media/usb/dvb-usb/nova-t-usb2.c @@ -74,22 +74,31 @@ static struct rc_map_table rc_map_haupp_table[] = { */ static int nova_t_rc_query(struct dvb_usb_device *d, u32 *event, int *state) { - u8 key[5],cmd[2] = { DIBUSB_REQ_POLL_REMOTE, 0x35 }, data,toggle,custom; + u8 *buf, data, toggle, custom; u16 raw; - int i; + int i, ret; struct dibusb_device_state *st = d->priv; - dvb_usb_generic_rw(d,cmd,2,key,5,0); + buf = kmalloc(5, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf[0] = DIBUSB_REQ_POLL_REMOTE; + buf[1] = 0x35; + ret = dvb_usb_generic_rw(d, buf, 2, buf, 5, 0); + if (ret < 0) + goto ret; *state = REMOTE_NO_KEY_PRESSED; - switch (key[0]) { + switch (buf[0]) { case DIBUSB_RC_HAUPPAUGE_KEY_PRESSED: - raw = ((key[1] << 8) | key[2]) >> 3; + raw = ((buf[1] << 8) | buf[2]) >> 3; toggle = !!(raw & 0x800); data = raw & 0x3f; custom = (raw >> 6) & 0x1f; - deb_rc("raw key code 0x%02x, 0x%02x, 0x%02x to c: %02x d: %02x toggle: %d\n",key[1],key[2],key[3],custom,data,toggle); + deb_rc("raw key code 0x%02x, 0x%02x, 0x%02x to c: %02x d: %02x toggle: %d\n", + buf[1], buf[2], buf[3], custom, data, toggle); for (i = 0; i < ARRAY_SIZE(rc_map_haupp_table); i++) { if (rc5_data(&rc_map_haupp_table[i]) == data && @@ -117,7 +126,9 @@ static int nova_t_rc_query(struct dvb_usb_device *d, u32 *event, int *state) break; } - return 0; +ret: + kfree(buf); + return ret; } static int nova_t_read_mac_address (struct dvb_usb_device *d, u8 mac[6]) diff --git a/drivers/media/usb/dvb-usb/pctv452e.c b/drivers/media/usb/dvb-usb/pctv452e.c index c05de1b088a4..07fa08be9e99 100644 --- a/drivers/media/usb/dvb-usb/pctv452e.c +++ b/drivers/media/usb/dvb-usb/pctv452e.c @@ -97,48 +97,53 @@ struct pctv452e_state { u8 c; /* transaction counter, wraps around... */ u8 initialized; /* set to 1 if 0x15 has been sent */ u16 last_rc_key; + + unsigned char data[80]; }; static int tt3650_ci_msg(struct dvb_usb_device *d, u8 cmd, u8 *data, unsigned int write_len, unsigned int read_len) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; - u8 buf[64]; u8 id; unsigned int rlen; int ret; - BUG_ON(NULL == data && 0 != (write_len | read_len)); - BUG_ON(write_len > 64 - 4); - BUG_ON(read_len > 64 - 4); + if (!data || (write_len > 64 - 4) || (read_len > 64 - 4)) { + err("%s: transfer data invalid", __func__); + return -EIO; + } + mutex_lock(&state->ca_mutex); id = state->c++; - buf[0] = SYNC_BYTE_OUT; - buf[1] = id; - buf[2] = cmd; - buf[3] = write_len; + state->data[0] = SYNC_BYTE_OUT; + state->data[1] = id; + state->data[2] = cmd; + state->data[3] = write_len; - memcpy(buf + 4, data, write_len); + memcpy(state->data + 4, data, write_len); rlen = (read_len > 0) ? 64 : 0; - ret = dvb_usb_generic_rw(d, buf, 4 + write_len, - buf, rlen, /* delay_ms */ 0); + ret = dvb_usb_generic_rw(d, state->data, 4 + write_len, + state->data, rlen, /* delay_ms */ 0); if (0 != ret) goto failed; ret = -EIO; - if (SYNC_BYTE_IN != buf[0] || id != buf[1]) + if (SYNC_BYTE_IN != state->data[0] || id != state->data[1]) goto failed; - memcpy(data, buf + 4, read_len); + memcpy(data, state->data + 4, read_len); + mutex_unlock(&state->ca_mutex); return 0; failed: err("CI error %d; %02X %02X %02X -> %*ph.", - ret, SYNC_BYTE_OUT, id, cmd, 3, buf); + ret, SYNC_BYTE_OUT, id, cmd, 3, state->data); + mutex_unlock(&state->ca_mutex); return ret; } @@ -405,52 +410,53 @@ static int pctv452e_i2c_msg(struct dvb_usb_device *d, u8 addr, u8 *rcv_buf, u8 rcv_len) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; - u8 buf[64]; u8 id; int ret; + mutex_lock(&state->ca_mutex); id = state->c++; ret = -EINVAL; if (snd_len > 64 - 7 || rcv_len > 64 - 7) goto failed; - buf[0] = SYNC_BYTE_OUT; - buf[1] = id; - buf[2] = PCTV_CMD_I2C; - buf[3] = snd_len + 3; - buf[4] = addr << 1; - buf[5] = snd_len; - buf[6] = rcv_len; + state->data[0] = SYNC_BYTE_OUT; + state->data[1] = id; + state->data[2] = PCTV_CMD_I2C; + state->data[3] = snd_len + 3; + state->data[4] = addr << 1; + state->data[5] = snd_len; + state->data[6] = rcv_len; - memcpy(buf + 7, snd_buf, snd_len); + memcpy(state->data + 7, snd_buf, snd_len); - ret = dvb_usb_generic_rw(d, buf, 7 + snd_len, - buf, /* rcv_len */ 64, + ret = dvb_usb_generic_rw(d, state->data, 7 + snd_len, + state->data, /* rcv_len */ 64, /* delay_ms */ 0); if (ret < 0) goto failed; /* TT USB protocol error. */ ret = -EIO; - if (SYNC_BYTE_IN != buf[0] || id != buf[1]) + if (SYNC_BYTE_IN != state->data[0] || id != state->data[1]) goto failed; /* I2C device didn't respond as expected. */ ret = -EREMOTEIO; - if (buf[5] < snd_len || buf[6] < rcv_len) + if (state->data[5] < snd_len || state->data[6] < rcv_len) goto failed; - memcpy(rcv_buf, buf + 7, rcv_len); + memcpy(rcv_buf, state->data + 7, rcv_len); + mutex_unlock(&state->ca_mutex); return rcv_len; failed: - err("I2C error %d; %02X %02X %02X %02X %02X -> " - "%02X %02X %02X %02X %02X.", + err("I2C error %d; %02X %02X %02X %02X %02X -> %*ph", ret, SYNC_BYTE_OUT, id, addr << 1, snd_len, rcv_len, - buf[0], buf[1], buf[4], buf[5], buf[6]); + 7, state->data); + mutex_unlock(&state->ca_mutex); return ret; } @@ -499,8 +505,7 @@ static u32 pctv452e_i2c_func(struct i2c_adapter *adapter) static int pctv452e_power_ctrl(struct dvb_usb_device *d, int i) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; - u8 b0[] = { 0xaa, 0, PCTV_CMD_RESET, 1, 0 }; - u8 rx[PCTV_ANSWER_LEN]; + u8 *rx; int ret; info("%s: %d\n", __func__, i); @@ -511,6 +516,11 @@ static int pctv452e_power_ctrl(struct dvb_usb_device *d, int i) if (state->initialized) return 0; + rx = kmalloc(PCTV_ANSWER_LEN, GFP_KERNEL); + if (!rx) + return -ENOMEM; + + mutex_lock(&state->ca_mutex); /* hmm where shoud this should go? */ ret = usb_set_interface(d->udev, 0, ISOC_INTERFACE_ALTERNATIVE); if (ret != 0) @@ -518,65 +528,75 @@ static int pctv452e_power_ctrl(struct dvb_usb_device *d, int i) __func__, ret); /* this is a one-time initialization, dont know where to put */ - b0[1] = state->c++; + state->data[0] = 0xaa; + state->data[1] = state->c++; + state->data[2] = PCTV_CMD_RESET; + state->data[3] = 1; + state->data[4] = 0; /* reset board */ - ret = dvb_usb_generic_rw(d, b0, sizeof(b0), rx, PCTV_ANSWER_LEN, 0); + ret = dvb_usb_generic_rw(d, state->data, 5, rx, PCTV_ANSWER_LEN, 0); if (ret) - return ret; + goto ret; - b0[1] = state->c++; - b0[4] = 1; + state->data[1] = state->c++; + state->data[4] = 1; /* reset board (again?) */ - ret = dvb_usb_generic_rw(d, b0, sizeof(b0), rx, PCTV_ANSWER_LEN, 0); + ret = dvb_usb_generic_rw(d, state->data, 5, rx, PCTV_ANSWER_LEN, 0); if (ret) - return ret; + goto ret; state->initialized = 1; - return 0; +ret: + mutex_unlock(&state->ca_mutex); + kfree(rx); + return ret; } static int pctv452e_rc_query(struct dvb_usb_device *d) { struct pctv452e_state *state = (struct pctv452e_state *)d->priv; - u8 b[CMD_BUFFER_SIZE]; - u8 rx[PCTV_ANSWER_LEN]; int ret, i; - u8 id = state->c++; + u8 id; + + mutex_lock(&state->ca_mutex); + id = state->c++; /* prepare command header */ - b[0] = SYNC_BYTE_OUT; - b[1] = id; - b[2] = PCTV_CMD_IR; - b[3] = 0; + state->data[0] = SYNC_BYTE_OUT; + state->data[1] = id; + state->data[2] = PCTV_CMD_IR; + state->data[3] = 0; /* send ir request */ - ret = dvb_usb_generic_rw(d, b, 4, rx, PCTV_ANSWER_LEN, 0); + ret = dvb_usb_generic_rw(d, state->data, 4, + state->data, PCTV_ANSWER_LEN, 0); if (ret != 0) - return ret; + goto ret; if (debug > 3) { - info("%s: read: %2d: %*ph: ", __func__, ret, 3, rx); - for (i = 0; (i < rx[3]) && ((i+3) < PCTV_ANSWER_LEN); i++) - info(" %02x", rx[i+3]); + info("%s: read: %2d: %*ph: ", __func__, ret, 3, state->data); + for (i = 0; (i < state->data[3]) && ((i + 3) < PCTV_ANSWER_LEN); i++) + info(" %02x", state->data[i + 3]); info("\n"); } - if ((rx[3] == 9) && (rx[12] & 0x01)) { + if ((state->data[3] == 9) && (state->data[12] & 0x01)) { /* got a "press" event */ - state->last_rc_key = RC_SCANCODE_RC5(rx[7], rx[6]); + state->last_rc_key = RC_SCANCODE_RC5(state->data[7], state->data[6]); if (debug > 2) info("%s: cmd=0x%02x sys=0x%02x\n", - __func__, rx[6], rx[7]); + __func__, state->data[6], state->data[7]); rc_keydown(d->rc_dev, RC_TYPE_RC5, state->last_rc_key, 0); } else if (state->last_rc_key) { rc_keyup(d->rc_dev); state->last_rc_key = 0; } - - return 0; +ret: + mutex_unlock(&state->ca_mutex); + return ret; } static int pctv452e_read_mac_address(struct dvb_usb_device *d, u8 mac[6]) diff --git a/drivers/media/usb/dvb-usb/technisat-usb2.c b/drivers/media/usb/dvb-usb/technisat-usb2.c index d9f3262bf071..4706628a3ed5 100644 --- a/drivers/media/usb/dvb-usb/technisat-usb2.c +++ b/drivers/media/usb/dvb-usb/technisat-usb2.c @@ -89,9 +89,13 @@ struct technisat_usb2_state { static int technisat_usb2_i2c_access(struct usb_device *udev, u8 device_addr, u8 *tx, u8 txlen, u8 *rx, u8 rxlen) { - u8 b[64]; + u8 *b; int ret, actual_length; + b = kmalloc(64, GFP_KERNEL); + if (!b) + return -ENOMEM; + deb_i2c("i2c-access: %02x, tx: ", device_addr); debug_dump(tx, txlen, deb_i2c); deb_i2c(" "); @@ -123,7 +127,7 @@ static int technisat_usb2_i2c_access(struct usb_device *udev, if (ret < 0) { err("i2c-error: out failed %02x = %d", device_addr, ret); - return -ENODEV; + goto err; } ret = usb_bulk_msg(udev, @@ -131,7 +135,7 @@ static int technisat_usb2_i2c_access(struct usb_device *udev, b, 64, &actual_length, 1000); if (ret < 0) { err("i2c-error: in failed %02x = %d", device_addr, ret); - return -ENODEV; + goto err; } if (b[0] != I2C_STATUS_OK) { @@ -140,7 +144,7 @@ static int technisat_usb2_i2c_access(struct usb_device *udev, if (!(b[0] == I2C_STATUS_NAK && device_addr == 0x60 /* && device_is_technisat_usb2 */)) - return -ENODEV; + goto err; } deb_i2c("status: %d, ", b[0]); @@ -154,7 +158,9 @@ static int technisat_usb2_i2c_access(struct usb_device *udev, deb_i2c("\n"); - return 0; +err: + kfree(b); + return ret; } static int technisat_usb2_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msg, diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c index c3a0e87066eb..f7bb78c1873c 100644 --- a/drivers/media/usb/s2255/s2255drv.c +++ b/drivers/media/usb/s2255/s2255drv.c @@ -1901,19 +1901,30 @@ static long s2255_vendor_req(struct s2255_dev *dev, unsigned char Request, s32 TransferBufferLength, int bOut) { int r; + unsigned char *buf; + + buf = kmalloc(TransferBufferLength, GFP_KERNEL); + if (!buf) + return -ENOMEM; + if (!bOut) { r = usb_control_msg(dev->udev, usb_rcvctrlpipe(dev->udev, 0), Request, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN, - Value, Index, TransferBuffer, + Value, Index, buf, TransferBufferLength, HZ * 5); + + if (r >= 0) + memcpy(TransferBuffer, buf, TransferBufferLength); } else { + memcpy(buf, TransferBuffer, TransferBufferLength); r = usb_control_msg(dev->udev, usb_sndctrlpipe(dev->udev, 0), Request, USB_TYPE_VENDOR | USB_RECIP_DEVICE, - Value, Index, TransferBuffer, + Value, Index, buf, TransferBufferLength, HZ * 5); } + kfree(buf); return r; } diff --git a/drivers/media/usb/stkwebcam/stk-webcam.c b/drivers/media/usb/stkwebcam/stk-webcam.c index db200c9d796d..22a9aae16291 100644 --- a/drivers/media/usb/stkwebcam/stk-webcam.c +++ b/drivers/media/usb/stkwebcam/stk-webcam.c @@ -147,20 +147,26 @@ int stk_camera_write_reg(struct stk_camera *dev, u16 index, u8 value) int stk_camera_read_reg(struct stk_camera *dev, u16 index, int *value) { struct usb_device *udev = dev->udev; + unsigned char *buf; int ret; + buf = kmalloc(sizeof(u8), GFP_KERNEL); + if (!buf) + return -ENOMEM; + ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), 0x00, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0x00, index, - (u8 *) value, + buf, sizeof(u8), 500); - if (ret < 0) - return ret; - else - return 0; + if (ret >= 0) + memcpy(value, buf, sizeof(u8)); + + kfree(buf); + return ret; } static int stk_start_stream(struct stk_camera *dev) diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c index af23d7dfe752..2e5233b60971 100644 --- a/drivers/misc/cxl/api.c +++ b/drivers/misc/cxl/api.c @@ -247,7 +247,9 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed, cxl_ctx_get(); if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) { + put_pid(ctx->glpid); put_pid(ctx->pid); + ctx->glpid = ctx->pid = NULL; cxl_adapter_context_put(ctx->afu->adapter); cxl_ctx_put(); goto out; diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c index d0b421f49b39..77080cc5fa0a 100644 --- a/drivers/misc/cxl/file.c +++ b/drivers/misc/cxl/file.c @@ -194,6 +194,16 @@ static long afu_ioctl_start_work(struct cxl_context *ctx, ctx->mmio_err_ff = !!(work.flags & CXL_START_WORK_ERR_FF); /* + * Increment the mapped context count for adapter. This also checks + * if adapter_context_lock is taken. + */ + rc = cxl_adapter_context_get(ctx->afu->adapter); + if (rc) { + afu_release_irqs(ctx, ctx); + goto out; + } + + /* * We grab the PID here and not in the file open to allow for the case * where a process (master, some daemon, etc) has opened the chardev on * behalf of another process, so the AFU's mm gets bound to the process @@ -205,15 +215,6 @@ static long afu_ioctl_start_work(struct cxl_context *ctx, ctx->pid = get_task_pid(current, PIDTYPE_PID); ctx->glpid = get_task_pid(current->group_leader, PIDTYPE_PID); - /* - * Increment the mapped context count for adapter. This also checks - * if adapter_context_lock is taken. - */ - rc = cxl_adapter_context_get(ctx->afu->adapter); - if (rc) { - afu_release_irqs(ctx, ctx); - goto out; - } trace_cxl_attach(ctx, work.work_element_descriptor, work.num_interrupts, amr); @@ -221,6 +222,9 @@ static long afu_ioctl_start_work(struct cxl_context *ctx, amr))) { afu_release_irqs(ctx, ctx); cxl_adapter_context_put(ctx->afu->adapter); + put_pid(ctx->glpid); + put_pid(ctx->pid); + ctx->glpid = ctx->pid = NULL; goto out; } diff --git a/drivers/misc/genwqe/card_utils.c b/drivers/misc/genwqe/card_utils.c index 8a679ecc8fd1..fc2794b513fa 100644 --- a/drivers/misc/genwqe/card_utils.c +++ b/drivers/misc/genwqe/card_utils.c @@ -352,17 +352,27 @@ int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl, if (copy_from_user(sgl->lpage, user_addr + user_size - sgl->lpage_size, sgl->lpage_size)) { rc = -EFAULT; - goto err_out1; + goto err_out2; } } return 0; + err_out2: + __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage, + sgl->lpage_dma_addr); + sgl->lpage = NULL; + sgl->lpage_dma_addr = 0; err_out1: __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage, sgl->fpage_dma_addr); + sgl->fpage = NULL; + sgl->fpage_dma_addr = 0; err_out: __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl, sgl->sgl_dma_addr); + sgl->sgl = NULL; + sgl->sgl_dma_addr = 0; + sgl->sgl_size = 0; return -ENOMEM; } diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c index e6e5e55a12ed..60415a2bfcbd 100644 --- a/drivers/misc/mei/hw-txe.c +++ b/drivers/misc/mei/hw-txe.c @@ -981,11 +981,13 @@ static bool mei_txe_check_and_ack_intrs(struct mei_device *dev, bool do_ack) hisr = mei_txe_br_reg_read(hw, HISR_REG); aliveness = mei_txe_aliveness_get(dev); - if (hhisr & IPC_HHIER_SEC && aliveness) + if (hhisr & IPC_HHIER_SEC && aliveness) { ipc_isr = mei_txe_sec_reg_read_silent(hw, SEC_IPC_HOST_INT_STATUS_REG); - else + } else { ipc_isr = 0; + hhisr &= ~IPC_HHIER_SEC; + } generated = generated || (hisr & HISR_INT_STS_MSK) || diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 1525870f460a..33741ad4a74a 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -283,7 +283,7 @@ static void gru_unload_mm_tracker(struct gru_state *gru, spin_lock(&gru->gs_asid_lock); BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap); asids->mt_ctxbitmap ^= ctxbitmap; - gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n", + gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum %d, asidmap 0x%lx\n", gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]); spin_unlock(&gru->gs_asid_lock); spin_unlock(&gms->ms_asid_lock); diff --git a/drivers/misc/vmw_vmci/vmci_doorbell.c b/drivers/misc/vmw_vmci/vmci_doorbell.c index a8cee33ae8d2..b3fa738ae005 100644 --- a/drivers/misc/vmw_vmci/vmci_doorbell.c +++ b/drivers/misc/vmw_vmci/vmci_doorbell.c @@ -431,6 +431,12 @@ int vmci_doorbell_create(struct vmci_handle *handle, if (vmci_handle_is_invalid(*handle)) { u32 context_id = vmci_get_context_id(); + if (context_id == VMCI_INVALID_ID) { + pr_warn("Failed to get context ID\n"); + result = VMCI_ERROR_NO_RESOURCES; + goto free_mem; + } + /* Let resource code allocate a free ID for us */ new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID); } else { @@ -525,7 +531,7 @@ int vmci_doorbell_destroy(struct vmci_handle handle) entry = container_of(resource, struct dbell_entry, resource); - if (vmci_guest_code_active()) { + if (!hlist_unhashed(&entry->node)) { int result; dbell_index_table_remove(entry); diff --git a/drivers/misc/vmw_vmci/vmci_driver.c b/drivers/misc/vmw_vmci/vmci_driver.c index 896be150e28f..d7eaf1eb11e7 100644 --- a/drivers/misc/vmw_vmci/vmci_driver.c +++ b/drivers/misc/vmw_vmci/vmci_driver.c @@ -113,5 +113,5 @@ module_exit(vmci_drv_exit); MODULE_AUTHOR("VMware, Inc."); MODULE_DESCRIPTION("VMware Virtual Machine Communication Interface."); -MODULE_VERSION("1.1.4.0-k"); +MODULE_VERSION("1.1.5.0-k"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/mmc/host/dw_mmc-pltfm.c b/drivers/mmc/host/dw_mmc-pltfm.c index c0bb0c793e84..dbbc4303bdd0 100644 --- a/drivers/mmc/host/dw_mmc-pltfm.c +++ b/drivers/mmc/host/dw_mmc-pltfm.c @@ -46,12 +46,13 @@ int dw_mci_pltfm_register(struct platform_device *pdev, host->pdata = pdev->dev.platform_data; regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); - /* Get registers' physical base address */ - host->phy_regs = regs->start; host->regs = devm_ioremap_resource(&pdev->dev, regs); if (IS_ERR(host->regs)) return PTR_ERR(host->regs); + /* Get registers' physical base address */ + host->phy_regs = regs->start; + platform_set_drvdata(pdev, host); return dw_mci_probe(host); } diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c index 8ef44a2a2fd9..90ed2e12d345 100644 --- a/drivers/mmc/host/sdhci-msm.c +++ b/drivers/mmc/host/sdhci-msm.c @@ -647,6 +647,7 @@ static int sdhci_msm_probe(struct platform_device *pdev) if (msm_host->pwr_irq < 0) { dev_err(&pdev->dev, "Get pwr_irq failed (%d)\n", msm_host->pwr_irq); + ret = msm_host->pwr_irq; goto clk_disable; } diff --git a/drivers/mtd/nand/gpmi-nand/gpmi-lib.c b/drivers/mtd/nand/gpmi-nand/gpmi-lib.c index 0f68a99fc4ad..141bd70a49c2 100644 --- a/drivers/mtd/nand/gpmi-nand/gpmi-lib.c +++ b/drivers/mtd/nand/gpmi-nand/gpmi-lib.c @@ -161,7 +161,7 @@ int gpmi_init(struct gpmi_nand_data *this) ret = gpmi_enable_clk(this); if (ret) - goto err_out; + return ret; ret = gpmi_reset_block(r->gpmi_regs, false); if (ret) goto err_out; @@ -197,6 +197,7 @@ int gpmi_init(struct gpmi_nand_data *this) gpmi_disable_clk(this); return 0; err_out: + gpmi_disable_clk(this); return ret; } @@ -270,7 +271,7 @@ int bch_set_geometry(struct gpmi_nand_data *this) ret = gpmi_enable_clk(this); if (ret) - goto err_out; + return ret; /* * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this @@ -308,6 +309,7 @@ int bch_set_geometry(struct gpmi_nand_data *this) gpmi_disable_clk(this); return 0; err_out: + gpmi_disable_clk(this); return ret; } diff --git a/drivers/mtd/nand/mtk_ecc.c b/drivers/mtd/nand/mtk_ecc.c index d54f666417e1..dbf256217b3e 100644 --- a/drivers/mtd/nand/mtk_ecc.c +++ b/drivers/mtd/nand/mtk_ecc.c @@ -86,6 +86,8 @@ struct mtk_ecc { struct completion done; struct mutex lock; u32 sectors; + + u8 eccdata[112]; }; static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc, @@ -366,9 +368,8 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config, u8 *data, u32 bytes) { dma_addr_t addr; - u8 *p; - u32 len, i, val; - int ret = 0; + u32 len; + int ret; addr = dma_map_single(ecc->dev, data, bytes, DMA_TO_DEVICE); ret = dma_mapping_error(ecc->dev, addr); @@ -393,14 +394,12 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config, /* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */ len = (config->strength * ECC_PARITY_BITS + 7) >> 3; - p = data + bytes; - /* write the parity bytes generated by the ECC back to the OOB region */ - for (i = 0; i < len; i++) { - if ((i % 4) == 0) - val = readl(ecc->regs + ECC_ENCPAR(i / 4)); - p[i] = (val >> ((i % 4) * 8)) & 0xff; - } + /* write the parity bytes generated by the ECC back to temp buffer */ + __ioread32_copy(ecc->eccdata, ecc->regs + ECC_ENCPAR(0), round_up(len, 4)); + + /* copy into possibly unaligned OOB region with actual length */ + memcpy(data + bytes, ecc->eccdata, len); timeout: dma_unmap_single(ecc->dev, addr, bytes, DMA_TO_DEVICE); diff --git a/drivers/mtd/nand/nand_base.c b/drivers/mtd/nand/nand_base.c index e5718e5ecf92..3bde96a3f7bf 100644 --- a/drivers/mtd/nand/nand_base.c +++ b/drivers/mtd/nand/nand_base.c @@ -1095,10 +1095,11 @@ static void nand_release_data_interface(struct nand_chip *chip) /** * nand_reset - Reset and initialize a NAND device * @chip: The NAND chip + * @chipnr: Internal die id * * Returns 0 for success or negative error code otherwise */ -int nand_reset(struct nand_chip *chip) +int nand_reset(struct nand_chip *chip, int chipnr) { struct mtd_info *mtd = nand_to_mtd(chip); int ret; @@ -1107,9 +1108,17 @@ int nand_reset(struct nand_chip *chip) if (ret) return ret; + /* + * The CS line has to be released before we can apply the new NAND + * interface settings, hence this weird ->select_chip() dance. + */ + chip->select_chip(mtd, chipnr); chip->cmdfunc(mtd, NAND_CMD_RESET, -1, -1); + chip->select_chip(mtd, -1); + chip->select_chip(mtd, chipnr); ret = nand_setup_data_interface(chip); + chip->select_chip(mtd, -1); if (ret) return ret; @@ -1185,8 +1194,6 @@ int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) /* Shift to get chip number */ chipnr = ofs >> chip->chip_shift; - chip->select_chip(mtd, chipnr); - /* * Reset the chip. * If we want to check the WP through READ STATUS and check the bit 7 @@ -1194,7 +1201,9 @@ int nand_unlock(struct mtd_info *mtd, loff_t ofs, uint64_t len) * some operation can also clear the bit 7 of status register * eg. erase/program a locked block */ - nand_reset(chip); + nand_reset(chip, chipnr); + + chip->select_chip(mtd, chipnr); /* Check, if it is write protected */ if (nand_check_wp(mtd)) { @@ -1244,8 +1253,6 @@ int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) /* Shift to get chip number */ chipnr = ofs >> chip->chip_shift; - chip->select_chip(mtd, chipnr); - /* * Reset the chip. * If we want to check the WP through READ STATUS and check the bit 7 @@ -1253,7 +1260,9 @@ int nand_lock(struct mtd_info *mtd, loff_t ofs, uint64_t len) * some operation can also clear the bit 7 of status register * eg. erase/program a locked block */ - nand_reset(chip); + nand_reset(chip, chipnr); + + chip->select_chip(mtd, chipnr); /* Check, if it is write protected */ if (nand_check_wp(mtd)) { @@ -2940,10 +2949,6 @@ static int nand_do_write_oob(struct mtd_info *mtd, loff_t to, } chipnr = (int)(to >> chip->chip_shift); - chip->select_chip(mtd, chipnr); - - /* Shift to get page */ - page = (int)(to >> chip->page_shift); /* * Reset the chip. Some chips (like the Toshiba TC5832DC found in one @@ -2951,7 +2956,12 @@ static int nand_do_write_oob(struct mtd_info *mtd, loff_t to, * if we don't do this. I have no clue why, but I seem to have 'fixed' * it in the doc2000 driver in August 1999. dwmw2. */ - nand_reset(chip); + nand_reset(chip, chipnr); + + chip->select_chip(mtd, chipnr); + + /* Shift to get page */ + page = (int)(to >> chip->page_shift); /* Check, if it is write protected */ if (nand_check_wp(mtd)) { @@ -3984,14 +3994,14 @@ static struct nand_flash_dev *nand_get_flash_type(struct mtd_info *mtd, int i, maf_idx; u8 id_data[8]; - /* Select the device */ - chip->select_chip(mtd, 0); - /* * Reset the chip, required by some chips (e.g. Micron MT29FxGxxxxx) * after power-up. */ - nand_reset(chip); + nand_reset(chip, 0); + + /* Select the device */ + chip->select_chip(mtd, 0); /* Send the command for reading device ID */ chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1); @@ -4329,17 +4339,31 @@ int nand_scan_ident(struct mtd_info *mtd, int maxchips, return PTR_ERR(type); } + /* Initialize the ->data_interface field. */ ret = nand_init_data_interface(chip); if (ret) return ret; + /* + * Setup the data interface correctly on the chip and controller side. + * This explicit call to nand_setup_data_interface() is only required + * for the first die, because nand_reset() has been called before + * ->data_interface and ->default_onfi_timing_mode were set. + * For the other dies, nand_reset() will automatically switch to the + * best mode for us. + */ + ret = nand_setup_data_interface(chip); + if (ret) + return ret; + chip->select_chip(mtd, -1); /* Check for a chip array */ for (i = 1; i < maxchips; i++) { - chip->select_chip(mtd, i); /* See comment in nand_get_flash_type for reset */ - nand_reset(chip); + nand_reset(chip, i); + + chip->select_chip(mtd, i); /* Send the command for reading device ID */ chip->cmdfunc(mtd, NAND_CMD_READID, 0x00, -1); /* Read manufacturer and device IDs */ diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c index 2ff62157d3bb..c1f5c29e458e 100644 --- a/drivers/mtd/ubi/fastmap.c +++ b/drivers/mtd/ubi/fastmap.c @@ -707,11 +707,11 @@ static int ubi_attach_fastmap(struct ubi_device *ubi, fmvhdr->vol_type, be32_to_cpu(fmvhdr->last_eb_bytes)); - if (!av) - goto fail_bad; - if (PTR_ERR(av) == -EINVAL) { - ubi_err(ubi, "volume (ID %i) already exists", - fmvhdr->vol_id); + if (IS_ERR(av)) { + if (PTR_ERR(av) == -EEXIST) + ubi_err(ubi, "volume (ID %i) already exists", + fmvhdr->vol_id); + goto fail_bad; } diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c index 76fb8552c9d9..ef63d24fef81 100644 --- a/drivers/net/dsa/b53/b53_mmap.c +++ b/drivers/net/dsa/b53/b53_mmap.c @@ -256,6 +256,7 @@ static const struct of_device_id b53_mmap_of_table[] = { { .compatible = "brcm,bcm63xx-switch" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, b53_mmap_of_table); static struct platform_driver b53_mmap_driver = { .probe = b53_mmap_probe, diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c index e218887f18b7..e3ee27ce13dd 100644 --- a/drivers/net/dsa/bcm_sf2.c +++ b/drivers/net/dsa/bcm_sf2.c @@ -1133,6 +1133,20 @@ static int bcm_sf2_sw_remove(struct platform_device *pdev) return 0; } +static void bcm_sf2_sw_shutdown(struct platform_device *pdev) +{ + struct bcm_sf2_priv *priv = platform_get_drvdata(pdev); + + /* For a kernel about to be kexec'd we want to keep the GPHY on for a + * successful MDIO bus scan to occur. If we did turn off the GPHY + * before (e.g: port_disable), this will also power it back on. + * + * Do not rely on kexec_in_progress, just power the PHY on. + */ + if (priv->hw_params.num_gphy == 1) + bcm_sf2_gphy_enable_set(priv->dev->ds, true); +} + #ifdef CONFIG_PM_SLEEP static int bcm_sf2_suspend(struct device *dev) { @@ -1158,10 +1172,12 @@ static const struct of_device_id bcm_sf2_of_match[] = { { .compatible = "brcm,bcm7445-switch-v4.0" }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, bcm_sf2_of_match); static struct platform_driver bcm_sf2_driver = { .probe = bcm_sf2_sw_probe, .remove = bcm_sf2_sw_remove, + .shutdown = bcm_sf2_sw_shutdown, .driver = { .name = "brcm-sf2", .of_match_table = bcm_sf2_of_match, diff --git a/drivers/net/ethernet/aurora/nb8800.c b/drivers/net/ethernet/aurora/nb8800.c index b047fd607b83..00c38bf151e6 100644 --- a/drivers/net/ethernet/aurora/nb8800.c +++ b/drivers/net/ethernet/aurora/nb8800.c @@ -1358,6 +1358,7 @@ static const struct of_device_id nb8800_dt_ids[] = { }, { } }; +MODULE_DEVICE_TABLE(of, nb8800_dt_ids); static int nb8800_probe(struct platform_device *pdev) { diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c index ae364c74baf3..537090952c45 100644 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c @@ -1126,7 +1126,8 @@ out_freeirq: free_irq(dev->irq, dev); out_phy_disconnect: - phy_disconnect(phydev); + if (priv->has_phy) + phy_disconnect(phydev); return ret; } diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c index 856379cbb402..31ca204b38d2 100644 --- a/drivers/net/ethernet/broadcom/bgmac.c +++ b/drivers/net/ethernet/broadcom/bgmac.c @@ -1449,7 +1449,7 @@ static int bgmac_phy_connect(struct bgmac *bgmac) phy_dev = phy_connect(bgmac->net_dev, bus_id, &bgmac_adjust_link, PHY_INTERFACE_MODE_MII); if (IS_ERR(phy_dev)) { - dev_err(bgmac->dev, "PHY connecton failed\n"); + dev_err(bgmac->dev, "PHY connection failed\n"); return PTR_ERR(phy_dev); } diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index 27f11a5d5fe2..b3791b394715 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -271,22 +271,25 @@ static inline u32 bnx2_tx_avail(struct bnx2 *bp, struct bnx2_tx_ring_info *txr) static u32 bnx2_reg_rd_ind(struct bnx2 *bp, u32 offset) { + unsigned long flags; u32 val; - spin_lock_bh(&bp->indirect_lock); + spin_lock_irqsave(&bp->indirect_lock, flags); BNX2_WR(bp, BNX2_PCICFG_REG_WINDOW_ADDRESS, offset); val = BNX2_RD(bp, BNX2_PCICFG_REG_WINDOW); - spin_unlock_bh(&bp->indirect_lock); + spin_unlock_irqrestore(&bp->indirect_lock, flags); return val; } static void bnx2_reg_wr_ind(struct bnx2 *bp, u32 offset, u32 val) { - spin_lock_bh(&bp->indirect_lock); + unsigned long flags; + + spin_lock_irqsave(&bp->indirect_lock, flags); BNX2_WR(bp, BNX2_PCICFG_REG_WINDOW_ADDRESS, offset); BNX2_WR(bp, BNX2_PCICFG_REG_WINDOW, val); - spin_unlock_bh(&bp->indirect_lock); + spin_unlock_irqrestore(&bp->indirect_lock, flags); } static void @@ -304,8 +307,10 @@ bnx2_shmem_rd(struct bnx2 *bp, u32 offset) static void bnx2_ctx_wr(struct bnx2 *bp, u32 cid_addr, u32 offset, u32 val) { + unsigned long flags; + offset += cid_addr; - spin_lock_bh(&bp->indirect_lock); + spin_lock_irqsave(&bp->indirect_lock, flags); if (BNX2_CHIP(bp) == BNX2_CHIP_5709) { int i; @@ -322,7 +327,7 @@ bnx2_ctx_wr(struct bnx2 *bp, u32 cid_addr, u32 offset, u32 val) BNX2_WR(bp, BNX2_CTX_DATA_ADR, offset); BNX2_WR(bp, BNX2_CTX_DATA, val); } - spin_unlock_bh(&bp->indirect_lock); + spin_unlock_irqrestore(&bp->indirect_lock, flags); } #ifdef BCM_CNIC diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 20fe6a8c35c1..0cee4c0283f9 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -15241,7 +15241,7 @@ static void bnx2x_init_cyclecounter(struct bnx2x *bp) memset(&bp->cyclecounter, 0, sizeof(bp->cyclecounter)); bp->cyclecounter.read = bnx2x_cyclecounter_read; bp->cyclecounter.mask = CYCLECOUNTER_MASK(64); - bp->cyclecounter.shift = 1; + bp->cyclecounter.shift = 0; bp->cyclecounter.mult = 1; } diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index f320497368f4..57eb4e1345cb 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -4057,7 +4057,7 @@ static void cfg_queues(struct adapter *adap) * capped by the number of available cores. */ if (n10g) { - i = num_online_cpus(); + i = min_t(int, MAX_OFLD_QSETS, num_online_cpus()); s->ofldqsets = roundup(i, adap->params.nports); } else { s->ofldqsets = adap->params.nports; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c index 0945fa49a5dd..2471ff465d5c 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c @@ -135,15 +135,17 @@ static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp, } static int alloc_uld_rxqs(struct adapter *adap, - struct sge_uld_rxq_info *rxq_info, - unsigned int nq, unsigned int offset, bool lro) + struct sge_uld_rxq_info *rxq_info, bool lro) { struct sge *s = &adap->sge; - struct sge_ofld_rxq *q = rxq_info->uldrxq + offset; - unsigned short *ids = rxq_info->rspq_id + offset; - unsigned int per_chan = nq / adap->params.nports; + unsigned int nq = rxq_info->nrxq + rxq_info->nciq; + struct sge_ofld_rxq *q = rxq_info->uldrxq; + unsigned short *ids = rxq_info->rspq_id; unsigned int bmap_idx = 0; - int i, err, msi_idx; + unsigned int per_chan; + int i, err, msi_idx, que_idx = 0; + + per_chan = rxq_info->nrxq / adap->params.nports; if (adap->flags & USING_MSIX) msi_idx = 1; @@ -151,12 +153,18 @@ static int alloc_uld_rxqs(struct adapter *adap, msi_idx = -((int)s->intrq.abs_id + 1); for (i = 0; i < nq; i++, q++) { + if (i == rxq_info->nrxq) { + /* start allocation of concentrator queues */ + per_chan = rxq_info->nciq / adap->params.nports; + que_idx = 0; + } + if (msi_idx >= 0) { bmap_idx = get_msix_idx_from_bmap(adap); msi_idx = adap->msix_info_ulds[bmap_idx].idx; } err = t4_sge_alloc_rxq(adap, &q->rspq, false, - adap->port[i / per_chan], + adap->port[que_idx++ / per_chan], msi_idx, q->fl.size ? &q->fl : NULL, uldrx_handler, @@ -165,29 +173,19 @@ static int alloc_uld_rxqs(struct adapter *adap, if (err) goto freeout; if (msi_idx >= 0) - rxq_info->msix_tbl[i + offset] = bmap_idx; + rxq_info->msix_tbl[i] = bmap_idx; memset(&q->stats, 0, sizeof(q->stats)); if (ids) ids[i] = q->rspq.abs_id; } return 0; freeout: - q = rxq_info->uldrxq + offset; + q = rxq_info->uldrxq; for ( ; i; i--, q++) { if (q->rspq.desc) free_rspq_fl(adap, &q->rspq, q->fl.size ? &q->fl : NULL); } - - /* We need to free rxq also in case of ciq allocation failure */ - if (offset) { - q = rxq_info->uldrxq + offset; - for ( ; i; i--, q++) { - if (q->rspq.desc) - free_rspq_fl(adap, &q->rspq, - q->fl.size ? &q->fl : NULL); - } - } return err; } @@ -205,9 +203,7 @@ setup_sge_queues_uld(struct adapter *adap, unsigned int uld_type, bool lro) return -ENOMEM; } - ret = !(!alloc_uld_rxqs(adap, rxq_info, rxq_info->nrxq, 0, lro) && - !alloc_uld_rxqs(adap, rxq_info, rxq_info->nciq, - rxq_info->nrxq, lro)); + ret = !(!alloc_uld_rxqs(adap, rxq_info, lro)); /* Tell uP to route control queue completions to rdma rspq */ if (adap->flags & FULL_INIT_DONE && diff --git a/drivers/net/ethernet/chelsio/cxgb4/sched.c b/drivers/net/ethernet/chelsio/cxgb4/sched.c index 539de764bbd3..cbd68a8fe2e4 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sched.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sched.c @@ -210,8 +210,10 @@ static int t4_sched_queue_bind(struct port_info *pi, struct ch_sched_queue *p) /* Unbind queue from any existing class */ err = t4_sched_queue_unbind(pi, p); - if (err) + if (err) { + t4_free_mem(qe); goto out; + } /* Bind queue to specified class */ memset(qe, 0, sizeof(*qe)); diff --git a/drivers/net/ethernet/cisco/enic/vnic_rq.c b/drivers/net/ethernet/cisco/enic/vnic_rq.c index e572a527b18d..36bc2c71fba9 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_rq.c +++ b/drivers/net/ethernet/cisco/enic/vnic_rq.c @@ -169,19 +169,28 @@ int vnic_rq_disable(struct vnic_rq *rq) { unsigned int wait; struct vnic_dev *vdev = rq->vdev; + int i; - iowrite32(0, &rq->ctrl->enable); + /* Due to a race condition with clearing RQ "mini-cache" in hw, we need + * to disable the RQ twice to guarantee that stale descriptors are not + * used when this RQ is re-enabled. + */ + for (i = 0; i < 2; i++) { + iowrite32(0, &rq->ctrl->enable); - /* Wait for HW to ACK disable request */ - for (wait = 0; wait < 1000; wait++) { - if (!(ioread32(&rq->ctrl->running))) - return 0; - udelay(10); - } + /* Wait for HW to ACK disable request */ + for (wait = 20000; wait > 0; wait--) + if (!ioread32(&rq->ctrl->running)) + break; + if (!wait) { + vdev_neterr(vdev, "Failed to disable RQ[%d]\n", + rq->index); - vdev_neterr(vdev, "Failed to disable RQ[%d]\n", rq->index); + return -ETIMEDOUT; + } + } - return -ETIMEDOUT; + return 0; } void vnic_rq_clean(struct vnic_rq *rq, @@ -212,6 +221,11 @@ void vnic_rq_clean(struct vnic_rq *rq, [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; iowrite32(fetch_index, &rq->ctrl->posted_index); + /* Anytime we write fetch_index, we need to re-write 0 to rq->enable + * to re-sync internal VIC state. + */ + iowrite32(0, &rq->ctrl->enable); + vnic_dev_clear_desc_ring(&rq->ring); } diff --git a/drivers/net/ethernet/ezchip/nps_enet.c b/drivers/net/ethernet/ezchip/nps_enet.c index f928e6f79c89..223f35cc034c 100644 --- a/drivers/net/ethernet/ezchip/nps_enet.c +++ b/drivers/net/ethernet/ezchip/nps_enet.c @@ -669,6 +669,7 @@ static const struct of_device_id nps_enet_dt_ids[] = { { .compatible = "ezchip,nps-mgt-enet" }, { /* Sentinel */ } }; +MODULE_DEVICE_TABLE(of, nps_enet_dt_ids); static struct platform_driver nps_enet_driver = { .probe = nps_enet_probe, diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 48a033e64423..5aa9d4ded214 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -1430,14 +1430,14 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id) skb_put(skb, pkt_len - 4); data = skb->data; + if (!is_copybreak && need_swap) + swap_buffer(data, pkt_len); + #if !defined(CONFIG_M5272) if (fep->quirks & FEC_QUIRK_HAS_RACC) data = skb_pull_inline(skb, 2); #endif - if (!is_copybreak && need_swap) - swap_buffer(data, pkt_len); - /* Extract the enhanced buffer descriptor */ ebdp = NULL; if (fep->bufdesc_ex) diff --git a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c index 8d70377f6624..8ea3d95fa483 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_dsaf_main.c @@ -2751,6 +2751,7 @@ static const struct of_device_id g_dsaf_match[] = { {.compatible = "hisilicon,hns-dsaf-v2"}, {} }; +MODULE_DEVICE_TABLE(of, g_dsaf_match); static struct platform_driver g_dsaf_driver = { .probe = hns_dsaf_probe, diff --git a/drivers/net/ethernet/hisilicon/hns_mdio.c b/drivers/net/ethernet/hisilicon/hns_mdio.c index 33f4c483af0f..501eb2090ca6 100644 --- a/drivers/net/ethernet/hisilicon/hns_mdio.c +++ b/drivers/net/ethernet/hisilicon/hns_mdio.c @@ -563,6 +563,7 @@ static const struct of_device_id hns_mdio_match[] = { {.compatible = "hisilicon,hns-mdio"}, {} }; +MODULE_DEVICE_TABLE(of, hns_mdio_match); static const struct acpi_device_id hns_mdio_acpi_match[] = { { "HISI0141", 0 }, diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index bfe17d9c022d..5f44c5520fbc 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -1190,7 +1190,7 @@ static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter if (!scrq) return NULL; - scrq->msgs = (union sub_crq *)__get_free_pages(GFP_KERNEL, 2); + scrq->msgs = (union sub_crq *)__get_free_pages(GFP_ATOMIC, 2); memset(scrq->msgs, 0, 4 * PAGE_SIZE); if (!scrq->msgs) { dev_warn(dev, "Couldn't allocate crq queue messages page\n"); @@ -1461,14 +1461,16 @@ static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter) return rc; req_rx_irq_failed: - for (j = 0; j < i; j++) + for (j = 0; j < i; j++) { free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]); irq_dispose_mapping(adapter->rx_scrq[j]->irq); + } i = adapter->req_tx_queues; req_tx_irq_failed: - for (j = 0; j < i; j++) + for (j = 0; j < i; j++) { free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]); irq_dispose_mapping(adapter->rx_scrq[j]->irq); + } release_sub_crqs_no_irqs(adapter); return rc; } @@ -3232,6 +3234,27 @@ static void ibmvnic_free_inflight(struct ibmvnic_adapter *adapter) spin_unlock_irqrestore(&adapter->inflight_lock, flags); } +static void ibmvnic_xport_event(struct work_struct *work) +{ + struct ibmvnic_adapter *adapter = container_of(work, + struct ibmvnic_adapter, + ibmvnic_xport); + struct device *dev = &adapter->vdev->dev; + long rc; + + ibmvnic_free_inflight(adapter); + release_sub_crqs(adapter); + if (adapter->migrated) { + rc = ibmvnic_reenable_crq_queue(adapter); + if (rc) + dev_err(dev, "Error after enable rc=%ld\n", rc); + adapter->migrated = false; + rc = ibmvnic_send_crq_init(adapter); + if (rc) + dev_err(dev, "Error sending init rc=%ld\n", rc); + } +} + static void ibmvnic_handle_crq(union ibmvnic_crq *crq, struct ibmvnic_adapter *adapter) { @@ -3267,15 +3290,7 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq, if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) { dev_info(dev, "Re-enabling adapter\n"); adapter->migrated = true; - ibmvnic_free_inflight(adapter); - release_sub_crqs(adapter); - rc = ibmvnic_reenable_crq_queue(adapter); - if (rc) - dev_err(dev, "Error after enable rc=%ld\n", rc); - adapter->migrated = false; - rc = ibmvnic_send_crq_init(adapter); - if (rc) - dev_err(dev, "Error sending init rc=%ld\n", rc); + schedule_work(&adapter->ibmvnic_xport); } else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) { dev_info(dev, "Backing device failover detected\n"); netif_carrier_off(netdev); @@ -3284,8 +3299,7 @@ static void ibmvnic_handle_crq(union ibmvnic_crq *crq, /* The adapter lost the connection */ dev_err(dev, "Virtual Adapter failed (rc=%d)\n", gen_crq->cmd); - ibmvnic_free_inflight(adapter); - release_sub_crqs(adapter); + schedule_work(&adapter->ibmvnic_xport); } return; case IBMVNIC_CRQ_CMD_RSP: @@ -3654,6 +3668,7 @@ static void handle_crq_init_rsp(struct work_struct *work) goto task_failed; netdev->real_num_tx_queues = adapter->req_tx_queues; + netdev->mtu = adapter->req_mtu; if (adapter->failover) { adapter->failover = false; @@ -3725,6 +3740,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) SET_NETDEV_DEV(netdev, &dev->dev); INIT_WORK(&adapter->vnic_crq_init, handle_crq_init_rsp); + INIT_WORK(&adapter->ibmvnic_xport, ibmvnic_xport_event); spin_lock_init(&adapter->stats_lock); @@ -3792,6 +3808,7 @@ static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id) } netdev->real_num_tx_queues = adapter->req_tx_queues; + netdev->mtu = adapter->req_mtu; rc = register_netdev(netdev); if (rc) { diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h index bfc84c7d0e11..dd775d951b73 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.h +++ b/drivers/net/ethernet/ibm/ibmvnic.h @@ -27,7 +27,7 @@ /**************************************************************************/ #define IBMVNIC_NAME "ibmvnic" -#define IBMVNIC_DRIVER_VERSION "1.0" +#define IBMVNIC_DRIVER_VERSION "1.0.1" #define IBMVNIC_INVALID_MAP -1 #define IBMVNIC_STATS_TIMEOUT 1 /* basic structures plus 100 2k buffers */ @@ -1048,5 +1048,6 @@ struct ibmvnic_adapter { u8 map_id; struct work_struct vnic_crq_init; + struct work_struct ibmvnic_xport; bool failover; }; diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 2030d7c1dc94..6d61e443bdf8 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -92,6 +92,7 @@ #define I40E_AQ_LEN 256 #define I40E_AQ_WORK_LIMIT 66 /* max number of VFs + a little */ #define I40E_MAX_USER_PRIORITY 8 +#define I40E_DEFAULT_TRAFFIC_CLASS BIT(0) #define I40E_DEFAULT_MSG_ENABLE 4 #define I40E_QUEUE_WAIT_RETRY_LIMIT 10 #define I40E_INT_NAME_STR_LEN (IFNAMSIZ + 16) diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index ac1faee2a5b8..31c97e3937a4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -4641,29 +4641,6 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) } /** - * i40e_pf_get_default_tc - Get bitmap for first enabled TC - * @pf: PF being queried - * - * Return a bitmap for first enabled traffic class for this PF. - **/ -static u8 i40e_pf_get_default_tc(struct i40e_pf *pf) -{ - u8 enabled_tc = pf->hw.func_caps.enabled_tcmap; - u8 i = 0; - - if (!enabled_tc) - return 0x1; /* TC0 */ - - /* Find the first enabled TC */ - for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { - if (enabled_tc & BIT(i)) - break; - } - - return BIT(i); -} - -/** * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes * @pf: PF being queried * @@ -4673,7 +4650,7 @@ static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) { /* If DCB is not enabled for this PF then just return default TC */ if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) - return i40e_pf_get_default_tc(pf); + return I40E_DEFAULT_TRAFFIC_CLASS; /* SFP mode we want PF to be enabled for all TCs */ if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) @@ -4683,7 +4660,7 @@ static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) if (pf->hw.func_caps.iscsi) return i40e_get_iscsi_tc_map(pf); else - return i40e_pf_get_default_tc(pf); + return I40E_DEFAULT_TRAFFIC_CLASS; } /** @@ -5029,7 +5006,7 @@ static void i40e_dcb_reconfigure(struct i40e_pf *pf) if (v == pf->lan_vsi) tc_map = i40e_pf_get_tc_map(pf); else - tc_map = i40e_pf_get_default_tc(pf); + tc_map = I40E_DEFAULT_TRAFFIC_CLASS; #ifdef I40E_FCOE if (pf->vsi[v]->type == I40E_VSI_FCOE) tc_map = i40e_get_fcoe_tc_map(pf); @@ -5717,7 +5694,7 @@ static int i40e_handle_lldp_event(struct i40e_pf *pf, u8 type; /* Not DCB capable or capability disabled */ - if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) + if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) return ret; /* Ignore if event is not for Nearest Bridge */ @@ -7707,6 +7684,7 @@ static int i40e_init_msix(struct i40e_pf *pf) pf->flags &= ~I40E_FLAG_MSIX_ENABLED; kfree(pf->msix_entries); pf->msix_entries = NULL; + pci_disable_msix(pf->pdev); return -ENODEV; } else if (v_actual == I40E_MIN_MSIX) { @@ -9056,7 +9034,7 @@ static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, return 0; return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, - nlflags, 0, 0, filter_mask, NULL); + 0, 0, nlflags, filter_mask, NULL); } /* Hardware supports L4 tunnel length of 128B (=2^7) which includes diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index a244d9a67264..bd93d823cc25 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -9135,10 +9135,14 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) goto fwd_add_err; fwd_adapter->pool = pool; fwd_adapter->real_adapter = adapter; - err = ixgbe_fwd_ring_up(vdev, fwd_adapter); - if (err) - goto fwd_add_err; - netif_tx_start_all_queues(vdev); + + if (netif_running(pdev)) { + err = ixgbe_fwd_ring_up(vdev, fwd_adapter); + if (err) + goto fwd_add_err; + netif_tx_start_all_queues(vdev); + } + return fwd_adapter; fwd_add_err: /* unwind counter and free adapter struct */ diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index 55831188bc32..bf5cc55ba24c 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -2968,6 +2968,22 @@ static void set_params(struct mv643xx_eth_private *mp, mp->txq_count = pd->tx_queue_count ? : 1; } +static int get_phy_mode(struct mv643xx_eth_private *mp) +{ + struct device *dev = mp->dev->dev.parent; + int iface = -1; + + if (dev->of_node) + iface = of_get_phy_mode(dev->of_node); + + /* Historical default if unspecified. We could also read/write + * the interface state in the PSC1 + */ + if (iface < 0) + iface = PHY_INTERFACE_MODE_GMII; + return iface; +} + static struct phy_device *phy_scan(struct mv643xx_eth_private *mp, int phy_addr) { @@ -2994,7 +3010,7 @@ static struct phy_device *phy_scan(struct mv643xx_eth_private *mp, "orion-mdio-mii", addr); phydev = phy_connect(mp->dev, phy_id, mv643xx_eth_adjust_link, - PHY_INTERFACE_MODE_GMII); + get_phy_mode(mp)); if (!IS_ERR(phydev)) { phy_addr_set(mp, addr); break; @@ -3090,6 +3106,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) if (!dev) return -ENOMEM; + SET_NETDEV_DEV(dev, &pdev->dev); mp = netdev_priv(dev); platform_set_drvdata(pdev, mp); @@ -3129,7 +3146,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) if (pd->phy_node) { mp->phy = of_phy_connect(mp->dev, pd->phy_node, mv643xx_eth_adjust_link, 0, - PHY_INTERFACE_MODE_GMII); + get_phy_mode(mp)); if (!mp->phy) err = -ENODEV; else @@ -3187,8 +3204,6 @@ static int mv643xx_eth_probe(struct platform_device *pdev) dev->priv_flags |= IFF_UNICAST_FLT; dev->gso_max_segs = MV643XX_MAX_TSO_SEGS; - SET_NETDEV_DEV(dev, &pdev->dev); - if (mp->shared->win_protect) wrl(mp, WINDOW_PROTECT(mp->port_num), mp->shared->win_protect); diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index b1cef7a0f7ca..e36bebcab3f2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -2469,6 +2469,7 @@ err_comm_admin: kfree(priv->mfunc.master.slave_state); err_comm: iounmap(priv->mfunc.comm); + priv->mfunc.comm = NULL; err_vhcr: dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE, priv->mfunc.vhcr, @@ -2537,6 +2538,13 @@ void mlx4_report_internal_err_comm_event(struct mlx4_dev *dev) int slave; u32 slave_read; + /* If the comm channel has not yet been initialized, + * skip reporting the internal error event to all + * the communication channels. + */ + if (!priv->mfunc.comm) + return; + /* Report an internal error event to all * communication channels. */ @@ -2571,6 +2579,7 @@ void mlx4_multi_func_cleanup(struct mlx4_dev *dev) } iounmap(priv->mfunc.comm); + priv->mfunc.comm = NULL; } void mlx4_cmd_cleanup(struct mlx4_dev *dev, int cleanup_mask) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_clock.c b/drivers/net/ethernet/mellanox/mlx4/en_clock.c index 08fc5fc56d43..a5fc46bbcbe2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_clock.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_clock.c @@ -245,8 +245,11 @@ static u32 freq_to_shift(u16 freq) { u32 freq_khz = freq * 1000; u64 max_val_cycles = freq_khz * 1000 * MLX4_EN_WRAP_AROUND_SEC; + u64 tmp_rounded = + roundup_pow_of_two(max_val_cycles) > max_val_cycles ? + roundup_pow_of_two(max_val_cycles) - 1 : UINT_MAX; u64 max_val_cycles_rounded = is_power_of_2(max_val_cycles + 1) ? - max_val_cycles : roundup_pow_of_two(max_val_cycles) - 1; + max_val_cycles : tmp_rounded; /* calculate max possible multiplier in order to fit in 64bit */ u64 max_mul = div_u64(0xffffffffffffffffULL, max_val_cycles_rounded); diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c index 132cea655920..e3be7e44ff51 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c @@ -127,7 +127,15 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, /* For TX we use the same irq per ring we assigned for the RX */ struct mlx4_en_cq *rx_cq; - + int xdp_index; + + /* The xdp tx irq must align with the rx ring that forwards to + * it, so reindex these from 0. This should only happen when + * tx_ring_num is not a multiple of rx_ring_num. + */ + xdp_index = (priv->xdp_ring_num - priv->tx_ring_num) + cq_idx; + if (xdp_index >= 0) + cq_idx = xdp_index; cq_idx = cq_idx % priv->rx_ring_num; rx_cq = priv->rx_cq[cq_idx]; cq->vector = rx_cq->vector; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c index 7e703bed7b82..12c99a2655f2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c @@ -1733,6 +1733,13 @@ int mlx4_en_start_port(struct net_device *dev) udp_tunnel_get_rx_info(dev); priv->port_up = true; + + /* Process all completions if exist to prevent + * the queues freezing if they are full + */ + for (i = 0; i < priv->rx_ring_num; i++) + napi_schedule(&priv->rx_cq[i]->napi); + netif_tx_start_all_queues(dev); netif_device_attach(dev); @@ -1910,8 +1917,9 @@ static void mlx4_en_clear_stats(struct net_device *dev) struct mlx4_en_dev *mdev = priv->mdev; int i; - if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) - en_dbg(HW, priv, "Failed dumping statistics\n"); + if (!mlx4_is_slave(mdev->dev)) + if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) + en_dbg(HW, priv, "Failed dumping statistics\n"); memset(&priv->pstats, 0, sizeof(priv->pstats)); memset(&priv->pkstats, 0, sizeof(priv->pkstats)); @@ -2194,6 +2202,7 @@ void mlx4_en_destroy_netdev(struct net_device *dev) if (!shutdown) free_netdev(dev); + dev->ethtool_ops = NULL; } static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_port.c b/drivers/net/ethernet/mellanox/mlx4/en_port.c index 5aa8b751f417..59473a0ebcdf 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_port.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_port.c @@ -166,7 +166,7 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) return PTR_ERR(mailbox); err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, in_mod, 0, MLX4_CMD_DUMP_ETH_STATS, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_WRAPPED); + MLX4_CMD_NATIVE); if (err) goto out; @@ -322,7 +322,7 @@ int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset) err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, in_mod | MLX4_DUMP_ETH_STATS_FLOW_CONTROL, 0, MLX4_CMD_DUMP_ETH_STATS, - MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); + MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE); if (err) goto out; } diff --git a/drivers/net/ethernet/mellanox/mlx4/en_selftest.c b/drivers/net/ethernet/mellanox/mlx4/en_selftest.c index b66e03d9711f..c06346a82496 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_selftest.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_selftest.c @@ -118,6 +118,29 @@ mlx4_en_test_loopback_exit: return !loopback_ok; } +static int mlx4_en_test_interrupts(struct mlx4_en_priv *priv) +{ + struct mlx4_en_dev *mdev = priv->mdev; + int err = 0; + int i = 0; + + err = mlx4_test_async(mdev->dev); + /* When not in MSI_X or slave, test only async */ + if (!(mdev->dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(mdev->dev)) + return err; + + /* A loop over all completion vectors of current port, + * for each vector check whether it works by mapping command + * completions to that vector and performing a NOP command + */ + for (i = 0; i < priv->rx_ring_num; i++) { + err = mlx4_test_interrupt(mdev->dev, priv->rx_cq[i]->vector); + if (err) + break; + } + + return err; +} static int mlx4_en_test_link(struct mlx4_en_priv *priv) { @@ -151,7 +174,6 @@ static int mlx4_en_test_speed(struct mlx4_en_priv *priv) void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf) { struct mlx4_en_priv *priv = netdev_priv(dev); - struct mlx4_en_dev *mdev = priv->mdev; int i, carrier_ok; memset(buf, 0, sizeof(u64) * MLX4_EN_NUM_SELF_TEST); @@ -177,7 +199,7 @@ void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf) netif_carrier_on(dev); } - buf[0] = mlx4_test_interrupts(mdev->dev); + buf[0] = mlx4_en_test_interrupts(priv); buf[1] = mlx4_en_test_link(priv); buf[2] = mlx4_en_test_speed(priv); diff --git a/drivers/net/ethernet/mellanox/mlx4/eq.c b/drivers/net/ethernet/mellanox/mlx4/eq.c index cf8f8a72a801..cd3638e6fe25 100644 --- a/drivers/net/ethernet/mellanox/mlx4/eq.c +++ b/drivers/net/ethernet/mellanox/mlx4/eq.c @@ -1361,53 +1361,49 @@ void mlx4_cleanup_eq_table(struct mlx4_dev *dev) kfree(priv->eq_table.uar_map); } -/* A test that verifies that we can accept interrupts on all - * the irq vectors of the device. +/* A test that verifies that we can accept interrupts + * on the vector allocated for asynchronous events + */ +int mlx4_test_async(struct mlx4_dev *dev) +{ + return mlx4_NOP(dev); +} +EXPORT_SYMBOL(mlx4_test_async); + +/* A test that verifies that we can accept interrupts + * on the given irq vector of the tested port. * Interrupts are checked using the NOP command. */ -int mlx4_test_interrupts(struct mlx4_dev *dev) +int mlx4_test_interrupt(struct mlx4_dev *dev, int vector) { struct mlx4_priv *priv = mlx4_priv(dev); - int i; int err; - err = mlx4_NOP(dev); - /* When not in MSI_X, there is only one irq to check */ - if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev)) - return err; - - /* A loop over all completion vectors, for each vector we will check - * whether it works by mapping command completions to that vector - * and performing a NOP command - */ - for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) { - /* Make sure request_irq was called */ - if (!priv->eq_table.eq[i].have_irq) - continue; - - /* Temporary use polling for command completions */ - mlx4_cmd_use_polling(dev); - - /* Map the new eq to handle all asynchronous events */ - err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, - priv->eq_table.eq[i].eqn); - if (err) { - mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); - mlx4_cmd_use_events(dev); - break; - } + /* Temporary use polling for command completions */ + mlx4_cmd_use_polling(dev); - /* Go back to using events */ - mlx4_cmd_use_events(dev); - err = mlx4_NOP(dev); + /* Map the new eq to handle all asynchronous events */ + err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, + priv->eq_table.eq[MLX4_CQ_TO_EQ_VECTOR(vector)].eqn); + if (err) { + mlx4_warn(dev, "Failed mapping eq for interrupt test\n"); + goto out; } + /* Go back to using events */ + mlx4_cmd_use_events(dev); + err = mlx4_NOP(dev); + /* Return to default */ + mlx4_cmd_use_polling(dev); +out: mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0, priv->eq_table.eq[MLX4_EQ_ASYNC].eqn); + mlx4_cmd_use_events(dev); + return err; } -EXPORT_SYMBOL(mlx4_test_interrupts); +EXPORT_SYMBOL(mlx4_test_interrupt); bool mlx4_is_eq_vector_valid(struct mlx4_dev *dev, u8 port, int vector) { diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c index c41ab31a39f8..84bab9f0732e 100644 --- a/drivers/net/ethernet/mellanox/mlx4/fw.c +++ b/drivers/net/ethernet/mellanox/mlx4/fw.c @@ -49,9 +49,9 @@ enum { extern void __buggy_use_of_MLX4_GET(void); extern void __buggy_use_of_MLX4_PUT(void); -static bool enable_qos = true; +static bool enable_qos; module_param(enable_qos, bool, 0444); -MODULE_PARM_DESC(enable_qos, "Enable Enhanced QoS support (default: on)"); +MODULE_PARM_DESC(enable_qos, "Enable Enhanced QoS support (default: off)"); #define MLX4_GET(dest, source, offset) \ do { \ diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index 7183ac4135d2..6f4e67bc3538 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -1102,6 +1102,14 @@ static int __set_port_type(struct mlx4_port_info *info, int i; int err = 0; + if ((port_type & mdev->caps.supported_type[info->port]) != port_type) { + mlx4_err(mdev, + "Requested port type for port %d is not supported on this HCA\n", + info->port); + err = -EINVAL; + goto err_sup; + } + mlx4_stop_sense(mdev); mutex_lock(&priv->port_mutex); info->tmp_type = port_type; @@ -1147,7 +1155,7 @@ static int __set_port_type(struct mlx4_port_info *info, out: mlx4_start_sense(mdev); mutex_unlock(&priv->port_mutex); - +err_sup: return err; } diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4.h b/drivers/net/ethernet/mellanox/mlx4/mlx4.h index e4878f31e45d..88ee7d8a5923 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4.h @@ -145,9 +145,10 @@ enum mlx4_resource { RES_MTT, RES_MAC, RES_VLAN, - RES_EQ, + RES_NPORT_ID, RES_COUNTER, RES_FS_RULE, + RES_EQ, MLX4_NUM_OF_RESOURCE_TYPE }; @@ -1329,8 +1330,6 @@ int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_cmd_info *cmd); int mlx4_common_set_vlan_fltr(struct mlx4_dev *dev, int function, int port, void *buf); -int mlx4_common_dump_eth_stats(struct mlx4_dev *dev, int slave, u32 in_mod, - struct mlx4_cmd_mailbox *outbox); int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, diff --git a/drivers/net/ethernet/mellanox/mlx4/port.c b/drivers/net/ethernet/mellanox/mlx4/port.c index c5b2064297a1..b656dd5772e5 100644 --- a/drivers/net/ethernet/mellanox/mlx4/port.c +++ b/drivers/net/ethernet/mellanox/mlx4/port.c @@ -1728,24 +1728,13 @@ int mlx4_SET_VLAN_FLTR_wrapper(struct mlx4_dev *dev, int slave, return err; } -int mlx4_common_dump_eth_stats(struct mlx4_dev *dev, int slave, - u32 in_mod, struct mlx4_cmd_mailbox *outbox) -{ - return mlx4_cmd_box(dev, 0, outbox->dma, in_mod, 0, - MLX4_CMD_DUMP_ETH_STATS, MLX4_CMD_TIME_CLASS_B, - MLX4_CMD_NATIVE); -} - int mlx4_DUMP_ETH_STATS_wrapper(struct mlx4_dev *dev, int slave, struct mlx4_vhcr *vhcr, struct mlx4_cmd_mailbox *inbox, struct mlx4_cmd_mailbox *outbox, struct mlx4_cmd_info *cmd) { - if (slave != dev->caps.function) - return 0; - return mlx4_common_dump_eth_stats(dev, slave, - vhcr->in_modifier, outbox); + return 0; } int mlx4_get_slave_from_roce_gid(struct mlx4_dev *dev, int port, u8 *gid, diff --git a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c index 84d7857ccc27..c548beaaf910 100644 --- a/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c +++ b/drivers/net/ethernet/mellanox/mlx4/resource_tracker.c @@ -1605,13 +1605,14 @@ static int eq_res_start_move_to(struct mlx4_dev *dev, int slave, int index, r->com.from_state = r->com.state; r->com.to_state = state; r->com.state = RES_EQ_BUSY; - if (eq) - *eq = r; } } spin_unlock_irq(mlx4_tlock(dev)); + if (!err && eq) + *eq = r; + return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c index 6cb38304669f..2c6e3c7b7417 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c @@ -41,6 +41,13 @@ #include "mlx5_core.h" +struct mlx5_db_pgdir { + struct list_head list; + unsigned long *bitmap; + __be32 *db_page; + dma_addr_t db_dma; +}; + /* Handling for queue buffers -- we allocate a bunch of memory and * register it in a memory region at HCA virtual address 0. */ @@ -102,17 +109,28 @@ EXPORT_SYMBOL_GPL(mlx5_buf_free); static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, int node) { + u32 db_per_page = PAGE_SIZE / cache_line_size(); struct mlx5_db_pgdir *pgdir; pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL); if (!pgdir) return NULL; - bitmap_fill(pgdir->bitmap, MLX5_DB_PER_PAGE); + pgdir->bitmap = kcalloc(BITS_TO_LONGS(db_per_page), + sizeof(unsigned long), + GFP_KERNEL); + + if (!pgdir->bitmap) { + kfree(pgdir); + return NULL; + } + + bitmap_fill(pgdir->bitmap, db_per_page); pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE, &pgdir->db_dma, node); if (!pgdir->db_page) { + kfree(pgdir->bitmap); kfree(pgdir); return NULL; } @@ -123,18 +141,19 @@ static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev, static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir, struct mlx5_db *db) { + u32 db_per_page = PAGE_SIZE / cache_line_size(); int offset; int i; - i = find_first_bit(pgdir->bitmap, MLX5_DB_PER_PAGE); - if (i >= MLX5_DB_PER_PAGE) + i = find_first_bit(pgdir->bitmap, db_per_page); + if (i >= db_per_page) return -ENOMEM; __clear_bit(i, pgdir->bitmap); db->u.pgdir = pgdir; db->index = i; - offset = db->index * L1_CACHE_BYTES; + offset = db->index * cache_line_size(); db->db = pgdir->db_page + offset / sizeof(*pgdir->db_page); db->dma = pgdir->db_dma + offset; @@ -181,14 +200,16 @@ EXPORT_SYMBOL_GPL(mlx5_db_alloc); void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db) { + u32 db_per_page = PAGE_SIZE / cache_line_size(); mutex_lock(&dev->priv.pgdir_mutex); __set_bit(db->index, db->u.pgdir->bitmap); - if (bitmap_full(db->u.pgdir->bitmap, MLX5_DB_PER_PAGE)) { + if (bitmap_full(db->u.pgdir->bitmap, db_per_page)) { dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE, db->u.pgdir->db_page, db->u.pgdir->db_dma); list_del(&db->u.pgdir->list); + kfree(db->u.pgdir->bitmap); kfree(db->u.pgdir); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 460363b66cb1..7a43502a89cc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -85,6 +85,9 @@ #define MLX5_MPWRQ_SMALL_PACKET_THRESHOLD (128) #define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024) +#define MLX5E_DEFAULT_LRO_TIMEOUT 32 +#define MLX5E_LRO_TIMEOUT_ARR_SIZE 4 + #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC 0x10 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE 0x3 #define MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS 0x20 @@ -221,6 +224,7 @@ struct mlx5e_params { struct ieee_ets ets; #endif bool rx_am_enabled; + u32 lro_timeout; }; struct mlx5e_tstamp { @@ -888,5 +892,6 @@ int mlx5e_attach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev); void mlx5e_detach_netdev(struct mlx5_core_dev *mdev, struct net_device *netdev); struct rtnl_link_stats64 * mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); +u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout); #endif /* __MLX5_EN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 7eaf38020a8f..f4c687ce4c59 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -1971,9 +1971,7 @@ static void mlx5e_build_tir_ctx_lro(void *tirc, struct mlx5e_priv *priv) MLX5_SET(tirc, tirc, lro_max_ip_payload_size, (priv->params.lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); - MLX5_SET(tirc, tirc, lro_timeout_period_usecs, - MLX5_CAP_ETH(priv->mdev, - lro_timer_supported_periods[2])); + MLX5_SET(tirc, tirc, lro_timeout_period_usecs, priv->params.lro_timeout); } void mlx5e_build_tir_ctx_hash(void *tirc, struct mlx5e_priv *priv) @@ -3401,6 +3399,18 @@ static void mlx5e_query_min_inline(struct mlx5_core_dev *mdev, } } +u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) +{ + int i; + + /* The supported periods are organized in ascending order */ + for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++) + if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout) + break; + + return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]); +} + static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev, struct net_device *netdev, const struct mlx5e_profile *profile, @@ -3419,6 +3429,9 @@ static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev, priv->profile = profile; priv->ppriv = ppriv; + priv->params.lro_timeout = + mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); + priv->params.log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; /* set CQE compression */ @@ -4035,7 +4048,6 @@ void mlx5e_destroy_netdev(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv) const struct mlx5e_profile *profile = priv->profile; struct net_device *netdev = priv->netdev; - unregister_netdev(netdev); destroy_workqueue(priv->wq); if (profile->cleanup) profile->cleanup(priv); @@ -4052,6 +4064,7 @@ static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv) for (vport = 1; vport < total_vfs; vport++) mlx5_eswitch_unregister_vport_rep(esw, vport); + unregister_netdev(priv->netdev); mlx5e_detach(mdev, vpriv); mlx5e_destroy_netdev(mdev, priv); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 3c97da103d30..7fe6559e4ab3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -457,6 +457,7 @@ void mlx5e_vport_rep_unload(struct mlx5_eswitch *esw, struct mlx5e_priv *priv = rep->priv_data; struct net_device *netdev = priv->netdev; + unregister_netdev(netdev); mlx5e_detach_netdev(esw->dev, netdev); mlx5e_destroy_netdev(esw->dev, priv); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index abbf2c369923..be1f7333ab7f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -931,8 +931,8 @@ static void esw_vport_change_handler(struct work_struct *work) mutex_unlock(&esw->state_lock); } -static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +static int esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *vlan_grp = NULL; @@ -949,9 +949,11 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, int table_size = 2; int err = 0; - if (!MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support) || - !IS_ERR_OR_NULL(vport->egress.acl)) - return; + if (!MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) + return -EOPNOTSUPP; + + if (!IS_ERR_OR_NULL(vport->egress.acl)) + return 0; esw_debug(dev, "Create vport[%d] egress ACL log_max_size(%d)\n", vport->vport, MLX5_CAP_ESW_EGRESS_ACL(dev, log_max_ft_size)); @@ -959,12 +961,12 @@ static void esw_vport_enable_egress_acl(struct mlx5_eswitch *esw, root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_EGRESS); if (!root_ns) { esw_warn(dev, "Failed to get E-Switch egress flow namespace\n"); - return; + return -EIO; } flow_group_in = mlx5_vzalloc(inlen); if (!flow_group_in) - return; + return -ENOMEM; acl = mlx5_create_vport_flow_table(root_ns, 0, table_size, 0, vport->vport); if (IS_ERR(acl)) { @@ -1009,6 +1011,7 @@ out: mlx5_destroy_flow_group(vlan_grp); if (err && !IS_ERR_OR_NULL(acl)) mlx5_destroy_flow_table(acl); + return err; } static void esw_vport_cleanup_egress_rules(struct mlx5_eswitch *esw, @@ -1041,8 +1044,8 @@ static void esw_vport_disable_egress_acl(struct mlx5_eswitch *esw, vport->egress.acl = NULL; } -static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, - struct mlx5_vport *vport) +static int esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_core_dev *dev = esw->dev; @@ -1063,9 +1066,11 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, int table_size = 4; int err = 0; - if (!MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support) || - !IS_ERR_OR_NULL(vport->ingress.acl)) - return; + if (!MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) + return -EOPNOTSUPP; + + if (!IS_ERR_OR_NULL(vport->ingress.acl)) + return 0; esw_debug(dev, "Create vport[%d] ingress ACL log_max_size(%d)\n", vport->vport, MLX5_CAP_ESW_INGRESS_ACL(dev, log_max_ft_size)); @@ -1073,12 +1078,12 @@ static void esw_vport_enable_ingress_acl(struct mlx5_eswitch *esw, root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_ESW_INGRESS); if (!root_ns) { esw_warn(dev, "Failed to get E-Switch ingress flow namespace\n"); - return; + return -EIO; } flow_group_in = mlx5_vzalloc(inlen); if (!flow_group_in) - return; + return -ENOMEM; acl = mlx5_create_vport_flow_table(root_ns, 0, table_size, 0, vport->vport); if (IS_ERR(acl)) { @@ -1167,6 +1172,7 @@ out: } kvfree(flow_group_in); + return err; } static void esw_vport_cleanup_ingress_rules(struct mlx5_eswitch *esw, @@ -1225,7 +1231,13 @@ static int esw_vport_ingress_config(struct mlx5_eswitch *esw, return 0; } - esw_vport_enable_ingress_acl(esw, vport); + err = esw_vport_enable_ingress_acl(esw, vport); + if (err) { + mlx5_core_warn(esw->dev, + "failed to enable ingress acl (%d) on vport[%d]\n", + err, vport->vport); + return err; + } esw_debug(esw->dev, "vport[%d] configure ingress rules, vlan(%d) qos(%d)\n", @@ -1299,7 +1311,13 @@ static int esw_vport_egress_config(struct mlx5_eswitch *esw, return 0; } - esw_vport_enable_egress_acl(esw, vport); + err = esw_vport_enable_egress_acl(esw, vport); + if (err) { + mlx5_core_warn(esw->dev, + "failed to enable egress acl (%d) on vport[%d]\n", + err, vport->vport); + return err; + } esw_debug(esw->dev, "vport[%d] configure egress rules, vlan(%d) qos(%d)\n", diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 5da2cc878582..89696048b045 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -436,6 +436,9 @@ static void del_flow_group(struct fs_node *node) fs_get_obj(ft, fg->node.parent); dev = get_dev(&ft->node); + if (ft->autogroup.active) + ft->autogroup.num_groups--; + if (mlx5_cmd_destroy_flow_group(dev, ft, fg->id)) mlx5_core_warn(dev, "flow steering can't destroy fg %d of ft %d\n", fg->id, ft->id); @@ -879,7 +882,7 @@ static struct mlx5_flow_group *create_flow_group_common(struct mlx5_flow_table * tree_init_node(&fg->node, !is_auto_fg, del_flow_group); tree_add_node(&fg->node, &ft->node); /* Add node to group list */ - list_add(&fg->node.list, ft->node.children.prev); + list_add(&fg->node.list, prev_fg); return fg; } @@ -893,7 +896,7 @@ struct mlx5_flow_group *mlx5_create_flow_group(struct mlx5_flow_table *ft, return ERR_PTR(-EPERM); lock_ref_node(&ft->node); - fg = create_flow_group_common(ft, fg_in, &ft->node.children, false); + fg = create_flow_group_common(ft, fg_in, ft->node.children.prev, false); unlock_ref_node(&ft->node); return fg; @@ -1012,7 +1015,7 @@ static struct mlx5_flow_group *create_autogroup(struct mlx5_flow_table *ft, u32 *match_criteria) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - struct list_head *prev = &ft->node.children; + struct list_head *prev = ft->node.children.prev; unsigned int candidate_index = 0; struct mlx5_flow_group *fg; void *match_criteria_addr; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c index 3a9195b4169d..3b026c151cf2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c @@ -218,6 +218,7 @@ struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging) goto err_out; if (aging) { + counter->cache.lastuse = jiffies; counter->aging = true; spin_lock(&fc_stats->addlist_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 1a05fb965c8d..5bcf93422ee0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -61,10 +61,15 @@ enum { enum { MLX5_NIC_IFC_FULL = 0, MLX5_NIC_IFC_DISABLED = 1, - MLX5_NIC_IFC_NO_DRAM_NIC = 2 + MLX5_NIC_IFC_NO_DRAM_NIC = 2, + MLX5_NIC_IFC_INVALID = 3 }; -static u8 get_nic_interface(struct mlx5_core_dev *dev) +enum { + MLX5_DROP_NEW_HEALTH_WORK, +}; + +static u8 get_nic_state(struct mlx5_core_dev *dev) { return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; } @@ -97,7 +102,7 @@ static int in_fatal(struct mlx5_core_dev *dev) struct mlx5_core_health *health = &dev->priv.health; struct health_buffer __iomem *h = health->health; - if (get_nic_interface(dev) == MLX5_NIC_IFC_DISABLED) + if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) return 1; if (ioread32be(&h->fw_ver) == 0xffffffff) @@ -127,7 +132,7 @@ unlock: static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) { - u8 nic_interface = get_nic_interface(dev); + u8 nic_interface = get_nic_state(dev); switch (nic_interface) { case MLX5_NIC_IFC_FULL: @@ -149,8 +154,34 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) mlx5_disable_device(dev); } +static void health_recover(struct work_struct *work) +{ + struct mlx5_core_health *health; + struct delayed_work *dwork; + struct mlx5_core_dev *dev; + struct mlx5_priv *priv; + u8 nic_state; + + dwork = container_of(work, struct delayed_work, work); + health = container_of(dwork, struct mlx5_core_health, recover_work); + priv = container_of(health, struct mlx5_priv, health); + dev = container_of(priv, struct mlx5_core_dev, priv); + + nic_state = get_nic_state(dev); + if (nic_state == MLX5_NIC_IFC_INVALID) { + dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); + return; + } + + dev_err(&dev->pdev->dev, "starting health recovery flow\n"); + mlx5_recover_device(dev); +} + +/* How much time to wait until health resetting the driver (in msecs) */ +#define MLX5_RECOVERY_DELAY_MSECS 60000 static void health_care(struct work_struct *work) { + unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS); struct mlx5_core_health *health; struct mlx5_core_dev *dev; struct mlx5_priv *priv; @@ -160,6 +191,14 @@ static void health_care(struct work_struct *work) dev = container_of(priv, struct mlx5_core_dev, priv); mlx5_core_warn(dev, "handling bad device here\n"); mlx5_handle_bad_state(dev); + + spin_lock(&health->wq_lock); + if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) + schedule_delayed_work(&health->recover_work, recover_delay); + else + dev_err(&dev->pdev->dev, + "new health works are not permitted at this stage\n"); + spin_unlock(&health->wq_lock); } static const char *hsynd_str(u8 synd) @@ -272,7 +311,13 @@ static void poll_health(unsigned long data) if (in_fatal(dev) && !health->sick) { health->sick = true; print_health_info(dev); - schedule_work(&health->work); + spin_lock(&health->wq_lock); + if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) + queue_work(health->wq, &health->work); + else + dev_err(&dev->pdev->dev, + "new health works are not permitted at this stage\n"); + spin_unlock(&health->wq_lock); } } @@ -281,6 +326,8 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) struct mlx5_core_health *health = &dev->priv.health; init_timer(&health->timer); + health->sick = 0; + clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); health->health = &dev->iseg->health; health->health_counter = &dev->iseg->health_counter; @@ -297,11 +344,22 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev) del_timer_sync(&health->timer); } +void mlx5_drain_health_wq(struct mlx5_core_dev *dev) +{ + struct mlx5_core_health *health = &dev->priv.health; + + spin_lock(&health->wq_lock); + set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); + spin_unlock(&health->wq_lock); + cancel_delayed_work_sync(&health->recover_work); + cancel_work_sync(&health->work); +} + void mlx5_health_cleanup(struct mlx5_core_dev *dev) { struct mlx5_core_health *health = &dev->priv.health; - flush_work(&health->work); + destroy_workqueue(health->wq); } int mlx5_health_init(struct mlx5_core_dev *dev) @@ -316,9 +374,13 @@ int mlx5_health_init(struct mlx5_core_dev *dev) strcpy(name, "mlx5_health"); strcat(name, dev_name(&dev->pdev->dev)); + health->wq = create_singlethread_workqueue(name); kfree(name); - + if (!health->wq) + return -ENOMEM; + spin_lock_init(&health->wq_lock); INIT_WORK(&health->work, health_care); + INIT_DELAYED_WORK(&health->recover_work, health_recover); return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index d9c3c70b29e4..d5433c49b2b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -844,12 +844,6 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) struct pci_dev *pdev = dev->pdev; int err; - err = mlx5_query_hca_caps(dev); - if (err) { - dev_err(&pdev->dev, "query hca failed\n"); - goto out; - } - err = mlx5_query_board_id(dev); if (err) { dev_err(&pdev->dev, "query board id failed\n"); @@ -1023,6 +1017,12 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, mlx5_start_health_poll(dev); + err = mlx5_query_hca_caps(dev); + if (err) { + dev_err(&pdev->dev, "query hca failed\n"); + goto err_stop_poll; + } + if (boot && mlx5_init_once(dev, priv)) { dev_err(&pdev->dev, "sw objs init failed\n"); goto err_stop_poll; @@ -1313,10 +1313,16 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, struct mlx5_priv *priv = &dev->priv; dev_info(&pdev->dev, "%s was called\n", __func__); + mlx5_enter_error_state(dev); mlx5_unload_one(dev, priv, false); - pci_save_state(pdev); - mlx5_pci_disable_device(dev); + /* In case of kernel call save the pci state and drain health wq */ + if (state) { + pci_save_state(pdev); + mlx5_drain_health_wq(dev); + mlx5_pci_disable_device(dev); + } + return state == pci_channel_io_perm_failure ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; } @@ -1373,11 +1379,6 @@ static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) return PCI_ERS_RESULT_RECOVERED; } -void mlx5_disable_device(struct mlx5_core_dev *dev) -{ - mlx5_pci_err_detected(dev->pdev, 0); -} - static void mlx5_pci_resume(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); @@ -1427,6 +1428,18 @@ static const struct pci_device_id mlx5_core_pci_table[] = { MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); +void mlx5_disable_device(struct mlx5_core_dev *dev) +{ + mlx5_pci_err_detected(dev->pdev, 0); +} + +void mlx5_recover_device(struct mlx5_core_dev *dev) +{ + mlx5_pci_disable_device(dev); + if (mlx5_pci_slot_reset(dev->pdev) == PCI_ERS_RESULT_RECOVERED) + mlx5_pci_resume(dev->pdev); +} + static struct pci_driver mlx5_core_driver = { .name = DRIVER_NAME, .id_table = mlx5_core_pci_table, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 3d0cfb9f18f9..187662c8ea96 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -83,6 +83,7 @@ void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event, unsigned long param); void mlx5_enter_error_state(struct mlx5_core_dev *dev); void mlx5_disable_device(struct mlx5_core_dev *dev); +void mlx5_recover_device(struct mlx5_core_dev *dev); int mlx5_sriov_init(struct mlx5_core_dev *dev); void mlx5_sriov_cleanup(struct mlx5_core_dev *dev); int mlx5_sriov_attach(struct mlx5_core_dev *dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index cc4fd61914d3..a57d5a81eb05 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -209,6 +209,7 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr) static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) { struct page *page; + u64 zero_addr = 1; u64 addr; int err; int nid = dev_to_node(&dev->pdev->dev); @@ -218,26 +219,35 @@ static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) mlx5_core_warn(dev, "failed to allocate page\n"); return -ENOMEM; } +map: addr = dma_map_page(&dev->pdev->dev, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); if (dma_mapping_error(&dev->pdev->dev, addr)) { mlx5_core_warn(dev, "failed dma mapping page\n"); err = -ENOMEM; - goto out_alloc; + goto err_mapping; } + + /* Firmware doesn't support page with physical address 0 */ + if (addr == 0) { + zero_addr = addr; + goto map; + } + err = insert_page(dev, addr, page, func_id); if (err) { mlx5_core_err(dev, "failed to track allocated page\n"); - goto out_mapping; + dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); } - return 0; - -out_mapping: - dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); +err_mapping: + if (err) + __free_page(page); -out_alloc: - __free_page(page); + if (zero_addr == 0) + dma_unmap_page(&dev->pdev->dev, zero_addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); return err; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index e742bd4e8894..912f71f84209 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1838,11 +1838,17 @@ static const struct mlxsw_bus mlxsw_pci_bus = { .cmd_exec = mlxsw_pci_cmd_exec, }; -static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci) +static int mlxsw_pci_sw_reset(struct mlxsw_pci *mlxsw_pci, + const struct pci_device_id *id) { unsigned long end; mlxsw_pci_write32(mlxsw_pci, SW_RESET, MLXSW_PCI_SW_RESET_RST_BIT); + if (id->device == PCI_DEVICE_ID_MELLANOX_SWITCHX2) { + msleep(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); + return 0; + } + wmb(); /* reset needs to be written before we read control register */ end = jiffies + msecs_to_jiffies(MLXSW_PCI_SW_RESET_TIMEOUT_MSECS); do { @@ -1909,7 +1915,7 @@ static int mlxsw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) mlxsw_pci->pdev = pdev; pci_set_drvdata(pdev, mlxsw_pci); - err = mlxsw_pci_sw_reset(mlxsw_pci); + err = mlxsw_pci_sw_reset(mlxsw_pci, id); if (err) { dev_err(&pdev->dev, "Software reset failed\n"); goto err_sw_reset; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 78fc557d6dd7..4573da2c5560 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -320,6 +320,8 @@ mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp, lpm_tree); if (err) goto err_left_struct_set; + memcpy(&lpm_tree->prefix_usage, prefix_usage, + sizeof(lpm_tree->prefix_usage)); return lpm_tree; err_left_struct_set: @@ -343,7 +345,8 @@ mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp, for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) { lpm_tree = &mlxsw_sp->router.lpm_trees[i]; - if (lpm_tree->proto == proto && + if (lpm_tree->ref_count != 0 && + lpm_tree->proto == proto && mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage, prefix_usage)) goto inc_ref_count; @@ -1820,19 +1823,17 @@ err_fib_entry_insert: return err; } -static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, - struct fib_entry_notifier_info *fen_info) +static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, + struct fib_entry_notifier_info *fen_info) { struct mlxsw_sp_fib_entry *fib_entry; if (mlxsw_sp->router.aborted) - return 0; + return; fib_entry = mlxsw_sp_fib_entry_find(mlxsw_sp, fen_info); - if (!fib_entry) { - dev_warn(mlxsw_sp->bus_info->dev, "Failed to find FIB4 entry being removed.\n"); - return -ENOENT; - } + if (!fib_entry) + return; if (fib_entry->ref_count == 1) { mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry); @@ -1840,7 +1841,6 @@ static int mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp, } mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry); - return 0; } static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) @@ -1862,7 +1862,8 @@ static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp) if (err) return err; - mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4, 0); + mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4, + MLXSW_SP_LPM_TREE_MIN); err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c index c0c23e2f3275..92bda8703f87 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/switchx2.c +++ b/drivers/net/ethernet/mellanox/mlxsw/switchx2.c @@ -1088,6 +1088,7 @@ err_port_stp_state_set: err_port_admin_status_set: err_port_mtu_set: err_port_speed_set: + mlxsw_sx_port_swid_set(mlxsw_sx_port, MLXSW_PORT_SWID_DISABLED_PORT); err_port_swid_set: err_port_system_port_mapping_set: port_not_usable: diff --git a/drivers/net/ethernet/qlogic/Kconfig b/drivers/net/ethernet/qlogic/Kconfig index 1e8339a67f6e..32f2a45f4ab2 100644 --- a/drivers/net/ethernet/qlogic/Kconfig +++ b/drivers/net/ethernet/qlogic/Kconfig @@ -107,4 +107,7 @@ config QEDE ---help--- This enables the support for ... +config QED_RDMA + bool + endif # NET_VENDOR_QLOGIC diff --git a/drivers/net/ethernet/qlogic/qed/Makefile b/drivers/net/ethernet/qlogic/qed/Makefile index cda0af7fbc20..967acf322c09 100644 --- a/drivers/net/ethernet/qlogic/qed/Makefile +++ b/drivers/net/ethernet/qlogic/qed/Makefile @@ -5,4 +5,4 @@ qed-y := qed_cxt.o qed_dev.o qed_hw.o qed_init_fw_funcs.o qed_init_ops.o \ qed_selftest.o qed_dcbx.o qed_debug.o qed-$(CONFIG_QED_SRIOV) += qed_sriov.o qed_vf.o qed-$(CONFIG_QED_LL2) += qed_ll2.o -qed-$(CONFIG_INFINIBAND_QEDR) += qed_roce.o +qed-$(CONFIG_QED_RDMA) += qed_roce.o diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c index 82370a1a59ad..0c42c240b5cf 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c +++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c @@ -47,13 +47,8 @@ #define TM_ALIGN BIT(TM_SHIFT) #define TM_ELEM_SIZE 4 -/* ILT constants */ -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) /* For RoCE we configure to 64K to cover for RoCE max tasks 256K purpose. */ -#define ILT_DEFAULT_HW_P_SIZE 4 -#else -#define ILT_DEFAULT_HW_P_SIZE 3 -#endif +#define ILT_DEFAULT_HW_P_SIZE (IS_ENABLED(CONFIG_QED_RDMA) ? 4 : 3) #define ILT_PAGE_IN_BYTES(hw_p_size) (1U << ((hw_p_size) + 12)) #define ILT_CFG_REG(cli, reg) PSWRQ2_REG_ ## cli ## _ ## reg ## _RT_OFFSET @@ -349,14 +344,14 @@ static struct qed_tid_seg *qed_cxt_tid_seg_info(struct qed_hwfn *p_hwfn, return NULL; } -void qed_cxt_set_srq_count(struct qed_hwfn *p_hwfn, u32 num_srqs) +static void qed_cxt_set_srq_count(struct qed_hwfn *p_hwfn, u32 num_srqs) { struct qed_cxt_mngr *p_mgr = p_hwfn->p_cxt_mngr; p_mgr->srq_count = num_srqs; } -u32 qed_cxt_get_srq_count(struct qed_hwfn *p_hwfn) +static u32 qed_cxt_get_srq_count(struct qed_hwfn *p_hwfn) { struct qed_cxt_mngr *p_mgr = p_hwfn->p_cxt_mngr; @@ -1804,8 +1799,8 @@ int qed_cxt_get_cid_info(struct qed_hwfn *p_hwfn, struct qed_cxt_info *p_info) return 0; } -void qed_rdma_set_pf_params(struct qed_hwfn *p_hwfn, - struct qed_rdma_pf_params *p_params) +static void qed_rdma_set_pf_params(struct qed_hwfn *p_hwfn, + struct qed_rdma_pf_params *p_params) { u32 num_cons, num_tasks, num_qps, num_mrs, num_srqs; enum protocol_type proto; diff --git a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c index 130da1c0490b..a4789a93b692 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_dcbx.c +++ b/drivers/net/ethernet/qlogic/qed/qed_dcbx.c @@ -1190,6 +1190,7 @@ int qed_dcbx_get_config_params(struct qed_hwfn *p_hwfn, if (!dcbx_info) return -ENOMEM; + memset(dcbx_info, 0, sizeof(*dcbx_info)); rc = qed_dcbx_query_params(p_hwfn, dcbx_info, QED_DCBX_OPERATIONAL_MIB); if (rc) { kfree(dcbx_info); @@ -1225,6 +1226,7 @@ static struct qed_dcbx_get *qed_dcbnl_get_dcbx(struct qed_hwfn *hwfn, if (!dcbx_info) return NULL; + memset(dcbx_info, 0, sizeof(*dcbx_info)); if (qed_dcbx_query_params(hwfn, dcbx_info, type)) { kfree(dcbx_info); return NULL; diff --git a/drivers/net/ethernet/qlogic/qed/qed_debug.c b/drivers/net/ethernet/qlogic/qed/qed_debug.c index 88e7d5bef909..68f19ca57f96 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_debug.c +++ b/drivers/net/ethernet/qlogic/qed/qed_debug.c @@ -405,7 +405,7 @@ struct phy_defs { /***************************** Constant Arrays *******************************/ /* Debug arrays */ -static struct dbg_array s_dbg_arrays[MAX_BIN_DBG_BUFFER_TYPE] = { {0} }; +static struct dbg_array s_dbg_arrays[MAX_BIN_DBG_BUFFER_TYPE] = { {NULL} }; /* Chip constant definitions array */ static struct chip_defs s_chip_defs[MAX_CHIP_IDS] = { @@ -4028,10 +4028,10 @@ static enum dbg_status qed_mcp_trace_read_meta(struct qed_hwfn *p_hwfn, } /* Dump MCP Trace */ -enum dbg_status qed_mcp_trace_dump(struct qed_hwfn *p_hwfn, - struct qed_ptt *p_ptt, - u32 *dump_buf, - bool dump, u32 *num_dumped_dwords) +static enum dbg_status qed_mcp_trace_dump(struct qed_hwfn *p_hwfn, + struct qed_ptt *p_ptt, + u32 *dump_buf, + bool dump, u32 *num_dumped_dwords) { u32 trace_data_grc_addr, trace_data_size_bytes, trace_data_size_dwords; u32 trace_meta_size_dwords, running_bundle_id, offset = 0; @@ -4130,10 +4130,10 @@ enum dbg_status qed_mcp_trace_dump(struct qed_hwfn *p_hwfn, } /* Dump GRC FIFO */ -enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn, - struct qed_ptt *p_ptt, - u32 *dump_buf, - bool dump, u32 *num_dumped_dwords) +static enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn, + struct qed_ptt *p_ptt, + u32 *dump_buf, + bool dump, u32 *num_dumped_dwords) { u32 offset = 0, dwords_read, size_param_offset; bool fifo_has_data; @@ -4192,10 +4192,10 @@ enum dbg_status qed_reg_fifo_dump(struct qed_hwfn *p_hwfn, } /* Dump IGU FIFO */ -enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn, - struct qed_ptt *p_ptt, - u32 *dump_buf, - bool dump, u32 *num_dumped_dwords) +static enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn, + struct qed_ptt *p_ptt, + u32 *dump_buf, + bool dump, u32 *num_dumped_dwords) { u32 offset = 0, dwords_read, size_param_offset; bool fifo_has_data; @@ -4255,10 +4255,11 @@ enum dbg_status qed_igu_fifo_dump(struct qed_hwfn *p_hwfn, } /* Protection Override dump */ -enum dbg_status qed_protection_override_dump(struct qed_hwfn *p_hwfn, - struct qed_ptt *p_ptt, - u32 *dump_buf, - bool dump, u32 *num_dumped_dwords) +static enum dbg_status qed_protection_override_dump(struct qed_hwfn *p_hwfn, + struct qed_ptt *p_ptt, + u32 *dump_buf, + bool dump, + u32 *num_dumped_dwords) { u32 offset = 0, size_param_offset, override_window_dwords; @@ -6339,10 +6340,11 @@ enum dbg_status qed_print_fw_asserts_results(struct qed_hwfn *p_hwfn, } /* Wrapper for unifying the idle_chk and mcp_trace api */ -enum dbg_status qed_print_idle_chk_results_wrapper(struct qed_hwfn *p_hwfn, - u32 *dump_buf, - u32 num_dumped_dwords, - char *results_buf) +static enum dbg_status +qed_print_idle_chk_results_wrapper(struct qed_hwfn *p_hwfn, + u32 *dump_buf, + u32 num_dumped_dwords, + char *results_buf) { u32 num_errors, num_warnnings; @@ -6413,8 +6415,8 @@ static void qed_dbg_print_feature(u8 *p_text_buf, u32 text_size) #define QED_RESULTS_BUF_MIN_SIZE 16 /* Generic function for decoding debug feature info */ -enum dbg_status format_feature(struct qed_hwfn *p_hwfn, - enum qed_dbg_features feature_idx) +static enum dbg_status format_feature(struct qed_hwfn *p_hwfn, + enum qed_dbg_features feature_idx) { struct qed_dbg_feature *feature = &p_hwfn->cdev->dbg_params.features[feature_idx]; @@ -6480,8 +6482,9 @@ enum dbg_status format_feature(struct qed_hwfn *p_hwfn, } /* Generic function for performing the dump of a debug feature. */ -enum dbg_status qed_dbg_dump(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, - enum qed_dbg_features feature_idx) +static enum dbg_status qed_dbg_dump(struct qed_hwfn *p_hwfn, + struct qed_ptt *p_ptt, + enum qed_dbg_features feature_idx) { struct qed_dbg_feature *feature = &p_hwfn->cdev->dbg_params.features[feature_idx]; diff --git a/drivers/net/ethernet/qlogic/qed/qed_dev.c b/drivers/net/ethernet/qlogic/qed/qed_dev.c index 754f6a908858..edae5fc5fccd 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_dev.c +++ b/drivers/net/ethernet/qlogic/qed/qed_dev.c @@ -497,12 +497,13 @@ int qed_resc_alloc(struct qed_dev *cdev) if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) { num_cons = qed_cxt_get_proto_cid_count(p_hwfn, PROTOCOLID_ROCE, - 0) * 2; + NULL) * 2; n_eqes += num_cons + 2 * MAX_NUM_VFS_BB; } else if (p_hwfn->hw_info.personality == QED_PCI_ISCSI) { num_cons = qed_cxt_get_proto_cid_count(p_hwfn, - PROTOCOLID_ISCSI, 0); + PROTOCOLID_ISCSI, + NULL); n_eqes += 2 * num_cons; } @@ -1422,19 +1423,19 @@ static void qed_hw_set_feat(struct qed_hwfn *p_hwfn) u32 *feat_num = p_hwfn->hw_info.feat_num; int num_features = 1; -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) - /* Roce CNQ each requires: 1 status block + 1 CNQ. We divide the - * status blocks equally between L2 / RoCE but with consideration as - * to how many l2 queues / cnqs we have - */ - if (p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) { + if (IS_ENABLED(CONFIG_QED_RDMA) && + p_hwfn->hw_info.personality == QED_PCI_ETH_ROCE) { + /* Roce CNQ each requires: 1 status block + 1 CNQ. We divide + * the status blocks equally between L2 / RoCE but with + * consideration as to how many l2 queues / cnqs we have. + */ num_features++; feat_num[QED_RDMA_CNQ] = min_t(u32, RESC_NUM(p_hwfn, QED_SB) / num_features, RESC_NUM(p_hwfn, QED_RDMA_CNQ_RAM)); } -#endif + feat_num[QED_PF_L2_QUE] = min_t(u32, RESC_NUM(p_hwfn, QED_SB) / num_features, RESC_NUM(p_hwfn, QED_L2_QUEUE)); diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.c b/drivers/net/ethernet/qlogic/qed/qed_ll2.c index 02a8be2faed7..63e1a1b0ef8e 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ll2.c +++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.c @@ -38,6 +38,7 @@ #include "qed_mcp.h" #include "qed_reg_addr.h" #include "qed_sp.h" +#include "qed_roce.h" #define QED_LL2_RX_REGISTERED(ll2) ((ll2)->rx_queue.b_cb_registred) #define QED_LL2_TX_REGISTERED(ll2) ((ll2)->tx_queue.b_cb_registred) @@ -140,11 +141,11 @@ static void qed_ll2_kill_buffers(struct qed_dev *cdev) qed_ll2_dealloc_buffer(cdev, buffer); } -void qed_ll2b_complete_rx_packet(struct qed_hwfn *p_hwfn, - u8 connection_handle, - struct qed_ll2_rx_packet *p_pkt, - struct core_rx_fast_path_cqe *p_cqe, - bool b_last_packet) +static void qed_ll2b_complete_rx_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + struct qed_ll2_rx_packet *p_pkt, + struct core_rx_fast_path_cqe *p_cqe, + bool b_last_packet) { u16 packet_length = le16_to_cpu(p_cqe->packet_length); struct qed_ll2_buffer *buffer = p_pkt->cookie; @@ -515,7 +516,7 @@ static int qed_ll2_rxq_completion(struct qed_hwfn *p_hwfn, void *cookie) return rc; } -void qed_ll2_rxq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle) +static void qed_ll2_rxq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle) { struct qed_ll2_info *p_ll2_conn = NULL; struct qed_ll2_rx_packet *p_pkt = NULL; @@ -537,8 +538,7 @@ void qed_ll2_rxq_flush(struct qed_hwfn *p_hwfn, u8 connection_handle) if (!p_pkt) break; - list_del(&p_pkt->list_entry); - list_add_tail(&p_pkt->list_entry, &p_rx->free_descq); + list_move_tail(&p_pkt->list_entry, &p_rx->free_descq); rx_buf_addr = p_pkt->rx_buf_addr; cookie = p_pkt->cookie; @@ -992,9 +992,8 @@ static void qed_ll2_post_rx_buffer_notify_fw(struct qed_hwfn *p_hwfn, p_posting_packet = list_first_entry(&p_rx->posting_descq, struct qed_ll2_rx_packet, list_entry); - list_del(&p_posting_packet->list_entry); - list_add_tail(&p_posting_packet->list_entry, - &p_rx->active_descq); + list_move_tail(&p_posting_packet->list_entry, + &p_rx->active_descq); b_notify_fw = true; } @@ -1123,9 +1122,6 @@ static void qed_ll2_prepare_tx_packet_set_bd(struct qed_hwfn *p_hwfn, DMA_REGPAIR_LE(start_bd->addr, first_frag); start_bd->nbytes = cpu_to_le16(first_frag_len); - SET_FIELD(start_bd->bd_flags.as_bitfield, CORE_TX_BD_FLAGS_ROCE_FLAV, - type); - DP_VERBOSE(p_hwfn, (NETIF_MSG_TX_QUEUED | QED_MSG_LL2), "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Tx Producer at [0x%04x] - set with a %04x bytes %02x BDs buffer at %08x:%08x\n", @@ -1188,8 +1184,7 @@ static void qed_ll2_tx_packet_notify(struct qed_hwfn *p_hwfn, if (!p_pkt) break; - list_del(&p_pkt->list_entry); - list_add_tail(&p_pkt->list_entry, &p_tx->active_descq); + list_move_tail(&p_pkt->list_entry, &p_tx->active_descq); } SET_FIELD(db_msg.params, CORE_DB_DATA_DEST, DB_DEST_XCM); diff --git a/drivers/net/ethernet/qlogic/qed/qed_ll2.h b/drivers/net/ethernet/qlogic/qed/qed_ll2.h index 80a5dc2d652d..4e3d62a16cab 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_ll2.h +++ b/drivers/net/ethernet/qlogic/qed/qed_ll2.h @@ -293,24 +293,4 @@ void qed_ll2_setup(struct qed_hwfn *p_hwfn, */ void qed_ll2_free(struct qed_hwfn *p_hwfn, struct qed_ll2_info *p_ll2_connections); -void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn, - u8 connection_handle, - void *cookie, - dma_addr_t rx_buf_addr, - u16 data_length, - u8 data_length_error, - u16 parse_flags, - u16 vlan, - u32 src_mac_addr_hi, - u16 src_mac_addr_lo, bool b_last_packet); -void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn, - u8 connection_handle, - void *cookie, - dma_addr_t first_frag_addr, - bool b_last_fragment, bool b_last_packet); -void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn, - u8 connection_handle, - void *cookie, - dma_addr_t first_frag_addr, - bool b_last_fragment, bool b_last_packet); #endif diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c index 4ee3151e80c2..c418360ba02a 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_main.c +++ b/drivers/net/ethernet/qlogic/qed/qed_main.c @@ -33,10 +33,8 @@ #include "qed_hw.h" #include "qed_selftest.h" -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) #define QED_ROCE_QPS (8192) #define QED_ROCE_DPIS (8) -#endif static char version[] = "QLogic FastLinQ 4xxxx Core Module qed " DRV_MODULE_VERSION "\n"; @@ -682,9 +680,7 @@ static int qed_slowpath_setup_int(struct qed_dev *cdev, enum qed_int_mode int_mode) { struct qed_sb_cnt_info sb_cnt_info; -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) - int num_l2_queues; -#endif + int num_l2_queues = 0; int rc; int i; @@ -715,8 +711,9 @@ static int qed_slowpath_setup_int(struct qed_dev *cdev, cdev->int_params.fp_msix_cnt = cdev->int_params.out.num_vectors - cdev->num_hwfns; -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) - num_l2_queues = 0; + if (!IS_ENABLED(CONFIG_QED_RDMA)) + return 0; + for_each_hwfn(cdev, i) num_l2_queues += FEAT_NUM(&cdev->hwfns[i], QED_PF_L2_QUE); @@ -738,7 +735,6 @@ static int qed_slowpath_setup_int(struct qed_dev *cdev, DP_VERBOSE(cdev, QED_MSG_RDMA, "roce_msix_cnt=%d roce_msix_base=%d\n", cdev->int_params.rdma_msix_cnt, cdev->int_params.rdma_msix_base); -#endif return 0; } @@ -843,18 +839,20 @@ static void qed_update_pf_params(struct qed_dev *cdev, { int i; -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) - params->rdma_pf_params.num_qps = QED_ROCE_QPS; - params->rdma_pf_params.min_dpis = QED_ROCE_DPIS; - /* divide by 3 the MRs to avoid MF ILT overflow */ - params->rdma_pf_params.num_mrs = RDMA_MAX_TIDS; - params->rdma_pf_params.gl_pi = QED_ROCE_PROTOCOL_INDEX; -#endif for (i = 0; i < cdev->num_hwfns; i++) { struct qed_hwfn *p_hwfn = &cdev->hwfns[i]; p_hwfn->pf_params = *params; } + + if (!IS_ENABLED(CONFIG_QED_RDMA)) + return; + + params->rdma_pf_params.num_qps = QED_ROCE_QPS; + params->rdma_pf_params.min_dpis = QED_ROCE_DPIS; + /* divide by 3 the MRs to avoid MF ILT overflow */ + params->rdma_pf_params.num_mrs = RDMA_MAX_TIDS; + params->rdma_pf_params.gl_pi = QED_ROCE_PROTOCOL_INDEX; } static int qed_slowpath_start(struct qed_dev *cdev, @@ -880,6 +878,7 @@ static int qed_slowpath_start(struct qed_dev *cdev, } } + cdev->rx_coalesce_usecs = QED_DEFAULT_RX_USECS; rc = qed_nic_setup(cdev); if (rc) goto err; @@ -1432,7 +1431,7 @@ static int qed_set_led(struct qed_dev *cdev, enum qed_led_mode mode) return status; } -struct qed_selftest_ops qed_selftest_ops_pass = { +static struct qed_selftest_ops qed_selftest_ops_pass = { .selftest_memory = &qed_selftest_memory, .selftest_interrupt = &qed_selftest_interrupt, .selftest_register = &qed_selftest_register, diff --git a/drivers/net/ethernet/qlogic/qed/qed_roce.c b/drivers/net/ethernet/qlogic/qed/qed_roce.c index 76831a398bed..f3a825a8f8d5 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_roce.c +++ b/drivers/net/ethernet/qlogic/qed/qed_roce.c @@ -129,17 +129,12 @@ static void qed_bmap_release_id(struct qed_hwfn *p_hwfn, } } -u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id) +static u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id) { /* First sb id for RoCE is after all the l2 sb */ return FEAT_NUM((struct qed_hwfn *)p_hwfn, QED_PF_L2_QUE) + rel_sb_id; } -u32 qed_rdma_query_cau_timer_res(void *rdma_cxt) -{ - return QED_CAU_DEF_RX_TIMER_RES; -} - static int qed_rdma_alloc(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, struct qed_rdma_start_in_params *params) @@ -162,7 +157,8 @@ static int qed_rdma_alloc(struct qed_hwfn *p_hwfn, p_hwfn->p_rdma_info = p_rdma_info; p_rdma_info->proto = PROTOCOLID_ROCE; - num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, 0); + num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, + NULL); p_rdma_info->num_qps = num_cons / 2; @@ -275,7 +271,7 @@ free_rdma_info: return rc; } -void qed_rdma_resc_free(struct qed_hwfn *p_hwfn) +static void qed_rdma_resc_free(struct qed_hwfn *p_hwfn) { struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; @@ -527,6 +523,26 @@ static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn, return qed_spq_post(p_hwfn, p_ent, NULL); } +static int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid) +{ + struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; + int rc; + + DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n"); + + spin_lock_bh(&p_hwfn->p_rdma_info->lock); + rc = qed_rdma_bmap_alloc_id(p_hwfn, + &p_hwfn->p_rdma_info->tid_map, itid); + spin_unlock_bh(&p_hwfn->p_rdma_info->lock); + if (rc) + goto out; + + rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid); +out: + DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc); + return rc; +} + static int qed_rdma_reserve_lkey(struct qed_hwfn *p_hwfn) { struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; @@ -573,7 +589,7 @@ static int qed_rdma_setup(struct qed_hwfn *p_hwfn, return qed_rdma_start_fw(p_hwfn, params, p_ptt); } -int qed_rdma_stop(void *rdma_cxt) +static int qed_rdma_stop(void *rdma_cxt) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct rdma_close_func_ramrod_data *p_ramrod; @@ -629,8 +645,8 @@ out: return rc; } -int qed_rdma_add_user(void *rdma_cxt, - struct qed_rdma_add_user_out_params *out_params) +static int qed_rdma_add_user(void *rdma_cxt, + struct qed_rdma_add_user_out_params *out_params) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; u32 dpi_start_offset; @@ -664,7 +680,7 @@ int qed_rdma_add_user(void *rdma_cxt, return rc; } -struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt) +static struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct qed_rdma_port *p_port = p_hwfn->p_rdma_info->port; @@ -680,7 +696,7 @@ struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt) return p_port; } -struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt) +static struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; @@ -690,7 +706,7 @@ struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt) return p_hwfn->p_rdma_info->dev; } -void qed_rdma_free_tid(void *rdma_cxt, u32 itid) +static void qed_rdma_free_tid(void *rdma_cxt, u32 itid) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; @@ -701,27 +717,7 @@ void qed_rdma_free_tid(void *rdma_cxt, u32 itid) spin_unlock_bh(&p_hwfn->p_rdma_info->lock); } -int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid) -{ - struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; - int rc; - - DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n"); - - spin_lock_bh(&p_hwfn->p_rdma_info->lock); - rc = qed_rdma_bmap_alloc_id(p_hwfn, - &p_hwfn->p_rdma_info->tid_map, itid); - spin_unlock_bh(&p_hwfn->p_rdma_info->lock); - if (rc) - goto out; - - rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid); -out: - DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc); - return rc; -} - -void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod) +static void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod) { struct qed_hwfn *p_hwfn; u16 qz_num; @@ -816,7 +812,7 @@ static int qed_rdma_get_int(struct qed_dev *cdev, struct qed_int_info *info) return 0; } -int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd) +static int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; u32 returned_id; @@ -836,7 +832,7 @@ int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd) return rc; } -void qed_rdma_free_pd(void *rdma_cxt, u16 pd) +static void qed_rdma_free_pd(void *rdma_cxt, u16 pd) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; @@ -873,8 +869,9 @@ qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn *p_hwfn, u16 icid) return toggle_bit; } -int qed_rdma_create_cq(void *rdma_cxt, - struct qed_rdma_create_cq_in_params *params, u16 *icid) +static int qed_rdma_create_cq(void *rdma_cxt, + struct qed_rdma_create_cq_in_params *params, + u16 *icid) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; @@ -957,98 +954,10 @@ err: return rc; } -int qed_rdma_resize_cq(void *rdma_cxt, - struct qed_rdma_resize_cq_in_params *in_params, - struct qed_rdma_resize_cq_out_params *out_params) -{ - struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; - struct rdma_resize_cq_output_params *p_ramrod_res; - struct rdma_resize_cq_ramrod_data *p_ramrod; - enum qed_rdma_toggle_bit toggle_bit; - struct qed_sp_init_data init_data; - struct qed_spq_entry *p_ent; - dma_addr_t ramrod_res_phys; - u8 fw_return_code; - int rc = -ENOMEM; - - DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid); - - p_ramrod_res = - (struct rdma_resize_cq_output_params *) - dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, - sizeof(struct rdma_resize_cq_output_params), - &ramrod_res_phys, GFP_KERNEL); - if (!p_ramrod_res) { - DP_NOTICE(p_hwfn, - "qed resize cq failed: cannot allocate memory (ramrod)\n"); - return rc; - } - - /* Get SPQ entry */ - memset(&init_data, 0, sizeof(init_data)); - init_data.cid = in_params->icid; - init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; - init_data.comp_mode = QED_SPQ_MODE_EBLOCK; - - rc = qed_sp_init_request(p_hwfn, &p_ent, - RDMA_RAMROD_RESIZE_CQ, - p_hwfn->p_rdma_info->proto, &init_data); - if (rc) - goto err; - - p_ramrod = &p_ent->ramrod.rdma_resize_cq; - - p_ramrod->flags = 0; - - /* toggle the bit for every resize or create cq for a given icid */ - toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn, - in_params->icid); - - SET_FIELD(p_ramrod->flags, - RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT, toggle_bit); - - SET_FIELD(p_ramrod->flags, - RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL, - in_params->pbl_two_level); - - p_ramrod->pbl_log_page_size = in_params->pbl_page_size_log - 12; - p_ramrod->pbl_num_pages = cpu_to_le16(in_params->pbl_num_pages); - p_ramrod->max_cqes = cpu_to_le32(in_params->cq_size); - DMA_REGPAIR_LE(p_ramrod->pbl_addr, in_params->pbl_ptr); - DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); - - rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); - if (rc) - goto err; - - if (fw_return_code != RDMA_RETURN_OK) { - DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); - rc = -EINVAL; - goto err; - } - - out_params->prod = le32_to_cpu(p_ramrod_res->old_cq_prod); - out_params->cons = le32_to_cpu(p_ramrod_res->old_cq_cons); - - dma_free_coherent(&p_hwfn->cdev->pdev->dev, - sizeof(struct rdma_resize_cq_output_params), - p_ramrod_res, ramrod_res_phys); - - DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Resized CQ, rc = %d\n", rc); - - return rc; - -err: dma_free_coherent(&p_hwfn->cdev->pdev->dev, - sizeof(struct rdma_resize_cq_output_params), - p_ramrod_res, ramrod_res_phys); - DP_NOTICE(p_hwfn, "Resized CQ, Failed - rc = %d\n", rc); - - return rc; -} - -int qed_rdma_destroy_cq(void *rdma_cxt, - struct qed_rdma_destroy_cq_in_params *in_params, - struct qed_rdma_destroy_cq_out_params *out_params) +static int +qed_rdma_destroy_cq(void *rdma_cxt, + struct qed_rdma_destroy_cq_in_params *in_params, + struct qed_rdma_destroy_cq_out_params *out_params) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct rdma_destroy_cq_output_params *p_ramrod_res; @@ -1169,7 +1078,7 @@ static enum roce_flavor qed_roce_mode_to_flavor(enum roce_mode roce_mode) return flavor; } -int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid) +static int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid) { struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; u32 responder_icid; @@ -1793,9 +1702,9 @@ err: return rc; } -int qed_roce_query_qp(struct qed_hwfn *p_hwfn, - struct qed_rdma_qp *qp, - struct qed_rdma_query_qp_out_params *out_params) +static int qed_roce_query_qp(struct qed_hwfn *p_hwfn, + struct qed_rdma_qp *qp, + struct qed_rdma_query_qp_out_params *out_params) { struct roce_query_qp_resp_output_params *p_resp_ramrod_res; struct roce_query_qp_req_output_params *p_req_ramrod_res; @@ -1936,7 +1845,7 @@ err_resp: return rc; } -int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) +static int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) { u32 num_invalidated_mw = 0; u32 num_bound_mw = 0; @@ -1985,9 +1894,9 @@ int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) return 0; } -int qed_rdma_query_qp(void *rdma_cxt, - struct qed_rdma_qp *qp, - struct qed_rdma_query_qp_out_params *out_params) +static int qed_rdma_query_qp(void *rdma_cxt, + struct qed_rdma_qp *qp, + struct qed_rdma_query_qp_out_params *out_params) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; int rc; @@ -2022,7 +1931,7 @@ int qed_rdma_query_qp(void *rdma_cxt, return rc; } -int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) +static int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; int rc = 0; @@ -2038,7 +1947,7 @@ int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) return rc; } -struct qed_rdma_qp * +static struct qed_rdma_qp * qed_rdma_create_qp(void *rdma_cxt, struct qed_rdma_create_qp_in_params *in_params, struct qed_rdma_create_qp_out_params *out_params) @@ -2215,9 +2124,9 @@ static int qed_roce_modify_qp(struct qed_hwfn *p_hwfn, return rc; } -int qed_rdma_modify_qp(void *rdma_cxt, - struct qed_rdma_qp *qp, - struct qed_rdma_modify_qp_in_params *params) +static int qed_rdma_modify_qp(void *rdma_cxt, + struct qed_rdma_qp *qp, + struct qed_rdma_modify_qp_in_params *params) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; enum qed_roce_qp_state prev_state; @@ -2312,8 +2221,9 @@ int qed_rdma_modify_qp(void *rdma_cxt, return rc; } -int qed_rdma_register_tid(void *rdma_cxt, - struct qed_rdma_register_tid_in_params *params) +static int +qed_rdma_register_tid(void *rdma_cxt, + struct qed_rdma_register_tid_in_params *params) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct rdma_register_tid_ramrod_data *p_ramrod; @@ -2450,7 +2360,7 @@ int qed_rdma_register_tid(void *rdma_cxt, return rc; } -int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid) +static int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct rdma_deregister_tid_ramrod_data *p_ramrod; @@ -2561,7 +2471,8 @@ void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) qed_rdma_dpm_conf(p_hwfn, p_ptt); } -int qed_rdma_start(void *rdma_cxt, struct qed_rdma_start_in_params *params) +static int qed_rdma_start(void *rdma_cxt, + struct qed_rdma_start_in_params *params) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; struct qed_ptt *p_ptt; @@ -2601,7 +2512,7 @@ static int qed_rdma_init(struct qed_dev *cdev, return qed_rdma_start(QED_LEADING_HWFN(cdev), params); } -void qed_rdma_remove_user(void *rdma_cxt, u16 dpi) +static void qed_rdma_remove_user(void *rdma_cxt, u16 dpi) { struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; @@ -2809,11 +2720,6 @@ static int qed_roce_ll2_stop(struct qed_dev *cdev) struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; int rc; - if (!cdev) { - DP_ERR(cdev, "qed roce ll2 stop: invalid cdev\n"); - return -EINVAL; - } - if (roce_ll2->handle == QED_LL2_UNUSED_HANDLE) { DP_ERR(cdev, "qed roce ll2 stop: cannot stop an unused LL2\n"); return -EINVAL; @@ -2850,7 +2756,7 @@ static int qed_roce_ll2_tx(struct qed_dev *cdev, int rc; int i; - if (!cdev || !pkt || !params) { + if (!pkt || !params) { DP_ERR(cdev, "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n", cdev, pkt, params); diff --git a/drivers/net/ethernet/qlogic/qed/qed_roce.h b/drivers/net/ethernet/qlogic/qed/qed_roce.h index 2f091e8a0f40..279f342af8db 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_roce.h +++ b/drivers/net/ethernet/qlogic/qed/qed_roce.h @@ -95,26 +95,6 @@ struct qed_rdma_info { enum protocol_type proto; }; -struct qed_rdma_resize_cq_in_params { - u16 icid; - u32 cq_size; - bool pbl_two_level; - u64 pbl_ptr; - u16 pbl_num_pages; - u8 pbl_page_size_log; -}; - -struct qed_rdma_resize_cq_out_params { - u32 prod; - u32 cons; -}; - -struct qed_rdma_resize_cnq_in_params { - u32 cnq_id; - u32 pbl_page_size_log; - u64 pbl_ptr; -}; - struct qed_rdma_qp { struct regpair qp_handle; struct regpair qp_handle_async; @@ -181,36 +161,55 @@ struct qed_rdma_qp { dma_addr_t shared_queue_phys_addr; }; -int -qed_rdma_add_user(void *rdma_cxt, - struct qed_rdma_add_user_out_params *out_params); -int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd); -int qed_rdma_alloc_tid(void *rdma_cxt, u32 *tid); -int qed_rdma_deregister_tid(void *rdma_cxt, u32 tid); -void qed_rdma_free_tid(void *rdma_cxt, u32 tid); -struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt); -struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt); -int -qed_rdma_register_tid(void *rdma_cxt, - struct qed_rdma_register_tid_in_params *params); -void qed_rdma_remove_user(void *rdma_cxt, u16 dpi); -int qed_rdma_start(void *p_hwfn, struct qed_rdma_start_in_params *params); -int qed_rdma_stop(void *rdma_cxt); -u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id); -u32 qed_rdma_query_cau_timer_res(void *p_hwfn); -void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 cnq_index, u16 prod); -void qed_rdma_resc_free(struct qed_hwfn *p_hwfn); +#if IS_ENABLED(CONFIG_QED_RDMA) +void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt); void qed_async_roce_event(struct qed_hwfn *p_hwfn, struct event_ring_entry *p_eqe); -int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp); -int qed_rdma_modify_qp(void *rdma_cxt, struct qed_rdma_qp *qp, - struct qed_rdma_modify_qp_in_params *params); -int qed_rdma_query_qp(void *rdma_cxt, struct qed_rdma_qp *qp, - struct qed_rdma_query_qp_out_params *out_params); - -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) -void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt); +void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + void *cookie, + dma_addr_t first_frag_addr, + bool b_last_fragment, bool b_last_packet); +void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + void *cookie, + dma_addr_t first_frag_addr, + bool b_last_fragment, bool b_last_packet); +void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + void *cookie, + dma_addr_t rx_buf_addr, + u16 data_length, + u8 data_length_error, + u16 parse_flags, + u16 vlan, + u32 src_mac_addr_hi, + u16 src_mac_addr_lo, bool b_last_packet); #else -void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) {} +static inline void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) {} +static inline void qed_async_roce_event(struct qed_hwfn *p_hwfn, struct event_ring_entry *p_eqe) {} +static inline void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + void *cookie, + dma_addr_t first_frag_addr, + bool b_last_fragment, + bool b_last_packet) {} +static inline void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + void *cookie, + dma_addr_t first_frag_addr, + bool b_last_fragment, + bool b_last_packet) {} +static inline void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn, + u8 connection_handle, + void *cookie, + dma_addr_t rx_buf_addr, + u16 data_length, + u8 data_length_error, + u16 parse_flags, + u16 vlan, + u32 src_mac_addr_hi, + u16 src_mac_addr_lo, + bool b_last_packet) {} #endif #endif diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp.h b/drivers/net/ethernet/qlogic/qed/qed_sp.h index 652c90819758..b2c08e4d2a9b 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_sp.h +++ b/drivers/net/ethernet/qlogic/qed/qed_sp.h @@ -80,7 +80,6 @@ union ramrod_data { struct roce_destroy_qp_resp_ramrod_data roce_destroy_qp_resp; struct roce_destroy_qp_req_ramrod_data roce_destroy_qp_req; struct rdma_create_cq_ramrod_data rdma_create_cq; - struct rdma_resize_cq_ramrod_data rdma_resize_cq; struct rdma_destroy_cq_ramrod_data rdma_destroy_cq; struct rdma_srq_create_ramrod_data rdma_create_srq; struct rdma_srq_destroy_ramrod_data rdma_destroy_srq; diff --git a/drivers/net/ethernet/qlogic/qed/qed_spq.c b/drivers/net/ethernet/qlogic/qed/qed_spq.c index caff41544898..9fbaf9429fd0 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_spq.c +++ b/drivers/net/ethernet/qlogic/qed/qed_spq.c @@ -28,9 +28,7 @@ #include "qed_reg_addr.h" #include "qed_sp.h" #include "qed_sriov.h" -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) #include "qed_roce.h" -#endif /*************************************************************************** * Structures & Definitions @@ -240,11 +238,9 @@ qed_async_event_completion(struct qed_hwfn *p_hwfn, struct event_ring_entry *p_eqe) { switch (p_eqe->protocol_id) { -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) case PROTOCOLID_ROCE: qed_async_roce_event(p_hwfn, p_eqe); return 0; -#endif case PROTOCOLID_COMMON: return qed_sriov_eqe_event(p_hwfn, p_eqe->opcode, diff --git a/drivers/net/ethernet/qlogic/qede/Makefile b/drivers/net/ethernet/qlogic/qede/Makefile index 28dc58919c85..048a230c3ce0 100644 --- a/drivers/net/ethernet/qlogic/qede/Makefile +++ b/drivers/net/ethernet/qlogic/qede/Makefile @@ -2,4 +2,4 @@ obj-$(CONFIG_QEDE) := qede.o qede-y := qede_main.o qede_ethtool.o qede-$(CONFIG_DCB) += qede_dcbnl.o -qede-$(CONFIG_INFINIBAND_QEDR) += qede_roce.o +qede-$(CONFIG_QED_RDMA) += qede_roce.o diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index 28c0e9f42c9e..974689a13337 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -348,12 +348,13 @@ bool qede_has_rx_work(struct qede_rx_queue *rxq); int qede_txq_has_work(struct qede_tx_queue *txq); void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, struct qede_dev *edev, u8 count); +void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq); #define RX_RING_SIZE_POW 13 #define RX_RING_SIZE ((u16)BIT(RX_RING_SIZE_POW)) #define NUM_RX_BDS_MAX (RX_RING_SIZE - 1) #define NUM_RX_BDS_MIN 128 -#define NUM_RX_BDS_DEF NUM_RX_BDS_MAX +#define NUM_RX_BDS_DEF ((u16)BIT(10) - 1) #define TX_RING_SIZE_POW 13 #define TX_RING_SIZE ((u16)BIT(TX_RING_SIZE_POW)) diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 25a9b293ee8f..12251a1032d1 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -756,6 +756,8 @@ static void qede_get_channels(struct net_device *dev, struct qede_dev *edev = netdev_priv(dev); channels->max_combined = QEDE_MAX_RSS_CNT(edev); + channels->max_rx = QEDE_MAX_RSS_CNT(edev); + channels->max_tx = QEDE_MAX_RSS_CNT(edev); channels->combined_count = QEDE_QUEUE_CNT(edev) - edev->fp_num_tx - edev->fp_num_rx; channels->tx_count = edev->fp_num_tx; @@ -820,6 +822,13 @@ static int qede_set_channels(struct net_device *dev, edev->req_queues = count; edev->req_num_tx = channels->tx_count; edev->req_num_rx = channels->rx_count; + /* Reset the indirection table if rx queue count is updated */ + if ((edev->req_queues - edev->req_num_tx) != QEDE_RSS_COUNT(edev)) { + edev->rss_params_inited &= ~QEDE_RSS_INDIR_INITED; + memset(&edev->rss_params.rss_ind_table, 0, + sizeof(edev->rss_params.rss_ind_table)); + } + if (netif_running(dev)) qede_reload(edev, NULL, NULL); @@ -1053,6 +1062,12 @@ static int qede_set_rxfh(struct net_device *dev, const u32 *indir, struct qede_dev *edev = netdev_priv(dev); int i; + if (edev->dev_info.common.num_hwfns > 1) { + DP_INFO(edev, + "RSS configuration is not supported for 100G devices\n"); + return -EOPNOTSUPP; + } + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; @@ -1184,8 +1199,8 @@ static int qede_selftest_transmit_traffic(struct qede_dev *edev, } first_bd = (struct eth_tx_1st_bd *)qed_chain_consume(&txq->tx_pbl); - dma_unmap_page(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), - BD_UNMAP_LEN(first_bd), DMA_TO_DEVICE); + dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), + BD_UNMAP_LEN(first_bd), DMA_TO_DEVICE); txq->sw_tx_cons++; txq->sw_tx_ring[idx].skb = NULL; @@ -1199,8 +1214,8 @@ static int qede_selftest_receive_traffic(struct qede_dev *edev) struct qede_rx_queue *rxq = NULL; struct sw_rx_data *sw_rx_data; union eth_rx_cqe *cqe; + int i, rc = 0; u8 *data_ptr; - int i; for_each_queue(i) { if (edev->fp_array[i].type & QEDE_FASTPATH_RX) { @@ -1219,46 +1234,60 @@ static int qede_selftest_receive_traffic(struct qede_dev *edev) * queue and that the loopback traffic is not IP. */ for (i = 0; i < QEDE_SELFTEST_POLL_COUNT; i++) { - if (qede_has_rx_work(rxq)) + if (!qede_has_rx_work(rxq)) { + usleep_range(100, 200); + continue; + } + + hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr); + sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring); + + /* Memory barrier to prevent the CPU from doing speculative + * reads of CQE/BD before reading hw_comp_cons. If the CQE is + * read before it is written by FW, then FW writes CQE and SB, + * and then the CPU reads the hw_comp_cons, it will use an old + * CQE. + */ + rmb(); + + /* Get the CQE from the completion ring */ + cqe = (union eth_rx_cqe *)qed_chain_consume(&rxq->rx_comp_ring); + + /* Get the data from the SW ring */ + sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS_MAX; + sw_rx_data = &rxq->sw_rx_ring[sw_rx_index]; + fp_cqe = &cqe->fast_path_regular; + len = le16_to_cpu(fp_cqe->len_on_first_bd); + data_ptr = (u8 *)(page_address(sw_rx_data->data) + + fp_cqe->placement_offset + + sw_rx_data->page_offset); + if (ether_addr_equal(data_ptr, edev->ndev->dev_addr) && + ether_addr_equal(data_ptr + ETH_ALEN, + edev->ndev->dev_addr)) { + for (i = ETH_HLEN; i < len; i++) + if (data_ptr[i] != (unsigned char)(i & 0xff)) { + rc = -1; + break; + } + + qede_recycle_rx_bd_ring(rxq, edev, 1); + qed_chain_recycle_consumed(&rxq->rx_comp_ring); break; - usleep_range(100, 200); + } + + DP_INFO(edev, "Not the transmitted packet\n"); + qede_recycle_rx_bd_ring(rxq, edev, 1); + qed_chain_recycle_consumed(&rxq->rx_comp_ring); } - if (!qede_has_rx_work(rxq)) { + if (i == QEDE_SELFTEST_POLL_COUNT) { DP_NOTICE(edev, "Failed to receive the traffic\n"); return -1; } - hw_comp_cons = le16_to_cpu(*rxq->hw_cons_ptr); - sw_comp_cons = qed_chain_get_cons_idx(&rxq->rx_comp_ring); + qede_update_rx_prod(edev, rxq); - /* Memory barrier to prevent the CPU from doing speculative reads of CQE - * / BD before reading hw_comp_cons. If the CQE is read before it is - * written by FW, then FW writes CQE and SB, and then the CPU reads the - * hw_comp_cons, it will use an old CQE. - */ - rmb(); - - /* Get the CQE from the completion ring */ - cqe = (union eth_rx_cqe *)qed_chain_consume(&rxq->rx_comp_ring); - - /* Get the data from the SW ring */ - sw_rx_index = rxq->sw_rx_cons & NUM_RX_BDS_MAX; - sw_rx_data = &rxq->sw_rx_ring[sw_rx_index]; - fp_cqe = &cqe->fast_path_regular; - len = le16_to_cpu(fp_cqe->len_on_first_bd); - data_ptr = (u8 *)(page_address(sw_rx_data->data) + - fp_cqe->placement_offset + sw_rx_data->page_offset); - for (i = ETH_HLEN; i < len; i++) - if (data_ptr[i] != (unsigned char)(i & 0xff)) { - DP_NOTICE(edev, "Loopback test failed\n"); - qede_recycle_rx_bd_ring(rxq, edev, 1); - return -1; - } - - qede_recycle_rx_bd_ring(rxq, edev, 1); - - return 0; + return rc; } static int qede_selftest_run_loopback(struct qede_dev *edev, u32 loopback_mode) diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 343038ca047d..7def29aaf65c 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -313,8 +313,8 @@ static int qede_free_tx_pkt(struct qede_dev *edev, split_bd_len = BD_UNMAP_LEN(split); bds_consumed++; } - dma_unmap_page(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), - BD_UNMAP_LEN(first_bd) + split_bd_len, DMA_TO_DEVICE); + dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), + BD_UNMAP_LEN(first_bd) + split_bd_len, DMA_TO_DEVICE); /* Unmap the data of the skb frags */ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, bds_consumed++) { @@ -359,8 +359,8 @@ static void qede_free_failed_tx_pkt(struct qede_dev *edev, nbd--; } - dma_unmap_page(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), - BD_UNMAP_LEN(first_bd) + split_bd_len, DMA_TO_DEVICE); + dma_unmap_single(&edev->pdev->dev, BD_UNMAP_ADDR(first_bd), + BD_UNMAP_LEN(first_bd) + split_bd_len, DMA_TO_DEVICE); /* Unmap the data of the skb frags */ for (i = 0; i < nbd; i++) { @@ -943,8 +943,7 @@ static inline int qede_realloc_rx_buffer(struct qede_dev *edev, return 0; } -static inline void qede_update_rx_prod(struct qede_dev *edev, - struct qede_rx_queue *rxq) +void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq) { u16 bd_prod = qed_chain_get_prod_idx(&rxq->rx_bd_ring); u16 cqe_prod = qed_chain_get_prod_idx(&rxq->rx_comp_ring); @@ -2941,7 +2940,7 @@ static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) txq->num_tx_buffers = edev->q_num_tx_buffers; /* Allocate the parallel driver ring for Tx buffers */ - size = sizeof(*txq->sw_tx_ring) * NUM_TX_BDS_MAX; + size = sizeof(*txq->sw_tx_ring) * TX_RING_SIZE; txq->sw_tx_ring = kzalloc(size, GFP_KERNEL); if (!txq->sw_tx_ring) { DP_NOTICE(edev, "Tx buffers ring allocation failed\n"); @@ -2952,7 +2951,7 @@ static int qede_alloc_mem_txq(struct qede_dev *edev, struct qede_tx_queue *txq) QED_CHAIN_USE_TO_CONSUME_PRODUCE, QED_CHAIN_MODE_PBL, QED_CHAIN_CNT_TYPE_U16, - NUM_TX_BDS_MAX, + TX_RING_SIZE, sizeof(*p_virt), &txq->tx_pbl); if (rc) goto err; diff --git a/drivers/net/ethernet/qualcomm/emac/emac-mac.c b/drivers/net/ethernet/qualcomm/emac/emac-mac.c index e97968ed4b8f..6fb3bee904d3 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac-mac.c +++ b/drivers/net/ethernet/qualcomm/emac/emac-mac.c @@ -1021,14 +1021,18 @@ void emac_mac_down(struct emac_adapter *adpt) napi_disable(&adpt->rx_q.napi); phy_stop(adpt->phydev); - phy_disconnect(adpt->phydev); - /* disable mac irq */ + /* Interrupts must be disabled before the PHY is disconnected, to + * avoid a race condition where adjust_link is null when we get + * an interrupt. + */ writel(DIS_INT, adpt->base + EMAC_INT_STATUS); writel(0, adpt->base + EMAC_INT_MASK); synchronize_irq(adpt->irq.irq); free_irq(adpt->irq.irq, &adpt->irq); + phy_disconnect(adpt->phydev); + emac_mac_reset(adpt); emac_tx_q_descs_free(adpt); diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c index 9bf3b2b82e95..4fede4b86538 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac.c +++ b/drivers/net/ethernet/qualcomm/emac/emac.c @@ -575,6 +575,7 @@ static const struct of_device_id emac_dt_match[] = { }, {} }; +MODULE_DEVICE_TABLE(of, emac_dt_match); #if IS_ENABLED(CONFIG_ACPI) static const struct acpi_device_id emac_acpi_match[] = { diff --git a/drivers/net/ethernet/realtek/r8169.c b/drivers/net/ethernet/realtek/r8169.c index e55638c7505a..bf000d819a21 100644 --- a/drivers/net/ethernet/realtek/r8169.c +++ b/drivers/net/ethernet/realtek/r8169.c @@ -8273,7 +8273,8 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if ((sizeof(dma_addr_t) > 4) && (use_dac == 1 || (use_dac == -1 && pci_is_pcie(pdev) && tp->mac_version >= RTL_GIGA_MAC_VER_18)) && - !pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + !pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) && + !pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) { /* CPlusCmd Dual Access Cycle is only needed for non-PCIe */ if (!pci_is_pcie(pdev)) diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index 5424fb341613..24b746406bc7 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -1471,7 +1471,7 @@ static int rocker_world_check_init(struct rocker_port *rocker_port) if (rocker->wops) { if (rocker->wops->mode != mode) { dev_err(&rocker->pdev->dev, "hardware has ports in different worlds, which is not supported\n"); - return err; + return -EINVAL; } return 0; } diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c index 431a60804272..4ca461322d60 100644 --- a/drivers/net/ethernet/rocker/rocker_ofdpa.c +++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c @@ -1493,8 +1493,6 @@ static int ofdpa_port_ipv4_nh(struct ofdpa_port *ofdpa_port, spin_lock_irqsave(&ofdpa->neigh_tbl_lock, lock_flags); found = ofdpa_neigh_tbl_find(ofdpa, ip_addr); - if (found) - *index = found->index; updating = found && adding; removing = found && !adding; @@ -1508,9 +1506,11 @@ static int ofdpa_port_ipv4_nh(struct ofdpa_port *ofdpa_port, resolved = false; } else if (removing) { ofdpa_neigh_del(trans, found); + *index = found->index; } else if (updating) { ofdpa_neigh_update(found, trans, NULL, false); resolved = !is_zero_ether_addr(found->eth_dst); + *index = found->index; } else { err = -ENOENT; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index 4ec7397e7fb3..a1b17cd7886b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -347,10 +347,9 @@ static void dwmac4_display_ring(void *head, unsigned int size, bool rx) pr_info("%s descriptor ring:\n", rx ? "RX" : "TX"); for (i = 0; i < size; i++) { - if (p->des0) - pr_info("%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", - i, (unsigned int)virt_to_phys(p), - p->des0, p->des1, p->des2, p->des3); + pr_info("%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n", + i, (unsigned int)virt_to_phys(p), + p->des0, p->des1, p->des2, p->des3); p++; } } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 8dc9056c1001..b15fc55f1b96 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -145,7 +145,7 @@ int stmmac_mdio_register(struct net_device *ndev); int stmmac_mdio_reset(struct mii_bus *mii); void stmmac_set_ethtool_ops(struct net_device *netdev); -int stmmac_ptp_register(struct stmmac_priv *priv); +void stmmac_ptp_register(struct stmmac_priv *priv); void stmmac_ptp_unregister(struct stmmac_priv *priv); int stmmac_resume(struct device *dev); int stmmac_suspend(struct device *dev); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 6c85b61aaa0b..48e71fad4210 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -676,7 +676,9 @@ static int stmmac_init_ptp(struct stmmac_priv *priv) priv->hwts_tx_en = 0; priv->hwts_rx_en = 0; - return stmmac_ptp_register(priv); + stmmac_ptp_register(priv); + + return 0; } static void stmmac_release_ptp(struct stmmac_priv *priv) @@ -1710,7 +1712,7 @@ static int stmmac_hw_setup(struct net_device *dev, bool init_ptp) if (init_ptp) { ret = stmmac_init_ptp(priv); if (ret) - netdev_warn(priv->dev, "PTP support cannot init.\n"); + netdev_warn(priv->dev, "fail to init PTP.\n"); } #ifdef CONFIG_DEBUG_FS diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 289d52725a6c..1477471f8d44 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -177,7 +177,7 @@ static struct ptp_clock_info stmmac_ptp_clock_ops = { * Description: this function will register the ptp clock driver * to kernel. It also does some house keeping work. */ -int stmmac_ptp_register(struct stmmac_priv *priv) +void stmmac_ptp_register(struct stmmac_priv *priv) { spin_lock_init(&priv->ptp_lock); priv->ptp_clock_ops = stmmac_ptp_clock_ops; @@ -185,15 +185,10 @@ int stmmac_ptp_register(struct stmmac_priv *priv) priv->ptp_clock = ptp_clock_register(&priv->ptp_clock_ops, priv->device); if (IS_ERR(priv->ptp_clock)) { + netdev_err(priv->dev, "ptp_clock_register failed\n"); priv->ptp_clock = NULL; - return PTR_ERR(priv->ptp_clock); - } - - spin_lock_init(&priv->ptp_lock); - - netdev_dbg(priv->dev, "Added PTP HW clock successfully\n"); - - return 0; + } else if (priv->ptp_clock) + netdev_info(priv->dev, "registered PTP clock\n"); } /** diff --git a/drivers/net/ethernet/synopsys/dwc_eth_qos.c b/drivers/net/ethernet/synopsys/dwc_eth_qos.c index 0d0053128542..5eedac495077 100644 --- a/drivers/net/ethernet/synopsys/dwc_eth_qos.c +++ b/drivers/net/ethernet/synopsys/dwc_eth_qos.c @@ -982,11 +982,13 @@ static int dwceqos_mii_probe(struct net_device *ndev) if (netif_msg_probe(lp)) phy_attached_info(phydev); - phydev->supported &= PHY_GBIT_FEATURES; + phydev->supported &= PHY_GBIT_FEATURES | SUPPORTED_Pause | + SUPPORTED_Asym_Pause; lp->link = 0; lp->speed = 0; lp->duplex = DUPLEX_UNKNOWN; + lp->flowcontrol.autoneg = AUTONEG_ENABLE; return 0; } diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 3c20e87bb761..42edd7b7902f 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -58,9 +58,9 @@ struct geneve_dev { struct hlist_node hlist; /* vni hash table */ struct net *net; /* netns for packet i/o */ struct net_device *dev; /* netdev for geneve tunnel */ - struct geneve_sock *sock4; /* IPv4 socket used for geneve tunnel */ + struct geneve_sock __rcu *sock4; /* IPv4 socket used for geneve tunnel */ #if IS_ENABLED(CONFIG_IPV6) - struct geneve_sock *sock6; /* IPv6 socket used for geneve tunnel */ + struct geneve_sock __rcu *sock6; /* IPv6 socket used for geneve tunnel */ #endif u8 vni[3]; /* virtual network ID for tunnel */ u8 ttl; /* TTL override */ @@ -453,7 +453,7 @@ static struct sk_buff **geneve_gro_receive(struct sock *sk, skb_gro_pull(skb, gh_len); skb_gro_postpull_rcsum(skb, gh, gh_len); - pp = ptype->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); flush = 0; out_unlock: @@ -543,9 +543,19 @@ static void __geneve_sock_release(struct geneve_sock *gs) static void geneve_sock_release(struct geneve_dev *geneve) { - __geneve_sock_release(geneve->sock4); + struct geneve_sock *gs4 = rtnl_dereference(geneve->sock4); #if IS_ENABLED(CONFIG_IPV6) - __geneve_sock_release(geneve->sock6); + struct geneve_sock *gs6 = rtnl_dereference(geneve->sock6); + + rcu_assign_pointer(geneve->sock6, NULL); +#endif + + rcu_assign_pointer(geneve->sock4, NULL); + synchronize_net(); + + __geneve_sock_release(gs4); +#if IS_ENABLED(CONFIG_IPV6) + __geneve_sock_release(gs6); #endif } @@ -586,10 +596,10 @@ out: gs->flags = geneve->flags; #if IS_ENABLED(CONFIG_IPV6) if (ipv6) - geneve->sock6 = gs; + rcu_assign_pointer(geneve->sock6, gs); else #endif - geneve->sock4 = gs; + rcu_assign_pointer(geneve->sock4, gs); hash = geneve_net_vni_hash(geneve->vni); hlist_add_head_rcu(&geneve->hlist, &gs->vni_list[hash]); @@ -603,9 +613,7 @@ static int geneve_open(struct net_device *dev) bool metadata = geneve->collect_md; int ret = 0; - geneve->sock4 = NULL; #if IS_ENABLED(CONFIG_IPV6) - geneve->sock6 = NULL; if (ipv6 || metadata) ret = geneve_sock_add(geneve, true); #endif @@ -720,6 +728,9 @@ static struct rtable *geneve_get_v4_rt(struct sk_buff *skb, struct rtable *rt = NULL; __u8 tos; + if (!rcu_dereference(geneve->sock4)) + return ERR_PTR(-EIO); + memset(fl4, 0, sizeof(*fl4)); fl4->flowi4_mark = skb->mark; fl4->flowi4_proto = IPPROTO_UDP; @@ -772,11 +783,15 @@ static struct dst_entry *geneve_get_v6_dst(struct sk_buff *skb, { bool use_cache = ip_tunnel_dst_cache_usable(skb, info); struct geneve_dev *geneve = netdev_priv(dev); - struct geneve_sock *gs6 = geneve->sock6; struct dst_entry *dst = NULL; struct dst_cache *dst_cache; + struct geneve_sock *gs6; __u8 prio; + gs6 = rcu_dereference(geneve->sock6); + if (!gs6) + return ERR_PTR(-EIO); + memset(fl6, 0, sizeof(*fl6)); fl6->flowi6_mark = skb->mark; fl6->flowi6_proto = IPPROTO_UDP; @@ -842,7 +857,7 @@ static netdev_tx_t geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev, struct ip_tunnel_info *info) { struct geneve_dev *geneve = netdev_priv(dev); - struct geneve_sock *gs4 = geneve->sock4; + struct geneve_sock *gs4; struct rtable *rt = NULL; const struct iphdr *iip; /* interior IP header */ int err = -EINVAL; @@ -853,6 +868,10 @@ static netdev_tx_t geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool xnet = !net_eq(geneve->net, dev_net(geneve->dev)); u32 flags = geneve->flags; + gs4 = rcu_dereference(geneve->sock4); + if (!gs4) + goto tx_error; + if (geneve->collect_md) { if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) { netdev_dbg(dev, "no tunnel metadata\n"); @@ -932,9 +951,9 @@ static netdev_tx_t geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev, struct ip_tunnel_info *info) { struct geneve_dev *geneve = netdev_priv(dev); - struct geneve_sock *gs6 = geneve->sock6; struct dst_entry *dst = NULL; const struct iphdr *iip; /* interior IP header */ + struct geneve_sock *gs6; int err = -EINVAL; struct flowi6 fl6; __u8 prio, ttl; @@ -943,6 +962,10 @@ static netdev_tx_t geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool xnet = !net_eq(geneve->net, dev_net(geneve->dev)); u32 flags = geneve->flags; + gs6 = rcu_dereference(geneve->sock6); + if (!gs6) + goto tx_error; + if (geneve->collect_md) { if (unlikely(!info || !(info->mode & IP_TUNNEL_INFO_TX))) { netdev_dbg(dev, "no tunnel metadata\n"); diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index f0919bd3a563..f6382150b16a 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -447,7 +447,7 @@ static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net) * Setup the sendside checksum offload only if this is not a * GSO packet. */ - if (skb_is_gso(skb)) { + if ((net_trans_info & (INFO_TCP | INFO_UDP)) && skb_is_gso(skb)) { struct ndis_tcp_lso_info *lso_info; rndis_msg_size += NDIS_LSO_PPI_SIZE; @@ -607,15 +607,18 @@ static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net, packet->total_data_buflen); skb->protocol = eth_type_trans(skb, net); - if (csum_info) { - /* We only look at the IP checksum here. - * Should we be dropping the packet if checksum - * failed? How do we deal with other checksums - TCP/UDP? - */ - if (csum_info->receive.ip_checksum_succeeded) + + /* skb is already created with CHECKSUM_NONE */ + skb_checksum_none_assert(skb); + + /* + * In Linux, the IP checksum is always checked. + * Do L4 checksum offload if enabled and present. + */ + if (csum_info && (net->features & NETIF_F_RXCSUM)) { + if (csum_info->receive.tcp_checksum_succeeded || + csum_info->receive.udp_checksum_succeeded) skb->ip_summed = CHECKSUM_UNNECESSARY; - else - skb->ip_summed = CHECKSUM_NONE; } if (vlan_tci & VLAN_TAG_PRESENT) @@ -696,12 +699,8 @@ int netvsc_recv_callback(struct hv_device *device_obj, static void netvsc_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *info) { - struct net_device_context *net_device_ctx = netdev_priv(net); - struct hv_device *dev = net_device_ctx->device_ctx; - strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); - strlcpy(info->bus_info, vmbus_dev_name(dev), sizeof(info->bus_info)); } static void netvsc_get_channels(struct net_device *net, diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 3ea47f28e143..d2e61e002926 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -397,6 +397,14 @@ static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb) #define DEFAULT_ENCRYPT false #define DEFAULT_ENCODING_SA 0 +static bool send_sci(const struct macsec_secy *secy) +{ + const struct macsec_tx_sc *tx_sc = &secy->tx_sc; + + return tx_sc->send_sci || + (secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb); +} + static sci_t make_sci(u8 *addr, __be16 port) { sci_t sci; @@ -437,15 +445,15 @@ static unsigned int macsec_extra_len(bool sci_present) /* Fill SecTAG according to IEEE 802.1AE-2006 10.5.3 */ static void macsec_fill_sectag(struct macsec_eth_header *h, - const struct macsec_secy *secy, u32 pn) + const struct macsec_secy *secy, u32 pn, + bool sci_present) { const struct macsec_tx_sc *tx_sc = &secy->tx_sc; - memset(&h->tci_an, 0, macsec_sectag_len(tx_sc->send_sci)); + memset(&h->tci_an, 0, macsec_sectag_len(sci_present)); h->eth.h_proto = htons(ETH_P_MACSEC); - if (tx_sc->send_sci || - (secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb)) { + if (sci_present) { h->tci_an |= MACSEC_TCI_SC; memcpy(&h->secure_channel_id, &secy->sci, sizeof(h->secure_channel_id)); @@ -650,6 +658,7 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb, struct macsec_tx_sc *tx_sc; struct macsec_tx_sa *tx_sa; struct macsec_dev *macsec = macsec_priv(dev); + bool sci_present; u32 pn; secy = &macsec->secy; @@ -687,7 +696,8 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb, unprotected_len = skb->len; eth = eth_hdr(skb); - hh = (struct macsec_eth_header *)skb_push(skb, macsec_extra_len(tx_sc->send_sci)); + sci_present = send_sci(secy); + hh = (struct macsec_eth_header *)skb_push(skb, macsec_extra_len(sci_present)); memmove(hh, eth, 2 * ETH_ALEN); pn = tx_sa_update_pn(tx_sa, secy); @@ -696,7 +706,7 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb, kfree_skb(skb); return ERR_PTR(-ENOLINK); } - macsec_fill_sectag(hh, secy, pn); + macsec_fill_sectag(hh, secy, pn, sci_present); macsec_set_shortlen(hh, unprotected_len - 2 * ETH_ALEN); skb_put(skb, secy->icv_len); @@ -726,10 +736,10 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb, skb_to_sgvec(skb, sg, 0, skb->len); if (tx_sc->encrypt) { - int len = skb->len - macsec_hdr_len(tx_sc->send_sci) - + int len = skb->len - macsec_hdr_len(sci_present) - secy->icv_len; aead_request_set_crypt(req, sg, sg, len, iv); - aead_request_set_ad(req, macsec_hdr_len(tx_sc->send_sci)); + aead_request_set_ad(req, macsec_hdr_len(sci_present)); } else { aead_request_set_crypt(req, sg, sg, 0, iv); aead_request_set_ad(req, skb->len - secy->icv_len); diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c index f279a897a5c7..a52b560e428b 100644 --- a/drivers/net/phy/at803x.c +++ b/drivers/net/phy/at803x.c @@ -42,19 +42,24 @@ #define AT803X_MMD_ACCESS_CONTROL 0x0D #define AT803X_MMD_ACCESS_CONTROL_DATA 0x0E #define AT803X_FUNC_DATA 0x4003 +#define AT803X_REG_CHIP_CONFIG 0x1f +#define AT803X_BT_BX_REG_SEL 0x8000 #define AT803X_DEBUG_ADDR 0x1D #define AT803X_DEBUG_DATA 0x1E +#define AT803X_MODE_CFG_MASK 0x0F +#define AT803X_MODE_CFG_SGMII 0x01 + +#define AT803X_PSSR 0x11 /*PHY-Specific Status Register*/ +#define AT803X_PSSR_MR_AN_COMPLETE 0x0200 + #define AT803X_DEBUG_REG_0 0x00 #define AT803X_DEBUG_RX_CLK_DLY_EN BIT(15) #define AT803X_DEBUG_REG_5 0x05 #define AT803X_DEBUG_TX_CLK_DLY_EN BIT(8) -#define AT803X_REG_CHIP_CONFIG 0x1f -#define AT803X_BT_BX_REG_SEL 0x8000 - #define ATH8030_PHY_ID 0x004dd076 #define ATH8031_PHY_ID 0x004dd074 #define ATH8035_PHY_ID 0x004dd072 @@ -209,7 +214,6 @@ static int at803x_suspend(struct phy_device *phydev) { int value; int wol_enabled; - int ccr; mutex_lock(&phydev->lock); @@ -225,16 +229,6 @@ static int at803x_suspend(struct phy_device *phydev) phy_write(phydev, MII_BMCR, value); - if (phydev->interface != PHY_INTERFACE_MODE_SGMII) - goto done; - - /* also power-down SGMII interface */ - ccr = phy_read(phydev, AT803X_REG_CHIP_CONFIG); - phy_write(phydev, AT803X_REG_CHIP_CONFIG, ccr & ~AT803X_BT_BX_REG_SEL); - phy_write(phydev, MII_BMCR, phy_read(phydev, MII_BMCR) | BMCR_PDOWN); - phy_write(phydev, AT803X_REG_CHIP_CONFIG, ccr | AT803X_BT_BX_REG_SEL); - -done: mutex_unlock(&phydev->lock); return 0; @@ -243,7 +237,6 @@ done: static int at803x_resume(struct phy_device *phydev) { int value; - int ccr; mutex_lock(&phydev->lock); @@ -251,17 +244,6 @@ static int at803x_resume(struct phy_device *phydev) value &= ~(BMCR_PDOWN | BMCR_ISOLATE); phy_write(phydev, MII_BMCR, value); - if (phydev->interface != PHY_INTERFACE_MODE_SGMII) - goto done; - - /* also power-up SGMII interface */ - ccr = phy_read(phydev, AT803X_REG_CHIP_CONFIG); - phy_write(phydev, AT803X_REG_CHIP_CONFIG, ccr & ~AT803X_BT_BX_REG_SEL); - value = phy_read(phydev, MII_BMCR) & ~(BMCR_PDOWN | BMCR_ISOLATE); - phy_write(phydev, MII_BMCR, value); - phy_write(phydev, AT803X_REG_CHIP_CONFIG, ccr | AT803X_BT_BX_REG_SEL); - -done: mutex_unlock(&phydev->lock); return 0; @@ -381,6 +363,36 @@ static void at803x_link_change_notify(struct phy_device *phydev) } } +static int at803x_aneg_done(struct phy_device *phydev) +{ + int ccr; + + int aneg_done = genphy_aneg_done(phydev); + if (aneg_done != BMSR_ANEGCOMPLETE) + return aneg_done; + + /* + * in SGMII mode, if copper side autoneg is successful, + * also check SGMII side autoneg result + */ + ccr = phy_read(phydev, AT803X_REG_CHIP_CONFIG); + if ((ccr & AT803X_MODE_CFG_MASK) != AT803X_MODE_CFG_SGMII) + return aneg_done; + + /* switch to SGMII/fiber page */ + phy_write(phydev, AT803X_REG_CHIP_CONFIG, ccr & ~AT803X_BT_BX_REG_SEL); + + /* check if the SGMII link is OK. */ + if (!(phy_read(phydev, AT803X_PSSR) & AT803X_PSSR_MR_AN_COMPLETE)) { + pr_warn("803x_aneg_done: SGMII link is not ok\n"); + aneg_done = 0; + } + /* switch back to copper page */ + phy_write(phydev, AT803X_REG_CHIP_CONFIG, ccr | AT803X_BT_BX_REG_SEL); + + return aneg_done; +} + static struct phy_driver at803x_driver[] = { { /* ATHEROS 8035 */ @@ -432,6 +444,7 @@ static struct phy_driver at803x_driver[] = { .flags = PHY_HAS_INTERRUPT, .config_aneg = genphy_config_aneg, .read_status = genphy_read_status, + .aneg_done = at803x_aneg_done, .ack_interrupt = &at803x_ack_interrupt, .config_intr = &at803x_config_intr, } }; diff --git a/drivers/net/phy/dp83848.c b/drivers/net/phy/dp83848.c index 03d54c4adc88..800b39f06279 100644 --- a/drivers/net/phy/dp83848.c +++ b/drivers/net/phy/dp83848.c @@ -19,6 +19,7 @@ #define TI_DP83848C_PHY_ID 0x20005ca0 #define NS_DP83848C_PHY_ID 0x20005c90 #define TLK10X_PHY_ID 0x2000a210 +#define TI_DP83822_PHY_ID 0x2000a240 /* Registers */ #define DP83848_MICR 0x11 /* MII Interrupt Control Register */ @@ -77,6 +78,7 @@ static struct mdio_device_id __maybe_unused dp83848_tbl[] = { { TI_DP83848C_PHY_ID, 0xfffffff0 }, { NS_DP83848C_PHY_ID, 0xfffffff0 }, { TLK10X_PHY_ID, 0xfffffff0 }, + { TI_DP83822_PHY_ID, 0xfffffff0 }, { } }; MODULE_DEVICE_TABLE(mdio, dp83848_tbl); @@ -105,6 +107,7 @@ static struct phy_driver dp83848_driver[] = { DP83848_PHY_DRIVER(TI_DP83848C_PHY_ID, "TI DP83848C 10/100 Mbps PHY"), DP83848_PHY_DRIVER(NS_DP83848C_PHY_ID, "NS DP83848C 10/100 Mbps PHY"), DP83848_PHY_DRIVER(TLK10X_PHY_ID, "TI TLK10X 10/100 Mbps PHY"), + DP83848_PHY_DRIVER(TI_DP83822_PHY_ID, "TI DP83822 10/100 Mbps PHY"), }; module_phy_driver(dp83848_driver); diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c index f79eb12c326a..125cff57c759 100644 --- a/drivers/net/usb/asix_common.c +++ b/drivers/net/usb/asix_common.c @@ -433,13 +433,13 @@ int asix_mdio_read(struct net_device *netdev, int phy_id, int loc) mutex_lock(&dev->phy_mutex); do { ret = asix_set_sw_mii(dev, 0); - if (ret == -ENODEV) + if (ret == -ENODEV || ret == -ETIMEDOUT) break; usleep_range(1000, 1100); ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG, 0, 0, 1, &smsr, 0); } while (!(smsr & AX_HOST_EN) && (i++ < 30) && (ret != -ENODEV)); - if (ret == -ENODEV) { + if (ret == -ENODEV || ret == -ETIMEDOUT) { mutex_unlock(&dev->phy_mutex); return ret; } @@ -497,13 +497,13 @@ int asix_mdio_read_nopm(struct net_device *netdev, int phy_id, int loc) mutex_lock(&dev->phy_mutex); do { ret = asix_set_sw_mii(dev, 1); - if (ret == -ENODEV) + if (ret == -ENODEV || ret == -ETIMEDOUT) break; usleep_range(1000, 1100); ret = asix_read_cmd(dev, AX_CMD_STATMNGSTS_REG, 0, 0, 1, &smsr, 1); } while (!(smsr & AX_HOST_EN) && (i++ < 30) && (ret != -ENODEV)); - if (ret == -ENODEV) { + if (ret == -ENODEV || ret == -ETIMEDOUT) { mutex_unlock(&dev->phy_mutex); return ret; } diff --git a/drivers/net/usb/kalmia.c b/drivers/net/usb/kalmia.c index 5662babf0583..3e37724d30ae 100644 --- a/drivers/net/usb/kalmia.c +++ b/drivers/net/usb/kalmia.c @@ -151,7 +151,7 @@ kalmia_bind(struct usbnet *dev, struct usb_interface *intf) status = kalmia_init_and_get_ethernet_addr(dev, ethernet_addr); - if (status < 0) { + if (status) { usb_set_intfdata(intf, NULL); usb_driver_release_interface(driver_of(intf), intf); return status; diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index b5554f2ebee4..ef83ae3b0a44 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -2279,6 +2279,7 @@ vmxnet3_set_mc(struct net_device *netdev) &adapter->shared->devRead.rxFilterConf; u8 *new_table = NULL; dma_addr_t new_table_pa = 0; + bool new_table_pa_valid = false; u32 new_mode = VMXNET3_RXM_UCAST; if (netdev->flags & IFF_PROMISC) { @@ -2307,13 +2308,15 @@ vmxnet3_set_mc(struct net_device *netdev) new_table, sz, PCI_DMA_TODEVICE); + if (!dma_mapping_error(&adapter->pdev->dev, + new_table_pa)) { + new_mode |= VMXNET3_RXM_MCAST; + new_table_pa_valid = true; + rxConf->mfTablePA = cpu_to_le64( + new_table_pa); + } } - - if (!dma_mapping_error(&adapter->pdev->dev, - new_table_pa)) { - new_mode |= VMXNET3_RXM_MCAST; - rxConf->mfTablePA = cpu_to_le64(new_table_pa); - } else { + if (!new_table_pa_valid) { netdev_info(netdev, "failed to copy mcast list, setting ALL_MULTI\n"); new_mode |= VMXNET3_RXM_ALL_MULTI; @@ -2338,7 +2341,7 @@ vmxnet3_set_mc(struct net_device *netdev) VMXNET3_CMD_UPDATE_MAC_FILTERS); spin_unlock_irqrestore(&adapter->cmd_lock, flags); - if (new_table_pa) + if (new_table_pa_valid) dma_unmap_single(&adapter->pdev->dev, new_table_pa, rxConf->mfTableLen, PCI_DMA_TODEVICE); kfree(new_table); diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index 85c271c70d42..820de6a9ddde 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -956,6 +956,7 @@ static struct sk_buff *vrf_ip6_rcv(struct net_device *vrf_dev, if (skb->pkt_type == PACKET_LOOPBACK) { skb->dev = vrf_dev; skb->skb_iif = vrf_dev->ifindex; + IP6CB(skb)->flags |= IP6SKB_L3SLAVE; skb->pkt_type = PACKET_HOST; goto out; } @@ -996,6 +997,7 @@ static struct sk_buff *vrf_ip_rcv(struct net_device *vrf_dev, { skb->dev = vrf_dev; skb->skb_iif = vrf_dev->ifindex; + IPCB(skb)->flags |= IPSKB_L3SLAVE; /* loopback traffic; do not push through packet taps again. * Reset pkt_type for upper layers to process skb diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index e7d16687538b..f3c2fa3ab0d5 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -583,7 +583,7 @@ static struct sk_buff **vxlan_gro_receive(struct sock *sk, } } - pp = eth_gro_receive(head, skb); + pp = call_gro_receive(eth_gro_receive, head, skb); flush = 0; out: @@ -943,17 +943,20 @@ static bool vxlan_snoop(struct net_device *dev, static bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev) { struct vxlan_dev *vxlan; + struct vxlan_sock *sock4; + struct vxlan_sock *sock6 = NULL; unsigned short family = dev->default_dst.remote_ip.sa.sa_family; + sock4 = rtnl_dereference(dev->vn4_sock); + /* The vxlan_sock is only used by dev, leaving group has * no effect on other vxlan devices. */ - if (family == AF_INET && dev->vn4_sock && - atomic_read(&dev->vn4_sock->refcnt) == 1) + if (family == AF_INET && sock4 && atomic_read(&sock4->refcnt) == 1) return false; #if IS_ENABLED(CONFIG_IPV6) - if (family == AF_INET6 && dev->vn6_sock && - atomic_read(&dev->vn6_sock->refcnt) == 1) + sock6 = rtnl_dereference(dev->vn6_sock); + if (family == AF_INET6 && sock6 && atomic_read(&sock6->refcnt) == 1) return false; #endif @@ -961,10 +964,12 @@ static bool vxlan_group_used(struct vxlan_net *vn, struct vxlan_dev *dev) if (!netif_running(vxlan->dev) || vxlan == dev) continue; - if (family == AF_INET && vxlan->vn4_sock != dev->vn4_sock) + if (family == AF_INET && + rtnl_dereference(vxlan->vn4_sock) != sock4) continue; #if IS_ENABLED(CONFIG_IPV6) - if (family == AF_INET6 && vxlan->vn6_sock != dev->vn6_sock) + if (family == AF_INET6 && + rtnl_dereference(vxlan->vn6_sock) != sock6) continue; #endif @@ -1005,22 +1010,25 @@ static bool __vxlan_sock_release_prep(struct vxlan_sock *vs) static void vxlan_sock_release(struct vxlan_dev *vxlan) { - bool ipv4 = __vxlan_sock_release_prep(vxlan->vn4_sock); + struct vxlan_sock *sock4 = rtnl_dereference(vxlan->vn4_sock); #if IS_ENABLED(CONFIG_IPV6) - bool ipv6 = __vxlan_sock_release_prep(vxlan->vn6_sock); + struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock); + + rcu_assign_pointer(vxlan->vn6_sock, NULL); #endif + rcu_assign_pointer(vxlan->vn4_sock, NULL); synchronize_net(); - if (ipv4) { - udp_tunnel_sock_release(vxlan->vn4_sock->sock); - kfree(vxlan->vn4_sock); + if (__vxlan_sock_release_prep(sock4)) { + udp_tunnel_sock_release(sock4->sock); + kfree(sock4); } #if IS_ENABLED(CONFIG_IPV6) - if (ipv6) { - udp_tunnel_sock_release(vxlan->vn6_sock->sock); - kfree(vxlan->vn6_sock); + if (__vxlan_sock_release_prep(sock6)) { + udp_tunnel_sock_release(sock6->sock); + kfree(sock6); } #endif } @@ -1036,18 +1044,21 @@ static int vxlan_igmp_join(struct vxlan_dev *vxlan) int ret = -EINVAL; if (ip->sa.sa_family == AF_INET) { + struct vxlan_sock *sock4 = rtnl_dereference(vxlan->vn4_sock); struct ip_mreqn mreq = { .imr_multiaddr.s_addr = ip->sin.sin_addr.s_addr, .imr_ifindex = ifindex, }; - sk = vxlan->vn4_sock->sock->sk; + sk = sock4->sock->sk; lock_sock(sk); ret = ip_mc_join_group(sk, &mreq); release_sock(sk); #if IS_ENABLED(CONFIG_IPV6) } else { - sk = vxlan->vn6_sock->sock->sk; + struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock); + + sk = sock6->sock->sk; lock_sock(sk); ret = ipv6_stub->ipv6_sock_mc_join(sk, ifindex, &ip->sin6.sin6_addr); @@ -1067,18 +1078,21 @@ static int vxlan_igmp_leave(struct vxlan_dev *vxlan) int ret = -EINVAL; if (ip->sa.sa_family == AF_INET) { + struct vxlan_sock *sock4 = rtnl_dereference(vxlan->vn4_sock); struct ip_mreqn mreq = { .imr_multiaddr.s_addr = ip->sin.sin_addr.s_addr, .imr_ifindex = ifindex, }; - sk = vxlan->vn4_sock->sock->sk; + sk = sock4->sock->sk; lock_sock(sk); ret = ip_mc_leave_group(sk, &mreq); release_sock(sk); #if IS_ENABLED(CONFIG_IPV6) } else { - sk = vxlan->vn6_sock->sock->sk; + struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock); + + sk = sock6->sock->sk; lock_sock(sk); ret = ipv6_stub->ipv6_sock_mc_drop(sk, ifindex, &ip->sin6.sin6_addr); @@ -1828,11 +1842,15 @@ static struct dst_entry *vxlan6_get_route(struct vxlan_dev *vxlan, struct dst_cache *dst_cache, const struct ip_tunnel_info *info) { + struct vxlan_sock *sock6 = rcu_dereference(vxlan->vn6_sock); bool use_cache = ip_tunnel_dst_cache_usable(skb, info); struct dst_entry *ndst; struct flowi6 fl6; int err; + if (!sock6) + return ERR_PTR(-EIO); + if (tos && !info) use_cache = false; if (use_cache) { @@ -1850,7 +1868,7 @@ static struct dst_entry *vxlan6_get_route(struct vxlan_dev *vxlan, fl6.flowi6_proto = IPPROTO_UDP; err = ipv6_stub->ipv6_dst_lookup(vxlan->net, - vxlan->vn6_sock->sock->sk, + sock6->sock->sk, &ndst, &fl6); if (err < 0) return ERR_PTR(err); @@ -1995,9 +2013,11 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, } if (dst->sa.sa_family == AF_INET) { - if (!vxlan->vn4_sock) + struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock); + + if (!sock4) goto drop; - sk = vxlan->vn4_sock->sock->sk; + sk = sock4->sock->sk; rt = vxlan_get_route(vxlan, skb, rdst ? rdst->remote_ifindex : 0, tos, @@ -2050,12 +2070,13 @@ static void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, src_port, dst_port, xnet, !udp_sum); #if IS_ENABLED(CONFIG_IPV6) } else { + struct vxlan_sock *sock6 = rcu_dereference(vxlan->vn6_sock); struct dst_entry *ndst; u32 rt6i_flags; - if (!vxlan->vn6_sock) + if (!sock6) goto drop; - sk = vxlan->vn6_sock->sock->sk; + sk = sock6->sock->sk; ndst = vxlan6_get_route(vxlan, skb, rdst ? rdst->remote_ifindex : 0, tos, @@ -2415,9 +2436,10 @@ static int vxlan_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) dport = info->key.tp_dst ? : vxlan->cfg.dst_port; if (ip_tunnel_info_af(info) == AF_INET) { + struct vxlan_sock *sock4 = rcu_dereference(vxlan->vn4_sock); struct rtable *rt; - if (!vxlan->vn4_sock) + if (!sock4) return -EINVAL; rt = vxlan_get_route(vxlan, skb, 0, info->key.tos, info->key.u.ipv4.dst, @@ -2429,8 +2451,6 @@ static int vxlan_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) #if IS_ENABLED(CONFIG_IPV6) struct dst_entry *ndst; - if (!vxlan->vn6_sock) - return -EINVAL; ndst = vxlan6_get_route(vxlan, skb, 0, info->key.tos, info->key.label, &info->key.u.ipv6.dst, &info->key.u.ipv6.src, NULL, info); @@ -2740,10 +2760,10 @@ static int __vxlan_sock_add(struct vxlan_dev *vxlan, bool ipv6) return PTR_ERR(vs); #if IS_ENABLED(CONFIG_IPV6) if (ipv6) - vxlan->vn6_sock = vs; + rcu_assign_pointer(vxlan->vn6_sock, vs); else #endif - vxlan->vn4_sock = vs; + rcu_assign_pointer(vxlan->vn4_sock, vs); vxlan_vs_add_dev(vs, vxlan); return 0; } @@ -2754,9 +2774,9 @@ static int vxlan_sock_add(struct vxlan_dev *vxlan) bool metadata = vxlan->flags & VXLAN_F_COLLECT_METADATA; int ret = 0; - vxlan->vn4_sock = NULL; + RCU_INIT_POINTER(vxlan->vn4_sock, NULL); #if IS_ENABLED(CONFIG_IPV6) - vxlan->vn6_sock = NULL; + RCU_INIT_POINTER(vxlan->vn6_sock, NULL); if (ipv6 || metadata) ret = __vxlan_sock_add(vxlan, true); #endif diff --git a/drivers/net/wan/Kconfig b/drivers/net/wan/Kconfig index 33ab3345d333..4e9fe75d7067 100644 --- a/drivers/net/wan/Kconfig +++ b/drivers/net/wan/Kconfig @@ -294,7 +294,7 @@ config FSL_UCC_HDLC config SLIC_DS26522 tristate "Slic Maxim ds26522 card support" depends on SPI - depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE + depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE || COMPILE_TEST help This module initializes and configures the slic maxim card in T1 or E1 mode. diff --git a/drivers/net/wan/slic_ds26522.c b/drivers/net/wan/slic_ds26522.c index d06a887a2352..b776a0ab106c 100644 --- a/drivers/net/wan/slic_ds26522.c +++ b/drivers/net/wan/slic_ds26522.c @@ -223,12 +223,19 @@ static int slic_ds26522_probe(struct spi_device *spi) return ret; } +static const struct spi_device_id slic_ds26522_id[] = { + { .name = "ds26522" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(spi, slic_ds26522_id); + static const struct of_device_id slic_ds26522_match[] = { { .compatible = "maxim,ds26522", }, {}, }; +MODULE_DEVICE_TABLE(of, slic_ds26522_match); static struct spi_driver slic_ds26522_driver = { .driver = { @@ -239,6 +246,7 @@ static struct spi_driver slic_ds26522_driver = { }, .probe = slic_ds26522_probe, .remove = slic_ds26522_remove, + .id_table = slic_ds26522_id, }; static int __init slic_ds26522_init(void) diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index dda49af1eb74..521f1c55c19e 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -450,6 +450,7 @@ struct ath10k_debug { u32 pktlog_filter; u32 reg_addr; u32 nf_cal_period; + void *cal_data; struct ath10k_fw_crash_data *fw_crash_data; }; diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c index 832da6ed9f13..82a4c67f3672 100644 --- a/drivers/net/wireless/ath/ath10k/debug.c +++ b/drivers/net/wireless/ath/ath10k/debug.c @@ -30,6 +30,8 @@ /* ms */ #define ATH10K_DEBUG_HTT_STATS_INTERVAL 1000 +#define ATH10K_DEBUG_CAL_DATA_LEN 12064 + #define ATH10K_FW_CRASH_DUMP_VERSION 1 /** @@ -1451,56 +1453,51 @@ static const struct file_operations fops_fw_dbglog = { .llseek = default_llseek, }; -static int ath10k_debug_cal_data_open(struct inode *inode, struct file *file) +static int ath10k_debug_cal_data_fetch(struct ath10k *ar) { - struct ath10k *ar = inode->i_private; - void *buf; u32 hi_addr; __le32 addr; int ret; - mutex_lock(&ar->conf_mutex); - - if (ar->state != ATH10K_STATE_ON && - ar->state != ATH10K_STATE_UTF) { - ret = -ENETDOWN; - goto err; - } + lockdep_assert_held(&ar->conf_mutex); - buf = vmalloc(ar->hw_params.cal_data_len); - if (!buf) { - ret = -ENOMEM; - goto err; - } + if (WARN_ON(ar->hw_params.cal_data_len > ATH10K_DEBUG_CAL_DATA_LEN)) + return -EINVAL; hi_addr = host_interest_item_address(HI_ITEM(hi_board_data)); ret = ath10k_hif_diag_read(ar, hi_addr, &addr, sizeof(addr)); if (ret) { - ath10k_warn(ar, "failed to read hi_board_data address: %d\n", ret); - goto err_vfree; + ath10k_warn(ar, "failed to read hi_board_data address: %d\n", + ret); + return ret; } - ret = ath10k_hif_diag_read(ar, le32_to_cpu(addr), buf, + ret = ath10k_hif_diag_read(ar, le32_to_cpu(addr), ar->debug.cal_data, ar->hw_params.cal_data_len); if (ret) { ath10k_warn(ar, "failed to read calibration data: %d\n", ret); - goto err_vfree; + return ret; } - file->private_data = buf; + return 0; +} - mutex_unlock(&ar->conf_mutex); +static int ath10k_debug_cal_data_open(struct inode *inode, struct file *file) +{ + struct ath10k *ar = inode->i_private; - return 0; + mutex_lock(&ar->conf_mutex); -err_vfree: - vfree(buf); + if (ar->state == ATH10K_STATE_ON || + ar->state == ATH10K_STATE_UTF) { + ath10k_debug_cal_data_fetch(ar); + } -err: + file->private_data = ar; mutex_unlock(&ar->conf_mutex); - return ret; + return 0; } static ssize_t ath10k_debug_cal_data_read(struct file *file, @@ -1508,18 +1505,16 @@ static ssize_t ath10k_debug_cal_data_read(struct file *file, size_t count, loff_t *ppos) { struct ath10k *ar = file->private_data; - void *buf = file->private_data; - return simple_read_from_buffer(user_buf, count, ppos, - buf, ar->hw_params.cal_data_len); -} + mutex_lock(&ar->conf_mutex); -static int ath10k_debug_cal_data_release(struct inode *inode, - struct file *file) -{ - vfree(file->private_data); + count = simple_read_from_buffer(user_buf, count, ppos, + ar->debug.cal_data, + ar->hw_params.cal_data_len); - return 0; + mutex_unlock(&ar->conf_mutex); + + return count; } static ssize_t ath10k_write_ani_enable(struct file *file, @@ -1580,7 +1575,6 @@ static const struct file_operations fops_ani_enable = { static const struct file_operations fops_cal_data = { .open = ath10k_debug_cal_data_open, .read = ath10k_debug_cal_data_read, - .release = ath10k_debug_cal_data_release, .owner = THIS_MODULE, .llseek = default_llseek, }; @@ -1932,6 +1926,8 @@ void ath10k_debug_stop(struct ath10k *ar) { lockdep_assert_held(&ar->conf_mutex); + ath10k_debug_cal_data_fetch(ar); + /* Must not use _sync to avoid deadlock, we do that in * ath10k_debug_destroy(). The check for htt_stats_mask is to avoid * warning from del_timer(). */ @@ -2344,6 +2340,10 @@ int ath10k_debug_create(struct ath10k *ar) if (!ar->debug.fw_crash_data) return -ENOMEM; + ar->debug.cal_data = vzalloc(ATH10K_DEBUG_CAL_DATA_LEN); + if (!ar->debug.cal_data) + return -ENOMEM; + INIT_LIST_HEAD(&ar->debug.fw_stats.pdevs); INIT_LIST_HEAD(&ar->debug.fw_stats.vdevs); INIT_LIST_HEAD(&ar->debug.fw_stats.peers); @@ -2357,6 +2357,9 @@ void ath10k_debug_destroy(struct ath10k *ar) vfree(ar->debug.fw_crash_data); ar->debug.fw_crash_data = NULL; + vfree(ar->debug.cal_data); + ar->debug.cal_data = NULL; + ath10k_debug_fw_stats_reset(ar); kfree(ar->debug.tpc_stats); diff --git a/drivers/net/wireless/ath/ath6kl/sdio.c b/drivers/net/wireless/ath/ath6kl/sdio.c index eab0ab976af2..76eb33679d4b 100644 --- a/drivers/net/wireless/ath/ath6kl/sdio.c +++ b/drivers/net/wireless/ath/ath6kl/sdio.c @@ -1401,6 +1401,7 @@ static const struct sdio_device_id ath6kl_sdio_devices[] = { {SDIO_DEVICE(MANUFACTURER_CODE, (MANUFACTURER_ID_AR6004_BASE | 0x0))}, {SDIO_DEVICE(MANUFACTURER_CODE, (MANUFACTURER_ID_AR6004_BASE | 0x1))}, {SDIO_DEVICE(MANUFACTURER_CODE, (MANUFACTURER_ID_AR6004_BASE | 0x2))}, + {SDIO_DEVICE(MANUFACTURER_CODE, (MANUFACTURER_ID_AR6004_BASE | 0x18))}, {}, }; diff --git a/drivers/net/wireless/ath/ath9k/ar9003_calib.c b/drivers/net/wireless/ath/ath9k/ar9003_calib.c index b6f064a8d264..7e27a06e5df1 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_calib.c +++ b/drivers/net/wireless/ath/ath9k/ar9003_calib.c @@ -33,7 +33,6 @@ struct coeff { enum ar9003_cal_types { IQ_MISMATCH_CAL = BIT(0), - TEMP_COMP_CAL = BIT(1), }; static void ar9003_hw_setup_calibration(struct ath_hw *ah, @@ -59,12 +58,6 @@ static void ar9003_hw_setup_calibration(struct ath_hw *ah, /* Kick-off cal */ REG_SET_BIT(ah, AR_PHY_TIMING4, AR_PHY_TIMING4_DO_CAL); break; - case TEMP_COMP_CAL: - ath_dbg(common, CALIBRATE, - "starting Temperature Compensation Calibration\n"); - REG_SET_BIT(ah, AR_CH0_THERM, AR_CH0_THERM_LOCAL); - REG_SET_BIT(ah, AR_CH0_THERM, AR_CH0_THERM_START); - break; default: ath_err(common, "Invalid calibration type\n"); break; @@ -93,8 +86,7 @@ static bool ar9003_hw_per_calibration(struct ath_hw *ah, /* * Accumulate cal measures for active chains */ - if (cur_caldata->calCollect) - cur_caldata->calCollect(ah); + cur_caldata->calCollect(ah); ah->cal_samples++; if (ah->cal_samples >= cur_caldata->calNumSamples) { @@ -107,8 +99,7 @@ static bool ar9003_hw_per_calibration(struct ath_hw *ah, /* * Process accumulated data */ - if (cur_caldata->calPostProc) - cur_caldata->calPostProc(ah, numChains); + cur_caldata->calPostProc(ah, numChains); /* Calibration has finished. */ caldata->CalValid |= cur_caldata->calType; @@ -323,16 +314,9 @@ static const struct ath9k_percal_data iq_cal_single_sample = { ar9003_hw_iqcalibrate }; -static const struct ath9k_percal_data temp_cal_single_sample = { - TEMP_COMP_CAL, - MIN_CAL_SAMPLES, - PER_MAX_LOG_COUNT, -}; - static void ar9003_hw_init_cal_settings(struct ath_hw *ah) { ah->iq_caldata.calData = &iq_cal_single_sample; - ah->temp_caldata.calData = &temp_cal_single_sample; if (AR_SREV_9300_20_OR_LATER(ah)) { ah->enabled_cals |= TX_IQ_CAL; @@ -340,7 +324,7 @@ static void ar9003_hw_init_cal_settings(struct ath_hw *ah) ah->enabled_cals |= TX_IQ_ON_AGC_CAL; } - ah->supp_cals = IQ_MISMATCH_CAL | TEMP_COMP_CAL; + ah->supp_cals = IQ_MISMATCH_CAL; } #define OFF_UPPER_LT 24 @@ -1399,9 +1383,6 @@ static void ar9003_hw_init_cal_common(struct ath_hw *ah) INIT_CAL(&ah->iq_caldata); INSERT_CAL(ah, &ah->iq_caldata); - INIT_CAL(&ah->temp_caldata); - INSERT_CAL(ah, &ah->temp_caldata); - /* Initialize current pointer to first element in list */ ah->cal_list_curr = ah->cal_list; diff --git a/drivers/net/wireless/ath/ath9k/hw.h b/drivers/net/wireless/ath/ath9k/hw.h index 2a5d3ad1169c..9cbca1229bac 100644 --- a/drivers/net/wireless/ath/ath9k/hw.h +++ b/drivers/net/wireless/ath/ath9k/hw.h @@ -830,7 +830,6 @@ struct ath_hw { /* Calibration */ u32 supp_cals; struct ath9k_cal_list iq_caldata; - struct ath9k_cal_list temp_caldata; struct ath9k_cal_list adcgain_caldata; struct ath9k_cal_list adcdc_caldata; struct ath9k_cal_list *cal_list; diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c index 94480123efa3..274dd5a1574a 100644 --- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c +++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c @@ -45,7 +45,7 @@ static int mwifiex_11n_dispatch_amsdu_pkt(struct mwifiex_private *priv, skb_trim(skb, le16_to_cpu(local_rx_pd->rx_pkt_length)); ieee80211_amsdu_to_8023s(skb, &list, priv->curr_addr, - priv->wdev.iftype, 0, false); + priv->wdev.iftype, 0, NULL, NULL); while (!skb_queue_empty(&list)) { struct rx_packet_hdr *rx_hdr; diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h index 1016628926d2..08d587a342d3 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h @@ -238,7 +238,7 @@ struct rtl8xxxu_rxdesc16 { u32 pattern1match:1; u32 pattern0match:1; #endif - __le32 tsfl; + u32 tsfl; #if 0 u32 bassn:12; u32 bavld:1; @@ -368,7 +368,7 @@ struct rtl8xxxu_rxdesc24 { u32 ldcp:1; u32 splcp:1; #endif - __le32 tsfl; + u32 tsfl; }; struct rtl8xxxu_txdesc32 { diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c index df54d27e7851..a793fedc3654 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8192e.c @@ -1461,7 +1461,9 @@ static int rtl8192eu_active_to_emu(struct rtl8xxxu_priv *priv) int count, ret = 0; /* Turn off RF */ - rtl8xxxu_write8(priv, REG_RF_CTRL, 0); + val8 = rtl8xxxu_read8(priv, REG_RF_CTRL); + val8 &= ~RF_ENABLE; + rtl8xxxu_write8(priv, REG_RF_CTRL, val8); /* Switch DPDT_SEL_P output from register 0x65[2] */ val8 = rtl8xxxu_read8(priv, REG_LEDCFG2); @@ -1593,6 +1595,10 @@ static void rtl8192e_enable_rf(struct rtl8xxxu_priv *priv) u32 val32; u8 val8; + val32 = rtl8xxxu_read32(priv, REG_RX_WAIT_CCA); + val32 |= (BIT(22) | BIT(23)); + rtl8xxxu_write32(priv, REG_RX_WAIT_CCA, val32); + val8 = rtl8xxxu_read8(priv, REG_GPIO_MUXCFG); val8 |= BIT(5); rtl8xxxu_write8(priv, REG_GPIO_MUXCFG, val8); diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c index 6c086b5657e9..02b8ddd98a95 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_8723b.c @@ -1498,6 +1498,10 @@ static void rtl8723b_enable_rf(struct rtl8xxxu_priv *priv) u32 val32; u8 val8; + val32 = rtl8xxxu_read32(priv, REG_RX_WAIT_CCA); + val32 |= (BIT(22) | BIT(23)); + rtl8xxxu_write32(priv, REG_RX_WAIT_CCA, val32); + /* * No indication anywhere as to what 0x0790 does. The 2 antenna * vendor code preserves bits 6-7 here. diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c index b2d7f6e69667..a5e6ec2152bf 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c @@ -5197,7 +5197,12 @@ int rtl8xxxu_parse_rxdesc16(struct rtl8xxxu_priv *priv, struct sk_buff *skb) pkt_offset = roundup(pkt_len + drvinfo_sz + desc_shift + sizeof(struct rtl8xxxu_rxdesc16), 128); - if (pkt_cnt > 1) + /* + * Only clone the skb if there's enough data at the end to + * at least cover the rx descriptor + */ + if (pkt_cnt > 1 && + urb_len > (pkt_offset + sizeof(struct rtl8xxxu_rxdesc16))) next_skb = skb_clone(skb, GFP_ATOMIC); rx_status = IEEE80211_SKB_RXCB(skb); @@ -5215,7 +5220,7 @@ int rtl8xxxu_parse_rxdesc16(struct rtl8xxxu_priv *priv, struct sk_buff *skb) rtl8xxxu_rx_parse_phystats(priv, rx_status, phy_stats, rx_desc->rxmcs); - rx_status->mactime = le32_to_cpu(rx_desc->tsfl); + rx_status->mactime = rx_desc->tsfl; rx_status->flag |= RX_FLAG_MACTIME_START; if (!rx_desc->swdec) @@ -5285,7 +5290,7 @@ int rtl8xxxu_parse_rxdesc24(struct rtl8xxxu_priv *priv, struct sk_buff *skb) rtl8xxxu_rx_parse_phystats(priv, rx_status, phy_stats, rx_desc->rxmcs); - rx_status->mactime = le32_to_cpu(rx_desc->tsfl); + rx_status->mactime = rx_desc->tsfl; rx_status->flag |= RX_FLAG_MACTIME_START; if (!rx_desc->swdec) diff --git a/drivers/net/wireless/realtek/rtlwifi/core.c b/drivers/net/wireless/realtek/rtlwifi/core.c index f95760c13c56..8e7f23c11680 100644 --- a/drivers/net/wireless/realtek/rtlwifi/core.c +++ b/drivers/net/wireless/realtek/rtlwifi/core.c @@ -111,7 +111,7 @@ static void rtl_fw_do_work(const struct firmware *firmware, void *context, if (!err) goto found_alt; } - pr_err("Firmware %s not available\n", rtlpriv->cfg->fw_name); + pr_err("Selected firmware is not available\n"); rtlpriv->max_fw_size = 0; return; } diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c index e7b11b40e68d..f361808def47 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8188ee/sw.c @@ -86,6 +86,7 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw) struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); u8 tid; + char *fw_name; rtl8188ee_bt_reg_init(hw); rtlpriv->dm.dm_initialgain_enable = 1; @@ -169,10 +170,10 @@ int rtl88e_init_sw_vars(struct ieee80211_hw *hw) return 1; } - rtlpriv->cfg->fw_name = "rtlwifi/rtl8188efw.bin"; + fw_name = "rtlwifi/rtl8188efw.bin"; rtlpriv->max_fw_size = 0x8000; - pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + pr_info("Using firmware %s\n", fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -284,7 +285,6 @@ static const struct rtl_hal_cfg rtl88ee_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl88e_pci", - .fw_name = "rtlwifi/rtl8188efw.bin", .ops = &rtl8188ee_hal_ops, .mod_params = &rtl88ee_mod_params, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c index 87aa209ae325..8b6e37ce3f66 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ce/sw.c @@ -96,6 +96,7 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); + char *fw_name = "rtlwifi/rtl8192cfwU.bin"; rtl8192ce_bt_reg_init(hw); @@ -167,15 +168,12 @@ int rtl92c_init_sw_vars(struct ieee80211_hw *hw) } /* request fw */ - if (IS_VENDOR_UMC_A_CUT(rtlhal->version) && - !IS_92C_SERIAL(rtlhal->version)) - rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cfwU.bin"; - else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) - rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cfwU_B.bin"; + if (IS_81XXC_VENDOR_UMC_B_CUT(rtlhal->version)) + fw_name = "rtlwifi/rtl8192cfwU_B.bin"; rtlpriv->max_fw_size = 0x4000; - pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + pr_info("Using firmware %s\n", fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -262,7 +260,6 @@ static const struct rtl_hal_cfg rtl92ce_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl92c_pci", - .fw_name = "rtlwifi/rtl8192cfw.bin", .ops = &rtl8192ce_hal_ops, .mod_params = &rtl92ce_mod_params, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c index 7c6f7f0d18c6..f953320f0e23 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192cu/sw.c @@ -59,6 +59,7 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw) { struct rtl_priv *rtlpriv = rtl_priv(hw); int err; + char *fw_name; rtlpriv->dm.dm_initialgain_enable = true; rtlpriv->dm.dm_flag = 0; @@ -77,18 +78,18 @@ static int rtl92cu_init_sw_vars(struct ieee80211_hw *hw) } if (IS_VENDOR_UMC_A_CUT(rtlpriv->rtlhal.version) && !IS_92C_SERIAL(rtlpriv->rtlhal.version)) { - rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_A.bin"; + fw_name = "rtlwifi/rtl8192cufw_A.bin"; } else if (IS_81XXC_VENDOR_UMC_B_CUT(rtlpriv->rtlhal.version)) { - rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_B.bin"; + fw_name = "rtlwifi/rtl8192cufw_B.bin"; } else { - rtlpriv->cfg->fw_name = "rtlwifi/rtl8192cufw_TMSC.bin"; + fw_name = "rtlwifi/rtl8192cufw_TMSC.bin"; } /* provide name of alternative file */ rtlpriv->cfg->alt_fw_name = "rtlwifi/rtl8192cufw.bin"; - pr_info("Loading firmware %s\n", rtlpriv->cfg->fw_name); + pr_info("Loading firmware %s\n", fw_name); rtlpriv->max_fw_size = 0x4000; err = request_firmware_nowait(THIS_MODULE, 1, - rtlpriv->cfg->fw_name, rtlpriv->io.dev, + fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); return err; } @@ -187,7 +188,6 @@ static struct rtl_hal_usbint_cfg rtl92cu_interface_cfg = { static struct rtl_hal_cfg rtl92cu_hal_cfg = { .name = "rtl92c_usb", - .fw_name = "rtlwifi/rtl8192cufw.bin", .ops = &rtl8192cu_hal_ops, .mod_params = &rtl92cu_mod_params, .usb_interface_cfg = &rtl92cu_interface_cfg, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c index 0538a4d09568..1ebfee18882f 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/sw.c @@ -92,6 +92,7 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw) u8 tid; struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); + char *fw_name = "rtlwifi/rtl8192defw.bin"; rtlpriv->dm.dm_initialgain_enable = true; rtlpriv->dm.dm_flag = 0; @@ -181,10 +182,10 @@ static int rtl92d_init_sw_vars(struct ieee80211_hw *hw) rtlpriv->max_fw_size = 0x8000; pr_info("Driver for Realtek RTL8192DE WLAN interface\n"); - pr_info("Loading firmware file %s\n", rtlpriv->cfg->fw_name); + pr_info("Loading firmware file %s\n", fw_name); /* request fw */ - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -266,7 +267,6 @@ static const struct rtl_hal_cfg rtl92de_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl8192de", - .fw_name = "rtlwifi/rtl8192defw.bin", .ops = &rtl8192de_hal_ops, .mod_params = &rtl92de_mod_params, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c index ac299cbe59b0..46b605de36e7 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192ee/sw.c @@ -91,6 +91,7 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw) struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); int err = 0; + char *fw_name; rtl92ee_bt_reg_init(hw); rtlpci->msi_support = rtlpriv->cfg->mod_params->msi_support; @@ -170,11 +171,11 @@ int rtl92ee_init_sw_vars(struct ieee80211_hw *hw) } /* request fw */ - rtlpriv->cfg->fw_name = "rtlwifi/rtl8192eefw.bin"; + fw_name = "rtlwifi/rtl8192eefw.bin"; rtlpriv->max_fw_size = 0x8000; - pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + pr_info("Using firmware %s\n", fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -266,7 +267,6 @@ static const struct rtl_hal_cfg rtl92ee_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl92ee_pci", - .fw_name = "rtlwifi/rtl8192eefw.bin", .ops = &rtl8192ee_hal_ops, .mod_params = &rtl92ee_mod_params, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c index 5e8e02d5de8a..3e1eaeac4fdc 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192se/sw.c @@ -89,12 +89,13 @@ static void rtl92se_fw_cb(const struct firmware *firmware, void *context) struct ieee80211_hw *hw = context; struct rtl_priv *rtlpriv = rtl_priv(hw); struct rt_firmware *pfirmware = NULL; + char *fw_name = "rtlwifi/rtl8192sefw.bin"; RT_TRACE(rtlpriv, COMP_ERR, DBG_LOUD, "Firmware callback routine entered!\n"); complete(&rtlpriv->firmware_loading_complete); if (!firmware) { - pr_err("Firmware %s not available\n", rtlpriv->cfg->fw_name); + pr_err("Firmware %s not available\n", fw_name); rtlpriv->max_fw_size = 0; return; } @@ -117,6 +118,7 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw) struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); int err = 0; u16 earlyrxthreshold = 7; + char *fw_name = "rtlwifi/rtl8192sefw.bin"; rtlpriv->dm.dm_initialgain_enable = true; rtlpriv->dm.dm_flag = 0; @@ -214,9 +216,9 @@ static int rtl92s_init_sw_vars(struct ieee80211_hw *hw) rtlpriv->max_fw_size = RTL8190_MAX_FIRMWARE_CODE_SIZE*2 + sizeof(struct fw_hdr); pr_info("Driver for Realtek RTL8192SE/RTL8191SE\n" - "Loading firmware %s\n", rtlpriv->cfg->fw_name); + "Loading firmware %s\n", fw_name); /* request fw */ - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl92se_fw_cb); if (err) { @@ -310,7 +312,6 @@ static const struct rtl_hal_cfg rtl92se_hal_cfg = { .bar_id = 1, .write_readback = false, .name = "rtl92s_pci", - .fw_name = "rtlwifi/rtl8192sefw.bin", .ops = &rtl8192se_hal_ops, .mod_params = &rtl92se_mod_params, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c index 89c828ad89f4..c51a9e8234e9 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723ae/sw.c @@ -94,6 +94,7 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw) struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); int err = 0; + char *fw_name = "rtlwifi/rtl8723fw.bin"; rtl8723e_bt_reg_init(hw); @@ -176,14 +177,12 @@ int rtl8723e_init_sw_vars(struct ieee80211_hw *hw) return 1; } - if (IS_VENDOR_8723_A_CUT(rtlhal->version)) - rtlpriv->cfg->fw_name = "rtlwifi/rtl8723fw.bin"; - else if (IS_81xxC_VENDOR_UMC_B_CUT(rtlhal->version)) - rtlpriv->cfg->fw_name = "rtlwifi/rtl8723fw_B.bin"; + if (IS_81xxC_VENDOR_UMC_B_CUT(rtlhal->version)) + fw_name = "rtlwifi/rtl8723fw_B.bin"; rtlpriv->max_fw_size = 0x6000; - pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + pr_info("Using firmware %s\n", fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -280,7 +279,6 @@ static const struct rtl_hal_cfg rtl8723e_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl8723e_pci", - .fw_name = "rtlwifi/rtl8723efw.bin", .ops = &rtl8723e_hal_ops, .mod_params = &rtl8723e_mod_params, .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c index 20b53f035483..847644d1f5f5 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8723be/sw.c @@ -91,6 +91,7 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw) struct rtl_priv *rtlpriv = rtl_priv(hw); struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); + char *fw_name = "rtlwifi/rtl8723befw.bin"; rtl8723be_bt_reg_init(hw); rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer(); @@ -184,8 +185,8 @@ int rtl8723be_init_sw_vars(struct ieee80211_hw *hw) } rtlpriv->max_fw_size = 0x8000; - pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + pr_info("Using firmware %s\n", fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -280,7 +281,6 @@ static const struct rtl_hal_cfg rtl8723be_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl8723be_pci", - .fw_name = "rtlwifi/rtl8723befw.bin", .ops = &rtl8723be_hal_ops, .mod_params = &rtl8723be_mod_params, .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c index 22f687b1f133..297938e0effd 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8821ae/sw.c @@ -93,6 +93,7 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) struct rtl_pci *rtlpci = rtl_pcidev(rtl_pcipriv(hw)); struct rtl_mac *mac = rtl_mac(rtl_priv(hw)); struct rtl_hal *rtlhal = rtl_hal(rtl_priv(hw)); + char *fw_name, *wowlan_fw_name; rtl8821ae_bt_reg_init(hw); rtlpriv->btcoexist.btc_ops = rtl_btc_get_ops_pointer(); @@ -203,17 +204,17 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) } if (rtlhal->hw_type == HARDWARE_TYPE_RTL8812AE) { - rtlpriv->cfg->fw_name = "rtlwifi/rtl8812aefw.bin"; - rtlpriv->cfg->wowlan_fw_name = "rtlwifi/rtl8812aefw_wowlan.bin"; + fw_name = "rtlwifi/rtl8812aefw.bin"; + wowlan_fw_name = "rtlwifi/rtl8812aefw_wowlan.bin"; } else { - rtlpriv->cfg->fw_name = "rtlwifi/rtl8821aefw.bin"; - rtlpriv->cfg->wowlan_fw_name = "rtlwifi/rtl8821aefw_wowlan.bin"; + fw_name = "rtlwifi/rtl8821aefw.bin"; + wowlan_fw_name = "rtlwifi/rtl8821aefw_wowlan.bin"; } rtlpriv->max_fw_size = 0x8000; /*load normal firmware*/ - pr_info("Using firmware %s\n", rtlpriv->cfg->fw_name); - err = request_firmware_nowait(THIS_MODULE, 1, rtlpriv->cfg->fw_name, + pr_info("Using firmware %s\n", fw_name); + err = request_firmware_nowait(THIS_MODULE, 1, fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_fw_cb); if (err) { @@ -222,9 +223,9 @@ int rtl8821ae_init_sw_vars(struct ieee80211_hw *hw) return 1; } /*load wowlan firmware*/ - pr_info("Using firmware %s\n", rtlpriv->cfg->wowlan_fw_name); + pr_info("Using firmware %s\n", wowlan_fw_name); err = request_firmware_nowait(THIS_MODULE, 1, - rtlpriv->cfg->wowlan_fw_name, + wowlan_fw_name, rtlpriv->io.dev, GFP_KERNEL, hw, rtl_wowlan_fw_cb); if (err) { @@ -320,7 +321,6 @@ static const struct rtl_hal_cfg rtl8821ae_hal_cfg = { .bar_id = 2, .write_readback = true, .name = "rtl8821ae_pci", - .fw_name = "rtlwifi/rtl8821aefw.bin", .ops = &rtl8821ae_hal_ops, .mod_params = &rtl8821ae_mod_params, .maps[SYS_ISO_CTRL] = REG_SYS_ISO_CTRL, diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h index 595f7d5d091a..dafe486f8448 100644 --- a/drivers/net/wireless/realtek/rtlwifi/wifi.h +++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h @@ -2278,9 +2278,7 @@ struct rtl_hal_cfg { u8 bar_id; bool write_readback; char *name; - char *fw_name; char *alt_fw_name; - char *wowlan_fw_name; struct rtl_hal_ops *ops; struct rtl_mod_params *mod_params; struct rtl_hal_usbint_cfg *usb_interface_cfg; diff --git a/drivers/net/wireless/ti/wlcore/sdio.c b/drivers/net/wireless/ti/wlcore/sdio.c index a6e94b1a12cb..47fe7f96a242 100644 --- a/drivers/net/wireless/ti/wlcore/sdio.c +++ b/drivers/net/wireless/ti/wlcore/sdio.c @@ -391,7 +391,6 @@ static void wl1271_remove(struct sdio_func *func) pm_runtime_get_noresume(&func->dev); platform_device_unregister(glue->core); - kfree(glue); } #ifdef CONFIG_PM diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig index 8b2b740d6679..124c2432ac9c 100644 --- a/drivers/nvdimm/Kconfig +++ b/drivers/nvdimm/Kconfig @@ -89,7 +89,7 @@ config NVDIMM_PFN Select Y if unsure config NVDIMM_DAX - tristate "NVDIMM DAX: Raw access to persistent memory" + bool "NVDIMM DAX: Raw access to persistent memory" default LIBNVDIMM depends on NVDIMM_PFN help diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index 3509cff68ef9..abe5c6bc756c 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -2176,12 +2176,14 @@ static struct device **scan_labels(struct nd_region *nd_region) return devs; err: - for (i = 0; devs[i]; i++) - if (is_nd_blk(&nd_region->dev)) - namespace_blk_release(devs[i]); - else - namespace_pmem_release(devs[i]); - kfree(devs); + if (devs) { + for (i = 0; devs[i]; i++) + if (is_nd_blk(&nd_region->dev)) + namespace_blk_release(devs[i]); + else + namespace_pmem_release(devs[i]); + kfree(devs); + } return NULL; } diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 42b3a8217073..24618431a14b 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -47,7 +47,7 @@ static struct nd_region *to_region(struct pmem_device *pmem) return to_nd_region(to_dev(pmem)->parent); } -static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, +static int pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, unsigned int len) { struct device *dev = to_dev(pmem); @@ -62,8 +62,12 @@ static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset, __func__, (unsigned long long) sector, cleared / 512, cleared / 512 > 1 ? "s" : ""); badblocks_clear(&pmem->bb, sector, cleared / 512); + } else { + return -EIO; } + invalidate_pmem(pmem->virt_addr + offset, len); + return 0; } static void write_pmem(void *pmem_addr, struct page *page, @@ -123,7 +127,7 @@ static int pmem_do_bvec(struct pmem_device *pmem, struct page *page, flush_dcache_page(page); write_pmem(pmem_addr, page, off, len); if (unlikely(bad_pmem)) { - pmem_clear_poison(pmem, pmem_off, len); + rc = pmem_clear_poison(pmem, pmem_off, len); write_pmem(pmem_addr, page, off, len); } } diff --git a/drivers/pci/host/pcie-designware.c b/drivers/pci/host/pcie-designware.c index 035f50c03281..bed19994c1e9 100644 --- a/drivers/pci/host/pcie-designware.c +++ b/drivers/pci/host/pcie-designware.c @@ -637,8 +637,6 @@ int dw_pcie_host_init(struct pcie_port *pp) } } - pp->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pp); - if (pp->ops->host_init) pp->ops->host_init(pp); @@ -809,6 +807,11 @@ void dw_pcie_setup_rc(struct pcie_port *pp) { u32 val; + /* get iATU unroll support */ + pp->iatu_unroll_enabled = dw_pcie_iatu_unroll_enabled(pp); + dev_dbg(pp->dev, "iATU unroll: %s\n", + pp->iatu_unroll_enabled ? "enabled" : "disabled"); + /* set the number of lanes */ val = dw_pcie_readl_rc(pp, PCIE_PORT_LINK_CONTROL); val &= ~PORT_LINK_MODE_MASK; diff --git a/drivers/pci/host/pcie-qcom.c b/drivers/pci/host/pcie-qcom.c index ef0a84c7a588..35936409b2d4 100644 --- a/drivers/pci/host/pcie-qcom.c +++ b/drivers/pci/host/pcie-qcom.c @@ -533,11 +533,11 @@ static int qcom_pcie_probe(struct platform_device *pdev) if (IS_ERR(pcie->phy)) return PTR_ERR(pcie->phy); + pp->dev = dev; ret = pcie->ops->get_resources(pcie); if (ret) return ret; - pp->dev = dev; pp->root_bus_nr = -1; pp->ops = &qcom_pcie_dw_ops; diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index bfdd0744b686..ad70507cfb56 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -610,6 +610,7 @@ static int msi_verify_entries(struct pci_dev *dev) * msi_capability_init - configure device's MSI capability structure * @dev: pointer to the pci_dev data structure of MSI device function * @nvec: number of interrupts to allocate + * @affinity: flag to indicate cpu irq affinity mask should be set * * Setup the MSI capability structure of the device with the requested * number of interrupts. A return value of zero indicates the successful @@ -752,6 +753,7 @@ static void msix_program_entries(struct pci_dev *dev, * @dev: pointer to the pci_dev data structure of MSI-X device function * @entries: pointer to an array of struct msix_entry entries * @nvec: number of @entries + * @affinity: flag to indicate cpu irq affinity mask should be set * * Setup the MSI-X capability structure of device function with a * single MSI-X irq. A return of zero indicates the successful setup of diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 67426c0477d3..5c1519b229e0 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -2754,7 +2754,7 @@ static int _regulator_set_voltage_time(struct regulator_dev *rdev, ramp_delay = rdev->desc->ramp_delay; if (ramp_delay == 0) { - rdev_warn(rdev, "ramp_delay not set\n"); + rdev_dbg(rdev, "ramp_delay not set\n"); return 0; } diff --git a/drivers/reset/reset-uniphier.c b/drivers/reset/reset-uniphier.c index 8b2558e7363e..968c3ae4535c 100644 --- a/drivers/reset/reset-uniphier.c +++ b/drivers/reset/reset-uniphier.c @@ -154,7 +154,7 @@ const struct uniphier_reset_data uniphier_sld3_mio_reset_data[] = { UNIPHIER_RESET_END, }; -const struct uniphier_reset_data uniphier_pro5_mio_reset_data[] = { +const struct uniphier_reset_data uniphier_pro5_sd_reset_data[] = { UNIPHIER_MIO_RESET_SD(0, 0), UNIPHIER_MIO_RESET_SD(1, 1), UNIPHIER_MIO_RESET_EMMC_HW_RESET(6, 1), @@ -360,7 +360,7 @@ static const struct of_device_id uniphier_reset_match[] = { .compatible = "socionext,uniphier-ld20-reset", .data = uniphier_ld20_sys_reset_data, }, - /* Media I/O reset */ + /* Media I/O reset, SD reset */ { .compatible = "socionext,uniphier-sld3-mio-reset", .data = uniphier_sld3_mio_reset_data, @@ -378,20 +378,20 @@ static const struct of_device_id uniphier_reset_match[] = { .data = uniphier_sld3_mio_reset_data, }, { - .compatible = "socionext,uniphier-pro5-mio-reset", - .data = uniphier_pro5_mio_reset_data, + .compatible = "socionext,uniphier-pro5-sd-reset", + .data = uniphier_pro5_sd_reset_data, }, { - .compatible = "socionext,uniphier-pxs2-mio-reset", - .data = uniphier_pro5_mio_reset_data, + .compatible = "socionext,uniphier-pxs2-sd-reset", + .data = uniphier_pro5_sd_reset_data, }, { .compatible = "socionext,uniphier-ld11-mio-reset", .data = uniphier_sld3_mio_reset_data, }, { - .compatible = "socionext,uniphier-ld20-mio-reset", - .data = uniphier_pro5_mio_reset_data, + .compatible = "socionext,uniphier-ld20-sd-reset", + .data = uniphier_pro5_sd_reset_data, }, /* Peripheral reset */ { diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 831935af7389..a7a88476e215 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1205,7 +1205,7 @@ static int verify_fcx_max_data(struct dasd_device *device, __u8 lpm) mdc, lpm); return mdc; } - fcx_max_data = mdc * FCX_MAX_DATA_FACTOR; + fcx_max_data = (u32)mdc * FCX_MAX_DATA_FACTOR; if (fcx_max_data < private->fcx_max_data) { dev_warn(&device->cdev->dev, "The maximum data size for zHPF requests %u " @@ -1675,7 +1675,7 @@ static u32 get_fcx_max_data(struct dasd_device *device) " data size for zHPF requests failed\n"); return 0; } else - return mdc * FCX_MAX_DATA_FACTOR; + return (u32)mdc * FCX_MAX_DATA_FACTOR; } /* diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index 46be25c7461e..876c7e6e3a99 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -780,7 +780,7 @@ static int cfg_wait_idle(void) static int __init chp_init(void) { struct chp_id chpid; - int ret; + int state, ret; ret = crw_register_handler(CRW_RSC_CPATH, chp_process_crw); if (ret) @@ -791,7 +791,9 @@ static int __init chp_init(void) return 0; /* Register available channel-paths. */ chp_id_for_each(&chpid) { - if (chp_info_get_status(chpid) != CHP_STATUS_NOT_RECOGNIZED) + state = chp_info_get_status(chpid); + if (state == CHP_STATUS_CONFIGURED || + state == CHP_STATUS_STANDBY) chp_new(chpid); } diff --git a/drivers/scsi/NCR5380.c b/drivers/scsi/NCR5380.c index db2739079cbb..790babc5ef66 100644 --- a/drivers/scsi/NCR5380.c +++ b/drivers/scsi/NCR5380.c @@ -353,7 +353,7 @@ static void NCR5380_print_phase(struct Scsi_Host *instance) #endif -static int probe_irq __initdata; +static int probe_irq; /** * probe_intr - helper for IRQ autoprobe @@ -365,7 +365,7 @@ static int probe_irq __initdata; * used by the IRQ probe code. */ -static irqreturn_t __init probe_intr(int irq, void *dev_id) +static irqreturn_t probe_intr(int irq, void *dev_id) { probe_irq = irq; return IRQ_HANDLED; @@ -380,7 +380,7 @@ static irqreturn_t __init probe_intr(int irq, void *dev_id) * and then looking to see what interrupt actually turned up. */ -static int __init __maybe_unused NCR5380_probe_irq(struct Scsi_Host *instance, +static int __maybe_unused NCR5380_probe_irq(struct Scsi_Host *instance, int possible) { struct NCR5380_hostdata *hostdata = shost_priv(instance); diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c index 3d53d636b17b..f0cfb0451757 100644 --- a/drivers/scsi/arcmsr/arcmsr_hba.c +++ b/drivers/scsi/arcmsr/arcmsr_hba.c @@ -2636,18 +2636,9 @@ static int arcmsr_queue_command_lck(struct scsi_cmnd *cmd, struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata; struct CommandControlBlock *ccb; int target = cmd->device->id; - int lun = cmd->device->lun; - uint8_t scsicmd = cmd->cmnd[0]; cmd->scsi_done = done; cmd->host_scribble = NULL; cmd->result = 0; - if ((scsicmd == SYNCHRONIZE_CACHE) ||(scsicmd == SEND_DIAGNOSTIC)){ - if(acb->devstate[target][lun] == ARECA_RAID_GONE) { - cmd->result = (DID_NO_CONNECT << 16); - } - cmd->scsi_done(cmd); - return 0; - } if (target == 16) { /* virtual device for iop message transfer */ arcmsr_handle_virtual_command(acb, cmd); diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c index 68138a647dfc..d9239c2d49b1 100644 --- a/drivers/scsi/be2iscsi/be_main.c +++ b/drivers/scsi/be2iscsi/be_main.c @@ -900,8 +900,9 @@ void hwi_ring_cq_db(struct beiscsi_hba *phba, static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) { struct sgl_handle *psgl_handle; + unsigned long flags; - spin_lock_bh(&phba->io_sgl_lock); + spin_lock_irqsave(&phba->io_sgl_lock, flags); if (phba->io_sgl_hndl_avbl) { beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, "BM_%d : In alloc_io_sgl_handle," @@ -919,14 +920,16 @@ static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba) phba->io_sgl_alloc_index++; } else psgl_handle = NULL; - spin_unlock_bh(&phba->io_sgl_lock); + spin_unlock_irqrestore(&phba->io_sgl_lock, flags); return psgl_handle; } static void free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) { - spin_lock_bh(&phba->io_sgl_lock); + unsigned long flags; + + spin_lock_irqsave(&phba->io_sgl_lock, flags); beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO, "BM_%d : In free_,io_sgl_free_index=%d\n", phba->io_sgl_free_index); @@ -941,7 +944,7 @@ free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) "value there=%p\n", phba->io_sgl_free_index, phba->io_sgl_hndl_base [phba->io_sgl_free_index]); - spin_unlock_bh(&phba->io_sgl_lock); + spin_unlock_irqrestore(&phba->io_sgl_lock, flags); return; } phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle; @@ -950,7 +953,7 @@ free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) phba->io_sgl_free_index = 0; else phba->io_sgl_free_index++; - spin_unlock_bh(&phba->io_sgl_lock); + spin_unlock_irqrestore(&phba->io_sgl_lock, flags); } static inline struct wrb_handle * @@ -958,15 +961,16 @@ beiscsi_get_wrb_handle(struct hwi_wrb_context *pwrb_context, unsigned int wrbs_per_cxn) { struct wrb_handle *pwrb_handle; + unsigned long flags; - spin_lock_bh(&pwrb_context->wrb_lock); + spin_lock_irqsave(&pwrb_context->wrb_lock, flags); pwrb_handle = pwrb_context->pwrb_handle_base[pwrb_context->alloc_index]; pwrb_context->wrb_handles_available--; if (pwrb_context->alloc_index == (wrbs_per_cxn - 1)) pwrb_context->alloc_index = 0; else pwrb_context->alloc_index++; - spin_unlock_bh(&pwrb_context->wrb_lock); + spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags); if (pwrb_handle) memset(pwrb_handle->pwrb, 0, sizeof(*pwrb_handle->pwrb)); @@ -1001,14 +1005,16 @@ beiscsi_put_wrb_handle(struct hwi_wrb_context *pwrb_context, struct wrb_handle *pwrb_handle, unsigned int wrbs_per_cxn) { - spin_lock_bh(&pwrb_context->wrb_lock); + unsigned long flags; + + spin_lock_irqsave(&pwrb_context->wrb_lock, flags); pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle; pwrb_context->wrb_handles_available++; if (pwrb_context->free_index == (wrbs_per_cxn - 1)) pwrb_context->free_index = 0; else pwrb_context->free_index++; - spin_unlock_bh(&pwrb_context->wrb_lock); + spin_unlock_irqrestore(&pwrb_context->wrb_lock, flags); } /** @@ -1037,8 +1043,9 @@ free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context, static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) { struct sgl_handle *psgl_handle; + unsigned long flags; - spin_lock_bh(&phba->mgmt_sgl_lock); + spin_lock_irqsave(&phba->mgmt_sgl_lock, flags); if (phba->eh_sgl_hndl_avbl) { psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index]; phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL; @@ -1056,14 +1063,16 @@ static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba) phba->eh_sgl_alloc_index++; } else psgl_handle = NULL; - spin_unlock_bh(&phba->mgmt_sgl_lock); + spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags); return psgl_handle; } void free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) { - spin_lock_bh(&phba->mgmt_sgl_lock); + unsigned long flags; + + spin_lock_irqsave(&phba->mgmt_sgl_lock, flags); beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG, "BM_%d : In free_mgmt_sgl_handle," "eh_sgl_free_index=%d\n", @@ -1078,7 +1087,7 @@ free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) "BM_%d : Double Free in eh SGL ," "eh_sgl_free_index=%d\n", phba->eh_sgl_free_index); - spin_unlock_bh(&phba->mgmt_sgl_lock); + spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags); return; } phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle; @@ -1088,7 +1097,7 @@ free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle) phba->eh_sgl_free_index = 0; else phba->eh_sgl_free_index++; - spin_unlock_bh(&phba->mgmt_sgl_lock); + spin_unlock_irqrestore(&phba->mgmt_sgl_lock, flags); } static void diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c index c051694bfcb0..f9b6fba689ff 100644 --- a/drivers/scsi/libiscsi.c +++ b/drivers/scsi/libiscsi.c @@ -791,9 +791,9 @@ __iscsi_conn_send_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr, free_task: /* regular RX path uses back_lock */ - spin_lock_bh(&session->back_lock); + spin_lock(&session->back_lock); __iscsi_put_task(task); - spin_unlock_bh(&session->back_lock); + spin_unlock(&session->back_lock); return NULL; } diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index 9ff57dee72d7..d8b1fbd4c8aa 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -1700,16 +1700,13 @@ megasas_queue_command(struct Scsi_Host *shost, struct scsi_cmnd *scmd) goto out_done; } - switch (scmd->cmnd[0]) { - case SYNCHRONIZE_CACHE: - /* - * FW takes care of flush cache on its own - * No need to send it down - */ + /* + * FW takes care of flush cache on its own for Virtual Disk. + * No need to send it down for VD. For JBOD send SYNCHRONIZE_CACHE to FW. + */ + if ((scmd->cmnd[0] == SYNCHRONIZE_CACHE) && MEGASAS_IS_LOGICAL(scmd)) { scmd->result = DID_OK << 16; goto out_done; - default: - break; } return instance->instancet->build_and_issue_cmd(instance, scmd); diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index c905709707f0..cf04a364fd8b 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -5134,6 +5134,7 @@ static void __exit scsi_debug_exit(void) bus_unregister(&pseudo_lld_bus); root_device_unregister(pseudo_primary); + vfree(map_storep); vfree(dif_storep); vfree(fake_storep); kfree(sdebug_q_arr); diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c index 35c0dd945668..a67b0ff6a362 100644 --- a/drivers/spi/spi-fsl-dspi.c +++ b/drivers/spi/spi-fsl-dspi.c @@ -70,6 +70,7 @@ #define SPI_SR 0x2c #define SPI_SR_EOQF 0x10000000 #define SPI_SR_TCFQF 0x80000000 +#define SPI_SR_CLEAR 0xdaad0000 #define SPI_RSER 0x30 #define SPI_RSER_EOQFE 0x10000000 @@ -646,6 +647,11 @@ static const struct regmap_config dspi_regmap_config = { .max_register = 0x88, }; +static void dspi_init(struct fsl_dspi *dspi) +{ + regmap_write(dspi->regmap, SPI_SR, SPI_SR_CLEAR); +} + static int dspi_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -709,6 +715,7 @@ static int dspi_probe(struct platform_device *pdev) return PTR_ERR(dspi->regmap); } + dspi_init(dspi); dspi->irq = platform_get_irq(pdev, 0); if (dspi->irq < 0) { dev_err(&pdev->dev, "can't get platform irq\n"); diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c index 7451585a080e..2c175b9495f7 100644 --- a/drivers/spi/spi-fsl-espi.c +++ b/drivers/spi/spi-fsl-espi.c @@ -458,7 +458,7 @@ static void fsl_espi_cpu_irq(struct mpc8xxx_spi *mspi, u32 events) mspi->len -= rx_nr_bytes; - if (mspi->rx) + if (rx_nr_bytes && mspi->rx) mspi->get_rx(rx_data, mspi); } diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 5787b723b593..838783c3fed0 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -1618,9 +1618,11 @@ static void of_register_spi_devices(struct spi_master *master) if (of_node_test_and_set_flag(nc, OF_POPULATED)) continue; spi = of_register_spi_device(master, nc); - if (IS_ERR(spi)) + if (IS_ERR(spi)) { dev_warn(&master->dev, "Failed to create SPI device for %s\n", nc->full_name); + of_node_clear_flag(nc, OF_POPULATED); + } } } #else @@ -3131,6 +3133,7 @@ static int of_spi_notify(struct notifier_block *nb, unsigned long action, if (IS_ERR(spi)) { pr_err("%s: failed to create for '%s'\n", __func__, rd->dn->full_name); + of_node_clear_flag(rd->dn, OF_POPULATED); return notifier_from_errno(PTR_ERR(spi)); } break; diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c index 396ded52ab70..209a8f7ef02b 100644 --- a/drivers/staging/android/ion/ion.c +++ b/drivers/staging/android/ion/ion.c @@ -1187,8 +1187,10 @@ int ion_query_heaps(struct ion_client *client, struct ion_heap_query *query) hdata.type = heap->type; hdata.heap_id = heap->id; - ret = copy_to_user(&buffer[cnt], - &hdata, sizeof(hdata)); + if (copy_to_user(&buffer[cnt], &hdata, sizeof(hdata))) { + ret = -EFAULT; + goto out; + } cnt++; if (cnt >= max_cnt) diff --git a/drivers/staging/android/ion/ion_of.c b/drivers/staging/android/ion/ion_of.c index 15bac92b7f04..46b2bb99bfd6 100644 --- a/drivers/staging/android/ion/ion_of.c +++ b/drivers/staging/android/ion/ion_of.c @@ -107,7 +107,7 @@ struct ion_platform_data *ion_parse_dt(struct platform_device *pdev, heap_pdev = of_platform_device_create(node, heaps[i].name, &pdev->dev); - if (!pdev) + if (!heap_pdev) return ERR_PTR(-ENOMEM); heap_pdev->dev.platform_data = &heaps[i]; diff --git a/drivers/staging/greybus/arche-platform.c b/drivers/staging/greybus/arche-platform.c index e36ee984485b..34307ac3f255 100644 --- a/drivers/staging/greybus/arche-platform.c +++ b/drivers/staging/greybus/arche-platform.c @@ -128,6 +128,7 @@ int arche_platform_change_state(enum arche_platform_state state, pdev = of_find_device_by_node(np); if (!pdev) { pr_err("arche-platform device not found\n"); + of_node_put(np); return -ENODEV; } diff --git a/drivers/staging/greybus/es2.c b/drivers/staging/greybus/es2.c index 071bb1cfd3ae..baab460eeaa3 100644 --- a/drivers/staging/greybus/es2.c +++ b/drivers/staging/greybus/es2.c @@ -1548,7 +1548,8 @@ static int ap_probe(struct usb_interface *interface, INIT_LIST_HEAD(&es2->arpcs); spin_lock_init(&es2->arpc_lock); - if (es2_arpc_in_enable(es2)) + retval = es2_arpc_in_enable(es2); + if (retval) goto error; retval = gb_hd_add(hd); diff --git a/drivers/staging/greybus/gpio.c b/drivers/staging/greybus/gpio.c index 5e06e4229e42..250caa00de5e 100644 --- a/drivers/staging/greybus/gpio.c +++ b/drivers/staging/greybus/gpio.c @@ -702,15 +702,13 @@ static int gb_gpio_probe(struct gbphy_device *gbphy_dev, ret = gb_gpio_irqchip_add(gpio, irqc, 0, handle_level_irq, IRQ_TYPE_NONE); if (ret) { - dev_err(&connection->bundle->dev, - "failed to add irq chip: %d\n", ret); + dev_err(&gbphy_dev->dev, "failed to add irq chip: %d\n", ret); goto exit_line_free; } ret = gpiochip_add(gpio); if (ret) { - dev_err(&connection->bundle->dev, - "failed to add gpio chip: %d\n", ret); + dev_err(&gbphy_dev->dev, "failed to add gpio chip: %d\n", ret); goto exit_gpio_irqchip_remove; } diff --git a/drivers/staging/greybus/module.c b/drivers/staging/greybus/module.c index 69f67ddbd4a3..660b4674a76f 100644 --- a/drivers/staging/greybus/module.c +++ b/drivers/staging/greybus/module.c @@ -127,7 +127,7 @@ struct gb_module *gb_module_create(struct gb_host_device *hd, u8 module_id, return module; err_put_interfaces: - for (--i; i > 0; --i) + for (--i; i >= 0; --i) gb_interface_put(module->interfaces[i]); put_device(&module->dev); diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index 5ee7954bd9f9..2633d2bfb1b4 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -888,7 +888,7 @@ static int gb_uart_probe(struct gbphy_device *gbphy_dev, minor = alloc_minor(gb_tty); if (minor < 0) { if (minor == -ENOSPC) { - dev_err(&connection->bundle->dev, + dev_err(&gbphy_dev->dev, "no more free minor numbers\n"); retval = -ENODEV; } else { diff --git a/drivers/staging/iio/accel/sca3000_core.c b/drivers/staging/iio/accel/sca3000_core.c index d626125d7af9..564b36d4f648 100644 --- a/drivers/staging/iio/accel/sca3000_core.c +++ b/drivers/staging/iio/accel/sca3000_core.c @@ -468,6 +468,8 @@ static inline int __sca3000_get_base_freq(struct sca3000_state *st, case SCA3000_MEAS_MODE_OP_2: *base_freq = info->option_mode_2_freq; break; + default: + ret = -EINVAL; } error_ret: return ret; diff --git a/drivers/staging/lustre/lustre/llite/lproc_llite.c b/drivers/staging/lustre/lustre/llite/lproc_llite.c index 6eae60595905..23fda9d98bff 100644 --- a/drivers/staging/lustre/lustre/llite/lproc_llite.c +++ b/drivers/staging/lustre/lustre/llite/lproc_llite.c @@ -871,12 +871,10 @@ static ssize_t xattr_cache_store(struct kobject *kobj, } LUSTRE_RW_ATTR(xattr_cache); -static ssize_t unstable_stats_show(struct kobject *kobj, - struct attribute *attr, - char *buf) +static int ll_unstable_stats_seq_show(struct seq_file *m, void *v) { - struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, - ll_kobj); + struct super_block *sb = m->private; + struct ll_sb_info *sbi = ll_s2sbi(sb); struct cl_client_cache *cache = sbi->ll_cache; long pages; int mb; @@ -884,19 +882,21 @@ static ssize_t unstable_stats_show(struct kobject *kobj, pages = atomic_long_read(&cache->ccc_unstable_nr); mb = (pages * PAGE_SIZE) >> 20; - return sprintf(buf, "unstable_check: %8d\n" - "unstable_pages: %12ld\n" - "unstable_mb: %8d\n", - cache->ccc_unstable_check, pages, mb); + seq_printf(m, + "unstable_check: %8d\n" + "unstable_pages: %12ld\n" + "unstable_mb: %8d\n", + cache->ccc_unstable_check, pages, mb); + + return 0; } -static ssize_t unstable_stats_store(struct kobject *kobj, - struct attribute *attr, - const char *buffer, - size_t count) +static ssize_t ll_unstable_stats_seq_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *off) { - struct ll_sb_info *sbi = container_of(kobj, struct ll_sb_info, - ll_kobj); + struct super_block *sb = ((struct seq_file *)file->private_data)->private; + struct ll_sb_info *sbi = ll_s2sbi(sb); char kernbuf[128]; int val, rc; @@ -922,7 +922,7 @@ static ssize_t unstable_stats_store(struct kobject *kobj, return count; } -LUSTRE_RW_ATTR(unstable_stats); +LPROC_SEQ_FOPS(ll_unstable_stats); static ssize_t root_squash_show(struct kobject *kobj, struct attribute *attr, char *buf) @@ -995,6 +995,7 @@ static struct lprocfs_vars lprocfs_llite_obd_vars[] = { /* { "filegroups", lprocfs_rd_filegroups, 0, 0 }, */ { "max_cached_mb", &ll_max_cached_mb_fops, NULL }, { "statahead_stats", &ll_statahead_stats_fops, NULL, 0 }, + { "unstable_stats", &ll_unstable_stats_fops, NULL }, { "sbi_flags", &ll_sbi_flags_fops, NULL, 0 }, { .name = "nosquash_nids", .fops = &ll_nosquash_nids_fops }, @@ -1026,7 +1027,6 @@ static struct attribute *llite_attrs[] = { &lustre_attr_max_easize.attr, &lustre_attr_default_easize.attr, &lustre_attr_xattr_cache.attr, - &lustre_attr_unstable_stats.attr, &lustre_attr_root_squash.attr, NULL, }; diff --git a/drivers/staging/media/bcm2048/radio-bcm2048.c b/drivers/staging/media/bcm2048/radio-bcm2048.c index ea15cc638097..4d9bd02ede47 100644 --- a/drivers/staging/media/bcm2048/radio-bcm2048.c +++ b/drivers/staging/media/bcm2048/radio-bcm2048.c @@ -482,6 +482,8 @@ static int bcm2048_set_rds_no_lock(struct bcm2048_device *bdev, u8 rds_on) flags); memset(&bdev->rds_info, 0, sizeof(bdev->rds_info)); } + if (err) + return err; return bcm2048_send_command(bdev, BCM2048_I2C_FM_RDS_SYSTEM, bdev->cache_fm_rds_system); diff --git a/drivers/staging/wilc1000/host_interface.c b/drivers/staging/wilc1000/host_interface.c index 78f5613e9467..6ab7443eabde 100644 --- a/drivers/staging/wilc1000/host_interface.c +++ b/drivers/staging/wilc1000/host_interface.c @@ -3388,7 +3388,6 @@ int wilc_init(struct net_device *dev, struct host_if_drv **hif_drv_handler) clients_count++; - destroy_workqueue(hif_workqueue); _fail_: return result; } diff --git a/drivers/thermal/intel_pch_thermal.c b/drivers/thermal/intel_pch_thermal.c index 9b4815e81b0d..19bf2028e508 100644 --- a/drivers/thermal/intel_pch_thermal.c +++ b/drivers/thermal/intel_pch_thermal.c @@ -20,10 +20,13 @@ #include <linux/types.h> #include <linux/init.h> #include <linux/pci.h> +#include <linux/acpi.h> #include <linux/thermal.h> #include <linux/pm.h> /* Intel PCH thermal Device IDs */ +#define PCH_THERMAL_DID_HSW_1 0x9C24 /* Haswell PCH */ +#define PCH_THERMAL_DID_HSW_2 0x8C24 /* Haswell PCH */ #define PCH_THERMAL_DID_WPT 0x9CA4 /* Wildcat Point */ #define PCH_THERMAL_DID_SKL 0x9D31 /* Skylake PCH */ @@ -66,9 +69,53 @@ struct pch_thermal_device { unsigned long crt_temp; int hot_trip_id; unsigned long hot_temp; + int psv_trip_id; + unsigned long psv_temp; bool bios_enabled; }; +#ifdef CONFIG_ACPI + +/* + * On some platforms, there is a companion ACPI device, which adds + * passive trip temperature using _PSV method. There is no specific + * passive temperature setting in MMIO interface of this PCI device. + */ +static void pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, + int *nr_trips) +{ + struct acpi_device *adev; + + ptd->psv_trip_id = -1; + + adev = ACPI_COMPANION(&ptd->pdev->dev); + if (adev) { + unsigned long long r; + acpi_status status; + + status = acpi_evaluate_integer(adev->handle, "_PSV", NULL, + &r); + if (ACPI_SUCCESS(status)) { + unsigned long trip_temp; + + trip_temp = DECI_KELVIN_TO_MILLICELSIUS(r); + if (trip_temp) { + ptd->psv_temp = trip_temp; + ptd->psv_trip_id = *nr_trips; + ++(*nr_trips); + } + } + } +} +#else +static void pch_wpt_add_acpi_psv_trip(struct pch_thermal_device *ptd, + int *nr_trips) +{ + ptd->psv_trip_id = -1; + +} +#endif + static int pch_wpt_init(struct pch_thermal_device *ptd, int *nr_trips) { u8 tsel; @@ -119,6 +166,8 @@ read_trips: ++(*nr_trips); } + pch_wpt_add_acpi_psv_trip(ptd, nr_trips); + return 0; } @@ -194,6 +243,8 @@ static int pch_get_trip_type(struct thermal_zone_device *tzd, int trip, *type = THERMAL_TRIP_CRITICAL; else if (ptd->hot_trip_id == trip) *type = THERMAL_TRIP_HOT; + else if (ptd->psv_trip_id == trip) + *type = THERMAL_TRIP_PASSIVE; else return -EINVAL; @@ -208,6 +259,8 @@ static int pch_get_trip_temp(struct thermal_zone_device *tzd, int trip, int *tem *temp = ptd->crt_temp; else if (ptd->hot_trip_id == trip) *temp = ptd->hot_temp; + else if (ptd->psv_trip_id == trip) + *temp = ptd->psv_temp; else return -EINVAL; @@ -242,6 +295,11 @@ static int intel_pch_thermal_probe(struct pci_dev *pdev, ptd->ops = &pch_dev_ops_wpt; dev_name = "pch_skylake"; break; + case PCH_THERMAL_DID_HSW_1: + case PCH_THERMAL_DID_HSW_2: + ptd->ops = &pch_dev_ops_wpt; + dev_name = "pch_haswell"; + break; default: dev_err(&pdev->dev, "unknown pch thermal device\n"); return -ENODEV; @@ -324,6 +382,8 @@ static int intel_pch_thermal_resume(struct device *device) static struct pci_device_id intel_pch_thermal_id[] = { { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_WPT) }, { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_SKL) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_1) }, + { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCH_THERMAL_DID_HSW_2) }, { 0, }, }; MODULE_DEVICE_TABLE(pci, intel_pch_thermal_id); diff --git a/drivers/thermal/intel_powerclamp.c b/drivers/thermal/intel_powerclamp.c index 0e4dc0afcfd2..7a223074df3d 100644 --- a/drivers/thermal/intel_powerclamp.c +++ b/drivers/thermal/intel_powerclamp.c @@ -669,20 +669,10 @@ static struct thermal_cooling_device_ops powerclamp_cooling_ops = { .set_cur_state = powerclamp_set_cur_state, }; -static const struct x86_cpu_id intel_powerclamp_ids[] __initconst = { - { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_MWAIT }, - { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_ARAT }, - { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_NONSTOP_TSC }, - { X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_CONSTANT_TSC}, - {} -}; -MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids); - static int __init powerclamp_probe(void) { - if (!x86_match_cpu(intel_powerclamp_ids)) { - pr_err("Intel powerclamp does not run on family %d model %d\n", - boot_cpu_data.x86, boot_cpu_data.x86_model); + if (!boot_cpu_has(X86_FEATURE_MWAIT)) { + pr_err("CPU does not support MWAIT"); return -ENODEV; } diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c index 886fcf37f291..b9923464599f 100644 --- a/drivers/tty/serial/8250/8250_lpss.c +++ b/drivers/tty/serial/8250/8250_lpss.c @@ -213,7 +213,7 @@ static int qrk_serial_setup(struct lpss8250 *lpss, struct uart_port *port) struct pci_dev *pdev = to_pci_dev(port->dev); int ret; - ret = pci_alloc_irq_vectors(pdev, 1, 1, 0); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); if (ret < 0) return ret; diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 1bfb6fdbaa20..1731b98d2471 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -83,7 +83,8 @@ static const struct serial8250_config uart_config[] = { .name = "16550A", .fifo_size = 16, .tx_loadsz = 16, - .fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10, + .fcr = UART_FCR_ENABLE_FIFO | UART_FCR_R_TRIG_10 | + UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT, .rxtrig_bytes = {1, 4, 8, 14}, .flags = UART_CAP_FIFO, }, diff --git a/drivers/tty/serial/8250/8250_uniphier.c b/drivers/tty/serial/8250/8250_uniphier.c index b8d9c8c9d02a..417d9e7038e1 100644 --- a/drivers/tty/serial/8250/8250_uniphier.c +++ b/drivers/tty/serial/8250/8250_uniphier.c @@ -99,7 +99,7 @@ static void uniphier_serial_out(struct uart_port *p, int offset, int value) case UART_LCR: valshift = UNIPHIER_UART_LCR_SHIFT; /* Divisor latch access bit does not exist. */ - value &= ~(UART_LCR_DLAB << valshift); + value &= ~UART_LCR_DLAB; /* fall through */ case UART_MCR: offset = UNIPHIER_UART_LCR_MCR; @@ -199,7 +199,7 @@ static int uniphier_uart_probe(struct platform_device *pdev) regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!regs) { - dev_err(dev, "failed to get memory resource"); + dev_err(dev, "failed to get memory resource\n"); return -EINVAL; } diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig index c7831407a882..25c1d7bc0100 100644 --- a/drivers/tty/serial/Kconfig +++ b/drivers/tty/serial/Kconfig @@ -1625,6 +1625,7 @@ config SERIAL_SPRD_CONSOLE config SERIAL_STM32 tristate "STMicroelectronics STM32 serial port support" select SERIAL_CORE + depends on HAS_DMA depends on ARM || COMPILE_TEST help This driver is for the on-chip Serial Controller on diff --git a/drivers/tty/serial/atmel_serial.c b/drivers/tty/serial/atmel_serial.c index fd8aa1f4ba78..168b10cad47b 100644 --- a/drivers/tty/serial/atmel_serial.c +++ b/drivers/tty/serial/atmel_serial.c @@ -2132,11 +2132,29 @@ static void atmel_set_termios(struct uart_port *port, struct ktermios *termios, mode |= ATMEL_US_USMODE_RS485; } else if (termios->c_cflag & CRTSCTS) { /* RS232 with hardware handshake (RTS/CTS) */ - if (atmel_use_dma_rx(port) && !atmel_use_fifo(port)) { - dev_info(port->dev, "not enabling hardware flow control because DMA is used"); - termios->c_cflag &= ~CRTSCTS; - } else { + if (atmel_use_fifo(port) && + !mctrl_gpio_to_gpiod(atmel_port->gpios, UART_GPIO_CTS)) { + /* + * with ATMEL_US_USMODE_HWHS set, the controller will + * be able to drive the RTS pin high/low when the RX + * FIFO is above RXFTHRES/below RXFTHRES2. + * It will also disable the transmitter when the CTS + * pin is high. + * This mode is not activated if CTS pin is a GPIO + * because in this case, the transmitter is always + * disabled (there must be an internal pull-up + * responsible for this behaviour). + * If the RTS pin is a GPIO, the controller won't be + * able to drive it according to the FIFO thresholds, + * but it will be handled by the driver. + */ mode |= ATMEL_US_USMODE_HWHS; + } else { + /* + * For platforms without FIFO, the flow control is + * handled by the driver. + */ + mode |= ATMEL_US_USMODE_NORMAL; } } else { /* RS232 without hadware handshake */ diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c index de9d5107c00a..76103f2c4a80 100644 --- a/drivers/tty/serial/fsl_lpuart.c +++ b/drivers/tty/serial/fsl_lpuart.c @@ -328,7 +328,7 @@ static void lpuart_dma_tx(struct lpuart_port *sport) sport->dma_tx_bytes = uart_circ_chars_pending(xmit); - if (xmit->tail < xmit->head) { + if (xmit->tail < xmit->head || xmit->head == 0) { sport->dma_tx_nents = 1; sg_init_one(sgl, xmit->buf + xmit->tail, sport->dma_tx_bytes); } else { @@ -359,7 +359,6 @@ static void lpuart_dma_tx(struct lpuart_port *sport) sport->dma_tx_in_progress = true; sport->dma_tx_cookie = dmaengine_submit(sport->dma_tx_desc); dma_async_issue_pending(sport->dma_tx_chan); - } static void lpuart_dma_tx_complete(void *arg) diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c index d391650b82e7..42caccb5e87e 100644 --- a/drivers/tty/serial/pch_uart.c +++ b/drivers/tty/serial/pch_uart.c @@ -419,6 +419,7 @@ static struct dmi_system_id pch_uart_dmi_table[] = { }, (void *)MINNOW_UARTCLK, }, + { } }; /* Return UART clock, checking for board specific clocks. */ diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index 2675792a8f59..fb0672554123 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -1130,9 +1130,13 @@ static int sc16is7xx_gpio_direction_output(struct gpio_chip *chip, { struct sc16is7xx_port *s = gpiochip_get_data(chip); struct uart_port *port = &s->p[0].port; + u8 state = sc16is7xx_port_read(port, SC16IS7XX_IOSTATE_REG); - sc16is7xx_port_update(port, SC16IS7XX_IOSTATE_REG, BIT(offset), - val ? BIT(offset) : 0); + if (val) + state |= BIT(offset); + else + state &= ~BIT(offset); + sc16is7xx_port_write(port, SC16IS7XX_IOSTATE_REG, state); sc16is7xx_port_update(port, SC16IS7XX_IODIR_REG, BIT(offset), BIT(offset)); diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index 6e4f63627479..f2303f390345 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -111,7 +111,7 @@ void uart_write_wakeup(struct uart_port *port) * closed. No cookie for you. */ BUG_ON(!state); - tty_wakeup(state->port.tty); + tty_port_tty_wakeup(&state->port); } static void uart_stop(struct tty_struct *tty) @@ -632,7 +632,7 @@ static void uart_flush_buffer(struct tty_struct *tty) if (port->ops->flush_buffer) port->ops->flush_buffer(port); uart_port_unlock(port, flags); - tty_wakeup(tty); + tty_port_tty_wakeup(&state->port); } /* @@ -2746,8 +2746,6 @@ int uart_add_one_port(struct uart_driver *drv, struct uart_port *uport) uport->cons = drv->cons; uport->minor = drv->tty_driver->minor_start + uport->line; - port->console = uart_console(uport); - /* * If this port is a console, then the spinlock is already * initialised. @@ -2761,6 +2759,8 @@ int uart_add_one_port(struct uart_driver *drv, struct uart_port *uport) uart_configure_port(drv, state, uport); + port->console = uart_console(uport); + num_groups = 2; if (uport->attr_group) num_groups++; diff --git a/drivers/tty/serial/stm32-usart.h b/drivers/tty/serial/stm32-usart.h index 41d974923102..cd97ceb76e4f 100644 --- a/drivers/tty/serial/stm32-usart.h +++ b/drivers/tty/serial/stm32-usart.h @@ -31,7 +31,7 @@ struct stm32_usart_info { struct stm32_usart_config cfg; }; -#define UNDEF_REG ~0 +#define UNDEF_REG 0xff /* Register offsets */ struct stm32_usart_info stm32f4_info = { diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c index f37edaa5ac75..dd4c02fa4820 100644 --- a/drivers/tty/serial/xilinx_uartps.c +++ b/drivers/tty/serial/xilinx_uartps.c @@ -1200,6 +1200,7 @@ static int __init cdns_early_console_setup(struct earlycon_device *device, OF_EARLYCON_DECLARE(cdns, "xlnx,xuartps", cdns_early_console_setup); OF_EARLYCON_DECLARE(cdns, "cdns,uart-r1p8", cdns_early_console_setup); OF_EARLYCON_DECLARE(cdns, "cdns,uart-r1p12", cdns_early_console_setup); +OF_EARLYCON_DECLARE(cdns, "xlnx,zynqmp-uart", cdns_early_console_setup); /** * cdns_uart_console_write - perform write operation @@ -1438,6 +1439,7 @@ static const struct of_device_id cdns_uart_of_match[] = { { .compatible = "xlnx,xuartps", }, { .compatible = "cdns,uart-r1p8", }, { .compatible = "cdns,uart-r1p12", .data = &zynqmp_uart_def }, + { .compatible = "xlnx,zynqmp-uart", .data = &zynqmp_uart_def }, {} }; MODULE_DEVICE_TABLE(of, cdns_uart_of_match); diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c index 06fb39c1d6dd..8c3bf3d613c0 100644 --- a/drivers/tty/vt/vt.c +++ b/drivers/tty/vt/vt.c @@ -870,10 +870,15 @@ static int vc_do_resize(struct tty_struct *tty, struct vc_data *vc, if (new_cols == vc->vc_cols && new_rows == vc->vc_rows) return 0; + if (new_screen_size > (4 << 20)) + return -EINVAL; newscreen = kmalloc(new_screen_size, GFP_USER); if (!newscreen) return -ENOMEM; + if (vc == sel_cons) + clear_selection(); + old_rows = vc->vc_rows; old_row_size = vc->vc_size_row; @@ -1176,7 +1181,7 @@ static void csi_J(struct vc_data *vc, int vpar) break; case 3: /* erase scroll-back buffer (and whole display) */ scr_memsetw(vc->vc_screenbuf, vc->vc_video_erase_char, - vc->vc_screenbuf_size >> 1); + vc->vc_screenbuf_size); set_origin(vc); if (con_is_visible(vc)) update_screen(vc); diff --git a/drivers/usb/chipidea/host.c b/drivers/usb/chipidea/host.c index 96ae69502c86..111b0e0b8698 100644 --- a/drivers/usb/chipidea/host.c +++ b/drivers/usb/chipidea/host.c @@ -188,6 +188,8 @@ static void host_stop(struct ci_hdrc *ci) if (hcd) { usb_remove_hcd(hcd); + ci->role = CI_ROLE_END; + synchronize_irq(ci->irq); usb_put_hcd(hcd); if (ci->platdata->reg_vbus && !ci_otg_is_fsm_mode(ci) && (ci->platdata->flags & CI_HDRC_TURN_VBUS_EARLY_ON)) diff --git a/drivers/usb/dwc2/core.c b/drivers/usb/dwc2/core.c index fa9b26b91507..4c0fa0b17353 100644 --- a/drivers/usb/dwc2/core.c +++ b/drivers/usb/dwc2/core.c @@ -463,9 +463,18 @@ static void dwc2_clear_force_mode(struct dwc2_hsotg *hsotg) */ void dwc2_force_dr_mode(struct dwc2_hsotg *hsotg) { + bool ret; + switch (hsotg->dr_mode) { case USB_DR_MODE_HOST: - dwc2_force_mode(hsotg, true); + ret = dwc2_force_mode(hsotg, true); + /* + * NOTE: This is required for some rockchip soc based + * platforms on their host-only dwc2. + */ + if (!ret) + msleep(50); + break; case USB_DR_MODE_PERIPHERAL: dwc2_force_mode(hsotg, false); diff --git a/drivers/usb/dwc2/core.h b/drivers/usb/dwc2/core.h index aad4107ef927..2a21a0414b1d 100644 --- a/drivers/usb/dwc2/core.h +++ b/drivers/usb/dwc2/core.h @@ -259,6 +259,13 @@ enum dwc2_lx_state { DWC2_L3, /* Off state */ }; +/* + * Gadget periodic tx fifo sizes as used by legacy driver + * EP0 is not included + */ +#define DWC2_G_P_LEGACY_TX_FIFO_SIZE {256, 256, 256, 256, 768, 768, 768, \ + 768, 0, 0, 0, 0, 0, 0, 0} + /* Gadget ep0 states */ enum dwc2_ep0_state { DWC2_EP0_SETUP, diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 4cd6403a7566..24fbebc9b409 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -186,10 +186,9 @@ static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg, */ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) { - unsigned int fifo; + unsigned int ep; unsigned int addr; int timeout; - u32 dptxfsizn; u32 val; /* Reset fifo map if not correctly cleared during previous session */ @@ -217,16 +216,16 @@ static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) * them to endpoints dynamically according to maxpacket size value of * given endpoint. */ - for (fifo = 1; fifo < MAX_EPS_CHANNELS; fifo++) { - dptxfsizn = dwc2_readl(hsotg->regs + DPTXFSIZN(fifo)); - - val = (dptxfsizn & FIFOSIZE_DEPTH_MASK) | addr; - addr += dptxfsizn >> FIFOSIZE_DEPTH_SHIFT; - - if (addr > hsotg->fifo_mem) - break; + for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) { + if (!hsotg->g_tx_fifo_sz[ep]) + continue; + val = addr; + val |= hsotg->g_tx_fifo_sz[ep] << FIFOSIZE_DEPTH_SHIFT; + WARN_ONCE(addr + hsotg->g_tx_fifo_sz[ep] > hsotg->fifo_mem, + "insufficient fifo memory"); + addr += hsotg->g_tx_fifo_sz[ep]; - dwc2_writel(val, hsotg->regs + DPTXFSIZN(fifo)); + dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep)); } /* @@ -3807,10 +3806,36 @@ static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg) static void dwc2_hsotg_of_probe(struct dwc2_hsotg *hsotg) { struct device_node *np = hsotg->dev->of_node; + u32 len = 0; + u32 i = 0; /* Enable dma if requested in device tree */ hsotg->g_using_dma = of_property_read_bool(np, "g-use-dma"); + /* + * Register TX periodic fifo size per endpoint. + * EP0 is excluded since it has no fifo configuration. + */ + if (!of_find_property(np, "g-tx-fifo-size", &len)) + goto rx_fifo; + + len /= sizeof(u32); + + /* Read tx fifo sizes other than ep0 */ + if (of_property_read_u32_array(np, "g-tx-fifo-size", + &hsotg->g_tx_fifo_sz[1], len)) + goto rx_fifo; + + /* Add ep0 */ + len++; + + /* Make remaining TX fifos unavailable */ + if (len < MAX_EPS_CHANNELS) { + for (i = len; i < MAX_EPS_CHANNELS; i++) + hsotg->g_tx_fifo_sz[i] = 0; + } + +rx_fifo: /* Register RX fifo size */ of_property_read_u32(np, "g-rx-fifo-size", &hsotg->g_rx_fifo_sz); @@ -3832,10 +3857,13 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) struct device *dev = hsotg->dev; int epnum; int ret; + int i; + u32 p_tx_fifo[] = DWC2_G_P_LEGACY_TX_FIFO_SIZE; /* Initialize to legacy fifo configuration values */ hsotg->g_rx_fifo_sz = 2048; hsotg->g_np_g_tx_fifo_sz = 1024; + memcpy(&hsotg->g_tx_fifo_sz[1], p_tx_fifo, sizeof(p_tx_fifo)); /* Device tree specific probe */ dwc2_hsotg_of_probe(hsotg); @@ -3853,6 +3881,9 @@ int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n", hsotg->g_np_g_tx_fifo_sz); dev_dbg(dev, "RXFIFO size: %d\n", hsotg->g_rx_fifo_sz); + for (i = 0; i < MAX_EPS_CHANNELS; i++) + dev_dbg(dev, "Periodic TXFIFO%2d size: %d\n", i, + hsotg->g_tx_fifo_sz[i]); hsotg->gadget.max_speed = USB_SPEED_HIGH; hsotg->gadget.ops = &dwc2_hsotg_gadget_ops; diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 07cc8929f271..1dfa56a5f1c5 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -783,6 +783,7 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, req->trb = trb; req->trb_dma = dwc3_trb_dma_offset(dep, trb); req->first_trb_index = dep->trb_enqueue; + dep->queued_requests++; } dwc3_ep_inc_enq(dep); @@ -833,8 +834,6 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, trb->ctrl |= DWC3_TRB_CTRL_HWO; - dep->queued_requests++; - trace_dwc3_prepare_trb(dep, trb); } @@ -1074,9 +1073,17 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) list_add_tail(&req->list, &dep->pending_list); - if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && - dep->flags & DWC3_EP_PENDING_REQUEST) { - if (list_empty(&dep->started_list)) { + /* + * NOTICE: Isochronous endpoints should NEVER be prestarted. We must + * wait for a XferNotReady event so we will know what's the current + * (micro-)frame number. + * + * Without this trick, we are very, very likely gonna get Bus Expiry + * errors which will force us issue EndTransfer command. + */ + if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { + if ((dep->flags & DWC3_EP_PENDING_REQUEST) && + list_empty(&dep->started_list)) { dwc3_stop_active_transfer(dwc, dep->number, true); dep->flags = DWC3_EP_ENABLED; } @@ -1861,8 +1868,11 @@ static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep, unsigned int s_pkt = 0; unsigned int trb_status; - dep->queued_requests--; dwc3_ep_inc_deq(dep); + + if (req->trb == trb) + dep->queued_requests--; + trace_dwc3_complete_trb(dep, trb); /* @@ -2980,7 +2990,7 @@ err3: kfree(dwc->setup_buf); err2: - dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb), + dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2, dwc->ep0_trb, dwc->ep0_trb_addr); err1: @@ -3005,7 +3015,7 @@ void dwc3_gadget_exit(struct dwc3 *dwc) kfree(dwc->setup_buf); kfree(dwc->zlp_buf); - dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb), + dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2, dwc->ep0_trb, dwc->ep0_trb_addr); dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req), diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 54ad100af35b..e40d47d47d82 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -136,8 +136,60 @@ struct ffs_epfile { /* * Buffer for holding data from partial reads which may happen since * we’re rounding user read requests to a multiple of a max packet size. + * + * The pointer is initialised with NULL value and may be set by + * __ffs_epfile_read_data function to point to a temporary buffer. + * + * In normal operation, calls to __ffs_epfile_read_buffered will consume + * data from said buffer and eventually free it. Importantly, while the + * function is using the buffer, it sets the pointer to NULL. This is + * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered + * can never run concurrently (they are synchronised by epfile->mutex) + * so the latter will not assign a new value to the pointer. + * + * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is + * valid) and sets the pointer to READ_BUFFER_DROP value. This special + * value is crux of the synchronisation between ffs_func_eps_disable and + * __ffs_epfile_read_data. + * + * Once __ffs_epfile_read_data is about to finish it will try to set the + * pointer back to its old value (as described above), but seeing as the + * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free + * the buffer. + * + * == State transitions == + * + * • ptr == NULL: (initial state) + * ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP + * ◦ __ffs_epfile_read_buffered: nop + * ◦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf + * ◦ reading finishes: n/a, not in ‘and reading’ state + * • ptr == DROP: + * ◦ __ffs_epfile_read_buffer_free: nop + * ◦ __ffs_epfile_read_buffered: go to ptr == NULL + * ◦ __ffs_epfile_read_data allocates temp buffer: free buf, nop + * ◦ reading finishes: n/a, not in ‘and reading’ state + * • ptr == buf: + * ◦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP + * ◦ __ffs_epfile_read_buffered: go to ptr == NULL and reading + * ◦ __ffs_epfile_read_data: n/a, __ffs_epfile_read_buffered + * is always called first + * ◦ reading finishes: n/a, not in ‘and reading’ state + * • ptr == NULL and reading: + * ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading + * ◦ __ffs_epfile_read_buffered: n/a, mutex is held + * ◦ __ffs_epfile_read_data: n/a, mutex is held + * ◦ reading finishes and … + * … all data read: free buf, go to ptr == NULL + * … otherwise: go to ptr == buf and reading + * • ptr == DROP and reading: + * ◦ __ffs_epfile_read_buffer_free: nop + * ◦ __ffs_epfile_read_buffered: n/a, mutex is held + * ◦ __ffs_epfile_read_data: n/a, mutex is held + * ◦ reading finishes: free buf, go to ptr == DROP */ - struct ffs_buffer *read_buffer; /* P: epfile->mutex */ + struct ffs_buffer *read_buffer; +#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN)) char name[5]; @@ -736,25 +788,47 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep, schedule_work(&io_data->work); } +static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile) +{ + /* + * See comment in struct ffs_epfile for full read_buffer pointer + * synchronisation story. + */ + struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP); + if (buf && buf != READ_BUFFER_DROP) + kfree(buf); +} + /* Assumes epfile->mutex is held. */ static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile, struct iov_iter *iter) { - struct ffs_buffer *buf = epfile->read_buffer; + /* + * Null out epfile->read_buffer so ffs_func_eps_disable does not free + * the buffer while we are using it. See comment in struct ffs_epfile + * for full read_buffer pointer synchronisation story. + */ + struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL); ssize_t ret; - if (!buf) + if (!buf || buf == READ_BUFFER_DROP) return 0; ret = copy_to_iter(buf->data, buf->length, iter); if (buf->length == ret) { kfree(buf); - epfile->read_buffer = NULL; - } else if (unlikely(iov_iter_count(iter))) { + return ret; + } + + if (unlikely(iov_iter_count(iter))) { ret = -EFAULT; } else { buf->length -= ret; buf->data += ret; } + + if (cmpxchg(&epfile->read_buffer, NULL, buf)) + kfree(buf); + return ret; } @@ -783,7 +857,15 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile, buf->length = data_len; buf->data = buf->storage; memcpy(buf->storage, data + ret, data_len); - epfile->read_buffer = buf; + + /* + * At this point read_buffer is NULL or READ_BUFFER_DROP (if + * ffs_func_eps_disable has been called in the meanwhile). See comment + * in struct ffs_epfile for full read_buffer pointer synchronisation + * story. + */ + if (unlikely(cmpxchg(&epfile->read_buffer, NULL, buf))) + kfree(buf); return ret; } @@ -1097,8 +1179,7 @@ ffs_epfile_release(struct inode *inode, struct file *file) ENTER(); - kfree(epfile->read_buffer); - epfile->read_buffer = NULL; + __ffs_epfile_read_buffer_free(epfile); ffs_data_closed(epfile->ffs); return 0; @@ -1724,24 +1805,20 @@ static void ffs_func_eps_disable(struct ffs_function *func) unsigned count = func->ffs->eps_count; unsigned long flags; + spin_lock_irqsave(&func->ffs->eps_lock, flags); do { - if (epfile) - mutex_lock(&epfile->mutex); - spin_lock_irqsave(&func->ffs->eps_lock, flags); /* pending requests get nuked */ if (likely(ep->ep)) usb_ep_disable(ep->ep); ++ep; - spin_unlock_irqrestore(&func->ffs->eps_lock, flags); if (epfile) { epfile->ep = NULL; - kfree(epfile->read_buffer); - epfile->read_buffer = NULL; - mutex_unlock(&epfile->mutex); + __ffs_epfile_read_buffer_free(epfile); ++epfile; } } while (--count); + spin_unlock_irqrestore(&func->ffs->eps_lock, flags); } static int ffs_func_eps_enable(struct ffs_function *func) diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c index 9c8c9ed1dc9e..fe1811650dbc 100644 --- a/drivers/usb/gadget/function/u_ether.c +++ b/drivers/usb/gadget/function/u_ether.c @@ -590,8 +590,9 @@ static netdev_tx_t eth_start_xmit(struct sk_buff *skb, /* throttle high/super speed IRQ rate back slightly */ if (gadget_is_dualspeed(dev->gadget)) - req->no_interrupt = (dev->gadget->speed == USB_SPEED_HIGH || - dev->gadget->speed == USB_SPEED_SUPER) + req->no_interrupt = (((dev->gadget->speed == USB_SPEED_HIGH || + dev->gadget->speed == USB_SPEED_SUPER)) && + !list_empty(&dev->tx_reqs)) ? ((atomic_read(&dev->tx_qlen) % dev->qmult) != 0) : 0; diff --git a/drivers/usb/gadget/udc/atmel_usba_udc.c b/drivers/usb/gadget/udc/atmel_usba_udc.c index bb1f6c8f0f01..45bc997d0711 100644 --- a/drivers/usb/gadget/udc/atmel_usba_udc.c +++ b/drivers/usb/gadget/udc/atmel_usba_udc.c @@ -1978,7 +1978,7 @@ static struct usba_ep * atmel_udc_of_init(struct platform_device *pdev, dev_err(&pdev->dev, "of_probe: name error(%d)\n", ret); goto err; } - ep->ep.name = name; + ep->ep.name = kasprintf(GFP_KERNEL, "ep%d", ep->index); ep->ep_regs = udc->regs + USBA_EPT_BASE(i); ep->dma_regs = udc->regs + USBA_DMA_BASE(i); diff --git a/drivers/usb/host/ehci-platform.c b/drivers/usb/host/ehci-platform.c index 876dca4fc216..a268d9e8d6cf 100644 --- a/drivers/usb/host/ehci-platform.c +++ b/drivers/usb/host/ehci-platform.c @@ -39,7 +39,7 @@ #define DRIVER_DESC "EHCI generic platform driver" #define EHCI_MAX_CLKS 4 -#define EHCI_MAX_RSTS 3 +#define EHCI_MAX_RSTS 4 #define hcd_to_ehci_priv(h) ((struct ehci_platform_priv *)hcd_to_ehci(h)->priv) struct ehci_platform_priv { diff --git a/drivers/usb/host/ohci-at91.c b/drivers/usb/host/ohci-at91.c index 5b5880c0ae19..b38a228134df 100644 --- a/drivers/usb/host/ohci-at91.c +++ b/drivers/usb/host/ohci-at91.c @@ -221,6 +221,12 @@ static int usb_hcd_at91_probe(const struct hc_driver *driver, ohci->num_ports = board->ports; at91_start_hc(pdev); + /* + * The RemoteWakeupConnected bit has to be set explicitly + * before calling ohci_run. The reset value of this bit is 0. + */ + ohci->hc_control = OHCI_CTRL_RWC; + retval = usb_add_hcd(hcd, irq, IRQF_SHARED); if (retval == 0) { device_wakeup_enable(hcd->self.controller); @@ -677,9 +683,6 @@ ohci_hcd_at91_drv_suspend(struct device *dev) * REVISIT: some boards will be able to turn VBUS off... */ if (!ohci_at91->wakeup) { - ohci->hc_control = ohci_readl(ohci, &ohci->regs->control); - ohci->hc_control &= OHCI_CTRL_RWC; - ohci_writel(ohci, ohci->hc_control, &ohci->regs->control); ohci->rh_state = OHCI_RH_HALTED; /* flush the writes */ diff --git a/drivers/usb/host/ohci-hcd.c b/drivers/usb/host/ohci-hcd.c index 1700908b84ef..86612ac3fda2 100644 --- a/drivers/usb/host/ohci-hcd.c +++ b/drivers/usb/host/ohci-hcd.c @@ -72,7 +72,7 @@ static const char hcd_name [] = "ohci_hcd"; #define STATECHANGE_DELAY msecs_to_jiffies(300) -#define IO_WATCHDOG_DELAY msecs_to_jiffies(250) +#define IO_WATCHDOG_DELAY msecs_to_jiffies(275) #include "ohci.h" #include "pci-quirks.h" diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index 730b9fd26685..0ef16900efed 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -1166,7 +1166,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, xhci_set_link_state(xhci, port_array, wIndex, XDEV_RESUME); spin_unlock_irqrestore(&xhci->lock, flags); - msleep(20); + msleep(USB_RESUME_TIMEOUT); spin_lock_irqsave(&xhci->lock, flags); xhci_set_link_state(xhci, port_array, wIndex, XDEV_U0); @@ -1355,6 +1355,35 @@ int xhci_bus_suspend(struct usb_hcd *hcd) return 0; } +/* + * Workaround for missing Cold Attach Status (CAS) if device re-plugged in S3. + * warm reset a USB3 device stuck in polling or compliance mode after resume. + * See Intel 100/c230 series PCH specification update Doc #332692-006 Errata #8 + */ +static bool xhci_port_missing_cas_quirk(int port_index, + __le32 __iomem **port_array) +{ + u32 portsc; + + portsc = readl(port_array[port_index]); + + /* if any of these are set we are not stuck */ + if (portsc & (PORT_CONNECT | PORT_CAS)) + return false; + + if (((portsc & PORT_PLS_MASK) != XDEV_POLLING) && + ((portsc & PORT_PLS_MASK) != XDEV_COMP_MODE)) + return false; + + /* clear wakeup/change bits, and do a warm port reset */ + portsc &= ~(PORT_RWC_BITS | PORT_CEC | PORT_WAKE_BITS); + portsc |= PORT_WR; + writel(portsc, port_array[port_index]); + /* flush write */ + readl(port_array[port_index]); + return true; +} + int xhci_bus_resume(struct usb_hcd *hcd) { struct xhci_hcd *xhci = hcd_to_xhci(hcd); @@ -1392,6 +1421,14 @@ int xhci_bus_resume(struct usb_hcd *hcd) u32 temp; temp = readl(port_array[port_index]); + + /* warm reset CAS limited ports stuck in polling/compliance */ + if ((xhci->quirks & XHCI_MISSING_CAS) && + (hcd->speed >= HCD_USB3) && + xhci_port_missing_cas_quirk(port_index, port_array)) { + xhci_dbg(xhci, "reset stuck port %d\n", port_index); + continue; + } if (DEV_SUPERSPEED_ANY(temp)) temp &= ~(PORT_RWC_BITS | PORT_CEC | PORT_WAKE_BITS); else @@ -1410,7 +1447,7 @@ int xhci_bus_resume(struct usb_hcd *hcd) if (need_usb2_u3_exit) { spin_unlock_irqrestore(&xhci->lock, flags); - msleep(20); + msleep(USB_RESUME_TIMEOUT); spin_lock_irqsave(&xhci->lock, flags); } diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index d7b0f97abbad..e96ae80d107e 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -45,11 +45,13 @@ #define PCI_DEVICE_ID_INTEL_LYNXPOINT_XHCI 0x8c31 #define PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI 0x9c31 +#define PCI_DEVICE_ID_INTEL_WILDCATPOINT_LP_XHCI 0x9cb1 #define PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI 0x22b5 #define PCI_DEVICE_ID_INTEL_SUNRISEPOINT_H_XHCI 0xa12f #define PCI_DEVICE_ID_INTEL_SUNRISEPOINT_LP_XHCI 0x9d2f #define PCI_DEVICE_ID_INTEL_BROXTON_M_XHCI 0x0aa8 #define PCI_DEVICE_ID_INTEL_BROXTON_B_XHCI 0x1aa8 +#define PCI_DEVICE_ID_INTEL_APL_XHCI 0x5aa8 static const char hcd_name[] = "xhci_hcd"; @@ -153,7 +155,8 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) xhci->quirks |= XHCI_SPURIOUS_REBOOT; } if (pdev->vendor == PCI_VENDOR_ID_INTEL && - pdev->device == PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI) { + (pdev->device == PCI_DEVICE_ID_INTEL_LYNXPOINT_LP_XHCI || + pdev->device == PCI_DEVICE_ID_INTEL_WILDCATPOINT_LP_XHCI)) { xhci->quirks |= XHCI_SPURIOUS_REBOOT; xhci->quirks |= XHCI_SPURIOUS_WAKEUP; } @@ -169,6 +172,11 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI) { xhci->quirks |= XHCI_SSIC_PORT_UNUSED; } + if (pdev->vendor == PCI_VENDOR_ID_INTEL && + (pdev->device == PCI_DEVICE_ID_INTEL_CHERRYVIEW_XHCI || + pdev->device == PCI_DEVICE_ID_INTEL_APL_XHCI)) + xhci->quirks |= XHCI_MISSING_CAS; + if (pdev->vendor == PCI_VENDOR_ID_ETRON && pdev->device == PCI_DEVICE_ID_EJ168) { xhci->quirks |= XHCI_RESET_ON_RESUME; diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index b2c1dc5dc0f3..f945380035d0 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -314,6 +314,8 @@ struct xhci_op_regs { #define XDEV_U2 (0x2 << 5) #define XDEV_U3 (0x3 << 5) #define XDEV_INACTIVE (0x6 << 5) +#define XDEV_POLLING (0x7 << 5) +#define XDEV_COMP_MODE (0xa << 5) #define XDEV_RESUME (0xf << 5) /* true: port has power (see HCC_PPC) */ #define PORT_POWER (1 << 9) @@ -1653,6 +1655,7 @@ struct xhci_hcd { #define XHCI_MTK_HOST (1 << 21) #define XHCI_SSIC_PORT_UNUSED (1 << 22) #define XHCI_NO_64BIT_SUPPORT (1 << 23) +#define XHCI_MISSING_CAS (1 << 24) unsigned int num_active_eps; unsigned int limit_active_eps; /* There are two roothubs to keep track of bus suspend info for */ diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c index bff4869a57cd..4042ea017985 100644 --- a/drivers/usb/musb/musb_gadget.c +++ b/drivers/usb/musb/musb_gadget.c @@ -1255,6 +1255,7 @@ static int musb_gadget_queue(struct usb_ep *ep, struct usb_request *req, map_dma_buffer(request, musb, musb_ep); + pm_runtime_get_sync(musb->controller); spin_lock_irqsave(&musb->lock, lockflags); /* don't queue if the ep is down */ @@ -1275,6 +1276,9 @@ static int musb_gadget_queue(struct usb_ep *ep, struct usb_request *req, unlock: spin_unlock_irqrestore(&musb->lock, lockflags); + pm_runtime_mark_last_busy(musb->controller); + pm_runtime_put_autosuspend(musb->controller); + return status; } diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c index 1ab6973d4f61..cc1225485509 100644 --- a/drivers/usb/musb/omap2430.c +++ b/drivers/usb/musb/omap2430.c @@ -287,6 +287,7 @@ static int omap2430_musb_init(struct musb *musb) } musb->isr = omap2430_musb_interrupt; phy_init(musb->phy); + phy_power_on(musb->phy); l = musb_readl(musb->mregs, OTG_INTERFSEL); @@ -323,8 +324,6 @@ static void omap2430_musb_enable(struct musb *musb) struct musb_hdrc_platform_data *pdata = dev_get_platdata(dev); struct omap_musb_board_data *data = pdata->board_data; - if (!WARN_ON(!musb->phy)) - phy_power_on(musb->phy); switch (glue->status) { @@ -361,9 +360,6 @@ static void omap2430_musb_disable(struct musb *musb) struct device *dev = musb->controller; struct omap2430_glue *glue = dev_get_drvdata(dev->parent); - if (!WARN_ON(!musb->phy)) - phy_power_off(musb->phy); - if (glue->status != MUSB_UNKNOWN) omap_control_usb_set_mode(glue->control_otghs, USB_MODE_DISCONNECT); @@ -375,6 +371,7 @@ static int omap2430_musb_exit(struct musb *musb) struct omap2430_glue *glue = dev_get_drvdata(dev->parent); omap2430_low_level_exit(musb); + phy_power_off(musb->phy); phy_exit(musb->phy); musb->phy = NULL; cancel_work_sync(&glue->omap_musb_mailbox_work); diff --git a/drivers/usb/renesas_usbhs/rcar3.c b/drivers/usb/renesas_usbhs/rcar3.c index 1d70add926f0..d544b331c9f2 100644 --- a/drivers/usb/renesas_usbhs/rcar3.c +++ b/drivers/usb/renesas_usbhs/rcar3.c @@ -9,6 +9,7 @@ * */ +#include <linux/delay.h> #include <linux/io.h> #include "common.h" #include "rcar3.h" @@ -35,10 +36,13 @@ static int usbhs_rcar3_power_ctrl(struct platform_device *pdev, usbhs_write32(priv, UGCTRL2, UGCTRL2_RESERVED_3 | UGCTRL2_USB0SEL_OTG); - if (enable) + if (enable) { usbhs_bset(priv, LPSTS, LPSTS_SUSPM, LPSTS_SUSPM); - else + /* The controller on R-Car Gen3 needs to wait up to 45 usec */ + udelay(45); + } else { usbhs_bset(priv, LPSTS, LPSTS_SUSPM, 0); + } return 0; } diff --git a/drivers/usb/serial/cp210x.c b/drivers/usb/serial/cp210x.c index 54a4de0efdba..f61477bed3a8 100644 --- a/drivers/usb/serial/cp210x.c +++ b/drivers/usb/serial/cp210x.c @@ -1077,7 +1077,9 @@ static int cp210x_tiocmget(struct tty_struct *tty) u8 control; int result; - cp210x_read_u8_reg(port, CP210X_GET_MDMSTS, &control); + result = cp210x_read_u8_reg(port, CP210X_GET_MDMSTS, &control); + if (result) + return result; result = ((control & CONTROL_DTR) ? TIOCM_DTR : 0) |((control & CONTROL_RTS) ? TIOCM_RTS : 0) diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index b2d767e743fc..0ff7f38d7800 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -986,7 +986,8 @@ static const struct usb_device_id id_table_combined[] = { /* ekey Devices */ { USB_DEVICE(FTDI_VID, FTDI_EKEY_CONV_USB_PID) }, /* Infineon Devices */ - { USB_DEVICE_INTERFACE_NUMBER(INFINEON_VID, INFINEON_TRIBOARD_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(INFINEON_VID, INFINEON_TRIBOARD_TC1798_PID, 1) }, + { USB_DEVICE_INTERFACE_NUMBER(INFINEON_VID, INFINEON_TRIBOARD_TC2X7_PID, 1) }, /* GE Healthcare devices */ { USB_DEVICE(GE_HEALTHCARE_VID, GE_HEALTHCARE_NEMO_TRACKER_PID) }, /* Active Research (Actisense) devices */ diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index f87a938cf005..21011c0a4c64 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -626,8 +626,9 @@ /* * Infineon Technologies */ -#define INFINEON_VID 0x058b -#define INFINEON_TRIBOARD_PID 0x0028 /* DAS JTAG TriBoard TC1798 V1.0 */ +#define INFINEON_VID 0x058b +#define INFINEON_TRIBOARD_TC1798_PID 0x0028 /* DAS JTAG TriBoard TC1798 V1.0 */ +#define INFINEON_TRIBOARD_TC2X7_PID 0x0043 /* DAS JTAG TriBoard TC2X7 V1.0 */ /* * Acton Research Corp. diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c index d213cf44a7e4..4a037b4a79cf 100644 --- a/drivers/usb/serial/usb-serial.c +++ b/drivers/usb/serial/usb-serial.c @@ -1078,7 +1078,8 @@ static int usb_serial_probe(struct usb_interface *interface, serial->disconnected = 0; - usb_serial_console_init(serial->port[0]->minor); + if (num_ports > 0) + usb_serial_console_init(serial->port[0]->minor); exit: module_put(type->driver.owner); return 0; diff --git a/drivers/usb/wusbcore/crypto.c b/drivers/usb/wusbcore/crypto.c index 79b2b628066d..79451f7ef1b7 100644 --- a/drivers/usb/wusbcore/crypto.c +++ b/drivers/usb/wusbcore/crypto.c @@ -133,6 +133,13 @@ static void bytewise_xor(void *_bo, const void *_bi1, const void *_bi2, bo[itr] = bi1[itr] ^ bi2[itr]; } +/* Scratch space for MAC calculations. */ +struct wusb_mac_scratch { + struct aes_ccm_b0 b0; + struct aes_ccm_b1 b1; + struct aes_ccm_a ax; +}; + /* * CC-MAC function WUSB1.0[6.5] * @@ -197,16 +204,15 @@ static void bytewise_xor(void *_bo, const void *_bi1, const void *_bi2, * what sg[4] is for. Maybe there is a smarter way to do this. */ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, - struct crypto_cipher *tfm_aes, void *mic, + struct crypto_cipher *tfm_aes, + struct wusb_mac_scratch *scratch, + void *mic, const struct aes_ccm_nonce *n, const struct aes_ccm_label *a, const void *b, size_t blen) { int result = 0; SKCIPHER_REQUEST_ON_STACK(req, tfm_cbc); - struct aes_ccm_b0 b0; - struct aes_ccm_b1 b1; - struct aes_ccm_a ax; struct scatterlist sg[4], sg_dst; void *dst_buf; size_t dst_size; @@ -218,16 +224,17 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, * These checks should be compile time optimized out * ensure @a fills b1's mac_header and following fields */ - WARN_ON(sizeof(*a) != sizeof(b1) - sizeof(b1.la)); - WARN_ON(sizeof(b0) != sizeof(struct aes_ccm_block)); - WARN_ON(sizeof(b1) != sizeof(struct aes_ccm_block)); - WARN_ON(sizeof(ax) != sizeof(struct aes_ccm_block)); + WARN_ON(sizeof(*a) != sizeof(scratch->b1) - sizeof(scratch->b1.la)); + WARN_ON(sizeof(scratch->b0) != sizeof(struct aes_ccm_block)); + WARN_ON(sizeof(scratch->b1) != sizeof(struct aes_ccm_block)); + WARN_ON(sizeof(scratch->ax) != sizeof(struct aes_ccm_block)); result = -ENOMEM; zero_padding = blen % sizeof(struct aes_ccm_block); if (zero_padding) zero_padding = sizeof(struct aes_ccm_block) - zero_padding; - dst_size = blen + sizeof(b0) + sizeof(b1) + zero_padding; + dst_size = blen + sizeof(scratch->b0) + sizeof(scratch->b1) + + zero_padding; dst_buf = kzalloc(dst_size, GFP_KERNEL); if (!dst_buf) goto error_dst_buf; @@ -235,9 +242,9 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, memset(iv, 0, sizeof(iv)); /* Setup B0 */ - b0.flags = 0x59; /* Format B0 */ - b0.ccm_nonce = *n; - b0.lm = cpu_to_be16(0); /* WUSB1.0[6.5] sez l(m) is 0 */ + scratch->b0.flags = 0x59; /* Format B0 */ + scratch->b0.ccm_nonce = *n; + scratch->b0.lm = cpu_to_be16(0); /* WUSB1.0[6.5] sez l(m) is 0 */ /* Setup B1 * @@ -246,12 +253,12 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, * 14'--after clarification, it means to use A's contents * for MAC Header, EO, sec reserved and padding. */ - b1.la = cpu_to_be16(blen + 14); - memcpy(&b1.mac_header, a, sizeof(*a)); + scratch->b1.la = cpu_to_be16(blen + 14); + memcpy(&scratch->b1.mac_header, a, sizeof(*a)); sg_init_table(sg, ARRAY_SIZE(sg)); - sg_set_buf(&sg[0], &b0, sizeof(b0)); - sg_set_buf(&sg[1], &b1, sizeof(b1)); + sg_set_buf(&sg[0], &scratch->b0, sizeof(scratch->b0)); + sg_set_buf(&sg[1], &scratch->b1, sizeof(scratch->b1)); sg_set_buf(&sg[2], b, blen); /* 0 if well behaved :) */ sg_set_buf(&sg[3], bzero, zero_padding); @@ -276,11 +283,12 @@ static int wusb_ccm_mac(struct crypto_skcipher *tfm_cbc, * POS Crypto API: size is assumed to be AES's block size. * Thanks for documenting it -- tip taken from airo.c */ - ax.flags = 0x01; /* as per WUSB 1.0 spec */ - ax.ccm_nonce = *n; - ax.counter = 0; - crypto_cipher_encrypt_one(tfm_aes, (void *)&ax, (void *)&ax); - bytewise_xor(mic, &ax, iv, 8); + scratch->ax.flags = 0x01; /* as per WUSB 1.0 spec */ + scratch->ax.ccm_nonce = *n; + scratch->ax.counter = 0; + crypto_cipher_encrypt_one(tfm_aes, (void *)&scratch->ax, + (void *)&scratch->ax); + bytewise_xor(mic, &scratch->ax, iv, 8); result = 8; error_cbc_crypt: kfree(dst_buf); @@ -303,6 +311,7 @@ ssize_t wusb_prf(void *out, size_t out_size, struct aes_ccm_nonce n = *_n; struct crypto_skcipher *tfm_cbc; struct crypto_cipher *tfm_aes; + struct wusb_mac_scratch *scratch; u64 sfn = 0; __le64 sfn_le; @@ -329,17 +338,25 @@ ssize_t wusb_prf(void *out, size_t out_size, printk(KERN_ERR "E: can't set AES key: %d\n", (int)result); goto error_setkey_aes; } + scratch = kmalloc(sizeof(*scratch), GFP_KERNEL); + if (!scratch) { + result = -ENOMEM; + goto error_alloc_scratch; + } for (bitr = 0; bitr < (len + 63) / 64; bitr++) { sfn_le = cpu_to_le64(sfn++); memcpy(&n.sfn, &sfn_le, sizeof(n.sfn)); /* n.sfn++... */ - result = wusb_ccm_mac(tfm_cbc, tfm_aes, out + bytes, + result = wusb_ccm_mac(tfm_cbc, tfm_aes, scratch, out + bytes, &n, a, b, blen); if (result < 0) goto error_ccm_mac; bytes += result; } result = bytes; + + kfree(scratch); +error_alloc_scratch: error_ccm_mac: error_setkey_aes: crypto_free_cipher(tfm_aes); diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index d624a527777f..031bc08d000d 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -829,8 +829,9 @@ static long vfio_pci_ioctl(void *device_data, } else if (cmd == VFIO_DEVICE_SET_IRQS) { struct vfio_irq_set hdr; + size_t size; u8 *data = NULL; - int ret = 0; + int max, ret = 0; minsz = offsetofend(struct vfio_irq_set, count); @@ -838,23 +839,31 @@ static long vfio_pci_ioctl(void *device_data, return -EFAULT; if (hdr.argsz < minsz || hdr.index >= VFIO_PCI_NUM_IRQS || + hdr.count >= (U32_MAX - hdr.start) || hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK | VFIO_IRQ_SET_ACTION_TYPE_MASK)) return -EINVAL; - if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) { - size_t size; - int max = vfio_pci_get_irq_count(vdev, hdr.index); + max = vfio_pci_get_irq_count(vdev, hdr.index); + if (hdr.start >= max || hdr.start + hdr.count > max) + return -EINVAL; - if (hdr.flags & VFIO_IRQ_SET_DATA_BOOL) - size = sizeof(uint8_t); - else if (hdr.flags & VFIO_IRQ_SET_DATA_EVENTFD) - size = sizeof(int32_t); - else - return -EINVAL; + switch (hdr.flags & VFIO_IRQ_SET_DATA_TYPE_MASK) { + case VFIO_IRQ_SET_DATA_NONE: + size = 0; + break; + case VFIO_IRQ_SET_DATA_BOOL: + size = sizeof(uint8_t); + break; + case VFIO_IRQ_SET_DATA_EVENTFD: + size = sizeof(int32_t); + break; + default: + return -EINVAL; + } - if (hdr.argsz - minsz < hdr.count * size || - hdr.start >= max || hdr.start + hdr.count > max) + if (size) { + if (hdr.argsz - minsz < hdr.count * size) return -EINVAL; data = memdup_user((void __user *)(arg + minsz), diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c index c2e60893cd09..1c46045b0e7f 100644 --- a/drivers/vfio/pci/vfio_pci_intrs.c +++ b/drivers/vfio/pci/vfio_pci_intrs.c @@ -256,7 +256,7 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix) if (!is_irq_none(vdev)) return -EINVAL; - vdev->ctx = kzalloc(nvec * sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); + vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); if (!vdev->ctx) return -ENOMEM; diff --git a/drivers/video/of_display_timing.c b/drivers/video/of_display_timing.c index 8a1076beecd3..db992c684f09 100644 --- a/drivers/video/of_display_timing.c +++ b/drivers/video/of_display_timing.c @@ -88,6 +88,15 @@ static int of_parse_display_timing(const struct device_node *np, dt->flags |= val ? DISPLAY_FLAGS_PIXDATA_POSEDGE : DISPLAY_FLAGS_PIXDATA_NEGEDGE; + if (!of_property_read_u32(np, "syncclk-active", &val)) + dt->flags |= val ? DISPLAY_FLAGS_SYNC_POSEDGE : + DISPLAY_FLAGS_SYNC_NEGEDGE; + else if (dt->flags & (DISPLAY_FLAGS_PIXDATA_POSEDGE | + DISPLAY_FLAGS_PIXDATA_NEGEDGE)) + dt->flags |= dt->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE ? + DISPLAY_FLAGS_SYNC_POSEDGE : + DISPLAY_FLAGS_SYNC_NEGEDGE; + if (of_property_read_bool(np, "interlaced")) dt->flags |= DISPLAY_FLAGS_INTERLACED; if (of_property_read_bool(np, "doublescan")) diff --git a/drivers/virtio/config.c b/drivers/virtio/config.c deleted file mode 100644 index f70bcd2ff98f..000000000000 --- a/drivers/virtio/config.c +++ /dev/null @@ -1,12 +0,0 @@ -/* Configuration space parsing helpers for virtio. - * - * The configuration is [type][len][... len bytes ...] fields. - * - * Copyright 2007 Rusty Russell, IBM Corporation. - * GPL v2 or later. - */ -#include <linux/err.h> -#include <linux/virtio.h> -#include <linux/virtio_config.h> -#include <linux/bug.h> - diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 4e7003db12c4..181793f07852 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -577,6 +577,8 @@ static int virtballoon_probe(struct virtio_device *vdev) virtio_device_ready(vdev); + if (towards_target(vb)) + virtballoon_changed(vdev); return 0; out_del_vqs: diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 8c4e61783441..6d9e5173d5fa 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -212,10 +212,18 @@ int virtio_pci_legacy_probe(struct virtio_pci_device *vp_dev) return -ENODEV; } - rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); - if (rc) - rc = dma_set_mask_and_coherent(&pci_dev->dev, - DMA_BIT_MASK(32)); + rc = dma_set_mask(&pci_dev->dev, DMA_BIT_MASK(64)); + if (rc) { + rc = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32)); + } else { + /* + * The virtio ring base address is expressed as a 32-bit PFN, + * with a page size of 1 << VIRTIO_PCI_QUEUE_ADDR_SHIFT. + */ + dma_set_coherent_mask(&pci_dev->dev, + DMA_BIT_MASK(32 + VIRTIO_PCI_QUEUE_ADDR_SHIFT)); + } + if (rc) dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n"); diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index ed9c9eeedfe5..489bfc61cf30 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -167,7 +167,7 @@ static bool vring_use_dma_api(struct virtio_device *vdev) * making all of the arch DMA ops work on the vring device itself * is a mess. For now, we use the parent device for DMA ops. */ -static struct device *vring_dma_dev(const struct vring_virtqueue *vq) +static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq) { return vq->vq.vdev->dev.parent; } @@ -732,7 +732,8 @@ void virtqueue_disable_cb(struct virtqueue *_vq) if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); } } @@ -764,7 +765,8 @@ unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq) * entry. Always do both to keep code simple. */ if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); } vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx); END_USE(vq); @@ -832,10 +834,11 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) * more to do. */ /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to * either clear the flags bit or point the event index at the next - * entry. Always do both to keep code simple. */ + * entry. Always update the event index to keep code simple. */ if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; - vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow); } /* TODO: tune this threshold */ bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4; @@ -953,7 +956,8 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, /* No callback? Tell other side not to bother us. */ if (!callback) { vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; - vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow); + if (!vq->event) + vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow); } /* Put everything in free lists. */ diff --git a/drivers/vme/vme.c b/drivers/vme/vme.c index 15b64076bc26..bdbadaa47ef3 100644 --- a/drivers/vme/vme.c +++ b/drivers/vme/vme.c @@ -156,12 +156,16 @@ size_t vme_get_size(struct vme_resource *resource) case VME_MASTER: retval = vme_master_get(resource, &enabled, &base, &size, &aspace, &cycle, &dwidth); + if (retval) + return 0; return size; break; case VME_SLAVE: retval = vme_slave_get(resource, &enabled, &base, &size, &buf_base, &aspace, &cycle); + if (retval) + return 0; return size; break; diff --git a/drivers/xen/manage.c b/drivers/xen/manage.c index e12bd3635f83..26e5e8507f03 100644 --- a/drivers/xen/manage.c +++ b/drivers/xen/manage.c @@ -168,7 +168,9 @@ out: #endif /* CONFIG_HIBERNATE_CALLBACKS */ struct shutdown_handler { - const char *command; +#define SHUTDOWN_CMD_SIZE 11 + const char command[SHUTDOWN_CMD_SIZE]; + bool flag; void (*cb)(void); }; @@ -206,22 +208,22 @@ static void do_reboot(void) ctrl_alt_del(); } +static struct shutdown_handler shutdown_handlers[] = { + { "poweroff", true, do_poweroff }, + { "halt", false, do_poweroff }, + { "reboot", true, do_reboot }, +#ifdef CONFIG_HIBERNATE_CALLBACKS + { "suspend", true, do_suspend }, +#endif +}; + static void shutdown_handler(struct xenbus_watch *watch, const char **vec, unsigned int len) { char *str; struct xenbus_transaction xbt; int err; - static struct shutdown_handler handlers[] = { - { "poweroff", do_poweroff }, - { "halt", do_poweroff }, - { "reboot", do_reboot }, -#ifdef CONFIG_HIBERNATE_CALLBACKS - { "suspend", do_suspend }, -#endif - {NULL, NULL}, - }; - static struct shutdown_handler *handler; + int idx; if (shutting_down != SHUTDOWN_INVALID) return; @@ -238,13 +240,13 @@ static void shutdown_handler(struct xenbus_watch *watch, return; } - for (handler = &handlers[0]; handler->command; handler++) { - if (strcmp(str, handler->command) == 0) + for (idx = 0; idx < ARRAY_SIZE(shutdown_handlers); idx++) { + if (strcmp(str, shutdown_handlers[idx].command) == 0) break; } /* Only acknowledge commands which we are prepared to handle. */ - if (handler->cb) + if (idx < ARRAY_SIZE(shutdown_handlers)) xenbus_write(xbt, "control", "shutdown", ""); err = xenbus_transaction_end(xbt, 0); @@ -253,8 +255,8 @@ static void shutdown_handler(struct xenbus_watch *watch, goto again; } - if (handler->cb) { - handler->cb(); + if (idx < ARRAY_SIZE(shutdown_handlers)) { + shutdown_handlers[idx].cb(); } else { pr_info("Ignoring shutdown request: %s\n", str); shutting_down = SHUTDOWN_INVALID; @@ -310,6 +312,9 @@ static struct notifier_block xen_reboot_nb = { static int setup_shutdown_watcher(void) { int err; + int idx; +#define FEATURE_PATH_SIZE (SHUTDOWN_CMD_SIZE + sizeof("feature-")) + char node[FEATURE_PATH_SIZE]; err = register_xenbus_watch(&shutdown_watch); if (err) { @@ -326,6 +331,14 @@ static int setup_shutdown_watcher(void) } #endif + for (idx = 0; idx < ARRAY_SIZE(shutdown_handlers); idx++) { + if (!shutdown_handlers[idx].flag) + continue; + snprintf(node, FEATURE_PATH_SIZE, "feature-%s", + shutdown_handlers[idx].command); + xenbus_printf(XBT_NIL, "control", node, "%u", 1); + } + return 0; } diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c index c1010f018bd8..1e8be12ebb55 100644 --- a/drivers/xen/xenbus/xenbus_dev_frontend.c +++ b/drivers/xen/xenbus/xenbus_dev_frontend.c @@ -364,7 +364,7 @@ out: static int xenbus_write_watch(unsigned msg_type, struct xenbus_file_priv *u) { - struct watch_adapter *watch, *tmp_watch; + struct watch_adapter *watch; char *path, *token; int err, rc; LIST_HEAD(staging_q); @@ -399,7 +399,7 @@ static int xenbus_write_watch(unsigned msg_type, struct xenbus_file_priv *u) } list_add(&watch->list, &u->watches); } else { - list_for_each_entry_safe(watch, tmp_watch, &u->watches, list) { + list_for_each_entry(watch, &u->watches, list) { if (!strcmp(watch->token, token) && !strcmp(watch->watch.node, path)) { unregister_xenbus_watch(&watch->watch); diff --git a/drivers/xen/xenbus/xenbus_probe_frontend.c b/drivers/xen/xenbus/xenbus_probe_frontend.c index 611a23119675..6d40a972ffb2 100644 --- a/drivers/xen/xenbus/xenbus_probe_frontend.c +++ b/drivers/xen/xenbus/xenbus_probe_frontend.c @@ -335,7 +335,9 @@ static int backend_state; static void xenbus_reset_backend_state_changed(struct xenbus_watch *w, const char **v, unsigned int l) { - xenbus_scanf(XBT_NIL, v[XS_WATCH_PATH], "", "%i", &backend_state); + if (xenbus_scanf(XBT_NIL, v[XS_WATCH_PATH], "", "%i", + &backend_state) != 1) + backend_state = XenbusStateUnknown; printk(KERN_DEBUG "XENBUS: backend %s %s\n", v[XS_WATCH_PATH], xenbus_strstate(backend_state)); wake_up(&backend_state_wq); diff --git a/fs/afs/cmservice.c b/fs/afs/cmservice.c index 2037e7a77a37..d764236072b1 100644 --- a/fs/afs/cmservice.c +++ b/fs/afs/cmservice.c @@ -91,11 +91,9 @@ static const struct afs_call_type afs_SRXCBTellMeAboutYourself = { */ bool afs_cm_incoming_call(struct afs_call *call) { - u32 operation_id = ntohl(call->operation_ID); + _enter("{CB.OP %u}", call->operation_ID); - _enter("{CB.OP %u}", operation_id); - - switch (operation_id) { + switch (call->operation_ID) { case CBCallBack: call->type = &afs_SRXCBCallBack; return true; diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c index 96f4d764d1a6..31c616ab9b40 100644 --- a/fs/afs/fsclient.c +++ b/fs/afs/fsclient.c @@ -364,7 +364,7 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) buffer = kmap(page); ret = afs_extract_data(call, buffer, call->count, true); - kunmap(buffer); + kunmap(page); if (ret < 0) return ret; } @@ -397,7 +397,7 @@ static int afs_deliver_fs_fetch_data(struct afs_call *call) page = call->reply3; buffer = kmap(page); memset(buffer + call->count, 0, PAGE_SIZE - call->count); - kunmap(buffer); + kunmap(page); } _leave(" = 0 [done]"); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index 5497c8496055..535a38d2c1d0 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -112,7 +112,7 @@ struct afs_call { bool need_attention; /* T if RxRPC poked us */ u16 service_id; /* RxRPC service ID to call */ __be16 port; /* target UDP port */ - __be32 operation_ID; /* operation ID for an incoming call */ + u32 operation_ID; /* operation ID for an incoming call */ u32 count; /* count for use in unmarshalling */ __be32 tmp; /* place to extract temporary data */ afs_dataversion_t store_version; /* updated version expected from store */ diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c index 477928b25940..25f05a8d21b1 100644 --- a/fs/afs/rxrpc.c +++ b/fs/afs/rxrpc.c @@ -676,10 +676,11 @@ static int afs_deliver_cm_op_id(struct afs_call *call) ASSERTCMP(call->offset, <, 4); /* the operation ID forms the first four bytes of the request data */ - ret = afs_extract_data(call, &call->operation_ID, 4, true); + ret = afs_extract_data(call, &call->tmp, 4, true); if (ret < 0) return ret; + call->operation_ID = ntohl(call->tmp); call->state = AFS_CALL_AWAIT_REQUEST; call->offset = 0; diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 210c94ac8818..4607af38c72e 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2647,7 +2647,10 @@ static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, btrfs_free_delayed_extent_op(extent_op); if (ret) { + spin_lock(&delayed_refs->lock); locked_ref->processing = 0; + delayed_refs->num_heads_ready++; + spin_unlock(&delayed_refs->lock); btrfs_delayed_ref_unlock(locked_ref); btrfs_put_delayed_ref(ref); btrfs_debug(fs_info, "run_one_delayed_ref returned %d", diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 66a755150056..8ed05d95584a 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5569,7 +5569,7 @@ void le_bitmap_set(u8 *map, unsigned int start, int len) *p |= mask_to_set; len -= bits_to_set; bits_to_set = BITS_PER_BYTE; - mask_to_set = ~(u8)0; + mask_to_set = ~0; p++; } if (len) { @@ -5589,7 +5589,7 @@ void le_bitmap_clear(u8 *map, unsigned int start, int len) *p &= ~mask_to_clear; len -= bits_to_clear; bits_to_clear = BITS_PER_BYTE; - mask_to_clear = ~(u8)0; + mask_to_clear = ~0; p++; } if (len) { @@ -5679,7 +5679,7 @@ void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start, kaddr[offset] |= mask_to_set; len -= bits_to_set; bits_to_set = BITS_PER_BYTE; - mask_to_set = ~(u8)0; + mask_to_set = ~0; if (++offset >= PAGE_SIZE && len > 0) { offset = 0; page = eb->pages[++i]; @@ -5721,7 +5721,7 @@ void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start, kaddr[offset] &= ~mask_to_clear; len -= bits_to_clear; bits_to_clear = BITS_PER_BYTE; - mask_to_clear = ~(u8)0; + mask_to_clear = ~0; if (++offset >= PAGE_SIZE && len > 0) { offset = 0; page = eb->pages[++i]; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 2b790bda7998..8e3a5a266917 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4605,8 +4605,8 @@ delete: BUG_ON(ret); if (btrfs_should_throttle_delayed_refs(trans, root)) btrfs_async_run_delayed_refs(root, - trans->transid, - trans->delayed_ref_updates * 2, 0); + trans->delayed_ref_updates * 2, + trans->transid, 0); if (be_nice) { if (truncate_space_check(trans, root, extent_num_bytes)) { @@ -8931,9 +8931,14 @@ again: * So even we call qgroup_free_data(), it won't decrease reserved * space. * 2) Not written to disk - * This means the reserved space should be freed here. + * This means the reserved space should be freed here. However, + * if a truncate invalidates the page (by clearing PageDirty) + * and the page is accounted for while allocating extent + * in btrfs_check_data_free_space() we let delayed_ref to + * free the entire extent. */ - btrfs_qgroup_free_data(inode, page_start, PAGE_SIZE); + if (PageDirty(page)) + btrfs_qgroup_free_data(inode, page_start, PAGE_SIZE); if (!inode_evicting) { clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED | EXTENT_DIRTY | diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 18e1aa0f85f5..7acbd2cf6192 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -3814,6 +3814,11 @@ process_slot: } btrfs_release_path(path); key.offset = next_key_min_offset; + + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } } ret = 0; diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 0ec8ffa37ab0..c4af0cdb783d 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2728,7 +2728,14 @@ static int do_relocation(struct btrfs_trans_handle *trans, bytenr = btrfs_node_blockptr(upper->eb, slot); if (lowest) { - BUG_ON(bytenr != node->bytenr); + if (bytenr != node->bytenr) { + btrfs_err(root->fs_info, + "lowest leaf/node mismatch: bytenr %llu node->bytenr %llu slot %d upper %llu", + bytenr, node->bytenr, slot, + upper->eb->start); + err = -EIO; + goto next; + } } else { if (node->eb->start == bytenr) goto next; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 01bc36cec26e..71261b459863 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -5805,6 +5805,64 @@ static int changed_extent(struct send_ctx *sctx, int ret = 0; if (sctx->cur_ino != sctx->cmp_key->objectid) { + + if (result == BTRFS_COMPARE_TREE_CHANGED) { + struct extent_buffer *leaf_l; + struct extent_buffer *leaf_r; + struct btrfs_file_extent_item *ei_l; + struct btrfs_file_extent_item *ei_r; + + leaf_l = sctx->left_path->nodes[0]; + leaf_r = sctx->right_path->nodes[0]; + ei_l = btrfs_item_ptr(leaf_l, + sctx->left_path->slots[0], + struct btrfs_file_extent_item); + ei_r = btrfs_item_ptr(leaf_r, + sctx->right_path->slots[0], + struct btrfs_file_extent_item); + + /* + * We may have found an extent item that has changed + * only its disk_bytenr field and the corresponding + * inode item was not updated. This case happens due to + * very specific timings during relocation when a leaf + * that contains file extent items is COWed while + * relocation is ongoing and its in the stage where it + * updates data pointers. So when this happens we can + * safely ignore it since we know it's the same extent, + * but just at different logical and physical locations + * (when an extent is fully replaced with a new one, we + * know the generation number must have changed too, + * since snapshot creation implies committing the current + * transaction, and the inode item must have been updated + * as well). + * This replacement of the disk_bytenr happens at + * relocation.c:replace_file_extents() through + * relocation.c:btrfs_reloc_cow_block(). + */ + if (btrfs_file_extent_generation(leaf_l, ei_l) == + btrfs_file_extent_generation(leaf_r, ei_r) && + btrfs_file_extent_ram_bytes(leaf_l, ei_l) == + btrfs_file_extent_ram_bytes(leaf_r, ei_r) && + btrfs_file_extent_compression(leaf_l, ei_l) == + btrfs_file_extent_compression(leaf_r, ei_r) && + btrfs_file_extent_encryption(leaf_l, ei_l) == + btrfs_file_extent_encryption(leaf_r, ei_r) && + btrfs_file_extent_other_encoding(leaf_l, ei_l) == + btrfs_file_extent_other_encoding(leaf_r, ei_r) && + btrfs_file_extent_type(leaf_l, ei_l) == + btrfs_file_extent_type(leaf_r, ei_r) && + btrfs_file_extent_disk_bytenr(leaf_l, ei_l) != + btrfs_file_extent_disk_bytenr(leaf_r, ei_r) && + btrfs_file_extent_disk_num_bytes(leaf_l, ei_l) == + btrfs_file_extent_disk_num_bytes(leaf_r, ei_r) && + btrfs_file_extent_offset(leaf_l, ei_l) == + btrfs_file_extent_offset(leaf_r, ei_r) && + btrfs_file_extent_num_bytes(leaf_l, ei_l) == + btrfs_file_extent_num_bytes(leaf_r, ei_r)) + return 0; + } + inconsistent_snapshot_error(sctx, result, "extent"); return -EIO; } diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 528cae123dc9..3d33c4e41e5f 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -2713,14 +2713,12 @@ static inline void btrfs_remove_all_log_ctxs(struct btrfs_root *root, int index, int error) { struct btrfs_log_ctx *ctx; + struct btrfs_log_ctx *safe; - if (!error) { - INIT_LIST_HEAD(&root->log_ctxs[index]); - return; - } - - list_for_each_entry(ctx, &root->log_ctxs[index], list) + list_for_each_entry_safe(ctx, safe, &root->log_ctxs[index], list) { + list_del_init(&ctx->list); ctx->log_ret = error; + } INIT_LIST_HEAD(&root->log_ctxs[index]); } @@ -2961,13 +2959,9 @@ int btrfs_sync_log(struct btrfs_trans_handle *trans, mutex_unlock(&root->log_mutex); out_wake_log_root: - /* - * We needn't get log_mutex here because we are sure all - * the other tasks are blocked. - */ + mutex_lock(&log_root_tree->log_mutex); btrfs_remove_all_log_ctxs(log_root_tree, index2, ret); - mutex_lock(&log_root_tree->log_mutex); log_root_tree->log_transid_committed++; atomic_set(&log_root_tree->log_commit[index2], 0); mutex_unlock(&log_root_tree->log_mutex); @@ -2978,10 +2972,8 @@ out_wake_log_root: if (waitqueue_active(&log_root_tree->log_commit_wait[index2])) wake_up(&log_root_tree->log_commit_wait[index2]); out: - /* See above. */ - btrfs_remove_all_log_ctxs(root, index1, ret); - mutex_lock(&root->log_mutex); + btrfs_remove_all_log_ctxs(root, index1, ret); root->log_transid_committed++; atomic_set(&root->log_commit[index1], 0); mutex_unlock(&root->log_mutex); diff --git a/fs/exofs/dir.c b/fs/exofs/dir.c index 79101651fe9e..42f9a0a0c4ca 100644 --- a/fs/exofs/dir.c +++ b/fs/exofs/dir.c @@ -137,7 +137,7 @@ Espan: bad_entry: EXOFS_ERR( "ERROR [exofs_check_page]: bad entry in directory(0x%lx): %s - " - "offset=%lu, inode=0x%llu, rec_len=%d, name_len=%d\n", + "offset=%lu, inode=0x%llx, rec_len=%d, name_len=%d\n", dir->i_ino, error, (page->index<<PAGE_SHIFT)+offs, _LLU(le64_to_cpu(p->inode_no)), rec_len, p->name_len); diff --git a/fs/iomap.c b/fs/iomap.c index 013d1d36fbbf..a8ee8c33ca78 100644 --- a/fs/iomap.c +++ b/fs/iomap.c @@ -433,8 +433,7 @@ iomap_page_mkwrite_actor(struct inode *inode, loff_t pos, loff_t length, struct page *page = data; int ret; - ret = __block_write_begin_int(page, pos & ~PAGE_MASK, length, - NULL, iomap); + ret = __block_write_begin_int(page, pos, length, NULL, iomap); if (ret) return ret; @@ -561,7 +560,7 @@ int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi, } while (len > 0) { - ret = iomap_apply(inode, start, len, 0, ops, &ctx, + ret = iomap_apply(inode, start, len, IOMAP_REPORT, ops, &ctx, iomap_fiemap_actor); /* inode with no (attribute) mapping will give ENOENT */ if (ret == -ENOENT) diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c index 2bcb86e6e6ca..78219d5644e9 100644 --- a/fs/kernfs/file.c +++ b/fs/kernfs/file.c @@ -911,6 +911,7 @@ const struct file_operations kernfs_file_fops = { .open = kernfs_fop_open, .release = kernfs_fop_release, .poll = kernfs_fop_poll, + .fsync = noop_fsync, }; /** diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index b10d557f9c9e..ee36efd5aece 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -84,6 +84,8 @@ struct nfsd_net { struct list_head client_lru; struct list_head close_lru; struct list_head del_recall_lru; + + /* protected by blocked_locks_lock */ struct list_head blocked_locks_lru; struct delayed_work laundromat_work; @@ -91,6 +93,9 @@ struct nfsd_net { /* client_lock protects the client lru list and session hash table */ spinlock_t client_lock; + /* protects blocked_locks_lru */ + spinlock_t blocked_locks_lock; + struct file *rec_file; bool in_grace; const struct nfsd4_client_tracking_ops *client_tracking_ops; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 9752beb78659..4b4beaaa4eaa 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -217,7 +217,7 @@ find_blocked_lock(struct nfs4_lockowner *lo, struct knfsd_fh *fh, { struct nfsd4_blocked_lock *cur, *found = NULL; - spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); list_for_each_entry(cur, &lo->lo_blocked, nbl_list) { if (fh_match(fh, &cur->nbl_fh)) { list_del_init(&cur->nbl_list); @@ -226,7 +226,7 @@ find_blocked_lock(struct nfs4_lockowner *lo, struct knfsd_fh *fh, break; } } - spin_unlock(&nn->client_lock); + spin_unlock(&nn->blocked_locks_lock); if (found) posix_unblock_lock(&found->nbl_lock); return found; @@ -1227,9 +1227,7 @@ static void put_ol_stateid_locked(struct nfs4_ol_stateid *stp, static bool unhash_lock_stateid(struct nfs4_ol_stateid *stp) { - struct nfs4_openowner *oo = openowner(stp->st_openstp->st_stateowner); - - lockdep_assert_held(&oo->oo_owner.so_client->cl_lock); + lockdep_assert_held(&stp->st_stid.sc_client->cl_lock); list_del_init(&stp->st_locks); nfs4_unhash_stid(&stp->st_stid); @@ -1238,12 +1236,12 @@ static bool unhash_lock_stateid(struct nfs4_ol_stateid *stp) static void release_lock_stateid(struct nfs4_ol_stateid *stp) { - struct nfs4_openowner *oo = openowner(stp->st_openstp->st_stateowner); + struct nfs4_client *clp = stp->st_stid.sc_client; bool unhashed; - spin_lock(&oo->oo_owner.so_client->cl_lock); + spin_lock(&clp->cl_lock); unhashed = unhash_lock_stateid(stp); - spin_unlock(&oo->oo_owner.so_client->cl_lock); + spin_unlock(&clp->cl_lock); if (unhashed) nfs4_put_stid(&stp->st_stid); } @@ -4665,7 +4663,7 @@ nfs4_laundromat(struct nfsd_net *nn) * indefinitely once the lock does become free. */ BUG_ON(!list_empty(&reaplist)); - spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); while (!list_empty(&nn->blocked_locks_lru)) { nbl = list_first_entry(&nn->blocked_locks_lru, struct nfsd4_blocked_lock, nbl_lru); @@ -4678,7 +4676,7 @@ nfs4_laundromat(struct nfsd_net *nn) list_move(&nbl->nbl_lru, &reaplist); list_del_init(&nbl->nbl_list); } - spin_unlock(&nn->client_lock); + spin_unlock(&nn->blocked_locks_lock); while (!list_empty(&reaplist)) { nbl = list_first_entry(&nn->blocked_locks_lru, @@ -5439,13 +5437,13 @@ nfsd4_lm_notify(struct file_lock *fl) bool queue = false; /* An empty list means that something else is going to be using it */ - spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); if (!list_empty(&nbl->nbl_list)) { list_del_init(&nbl->nbl_list); list_del_init(&nbl->nbl_lru); queue = true; } - spin_unlock(&nn->client_lock); + spin_unlock(&nn->blocked_locks_lock); if (queue) nfsd4_run_cb(&nbl->nbl_cb); @@ -5868,10 +5866,10 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (fl_flags & FL_SLEEP) { nbl->nbl_time = jiffies; - spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); list_add_tail(&nbl->nbl_list, &lock_sop->lo_blocked); list_add_tail(&nbl->nbl_lru, &nn->blocked_locks_lru); - spin_unlock(&nn->client_lock); + spin_unlock(&nn->blocked_locks_lock); } err = vfs_lock_file(filp, F_SETLK, file_lock, conflock); @@ -5900,10 +5898,10 @@ out: if (nbl) { /* dequeue it if we queued it before */ if (fl_flags & FL_SLEEP) { - spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); list_del_init(&nbl->nbl_list); list_del_init(&nbl->nbl_lru); - spin_unlock(&nn->client_lock); + spin_unlock(&nn->blocked_locks_lock); } free_blocked_lock(nbl); } @@ -6943,9 +6941,11 @@ static int nfs4_state_create_net(struct net *net) INIT_LIST_HEAD(&nn->client_lru); INIT_LIST_HEAD(&nn->close_lru); INIT_LIST_HEAD(&nn->del_recall_lru); - INIT_LIST_HEAD(&nn->blocked_locks_lru); spin_lock_init(&nn->client_lock); + spin_lock_init(&nn->blocked_locks_lock); + INIT_LIST_HEAD(&nn->blocked_locks_lru); + INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main); get_net(net); @@ -7063,14 +7063,14 @@ nfs4_state_shutdown_net(struct net *net) } BUG_ON(!list_empty(&reaplist)); - spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); while (!list_empty(&nn->blocked_locks_lru)) { nbl = list_first_entry(&nn->blocked_locks_lru, struct nfsd4_blocked_lock, nbl_lru); list_move(&nbl->nbl_lru, &reaplist); list_del_init(&nbl->nbl_list); } - spin_unlock(&nn->client_lock); + spin_unlock(&nn->blocked_locks_lock); while (!list_empty(&reaplist)) { nbl = list_first_entry(&nn->blocked_locks_lru, diff --git a/fs/orangefs/dcache.c b/fs/orangefs/dcache.c index 1e8fe844e69f..5355efba4bc8 100644 --- a/fs/orangefs/dcache.c +++ b/fs/orangefs/dcache.c @@ -73,7 +73,7 @@ static int orangefs_revalidate_lookup(struct dentry *dentry) } } - dentry->d_time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + orangefs_set_timeout(dentry); ret = 1; out_release_op: op_release(new_op); @@ -94,8 +94,9 @@ out_drop: static int orangefs_d_revalidate(struct dentry *dentry, unsigned int flags) { int ret; + unsigned long time = (unsigned long) dentry->d_fsdata; - if (time_before(jiffies, dentry->d_time)) + if (time_before(jiffies, time)) return 1; if (flags & LOOKUP_RCU) diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c index 66ea0cc37b18..02cc6139ec90 100644 --- a/fs/orangefs/file.c +++ b/fs/orangefs/file.c @@ -621,9 +621,9 @@ static int orangefs_file_release(struct inode *inode, struct file *file) * readahead cache (if any); this forces an expensive refresh of * data for the next caller of mmap (or 'get_block' accesses) */ - if (file->f_path.dentry->d_inode && - file->f_path.dentry->d_inode->i_mapping && - mapping_nrpages(&file->f_path.dentry->d_inode->i_data)) { + if (file_inode(file) && + file_inode(file)->i_mapping && + mapping_nrpages(&file_inode(file)->i_data)) { if (orangefs_features & ORANGEFS_FEATURE_READAHEAD) { gossip_debug(GOSSIP_INODE_DEBUG, "calling flush_racache on %pU\n", @@ -632,7 +632,7 @@ static int orangefs_file_release(struct inode *inode, struct file *file) gossip_debug(GOSSIP_INODE_DEBUG, "flush_racache finished\n"); } - truncate_inode_pages(file->f_path.dentry->d_inode->i_mapping, + truncate_inode_pages(file_inode(file)->i_mapping, 0); } return 0; @@ -648,7 +648,7 @@ static int orangefs_fsync(struct file *file, { int ret = -EINVAL; struct orangefs_inode_s *orangefs_inode = - ORANGEFS_I(file->f_path.dentry->d_inode); + ORANGEFS_I(file_inode(file)); struct orangefs_kernel_op_s *new_op = NULL; /* required call */ @@ -661,7 +661,7 @@ static int orangefs_fsync(struct file *file, ret = service_operation(new_op, "orangefs_fsync", - get_interruptible_flag(file->f_path.dentry->d_inode)); + get_interruptible_flag(file_inode(file))); gossip_debug(GOSSIP_FILE_DEBUG, "orangefs_fsync got return value of %d\n", @@ -669,7 +669,7 @@ static int orangefs_fsync(struct file *file, op_release(new_op); - orangefs_flush_inode(file->f_path.dentry->d_inode); + orangefs_flush_inode(file_inode(file)); return ret; } diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c index d15d3d2dba62..a290ff6ec756 100644 --- a/fs/orangefs/namei.c +++ b/fs/orangefs/namei.c @@ -72,7 +72,7 @@ static int orangefs_create(struct inode *dir, d_instantiate(dentry, inode); unlock_new_inode(inode); - dentry->d_time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + orangefs_set_timeout(dentry); ORANGEFS_I(inode)->getattr_time = jiffies - 1; gossip_debug(GOSSIP_NAME_DEBUG, @@ -183,7 +183,7 @@ static struct dentry *orangefs_lookup(struct inode *dir, struct dentry *dentry, goto out; } - dentry->d_time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + orangefs_set_timeout(dentry); inode = orangefs_iget(dir->i_sb, &new_op->downcall.resp.lookup.refn); if (IS_ERR(inode)) { @@ -322,7 +322,7 @@ static int orangefs_symlink(struct inode *dir, d_instantiate(dentry, inode); unlock_new_inode(inode); - dentry->d_time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + orangefs_set_timeout(dentry); ORANGEFS_I(inode)->getattr_time = jiffies - 1; gossip_debug(GOSSIP_NAME_DEBUG, @@ -386,7 +386,7 @@ static int orangefs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode d_instantiate(dentry, inode); unlock_new_inode(inode); - dentry->d_time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + orangefs_set_timeout(dentry); ORANGEFS_I(inode)->getattr_time = jiffies - 1; gossip_debug(GOSSIP_NAME_DEBUG, diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h index 0a82048f3aaf..3bf803d732c5 100644 --- a/fs/orangefs/orangefs-kernel.h +++ b/fs/orangefs/orangefs-kernel.h @@ -580,4 +580,11 @@ static inline void orangefs_i_size_write(struct inode *inode, loff_t i_size) #endif } +static inline void orangefs_set_timeout(struct dentry *dentry) +{ + unsigned long time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + + dentry->d_fsdata = (void *) time; +} + #endif /* __ORANGEFSKERNEL_H */ diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index aeb60f791418..36795eed40b0 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -178,6 +178,8 @@ static int ovl_copy_up_data(struct path *old, struct path *new, loff_t len) len -= bytes; } + if (!error) + error = vfs_fsync(new_file, 0); fput(new_file); out_fput: fput(old_file); diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c index c58f01babf30..7fb53d055537 100644 --- a/fs/overlayfs/inode.c +++ b/fs/overlayfs/inode.c @@ -270,9 +270,6 @@ struct posix_acl *ovl_get_acl(struct inode *inode, int type) if (!IS_ENABLED(CONFIG_FS_POSIX_ACL) || !IS_POSIXACL(realinode)) return NULL; - if (!realinode->i_op->get_acl) - return NULL; - old_cred = ovl_override_creds(inode->i_sb); acl = get_acl(realinode, type); revert_creds(old_cred); diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index bcf3965be819..edd46a0e951d 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1037,6 +1037,21 @@ ovl_posix_acl_xattr_set(const struct xattr_handler *handler, posix_acl_release(acl); + /* + * Check if sgid bit needs to be cleared (actual setacl operation will + * be done with mounter's capabilities and so that won't do it for us). + */ + if (unlikely(inode->i_mode & S_ISGID) && + handler->flags == ACL_TYPE_ACCESS && + !in_group_p(inode->i_gid) && + !capable_wrt_inode_uidgid(inode, CAP_FSETID)) { + struct iattr iattr = { .ia_valid = ATTR_KILL_SGID }; + + err = ovl_setattr(dentry, &iattr); + if (err) + return err; + } + err = ovl_xattr_set(dentry, handler->name, value, size, flags); if (!err) ovl_copyattr(ovl_inode_real(inode, NULL), inode); diff --git a/fs/proc/base.c b/fs/proc/base.c index 8e654468ab67..ca651ac00660 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -252,7 +252,7 @@ static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf, * Inherently racy -- command line shares address space * with code and data. */ - rv = access_remote_vm(mm, arg_end - 1, &c, 1, FOLL_FORCE); + rv = access_remote_vm(mm, arg_end - 1, &c, 1, 0); if (rv <= 0) goto out_free_page; @@ -270,8 +270,7 @@ static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf, int nr_read; _count = min3(count, len, PAGE_SIZE); - nr_read = access_remote_vm(mm, p, page, _count, - FOLL_FORCE); + nr_read = access_remote_vm(mm, p, page, _count, 0); if (nr_read < 0) rv = nr_read; if (nr_read <= 0) @@ -306,8 +305,7 @@ static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf, bool final; _count = min3(count, len, PAGE_SIZE); - nr_read = access_remote_vm(mm, p, page, _count, - FOLL_FORCE); + nr_read = access_remote_vm(mm, p, page, _count, 0); if (nr_read < 0) rv = nr_read; if (nr_read <= 0) @@ -356,8 +354,7 @@ skip_argv: bool final; _count = min3(count, len, PAGE_SIZE); - nr_read = access_remote_vm(mm, p, page, _count, - FOLL_FORCE); + nr_read = access_remote_vm(mm, p, page, _count, 0); if (nr_read < 0) rv = nr_read; if (nr_read <= 0) @@ -835,7 +832,7 @@ static ssize_t mem_rw(struct file *file, char __user *buf, unsigned long addr = *ppos; ssize_t copied; char *page; - unsigned int flags = FOLL_FORCE; + unsigned int flags; if (!mm) return 0; @@ -848,6 +845,8 @@ static ssize_t mem_rw(struct file *file, char __user *buf, if (!atomic_inc_not_zero(&mm->mm_users)) goto free; + /* Maybe we should limit FOLL_FORCE to actual ptrace users? */ + flags = FOLL_FORCE; if (write) flags |= FOLL_WRITE; @@ -971,8 +970,7 @@ static ssize_t environ_read(struct file *file, char __user *buf, max_len = min_t(size_t, PAGE_SIZE, count); this_len = min(max_len, this_len); - retval = access_remote_vm(mm, (env_start + src), - page, this_len, FOLL_FORCE); + retval = access_remote_vm(mm, (env_start + src), page, this_len, 0); if (retval <= 0) { ret = retval; @@ -1014,6 +1012,9 @@ static ssize_t auxv_read(struct file *file, char __user *buf, { struct mm_struct *mm = file->private_data; unsigned int nwords = 0; + + if (!mm) + return 0; do { nwords += 2; } while (mm->saved_auxv[nwords - 2] != 0); /* AT_NULL */ diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c index bd4a5e8ce441..ca16c5d7bab1 100644 --- a/fs/ubifs/dir.c +++ b/fs/ubifs/dir.c @@ -543,6 +543,14 @@ out: if (err != -ENOENT) ubifs_err(c, "cannot find next direntry, error %d", err); + else + /* + * -ENOENT is a non-fatal error in this context, the TNC uses + * it to indicate that the cursor moved past the current directory + * and readdir() has to stop. + */ + err = 0; + /* 2 is a special value indicating that there are no more direntries */ ctx->pos = 2; diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index c27344cf38e1..c6eb21940783 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3974,9 +3974,6 @@ xfs_bmap_remap_alloc( * allocating, so skip that check by pretending to be freeing. */ error = xfs_alloc_fix_freelist(&args, XFS_ALLOC_FLAG_FREEING); - if (error) - goto error0; -error0: xfs_perag_put(args.pag); if (error) trace_xfs_bmap_remap_alloc_error(ap->ip, error, _RET_IP_); @@ -3999,6 +3996,39 @@ xfs_bmap_alloc( return xfs_bmap_btalloc(ap); } +/* Trim extent to fit a logical block range. */ +void +xfs_trim_extent( + struct xfs_bmbt_irec *irec, + xfs_fileoff_t bno, + xfs_filblks_t len) +{ + xfs_fileoff_t distance; + xfs_fileoff_t end = bno + len; + + if (irec->br_startoff + irec->br_blockcount <= bno || + irec->br_startoff >= end) { + irec->br_blockcount = 0; + return; + } + + if (irec->br_startoff < bno) { + distance = bno - irec->br_startoff; + if (isnullstartblock(irec->br_startblock)) + irec->br_startblock = DELAYSTARTBLOCK; + if (irec->br_startblock != DELAYSTARTBLOCK && + irec->br_startblock != HOLESTARTBLOCK) + irec->br_startblock += distance; + irec->br_startoff += distance; + irec->br_blockcount -= distance; + } + + if (end < irec->br_startoff + irec->br_blockcount) { + distance = irec->br_startoff + irec->br_blockcount - end; + irec->br_blockcount -= distance; + } +} + /* * Trim the returned map to the required bounds */ @@ -4829,6 +4859,219 @@ xfs_bmap_split_indlen( return stolen; } +int +xfs_bmap_del_extent_delay( + struct xfs_inode *ip, + int whichfork, + xfs_extnum_t *idx, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); + struct xfs_bmbt_irec new; + int64_t da_old, da_new, da_diff = 0; + xfs_fileoff_t del_endoff, got_endoff; + xfs_filblks_t got_indlen, new_indlen, stolen; + int error = 0, state = 0; + bool isrt; + + XFS_STATS_INC(mp, xs_del_exlist); + + isrt = (whichfork == XFS_DATA_FORK) && XFS_IS_REALTIME_INODE(ip); + del_endoff = del->br_startoff + del->br_blockcount; + got_endoff = got->br_startoff + got->br_blockcount; + da_old = startblockval(got->br_startblock); + da_new = 0; + + ASSERT(*idx >= 0); + ASSERT(*idx < ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(del->br_blockcount > 0); + ASSERT(got->br_startoff <= del->br_startoff); + ASSERT(got_endoff >= del_endoff); + + if (isrt) { + int64_t rtexts = XFS_FSB_TO_B(mp, del->br_blockcount); + + do_div(rtexts, mp->m_sb.sb_rextsize); + xfs_mod_frextents(mp, rtexts); + } + + /* + * Update the inode delalloc counter now and wait to update the + * sb counters as we might have to borrow some blocks for the + * indirect block accounting. + */ + xfs_trans_reserve_quota_nblks(NULL, ip, -((long)del->br_blockcount), 0, + isrt ? XFS_QMOPT_RES_RTBLKS : XFS_QMOPT_RES_REGBLKS); + ip->i_delayed_blks -= del->br_blockcount; + + if (whichfork == XFS_COW_FORK) + state |= BMAP_COWFORK; + + if (got->br_startoff == del->br_startoff) + state |= BMAP_LEFT_CONTIG; + if (got_endoff == del_endoff) + state |= BMAP_RIGHT_CONTIG; + + switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) { + case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + /* + * Matches the whole extent. Delete the entry. + */ + xfs_iext_remove(ip, *idx, 1, state); + --*idx; + break; + case BMAP_LEFT_CONTIG: + /* + * Deleting the first part of the extent. + */ + trace_xfs_bmap_pre_update(ip, *idx, state, _THIS_IP_); + got->br_startoff = del_endoff; + got->br_blockcount -= del->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, + got->br_blockcount), da_old); + got->br_startblock = nullstartblock((int)da_new); + xfs_bmbt_set_all(xfs_iext_get_ext(ifp, *idx), got); + trace_xfs_bmap_post_update(ip, *idx, state, _THIS_IP_); + break; + case BMAP_RIGHT_CONTIG: + /* + * Deleting the last part of the extent. + */ + trace_xfs_bmap_pre_update(ip, *idx, state, _THIS_IP_); + got->br_blockcount = got->br_blockcount - del->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, + got->br_blockcount), da_old); + got->br_startblock = nullstartblock((int)da_new); + xfs_bmbt_set_all(xfs_iext_get_ext(ifp, *idx), got); + trace_xfs_bmap_post_update(ip, *idx, state, _THIS_IP_); + break; + case 0: + /* + * Deleting the middle of the extent. + * + * Distribute the original indlen reservation across the two new + * extents. Steal blocks from the deleted extent if necessary. + * Stealing blocks simply fudges the fdblocks accounting below. + * Warn if either of the new indlen reservations is zero as this + * can lead to delalloc problems. + */ + trace_xfs_bmap_pre_update(ip, *idx, state, _THIS_IP_); + + got->br_blockcount = del->br_startoff - got->br_startoff; + got_indlen = xfs_bmap_worst_indlen(ip, got->br_blockcount); + + new.br_blockcount = got_endoff - del_endoff; + new_indlen = xfs_bmap_worst_indlen(ip, new.br_blockcount); + + WARN_ON_ONCE(!got_indlen || !new_indlen); + stolen = xfs_bmap_split_indlen(da_old, &got_indlen, &new_indlen, + del->br_blockcount); + + got->br_startblock = nullstartblock((int)got_indlen); + xfs_bmbt_set_all(xfs_iext_get_ext(ifp, *idx), got); + trace_xfs_bmap_post_update(ip, *idx, 0, _THIS_IP_); + + new.br_startoff = del_endoff; + new.br_state = got->br_state; + new.br_startblock = nullstartblock((int)new_indlen); + + ++*idx; + xfs_iext_insert(ip, *idx, 1, &new, state); + + da_new = got_indlen + new_indlen - stolen; + del->br_blockcount -= stolen; + break; + } + + ASSERT(da_old >= da_new); + da_diff = da_old - da_new; + if (!isrt) + da_diff += del->br_blockcount; + if (da_diff) + xfs_mod_fdblocks(mp, da_diff, false); + return error; +} + +void +xfs_bmap_del_extent_cow( + struct xfs_inode *ip, + xfs_extnum_t *idx, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); + struct xfs_bmbt_irec new; + xfs_fileoff_t del_endoff, got_endoff; + int state = BMAP_COWFORK; + + XFS_STATS_INC(mp, xs_del_exlist); + + del_endoff = del->br_startoff + del->br_blockcount; + got_endoff = got->br_startoff + got->br_blockcount; + + ASSERT(*idx >= 0); + ASSERT(*idx < ifp->if_bytes / sizeof(struct xfs_bmbt_rec)); + ASSERT(del->br_blockcount > 0); + ASSERT(got->br_startoff <= del->br_startoff); + ASSERT(got_endoff >= del_endoff); + ASSERT(!isnullstartblock(got->br_startblock)); + + if (got->br_startoff == del->br_startoff) + state |= BMAP_LEFT_CONTIG; + if (got_endoff == del_endoff) + state |= BMAP_RIGHT_CONTIG; + + switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) { + case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + /* + * Matches the whole extent. Delete the entry. + */ + xfs_iext_remove(ip, *idx, 1, state); + --*idx; + break; + case BMAP_LEFT_CONTIG: + /* + * Deleting the first part of the extent. + */ + trace_xfs_bmap_pre_update(ip, *idx, state, _THIS_IP_); + got->br_startoff = del_endoff; + got->br_blockcount -= del->br_blockcount; + got->br_startblock = del->br_startblock + del->br_blockcount; + xfs_bmbt_set_all(xfs_iext_get_ext(ifp, *idx), got); + trace_xfs_bmap_post_update(ip, *idx, state, _THIS_IP_); + break; + case BMAP_RIGHT_CONTIG: + /* + * Deleting the last part of the extent. + */ + trace_xfs_bmap_pre_update(ip, *idx, state, _THIS_IP_); + got->br_blockcount -= del->br_blockcount; + xfs_bmbt_set_all(xfs_iext_get_ext(ifp, *idx), got); + trace_xfs_bmap_post_update(ip, *idx, state, _THIS_IP_); + break; + case 0: + /* + * Deleting the middle of the extent. + */ + trace_xfs_bmap_pre_update(ip, *idx, state, _THIS_IP_); + got->br_blockcount = del->br_startoff - got->br_startoff; + xfs_bmbt_set_all(xfs_iext_get_ext(ifp, *idx), got); + trace_xfs_bmap_post_update(ip, *idx, state, _THIS_IP_); + + new.br_startoff = del_endoff; + new.br_blockcount = got_endoff - del_endoff; + new.br_state = got->br_state; + new.br_startblock = del->br_startblock + del->br_blockcount; + + ++*idx; + xfs_iext_insert(ip, *idx, 1, &new, state); + break; + } +} + /* * Called by xfs_bmapi to update file extent records and the btree * after removing space (or undoing a delayed allocation). @@ -5171,175 +5414,6 @@ done: return error; } -/* Remove an extent from the CoW fork. Similar to xfs_bmap_del_extent. */ -int -xfs_bunmapi_cow( - struct xfs_inode *ip, - struct xfs_bmbt_irec *del) -{ - xfs_filblks_t da_new; - xfs_filblks_t da_old; - xfs_fsblock_t del_endblock = 0; - xfs_fileoff_t del_endoff; - int delay; - struct xfs_bmbt_rec_host *ep; - int error; - struct xfs_bmbt_irec got; - xfs_fileoff_t got_endoff; - struct xfs_ifork *ifp; - struct xfs_mount *mp; - xfs_filblks_t nblks; - struct xfs_bmbt_irec new; - /* REFERENCED */ - uint qfield; - xfs_filblks_t temp; - xfs_filblks_t temp2; - int state = BMAP_COWFORK; - int eof; - xfs_extnum_t eidx; - - mp = ip->i_mount; - XFS_STATS_INC(mp, xs_del_exlist); - - ep = xfs_bmap_search_extents(ip, del->br_startoff, XFS_COW_FORK, &eof, - &eidx, &got, &new); - - ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); ifp = ifp; - ASSERT((eidx >= 0) && (eidx < ifp->if_bytes / - (uint)sizeof(xfs_bmbt_rec_t))); - ASSERT(del->br_blockcount > 0); - ASSERT(got.br_startoff <= del->br_startoff); - del_endoff = del->br_startoff + del->br_blockcount; - got_endoff = got.br_startoff + got.br_blockcount; - ASSERT(got_endoff >= del_endoff); - delay = isnullstartblock(got.br_startblock); - ASSERT(isnullstartblock(del->br_startblock) == delay); - qfield = 0; - error = 0; - /* - * If deleting a real allocation, must free up the disk space. - */ - if (!delay) { - nblks = del->br_blockcount; - qfield = XFS_TRANS_DQ_BCOUNT; - /* - * Set up del_endblock and cur for later. - */ - del_endblock = del->br_startblock + del->br_blockcount; - da_old = da_new = 0; - } else { - da_old = startblockval(got.br_startblock); - da_new = 0; - nblks = 0; - } - qfield = qfield; - nblks = nblks; - - /* - * Set flag value to use in switch statement. - * Left-contig is 2, right-contig is 1. - */ - switch (((got.br_startoff == del->br_startoff) << 1) | - (got_endoff == del_endoff)) { - case 3: - /* - * Matches the whole extent. Delete the entry. - */ - xfs_iext_remove(ip, eidx, 1, BMAP_COWFORK); - --eidx; - break; - - case 2: - /* - * Deleting the first part of the extent. - */ - trace_xfs_bmap_pre_update(ip, eidx, state, _THIS_IP_); - xfs_bmbt_set_startoff(ep, del_endoff); - temp = got.br_blockcount - del->br_blockcount; - xfs_bmbt_set_blockcount(ep, temp); - if (delay) { - temp = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp), - da_old); - xfs_bmbt_set_startblock(ep, nullstartblock((int)temp)); - trace_xfs_bmap_post_update(ip, eidx, state, _THIS_IP_); - da_new = temp; - break; - } - xfs_bmbt_set_startblock(ep, del_endblock); - trace_xfs_bmap_post_update(ip, eidx, state, _THIS_IP_); - break; - - case 1: - /* - * Deleting the last part of the extent. - */ - temp = got.br_blockcount - del->br_blockcount; - trace_xfs_bmap_pre_update(ip, eidx, state, _THIS_IP_); - xfs_bmbt_set_blockcount(ep, temp); - if (delay) { - temp = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp), - da_old); - xfs_bmbt_set_startblock(ep, nullstartblock((int)temp)); - trace_xfs_bmap_post_update(ip, eidx, state, _THIS_IP_); - da_new = temp; - break; - } - trace_xfs_bmap_post_update(ip, eidx, state, _THIS_IP_); - break; - - case 0: - /* - * Deleting the middle of the extent. - */ - temp = del->br_startoff - got.br_startoff; - trace_xfs_bmap_pre_update(ip, eidx, state, _THIS_IP_); - xfs_bmbt_set_blockcount(ep, temp); - new.br_startoff = del_endoff; - temp2 = got_endoff - del_endoff; - new.br_blockcount = temp2; - new.br_state = got.br_state; - if (!delay) { - new.br_startblock = del_endblock; - } else { - temp = xfs_bmap_worst_indlen(ip, temp); - xfs_bmbt_set_startblock(ep, nullstartblock((int)temp)); - temp2 = xfs_bmap_worst_indlen(ip, temp2); - new.br_startblock = nullstartblock((int)temp2); - da_new = temp + temp2; - while (da_new > da_old) { - if (temp) { - temp--; - da_new--; - xfs_bmbt_set_startblock(ep, - nullstartblock((int)temp)); - } - if (da_new == da_old) - break; - if (temp2) { - temp2--; - da_new--; - new.br_startblock = - nullstartblock((int)temp2); - } - } - } - trace_xfs_bmap_post_update(ip, eidx, state, _THIS_IP_); - xfs_iext_insert(ip, eidx + 1, 1, &new, state); - ++eidx; - break; - } - - /* - * Account for change in delayed indirect blocks. - * Nothing to do for disk quota accounting here. - */ - ASSERT(da_old >= da_new); - if (da_old > da_new) - xfs_mod_fdblocks(mp, (int64_t)(da_old - da_new), false); - - return error; -} - /* * Unmap (remove) blocks from a file. * If nexts is nonzero then the number of extents to remove is limited to diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index f97db7132564..7cae6ec27fa6 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -190,6 +190,8 @@ void xfs_bmap_trace_exlist(struct xfs_inode *ip, xfs_extnum_t cnt, #define XFS_BMAP_TRACE_EXLIST(ip,c,w) #endif +void xfs_trim_extent(struct xfs_bmbt_irec *irec, xfs_fileoff_t bno, + xfs_filblks_t len); int xfs_bmap_add_attrfork(struct xfs_inode *ip, int size, int rsvd); void xfs_bmap_local_to_extents_empty(struct xfs_inode *ip, int whichfork); void xfs_bmap_add_free(struct xfs_mount *mp, struct xfs_defer_ops *dfops, @@ -221,7 +223,11 @@ int xfs_bunmapi(struct xfs_trans *tp, struct xfs_inode *ip, xfs_fileoff_t bno, xfs_filblks_t len, int flags, xfs_extnum_t nexts, xfs_fsblock_t *firstblock, struct xfs_defer_ops *dfops, int *done); -int xfs_bunmapi_cow(struct xfs_inode *ip, struct xfs_bmbt_irec *del); +int xfs_bmap_del_extent_delay(struct xfs_inode *ip, int whichfork, + xfs_extnum_t *idx, struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del); +void xfs_bmap_del_extent_cow(struct xfs_inode *ip, xfs_extnum_t *idx, + struct xfs_bmbt_irec *got, struct xfs_bmbt_irec *del); int xfs_check_nostate_extents(struct xfs_ifork *ifp, xfs_extnum_t idx, xfs_extnum_t num); uint xfs_default_attroffset(struct xfs_inode *ip); diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c index 5c8e6f2ce44f..0e80993c8a59 100644 --- a/fs/xfs/libxfs/xfs_btree.c +++ b/fs/xfs/libxfs/xfs_btree.c @@ -4826,7 +4826,7 @@ xfs_btree_calc_size( return rval; } -int +static int xfs_btree_count_blocks_helper( struct xfs_btree_cur *cur, int level, diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c index 3cc3cf767474..ac9a003dd29a 100644 --- a/fs/xfs/libxfs/xfs_dquot_buf.c +++ b/fs/xfs/libxfs/xfs_dquot_buf.c @@ -191,8 +191,7 @@ xfs_dquot_buf_verify_crc( if (mp->m_quotainfo) ndquots = mp->m_quotainfo->qi_dqperchunk; else - ndquots = xfs_calc_dquots_per_chunk( - XFS_BB_TO_FSB(mp, bp->b_length)); + ndquots = xfs_calc_dquots_per_chunk(bp->b_length); for (i = 0; i < ndquots; i++, d++) { if (!xfs_verify_cksum((char *)d, sizeof(struct xfs_dqblk), diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index f6547fc5e016..6b7579e7b60a 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -865,7 +865,6 @@ typedef struct xfs_timestamp { * padding field for v3 inodes. */ #define XFS_DINODE_MAGIC 0x494e /* 'IN' */ -#define XFS_DINODE_GOOD_VERSION(v) ((v) >= 1 && (v) <= 3) typedef struct xfs_dinode { __be16 di_magic; /* inode magic # = XFS_DINODE_MAGIC */ __be16 di_mode; /* mode and type of file */ diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 8de9a3a29589..134424fac434 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -57,6 +57,17 @@ xfs_inobp_check( } #endif +bool +xfs_dinode_good_version( + struct xfs_mount *mp, + __u8 version) +{ + if (xfs_sb_version_hascrc(&mp->m_sb)) + return version == 3; + + return version == 1 || version == 2; +} + /* * If we are doing readahead on an inode buffer, we might be in log recovery * reading an inode allocation buffer that hasn't yet been replayed, and hence @@ -91,7 +102,7 @@ xfs_inode_buf_verify( dip = xfs_buf_offset(bp, (i << mp->m_sb.sb_inodelog)); di_ok = dip->di_magic == cpu_to_be16(XFS_DINODE_MAGIC) && - XFS_DINODE_GOOD_VERSION(dip->di_version); + xfs_dinode_good_version(mp, dip->di_version); if (unlikely(XFS_TEST_ERROR(!di_ok, mp, XFS_ERRTAG_ITOBP_INOTOBP, XFS_RANDOM_ITOBP_INOTOBP))) { diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index 62d9d4681c8c..3cfe12a4f58a 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -74,6 +74,8 @@ void xfs_inode_from_disk(struct xfs_inode *ip, struct xfs_dinode *from); void xfs_log_dinode_to_disk(struct xfs_log_dinode *from, struct xfs_dinode *to); +bool xfs_dinode_good_version(struct xfs_mount *mp, __u8 version); + #if defined(DEBUG) void xfs_inobp_check(struct xfs_mount *, struct xfs_buf *); #else diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index a314fc7b56fa..6e4f7f900fea 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -249,6 +249,7 @@ xfs_file_dio_aio_read( struct xfs_inode *ip = XFS_I(inode); loff_t isize = i_size_read(inode); size_t count = iov_iter_count(to); + loff_t end = iocb->ki_pos + count - 1; struct iov_iter data; struct xfs_buftarg *target; ssize_t ret = 0; @@ -272,49 +273,21 @@ xfs_file_dio_aio_read( file_accessed(iocb->ki_filp); - /* - * Locking is a bit tricky here. If we take an exclusive lock for direct - * IO, we effectively serialise all new concurrent read IO to this file - * and block it behind IO that is currently in progress because IO in - * progress holds the IO lock shared. We only need to hold the lock - * exclusive to blow away the page cache, so only take lock exclusively - * if the page cache needs invalidation. This allows the normal direct - * IO case of no page cache pages to proceeed concurrently without - * serialisation. - */ xfs_rw_ilock(ip, XFS_IOLOCK_SHARED); if (mapping->nrpages) { - xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED); - xfs_rw_ilock(ip, XFS_IOLOCK_EXCL); + ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end); + if (ret) + goto out_unlock; /* - * The generic dio code only flushes the range of the particular - * I/O. Because we take an exclusive lock here, this whole - * sequence is considerably more expensive for us. This has a - * noticeable performance impact for any file with cached pages, - * even when outside of the range of the particular I/O. - * - * Hence, amortize the cost of the lock against a full file - * flush and reduce the chances of repeated iolock cycles going - * forward. + * Invalidate whole pages. This can return an error if we fail + * to invalidate a page, but this should never happen on XFS. + * Warn if it does fail. */ - if (mapping->nrpages) { - ret = filemap_write_and_wait(mapping); - if (ret) { - xfs_rw_iunlock(ip, XFS_IOLOCK_EXCL); - return ret; - } - - /* - * Invalidate whole pages. This can return an error if - * we fail to invalidate a page, but this should never - * happen on XFS. Warn if it does fail. - */ - ret = invalidate_inode_pages2(mapping); - WARN_ON_ONCE(ret); - ret = 0; - } - xfs_rw_ilock_demote(ip, XFS_IOLOCK_EXCL); + ret = invalidate_inode_pages2_range(mapping, + iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT); + WARN_ON_ONCE(ret); + ret = 0; } data = *to; @@ -324,8 +297,9 @@ xfs_file_dio_aio_read( iocb->ki_pos += ret; iov_iter_advance(to, ret); } - xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED); +out_unlock: + xfs_rw_iunlock(ip, XFS_IOLOCK_SHARED); return ret; } @@ -570,61 +544,49 @@ xfs_file_dio_aio_write( if ((iocb->ki_pos | count) & target->bt_logical_sectormask) return -EINVAL; - /* "unaligned" here means not aligned to a filesystem block */ - if ((iocb->ki_pos & mp->m_blockmask) || - ((iocb->ki_pos + count) & mp->m_blockmask)) - unaligned_io = 1; - /* - * We don't need to take an exclusive lock unless there page cache needs - * to be invalidated or unaligned IO is being executed. We don't need to - * consider the EOF extension case here because - * xfs_file_aio_write_checks() will relock the inode as necessary for - * EOF zeroing cases and fill out the new inode size as appropriate. + * Don't take the exclusive iolock here unless the I/O is unaligned to + * the file system block size. We don't need to consider the EOF + * extension case here because xfs_file_aio_write_checks() will relock + * the inode as necessary for EOF zeroing cases and fill out the new + * inode size as appropriate. */ - if (unaligned_io || mapping->nrpages) + if ((iocb->ki_pos & mp->m_blockmask) || + ((iocb->ki_pos + count) & mp->m_blockmask)) { + unaligned_io = 1; iolock = XFS_IOLOCK_EXCL; - else + } else { iolock = XFS_IOLOCK_SHARED; - xfs_rw_ilock(ip, iolock); - - /* - * Recheck if there are cached pages that need invalidate after we got - * the iolock to protect against other threads adding new pages while - * we were waiting for the iolock. - */ - if (mapping->nrpages && iolock == XFS_IOLOCK_SHARED) { - xfs_rw_iunlock(ip, iolock); - iolock = XFS_IOLOCK_EXCL; - xfs_rw_ilock(ip, iolock); } + xfs_rw_ilock(ip, iolock); + ret = xfs_file_aio_write_checks(iocb, from, &iolock); if (ret) goto out; count = iov_iter_count(from); end = iocb->ki_pos + count - 1; - /* - * See xfs_file_dio_aio_read() for why we do a full-file flush here. - */ if (mapping->nrpages) { - ret = filemap_write_and_wait(VFS_I(ip)->i_mapping); + ret = filemap_write_and_wait_range(mapping, iocb->ki_pos, end); if (ret) goto out; + /* * Invalidate whole pages. This can return an error if we fail * to invalidate a page, but this should never happen on XFS. * Warn if it does fail. */ - ret = invalidate_inode_pages2(VFS_I(ip)->i_mapping); + ret = invalidate_inode_pages2_range(mapping, + iocb->ki_pos >> PAGE_SHIFT, end >> PAGE_SHIFT); WARN_ON_ONCE(ret); ret = 0; } /* * If we are doing unaligned IO, wait for all other IO to drain, - * otherwise demote the lock if we had to flush cached pages + * otherwise demote the lock if we had to take the exclusive lock + * for other reasons in xfs_file_aio_write_checks. */ if (unaligned_io) inode_dio_wait(inode); @@ -947,134 +909,6 @@ out_unlock: return error; } -/* - * Flush all file writes out to disk. - */ -static int -xfs_file_wait_for_io( - struct inode *inode, - loff_t offset, - size_t len) -{ - loff_t rounding; - loff_t ioffset; - loff_t iendoffset; - loff_t bs; - int ret; - - bs = inode->i_sb->s_blocksize; - inode_dio_wait(inode); - - rounding = max_t(xfs_off_t, bs, PAGE_SIZE); - ioffset = round_down(offset, rounding); - iendoffset = round_up(offset + len, rounding) - 1; - ret = filemap_write_and_wait_range(inode->i_mapping, ioffset, - iendoffset); - return ret; -} - -/* Hook up to the VFS reflink function */ -STATIC int -xfs_file_share_range( - struct file *file_in, - loff_t pos_in, - struct file *file_out, - loff_t pos_out, - u64 len, - bool is_dedupe) -{ - struct inode *inode_in; - struct inode *inode_out; - ssize_t ret; - loff_t bs; - loff_t isize; - int same_inode; - loff_t blen; - unsigned int flags = 0; - - inode_in = file_inode(file_in); - inode_out = file_inode(file_out); - bs = inode_out->i_sb->s_blocksize; - - /* Don't touch certain kinds of inodes */ - if (IS_IMMUTABLE(inode_out)) - return -EPERM; - if (IS_SWAPFILE(inode_in) || - IS_SWAPFILE(inode_out)) - return -ETXTBSY; - - /* Reflink only works within this filesystem. */ - if (inode_in->i_sb != inode_out->i_sb) - return -EXDEV; - same_inode = (inode_in->i_ino == inode_out->i_ino); - - /* Don't reflink dirs, pipes, sockets... */ - if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) - return -EISDIR; - if (S_ISFIFO(inode_in->i_mode) || S_ISFIFO(inode_out->i_mode)) - return -EINVAL; - if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) - return -EINVAL; - - /* Don't share DAX file data for now. */ - if (IS_DAX(inode_in) || IS_DAX(inode_out)) - return -EINVAL; - - /* Are we going all the way to the end? */ - isize = i_size_read(inode_in); - if (isize == 0) - return 0; - if (len == 0) - len = isize - pos_in; - - /* Ensure offsets don't wrap and the input is inside i_size */ - if (pos_in + len < pos_in || pos_out + len < pos_out || - pos_in + len > isize) - return -EINVAL; - - /* Don't allow dedupe past EOF in the dest file */ - if (is_dedupe) { - loff_t disize; - - disize = i_size_read(inode_out); - if (pos_out >= disize || pos_out + len > disize) - return -EINVAL; - } - - /* If we're linking to EOF, continue to the block boundary. */ - if (pos_in + len == isize) - blen = ALIGN(isize, bs) - pos_in; - else - blen = len; - - /* Only reflink if we're aligned to block boundaries */ - if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_in + blen, bs) || - !IS_ALIGNED(pos_out, bs) || !IS_ALIGNED(pos_out + blen, bs)) - return -EINVAL; - - /* Don't allow overlapped reflink within the same file */ - if (same_inode && pos_out + blen > pos_in && pos_out < pos_in + blen) - return -EINVAL; - - /* Wait for the completion of any pending IOs on srcfile */ - ret = xfs_file_wait_for_io(inode_in, pos_in, len); - if (ret) - goto out; - ret = xfs_file_wait_for_io(inode_out, pos_out, len); - if (ret) - goto out; - - if (is_dedupe) - flags |= XFS_REFLINK_DEDUPE; - ret = xfs_reflink_remap_range(XFS_I(inode_in), pos_in, XFS_I(inode_out), - pos_out, len, flags); - if (ret < 0) - goto out; - -out: - return ret; -} - STATIC ssize_t xfs_file_copy_range( struct file *file_in, @@ -1086,7 +920,7 @@ xfs_file_copy_range( { int error; - error = xfs_file_share_range(file_in, pos_in, file_out, pos_out, + error = xfs_reflink_remap_range(file_in, pos_in, file_out, pos_out, len, false); if (error) return error; @@ -1101,7 +935,7 @@ xfs_file_clone_range( loff_t pos_out, u64 len) { - return xfs_file_share_range(file_in, pos_in, file_out, pos_out, + return xfs_reflink_remap_range(file_in, pos_in, file_out, pos_out, len, false); } @@ -1124,7 +958,7 @@ xfs_file_dedupe_range( if (len > XFS_MAX_DEDUPE_LEN) len = XFS_MAX_DEDUPE_LEN; - error = xfs_file_share_range(src_file, loff, dst_file, dst_loff, + error = xfs_reflink_remap_range(src_file, loff, dst_file, dst_loff, len, true); if (error) return error; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 14796b744e0a..f295049db681 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1656,9 +1656,9 @@ void xfs_inode_set_cowblocks_tag( xfs_inode_t *ip) { - trace_xfs_inode_set_eofblocks_tag(ip); + trace_xfs_inode_set_cowblocks_tag(ip); return __xfs_inode_set_eofblocks_tag(ip, xfs_queue_cowblocks, - trace_xfs_perag_set_eofblocks, + trace_xfs_perag_set_cowblocks, XFS_ICI_COWBLOCKS_TAG); } @@ -1666,7 +1666,7 @@ void xfs_inode_clear_cowblocks_tag( xfs_inode_t *ip) { - trace_xfs_inode_clear_eofblocks_tag(ip); + trace_xfs_inode_clear_cowblocks_tag(ip); return __xfs_inode_clear_eofblocks_tag(ip, - trace_xfs_perag_clear_eofblocks, XFS_ICI_COWBLOCKS_TAG); + trace_xfs_perag_clear_cowblocks, XFS_ICI_COWBLOCKS_TAG); } diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index d907eb9f8ef3..436e109bb01e 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -566,6 +566,17 @@ xfs_file_iomap_begin_delay( xfs_bmap_search_extents(ip, offset_fsb, XFS_DATA_FORK, &eof, &idx, &got, &prev); if (!eof && got.br_startoff <= offset_fsb) { + if (xfs_is_reflink_inode(ip)) { + bool shared; + + end_fsb = min(XFS_B_TO_FSB(mp, offset + count), + maxbytes_fsb); + xfs_trim_extent(&got, offset_fsb, end_fsb - offset_fsb); + error = xfs_reflink_reserve_cow(ip, &got, &shared); + if (error) + goto out_unlock; + } + trace_xfs_iomap_found(ip, offset, count, 0, &got); goto done; } @@ -961,19 +972,13 @@ xfs_file_iomap_begin( struct xfs_mount *mp = ip->i_mount; struct xfs_bmbt_irec imap; xfs_fileoff_t offset_fsb, end_fsb; - bool shared, trimmed; int nimaps = 1, error = 0; + bool shared = false, trimmed = false; unsigned lockmode; if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; - if ((flags & (IOMAP_WRITE | IOMAP_ZERO)) && xfs_is_reflink_inode(ip)) { - error = xfs_reflink_reserve_cow_range(ip, offset, length); - if (error < 0) - return error; - } - if ((flags & IOMAP_WRITE) && !IS_DAX(inode) && !xfs_get_extsz_hint(ip)) { /* Reserve delalloc blocks for regular writeback. */ @@ -981,7 +986,16 @@ xfs_file_iomap_begin( iomap); } - lockmode = xfs_ilock_data_map_shared(ip); + /* + * COW writes will allocate delalloc space, so we need to make sure + * to take the lock exclusively here. + */ + if ((flags & (IOMAP_WRITE | IOMAP_ZERO)) && xfs_is_reflink_inode(ip)) { + lockmode = XFS_ILOCK_EXCL; + xfs_ilock(ip, XFS_ILOCK_EXCL); + } else { + lockmode = xfs_ilock_data_map_shared(ip); + } ASSERT(offset <= mp->m_super->s_maxbytes); if ((xfs_fsize_t)offset + length > mp->m_super->s_maxbytes) @@ -991,16 +1005,24 @@ xfs_file_iomap_begin( error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, &nimaps, 0); - if (error) { - xfs_iunlock(ip, lockmode); - return error; + if (error) + goto out_unlock; + + if (flags & IOMAP_REPORT) { + /* Trim the mapping to the nearest shared extent boundary. */ + error = xfs_reflink_trim_around_shared(ip, &imap, &shared, + &trimmed); + if (error) + goto out_unlock; } - /* Trim the mapping to the nearest shared extent boundary. */ - error = xfs_reflink_trim_around_shared(ip, &imap, &shared, &trimmed); - if (error) { - xfs_iunlock(ip, lockmode); - return error; + if ((flags & (IOMAP_WRITE | IOMAP_ZERO)) && xfs_is_reflink_inode(ip)) { + error = xfs_reflink_reserve_cow(ip, &imap, &shared); + if (error) + goto out_unlock; + + end_fsb = imap.br_startoff + imap.br_blockcount; + length = XFS_FSB_TO_B(mp, end_fsb) - offset; } if ((flags & IOMAP_WRITE) && imap_needs_alloc(inode, &imap, nimaps)) { @@ -1039,6 +1061,9 @@ xfs_file_iomap_begin( if (shared) iomap->flags |= IOMAP_F_SHARED; return 0; +out_unlock: + xfs_iunlock(ip, lockmode); + return error; } static int diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index fc7873942bea..b341f10cf481 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1009,6 +1009,7 @@ xfs_mountfs( out_quota: xfs_qm_unmount_quotas(mp); out_rtunmount: + mp->m_super->s_flags &= ~MS_ACTIVE; xfs_rtunmount_inodes(mp); out_rele_rip: IRELE(rip); diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c index 5965e9455d91..a279b4e7f5fe 100644 --- a/fs/xfs/xfs_reflink.c +++ b/fs/xfs/xfs_reflink.c @@ -182,7 +182,8 @@ xfs_reflink_trim_around_shared( if (!xfs_is_reflink_inode(ip) || ISUNWRITTEN(irec) || irec->br_startblock == HOLESTARTBLOCK || - irec->br_startblock == DELAYSTARTBLOCK) { + irec->br_startblock == DELAYSTARTBLOCK || + isnullstartblock(irec->br_startblock)) { *shared = false; return 0; } @@ -227,50 +228,54 @@ xfs_reflink_trim_around_shared( } } -/* Create a CoW reservation for a range of blocks within a file. */ -static int -__xfs_reflink_reserve_cow( +/* + * Trim the passed in imap to the next shared/unshared extent boundary, and + * if imap->br_startoff points to a shared extent reserve space for it in the + * COW fork. In this case *shared is set to true, else to false. + * + * Note that imap will always contain the block numbers for the existing blocks + * in the data fork, as the upper layers need them for read-modify-write + * operations. + */ +int +xfs_reflink_reserve_cow( struct xfs_inode *ip, - xfs_fileoff_t *offset_fsb, - xfs_fileoff_t end_fsb, - bool *skipped) + struct xfs_bmbt_irec *imap, + bool *shared) { - struct xfs_bmbt_irec got, prev, imap; - xfs_fileoff_t orig_end_fsb; - int nimaps, eof = 0, error = 0; - bool shared = false, trimmed = false; + struct xfs_bmbt_irec got, prev; + xfs_fileoff_t end_fsb, orig_end_fsb; + int eof = 0, error = 0; + bool trimmed; xfs_extnum_t idx; xfs_extlen_t align; - /* Already reserved? Skip the refcount btree access. */ - xfs_bmap_search_extents(ip, *offset_fsb, XFS_COW_FORK, &eof, &idx, + /* + * Search the COW fork extent list first. This serves two purposes: + * first this implement the speculative preallocation using cowextisze, + * so that we also unshared block adjacent to shared blocks instead + * of just the shared blocks themselves. Second the lookup in the + * extent list is generally faster than going out to the shared extent + * tree. + */ + xfs_bmap_search_extents(ip, imap->br_startoff, XFS_COW_FORK, &eof, &idx, &got, &prev); - if (!eof && got.br_startoff <= *offset_fsb) { - end_fsb = orig_end_fsb = got.br_startoff + got.br_blockcount; - trace_xfs_reflink_cow_found(ip, &got); - goto done; - } + if (!eof && got.br_startoff <= imap->br_startoff) { + trace_xfs_reflink_cow_found(ip, imap); + xfs_trim_extent(imap, got.br_startoff, got.br_blockcount); - /* Read extent from the source file. */ - nimaps = 1; - error = xfs_bmapi_read(ip, *offset_fsb, end_fsb - *offset_fsb, - &imap, &nimaps, 0); - if (error) - goto out_unlock; - ASSERT(nimaps == 1); + *shared = true; + return 0; + } /* Trim the mapping to the nearest shared extent boundary. */ - error = xfs_reflink_trim_around_shared(ip, &imap, &shared, &trimmed); + error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed); if (error) - goto out_unlock; - - end_fsb = orig_end_fsb = imap.br_startoff + imap.br_blockcount; + return error; /* Not shared? Just report the (potentially capped) extent. */ - if (!shared) { - *skipped = true; - goto done; - } + if (!*shared) + return 0; /* * Fork all the shared blocks from our write offset until the end of @@ -278,72 +283,38 @@ __xfs_reflink_reserve_cow( */ error = xfs_qm_dqattach_locked(ip, 0); if (error) - goto out_unlock; + return error; + + end_fsb = orig_end_fsb = imap->br_startoff + imap->br_blockcount; align = xfs_eof_alignment(ip, xfs_get_cowextsz_hint(ip)); if (align) end_fsb = roundup_64(end_fsb, align); retry: - error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, *offset_fsb, - end_fsb - *offset_fsb, &got, - &prev, &idx, eof); + error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff, + end_fsb - imap->br_startoff, &got, &prev, &idx, eof); switch (error) { case 0: break; case -ENOSPC: case -EDQUOT: /* retry without any preallocation */ - trace_xfs_reflink_cow_enospc(ip, &imap); + trace_xfs_reflink_cow_enospc(ip, imap); if (end_fsb != orig_end_fsb) { end_fsb = orig_end_fsb; goto retry; } /*FALLTHRU*/ default: - goto out_unlock; + return error; } if (end_fsb != orig_end_fsb) xfs_inode_set_cowblocks_tag(ip); trace_xfs_reflink_cow_alloc(ip, &got); -done: - *offset_fsb = end_fsb; -out_unlock: - return error; -} - -/* Create a CoW reservation for part of a file. */ -int -xfs_reflink_reserve_cow_range( - struct xfs_inode *ip, - xfs_off_t offset, - xfs_off_t count) -{ - struct xfs_mount *mp = ip->i_mount; - xfs_fileoff_t offset_fsb, end_fsb; - bool skipped = false; - int error; - - trace_xfs_reflink_reserve_cow_range(ip, offset, count); - - offset_fsb = XFS_B_TO_FSBT(mp, offset); - end_fsb = XFS_B_TO_FSB(mp, offset + count); - - xfs_ilock(ip, XFS_ILOCK_EXCL); - while (offset_fsb < end_fsb) { - error = __xfs_reflink_reserve_cow(ip, &offset_fsb, end_fsb, - &skipped); - if (error) { - trace_xfs_reflink_reserve_cow_range_error(ip, error, - _RET_IP_); - break; - } - } - xfs_iunlock(ip, XFS_ILOCK_EXCL); - - return error; + return 0; } /* Allocate all CoW reservations covering a range of blocks in a file. */ @@ -358,9 +329,8 @@ __xfs_reflink_allocate_cow( struct xfs_defer_ops dfops; struct xfs_trans *tp; xfs_fsblock_t first_block; - xfs_fileoff_t next_fsb; int nimaps = 1, error; - bool skipped = false; + bool shared; xfs_defer_init(&dfops, &first_block); @@ -371,33 +341,38 @@ __xfs_reflink_allocate_cow( xfs_ilock(ip, XFS_ILOCK_EXCL); - next_fsb = *offset_fsb; - error = __xfs_reflink_reserve_cow(ip, &next_fsb, end_fsb, &skipped); + /* Read extent from the source file. */ + nimaps = 1; + error = xfs_bmapi_read(ip, *offset_fsb, end_fsb - *offset_fsb, + &imap, &nimaps, 0); + if (error) + goto out_unlock; + ASSERT(nimaps == 1); + + error = xfs_reflink_reserve_cow(ip, &imap, &shared); if (error) goto out_trans_cancel; - if (skipped) { - *offset_fsb = next_fsb; + if (!shared) { + *offset_fsb = imap.br_startoff + imap.br_blockcount; goto out_trans_cancel; } xfs_trans_ijoin(tp, ip, 0); - error = xfs_bmapi_write(tp, ip, *offset_fsb, next_fsb - *offset_fsb, + error = xfs_bmapi_write(tp, ip, imap.br_startoff, imap.br_blockcount, XFS_BMAPI_COWFORK, &first_block, XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK), &imap, &nimaps, &dfops); if (error) goto out_trans_cancel; - /* We might not have been able to map the whole delalloc extent */ - *offset_fsb = min(*offset_fsb + imap.br_blockcount, next_fsb); - error = xfs_defer_finish(&tp, &dfops, NULL); if (error) goto out_trans_cancel; error = xfs_trans_commit(tp); + *offset_fsb = imap.br_startoff + imap.br_blockcount; out_unlock: xfs_iunlock(ip, XFS_ILOCK_EXCL); return error; @@ -536,58 +511,49 @@ xfs_reflink_cancel_cow_blocks( xfs_fileoff_t offset_fsb, xfs_fileoff_t end_fsb) { - struct xfs_bmbt_irec irec; - xfs_filblks_t count_fsb; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); + struct xfs_bmbt_irec got, prev, del; + xfs_extnum_t idx; xfs_fsblock_t firstfsb; struct xfs_defer_ops dfops; - int error = 0; - int nimaps; + int error = 0, eof = 0; if (!xfs_is_reflink_inode(ip)) return 0; - /* Go find the old extent in the CoW fork. */ - while (offset_fsb < end_fsb) { - nimaps = 1; - count_fsb = (xfs_filblks_t)(end_fsb - offset_fsb); - error = xfs_bmapi_read(ip, offset_fsb, count_fsb, &irec, - &nimaps, XFS_BMAPI_COWFORK); - if (error) - break; - ASSERT(nimaps == 1); - - trace_xfs_reflink_cancel_cow(ip, &irec); + xfs_bmap_search_extents(ip, offset_fsb, XFS_COW_FORK, &eof, &idx, + &got, &prev); + if (eof) + return 0; - if (irec.br_startblock == DELAYSTARTBLOCK) { - /* Free a delayed allocation. */ - xfs_mod_fdblocks(ip->i_mount, irec.br_blockcount, - false); - ip->i_delayed_blks -= irec.br_blockcount; + while (got.br_startoff < end_fsb) { + del = got; + xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); + trace_xfs_reflink_cancel_cow(ip, &del); - /* Remove the mapping from the CoW fork. */ - error = xfs_bunmapi_cow(ip, &irec); + if (isnullstartblock(del.br_startblock)) { + error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, + &idx, &got, &del); if (error) break; - } else if (irec.br_startblock == HOLESTARTBLOCK) { - /* empty */ } else { xfs_trans_ijoin(*tpp, ip, 0); xfs_defer_init(&dfops, &firstfsb); /* Free the CoW orphan record. */ error = xfs_refcount_free_cow_extent(ip->i_mount, - &dfops, irec.br_startblock, - irec.br_blockcount); + &dfops, del.br_startblock, + del.br_blockcount); if (error) break; xfs_bmap_add_free(ip->i_mount, &dfops, - irec.br_startblock, irec.br_blockcount, + del.br_startblock, del.br_blockcount, NULL); /* Update quota accounting */ xfs_trans_mod_dquot_byino(*tpp, ip, XFS_TRANS_DQ_BCOUNT, - -(long)irec.br_blockcount); + -(long)del.br_blockcount); /* Roll the transaction */ error = xfs_defer_finish(tpp, &dfops, ip); @@ -597,15 +563,18 @@ xfs_reflink_cancel_cow_blocks( } /* Remove the mapping from the CoW fork. */ - error = xfs_bunmapi_cow(ip, &irec); - if (error) - break; + xfs_bmap_del_extent_cow(ip, &idx, &got, &del); } - /* Roll on... */ - offset_fsb = irec.br_startoff + irec.br_blockcount; + if (++idx >= ifp->if_bytes / sizeof(struct xfs_bmbt_rec)) + break; + xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), &got); } + /* clear tag if cow fork is emptied */ + if (!ifp->if_bytes) + xfs_inode_clear_cowblocks_tag(ip); + return error; } @@ -668,25 +637,26 @@ xfs_reflink_end_cow( xfs_off_t offset, xfs_off_t count) { - struct xfs_bmbt_irec irec; - struct xfs_bmbt_irec uirec; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK); + struct xfs_bmbt_irec got, prev, del; struct xfs_trans *tp; xfs_fileoff_t offset_fsb; xfs_fileoff_t end_fsb; - xfs_filblks_t count_fsb; xfs_fsblock_t firstfsb; struct xfs_defer_ops dfops; - int error; + int error, eof = 0; unsigned int resblks; - xfs_filblks_t ilen; xfs_filblks_t rlen; - int nimaps; + xfs_extnum_t idx; trace_xfs_reflink_end_cow(ip, offset, count); + /* No COW extents? That's easy! */ + if (ifp->if_bytes == 0) + return 0; + offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); - count_fsb = (xfs_filblks_t)(end_fsb - offset_fsb); /* Start a rolling transaction to switch the mappings */ resblks = XFS_EXTENTADD_SPACE_RES(ip->i_mount, XFS_DATA_FORK); @@ -698,72 +668,65 @@ xfs_reflink_end_cow( xfs_ilock(ip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, ip, 0); - /* Go find the old extent in the CoW fork. */ - while (offset_fsb < end_fsb) { - /* Read extent from the source file */ - nimaps = 1; - count_fsb = (xfs_filblks_t)(end_fsb - offset_fsb); - error = xfs_bmapi_read(ip, offset_fsb, count_fsb, &irec, - &nimaps, XFS_BMAPI_COWFORK); - if (error) - goto out_cancel; - ASSERT(nimaps == 1); + xfs_bmap_search_extents(ip, end_fsb - 1, XFS_COW_FORK, &eof, &idx, + &got, &prev); - ASSERT(irec.br_startblock != DELAYSTARTBLOCK); - trace_xfs_reflink_cow_remap(ip, &irec); + /* If there is a hole at end_fsb - 1 go to the previous extent */ + if (eof || got.br_startoff > end_fsb) { + ASSERT(idx > 0); + xfs_bmbt_get_all(xfs_iext_get_ext(ifp, --idx), &got); + } - /* - * We can have a hole in the CoW fork if part of a directio - * write is CoW but part of it isn't. - */ - rlen = ilen = irec.br_blockcount; - if (irec.br_startblock == HOLESTARTBLOCK) + /* Walk backwards until we're out of the I/O range... */ + while (got.br_startoff + got.br_blockcount > offset_fsb) { + del = got; + xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); + + /* Extent delete may have bumped idx forward */ + if (!del.br_blockcount) { + idx--; goto next_extent; + } + + ASSERT(!isnullstartblock(got.br_startblock)); /* Unmap the old blocks in the data fork. */ - while (rlen) { - xfs_defer_init(&dfops, &firstfsb); - error = __xfs_bunmapi(tp, ip, irec.br_startoff, - &rlen, 0, 1, &firstfsb, &dfops); - if (error) - goto out_defer; - - /* - * Trim the extent to whatever got unmapped. - * Remember, bunmapi works backwards. - */ - uirec.br_startblock = irec.br_startblock + rlen; - uirec.br_startoff = irec.br_startoff + rlen; - uirec.br_blockcount = irec.br_blockcount - rlen; - irec.br_blockcount = rlen; - trace_xfs_reflink_cow_remap_piece(ip, &uirec); + xfs_defer_init(&dfops, &firstfsb); + rlen = del.br_blockcount; + error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1, + &firstfsb, &dfops); + if (error) + goto out_defer; - /* Free the CoW orphan record. */ - error = xfs_refcount_free_cow_extent(tp->t_mountp, - &dfops, uirec.br_startblock, - uirec.br_blockcount); - if (error) - goto out_defer; + /* Trim the extent to whatever got unmapped. */ + if (rlen) { + xfs_trim_extent(&del, del.br_startoff + rlen, + del.br_blockcount - rlen); + } + trace_xfs_reflink_cow_remap(ip, &del); - /* Map the new blocks into the data fork. */ - error = xfs_bmap_map_extent(tp->t_mountp, &dfops, - ip, &uirec); - if (error) - goto out_defer; + /* Free the CoW orphan record. */ + error = xfs_refcount_free_cow_extent(tp->t_mountp, &dfops, + del.br_startblock, del.br_blockcount); + if (error) + goto out_defer; - /* Remove the mapping from the CoW fork. */ - error = xfs_bunmapi_cow(ip, &uirec); - if (error) - goto out_defer; + /* Map the new blocks into the data fork. */ + error = xfs_bmap_map_extent(tp->t_mountp, &dfops, ip, &del); + if (error) + goto out_defer; - error = xfs_defer_finish(&tp, &dfops, ip); - if (error) - goto out_defer; - } + /* Remove the mapping from the CoW fork. */ + xfs_bmap_del_extent_cow(ip, &idx, &got, &del); + + error = xfs_defer_finish(&tp, &dfops, ip); + if (error) + goto out_defer; next_extent: - /* Roll on... */ - offset_fsb = irec.br_startoff + ilen; + if (idx < 0) + break; + xfs_bmbt_get_all(xfs_iext_get_ext(ifp, idx), &got); } error = xfs_trans_commit(tp); @@ -774,7 +737,6 @@ next_extent: out_defer: xfs_defer_cancel(&dfops); -out_cancel: xfs_trans_cancel(tp); xfs_iunlock(ip, XFS_ILOCK_EXCL); out: @@ -1312,19 +1274,26 @@ out_error: */ int xfs_reflink_remap_range( - struct xfs_inode *src, - xfs_off_t srcoff, - struct xfs_inode *dest, - xfs_off_t destoff, - xfs_off_t len, - unsigned int flags) + struct file *file_in, + loff_t pos_in, + struct file *file_out, + loff_t pos_out, + u64 len, + bool is_dedupe) { + struct inode *inode_in = file_inode(file_in); + struct xfs_inode *src = XFS_I(inode_in); + struct inode *inode_out = file_inode(file_out); + struct xfs_inode *dest = XFS_I(inode_out); struct xfs_mount *mp = src->i_mount; + loff_t bs = inode_out->i_sb->s_blocksize; + bool same_inode = (inode_in == inode_out); xfs_fileoff_t sfsbno, dfsbno; xfs_filblks_t fsblen; - int error; xfs_extlen_t cowextsize; - bool is_same; + loff_t isize; + ssize_t ret; + loff_t blen; if (!xfs_sb_version_hasreflink(&mp->m_sb)) return -EOPNOTSUPP; @@ -1332,17 +1301,8 @@ xfs_reflink_remap_range( if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; - /* Don't reflink realtime inodes */ - if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) - return -EINVAL; - - if (flags & ~XFS_REFLINK_ALL) - return -EINVAL; - - trace_xfs_reflink_remap_range(src, srcoff, len, dest, destoff); - /* Lock both files against IO */ - if (src->i_ino == dest->i_ino) { + if (same_inode) { xfs_ilock(src, XFS_IOLOCK_EXCL); xfs_ilock(src, XFS_MMAPLOCK_EXCL); } else { @@ -1350,39 +1310,126 @@ xfs_reflink_remap_range( xfs_lock_two_inodes(src, dest, XFS_MMAPLOCK_EXCL); } + /* Don't touch certain kinds of inodes */ + ret = -EPERM; + if (IS_IMMUTABLE(inode_out)) + goto out_unlock; + + ret = -ETXTBSY; + if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) + goto out_unlock; + + + /* Don't reflink dirs, pipes, sockets... */ + ret = -EISDIR; + if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) + goto out_unlock; + ret = -EINVAL; + if (S_ISFIFO(inode_in->i_mode) || S_ISFIFO(inode_out->i_mode)) + goto out_unlock; + if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) + goto out_unlock; + + /* Don't reflink realtime inodes */ + if (XFS_IS_REALTIME_INODE(src) || XFS_IS_REALTIME_INODE(dest)) + goto out_unlock; + + /* Don't share DAX file data for now. */ + if (IS_DAX(inode_in) || IS_DAX(inode_out)) + goto out_unlock; + + /* Are we going all the way to the end? */ + isize = i_size_read(inode_in); + if (isize == 0) { + ret = 0; + goto out_unlock; + } + + if (len == 0) + len = isize - pos_in; + + /* Ensure offsets don't wrap and the input is inside i_size */ + if (pos_in + len < pos_in || pos_out + len < pos_out || + pos_in + len > isize) + goto out_unlock; + + /* Don't allow dedupe past EOF in the dest file */ + if (is_dedupe) { + loff_t disize; + + disize = i_size_read(inode_out); + if (pos_out >= disize || pos_out + len > disize) + goto out_unlock; + } + + /* If we're linking to EOF, continue to the block boundary. */ + if (pos_in + len == isize) + blen = ALIGN(isize, bs) - pos_in; + else + blen = len; + + /* Only reflink if we're aligned to block boundaries */ + if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_in + blen, bs) || + !IS_ALIGNED(pos_out, bs) || !IS_ALIGNED(pos_out + blen, bs)) + goto out_unlock; + + /* Don't allow overlapped reflink within the same file */ + if (same_inode) { + if (pos_out + blen > pos_in && pos_out < pos_in + blen) + goto out_unlock; + } + + /* Wait for the completion of any pending IOs on both files */ + inode_dio_wait(inode_in); + if (!same_inode) + inode_dio_wait(inode_out); + + ret = filemap_write_and_wait_range(inode_in->i_mapping, + pos_in, pos_in + len - 1); + if (ret) + goto out_unlock; + + ret = filemap_write_and_wait_range(inode_out->i_mapping, + pos_out, pos_out + len - 1); + if (ret) + goto out_unlock; + + trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out); + /* * Check that the extents are the same. */ - if (flags & XFS_REFLINK_DEDUPE) { - is_same = false; - error = xfs_compare_extents(VFS_I(src), srcoff, VFS_I(dest), - destoff, len, &is_same); - if (error) - goto out_error; + if (is_dedupe) { + bool is_same = false; + + ret = xfs_compare_extents(inode_in, pos_in, inode_out, pos_out, + len, &is_same); + if (ret) + goto out_unlock; if (!is_same) { - error = -EBADE; - goto out_error; + ret = -EBADE; + goto out_unlock; } } - error = xfs_reflink_set_inode_flag(src, dest); - if (error) - goto out_error; + ret = xfs_reflink_set_inode_flag(src, dest); + if (ret) + goto out_unlock; /* * Invalidate the page cache so that we can clear any CoW mappings * in the destination file. */ - truncate_inode_pages_range(&VFS_I(dest)->i_data, destoff, - PAGE_ALIGN(destoff + len) - 1); + truncate_inode_pages_range(&inode_out->i_data, pos_out, + PAGE_ALIGN(pos_out + len) - 1); - dfsbno = XFS_B_TO_FSBT(mp, destoff); - sfsbno = XFS_B_TO_FSBT(mp, srcoff); + dfsbno = XFS_B_TO_FSBT(mp, pos_out); + sfsbno = XFS_B_TO_FSBT(mp, pos_in); fsblen = XFS_B_TO_FSB(mp, len); - error = xfs_reflink_remap_blocks(src, sfsbno, dest, dfsbno, fsblen, - destoff + len); - if (error) - goto out_error; + ret = xfs_reflink_remap_blocks(src, sfsbno, dest, dfsbno, fsblen, + pos_out + len); + if (ret) + goto out_unlock; /* * Carry the cowextsize hint from src to dest if we're sharing the @@ -1390,26 +1437,24 @@ xfs_reflink_remap_range( * has a cowextsize hint, and the destination file does not. */ cowextsize = 0; - if (srcoff == 0 && len == i_size_read(VFS_I(src)) && + if (pos_in == 0 && len == i_size_read(inode_in) && (src->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE) && - destoff == 0 && len >= i_size_read(VFS_I(dest)) && + pos_out == 0 && len >= i_size_read(inode_out) && !(dest->i_d.di_flags2 & XFS_DIFLAG2_COWEXTSIZE)) cowextsize = src->i_d.di_cowextsize; - error = xfs_reflink_update_dest(dest, destoff + len, cowextsize); - if (error) - goto out_error; + ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize); -out_error: +out_unlock: xfs_iunlock(src, XFS_MMAPLOCK_EXCL); xfs_iunlock(src, XFS_IOLOCK_EXCL); if (src->i_ino != dest->i_ino) { xfs_iunlock(dest, XFS_MMAPLOCK_EXCL); xfs_iunlock(dest, XFS_IOLOCK_EXCL); } - if (error) - trace_xfs_reflink_remap_range_error(dest, error, _RET_IP_); - return error; + if (ret) + trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_); + return ret; } /* diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h index 5dc3c8ac12aa..fad11607c9ad 100644 --- a/fs/xfs/xfs_reflink.h +++ b/fs/xfs/xfs_reflink.h @@ -26,8 +26,8 @@ extern int xfs_reflink_find_shared(struct xfs_mount *mp, xfs_agnumber_t agno, extern int xfs_reflink_trim_around_shared(struct xfs_inode *ip, struct xfs_bmbt_irec *irec, bool *shared, bool *trimmed); -extern int xfs_reflink_reserve_cow_range(struct xfs_inode *ip, - xfs_off_t offset, xfs_off_t count); +extern int xfs_reflink_reserve_cow(struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, bool *shared); extern int xfs_reflink_allocate_cow_range(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t count); extern bool xfs_reflink_find_cow_mapping(struct xfs_inode *ip, xfs_off_t offset, @@ -43,11 +43,8 @@ extern int xfs_reflink_cancel_cow_range(struct xfs_inode *ip, xfs_off_t offset, extern int xfs_reflink_end_cow(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t count); extern int xfs_reflink_recover_cow(struct xfs_mount *mp); -#define XFS_REFLINK_DEDUPE 1 /* only reflink if contents match */ -#define XFS_REFLINK_ALL (XFS_REFLINK_DEDUPE) -extern int xfs_reflink_remap_range(struct xfs_inode *src, xfs_off_t srcoff, - struct xfs_inode *dest, xfs_off_t destoff, xfs_off_t len, - unsigned int flags); +extern int xfs_reflink_remap_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, u64 len, bool is_dedupe); extern int xfs_reflink_clear_inode_flag(struct xfs_inode *ip, struct xfs_trans **tpp); extern int xfs_reflink_unshare(struct xfs_inode *ip, xfs_off_t offset, diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c index 5f8d55d29a11..276d3023d60f 100644 --- a/fs/xfs/xfs_sysfs.c +++ b/fs/xfs/xfs_sysfs.c @@ -512,13 +512,13 @@ static struct attribute *xfs_error_attrs[] = { }; -struct kobj_type xfs_error_cfg_ktype = { +static struct kobj_type xfs_error_cfg_ktype = { .release = xfs_sysfs_release, .sysfs_ops = &xfs_sysfs_ops, .default_attrs = xfs_error_attrs, }; -struct kobj_type xfs_error_ktype = { +static struct kobj_type xfs_error_ktype = { .release = xfs_sysfs_release, .sysfs_ops = &xfs_sysfs_ops, }; diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index ad188d3a83f3..0907752be62d 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -3346,7 +3346,7 @@ DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_alloc); DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_found); DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_enospc); -DEFINE_RW_EVENT(xfs_reflink_reserve_cow_range); +DEFINE_RW_EVENT(xfs_reflink_reserve_cow); DEFINE_RW_EVENT(xfs_reflink_allocate_cow_range); DEFINE_INODE_IREC_EVENT(xfs_reflink_bounce_dio_write); @@ -3356,9 +3356,7 @@ DEFINE_INODE_IREC_EVENT(xfs_reflink_trim_irec); DEFINE_SIMPLE_IO_EVENT(xfs_reflink_cancel_cow_range); DEFINE_SIMPLE_IO_EVENT(xfs_reflink_end_cow); DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap); -DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_piece); -DEFINE_INODE_ERROR_EVENT(xfs_reflink_reserve_cow_range_error); DEFINE_INODE_ERROR_EVENT(xfs_reflink_allocate_cow_range_error); DEFINE_INODE_ERROR_EVENT(xfs_reflink_cancel_cow_range_error); DEFINE_INODE_ERROR_EVENT(xfs_reflink_end_cow_error); diff --git a/include/asm-generic/export.h b/include/asm-generic/export.h index 43199a049da5..63554e9f6e0c 100644 --- a/include/asm-generic/export.h +++ b/include/asm-generic/export.h @@ -70,7 +70,7 @@ KSYM(__kcrctab_\name): #include <generated/autoksyms.h> #define __EXPORT_SYMBOL(sym, val, sec) \ - __cond_export_sym(sym, val, sec, config_enabled(__KSYM_##sym)) + __cond_export_sym(sym, val, sec, __is_defined(__KSYM_##sym)) #define __cond_export_sym(sym, val, sec, conf) \ ___cond_export_sym(sym, val, sec, conf) #define ___cond_export_sym(sym, val, sec, enabled) \ diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h index 63d4e5051936..5b38eb94783b 100644 --- a/include/drm/drm_plane.h +++ b/include/drm/drm_plane.h @@ -43,8 +43,14 @@ struct drm_printer; * @src_h: height of visible portion of plane (in 16.16) * @rotation: rotation of the plane * @zpos: priority of the given plane on crtc (optional) + * Note that multiple active planes on the same crtc can have an identical + * zpos value. The rule to solving the conflict is to compare the plane + * object IDs; the plane with a higher ID must be stacked on top of a + * plane with a lower ID. * @normalized_zpos: normalized value of zpos: unique, range from 0 to N-1 - * where N is the number of active planes for given crtc + * where N is the number of active planes for given crtc. Note that + * the driver must call drm_atomic_normalize_zpos() to update this before + * it can be trusted. * @src: clipped source coordinates of the plane (in 16.16) * @dst: clipped destination coordinates of the plane * @state: backpointer to global drm_atomic_state diff --git a/include/linux/acpi.h b/include/linux/acpi.h index ddbeda6dbdc8..689a8b9b9c8f 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -326,6 +326,7 @@ struct pci_dev; int acpi_pci_irq_enable (struct pci_dev *dev); void acpi_penalize_isa_irq(int irq, int active); bool acpi_isa_irq_available(int irq); +void acpi_penalize_sci_irq(int irq, int trigger, int polarity); void acpi_pci_irq_disable (struct pci_dev *dev); extern int ec_read(u8 addr, u8 *val); diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index af596381fa0f..a428aec36ace 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -785,7 +785,7 @@ extern struct of_device_id __clk_of_table; * routines, one at of_clk_init(), and one at platform device probe */ #define CLK_OF_DECLARE_DRIVER(name, compat, fn) \ - static void name##_of_clk_init_driver(struct device_node *np) \ + static void __init name##_of_clk_init_driver(struct device_node *np) \ { \ of_node_clear_flag(np, OF_POPULATED); \ fn(np); \ diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 6824556d37ed..cd184bdca58f 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1169,13 +1169,6 @@ int __must_check __vmbus_driver_register(struct hv_driver *hv_driver, const char *mod_name); void vmbus_driver_unregister(struct hv_driver *hv_driver); -static inline const char *vmbus_dev_name(const struct hv_device *device_obj) -{ - const struct kobject *kobj = &device_obj->device.kobj; - - return kobj->name; -} - void vmbus_hvsock_device_unregister(struct vmbus_channel *channel); int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, diff --git a/include/linux/io.h b/include/linux/io.h index e2c8419278c1..82ef36eac8a1 100644 --- a/include/linux/io.h +++ b/include/linux/io.h @@ -141,4 +141,26 @@ enum { void *memremap(resource_size_t offset, size_t size, unsigned long flags); void memunmap(void *addr); +/* + * On x86 PAT systems we have memory tracking that keeps track of + * the allowed mappings on memory ranges. This tracking works for + * all the in-kernel mapping APIs (ioremap*), but where the user + * wishes to map a range from a physical device into user memory + * the tracking won't be updated. This API is to be used by + * drivers which remap physical device pages into userspace, + * and wants to make sure they are mapped WC and not UC. + */ +#ifndef arch_io_reserve_memtype_wc +static inline int arch_io_reserve_memtype_wc(resource_size_t base, + resource_size_t size) +{ + return 0; +} + +static inline void arch_io_free_memtype_wc(resource_size_t base, + resource_size_t size) +{ +} +#endif + #endif /* _LINUX_IO_H */ diff --git a/include/linux/iomap.h b/include/linux/iomap.h index e63e288dee83..7892f55a1866 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -19,11 +19,15 @@ struct vm_fault; #define IOMAP_UNWRITTEN 0x04 /* blocks allocated @blkno in unwritten state */ /* - * Flags for iomap mappings: + * Flags for all iomap mappings: */ -#define IOMAP_F_MERGED 0x01 /* contains multiple blocks/extents */ -#define IOMAP_F_SHARED 0x02 /* block shared with another file */ -#define IOMAP_F_NEW 0x04 /* blocks have been newly allocated */ +#define IOMAP_F_NEW 0x01 /* blocks have been newly allocated */ + +/* + * Flags that only need to be reported for IOMAP_REPORT requests: + */ +#define IOMAP_F_MERGED 0x10 /* contains multiple blocks/extents */ +#define IOMAP_F_SHARED 0x20 /* block shared with another file */ /* * Magic value for blkno: @@ -42,8 +46,9 @@ struct iomap { /* * Flags for iomap_begin / iomap_end. No flag implies a read. */ -#define IOMAP_WRITE (1 << 0) -#define IOMAP_ZERO (1 << 1) +#define IOMAP_WRITE (1 << 0) /* writing, must allocate blocks */ +#define IOMAP_ZERO (1 << 1) /* zeroing operation, may skip holes */ +#define IOMAP_REPORT (1 << 2) /* report extent status, e.g. FIEMAP */ struct iomap_ops { /* diff --git a/include/linux/ipv6.h b/include/linux/ipv6.h index 7e9a789be5e0..ca1ad9ebbc92 100644 --- a/include/linux/ipv6.h +++ b/include/linux/ipv6.h @@ -123,12 +123,12 @@ struct inet6_skb_parm { }; #if defined(CONFIG_NET_L3_MASTER_DEV) -static inline bool skb_l3mdev_slave(__u16 flags) +static inline bool ipv6_l3mdev_skb(__u16 flags) { return flags & IP6SKB_L3SLAVE; } #else -static inline bool skb_l3mdev_slave(__u16 flags) +static inline bool ipv6_l3mdev_skb(__u16 flags) { return false; } @@ -139,11 +139,22 @@ static inline bool skb_l3mdev_slave(__u16 flags) static inline int inet6_iif(const struct sk_buff *skb) { - bool l3_slave = skb_l3mdev_slave(IP6CB(skb)->flags); + bool l3_slave = ipv6_l3mdev_skb(IP6CB(skb)->flags); return l3_slave ? skb->skb_iif : IP6CB(skb)->iif; } +/* can not be used in TCP layer after tcp_v6_fill_cb */ +static inline bool inet6_exact_dif_match(struct net *net, struct sk_buff *skb) +{ +#if defined(CONFIG_NET_L3_MASTER_DEV) + if (!net->ipv4.sysctl_tcp_l3mdev_accept && + ipv6_l3mdev_skb(IP6CB(skb)->flags)) + return true; +#endif + return false; +} + struct tcp6_request_sock { struct tcp_request_sock tcp6rsk_tcp; }; diff --git a/include/linux/kconfig.h b/include/linux/kconfig.h index 15ec117ec537..8f2e059e4d45 100644 --- a/include/linux/kconfig.h +++ b/include/linux/kconfig.h @@ -31,7 +31,6 @@ * When CONFIG_BOOGER is not defined, we generate a (... 1, 0) pair, and when * the last step cherry picks the 2nd arg, we get a zero. */ -#define config_enabled(cfg) ___is_defined(cfg) #define __is_defined(x) ___is_defined(x) #define ___is_defined(val) ____is_defined(__ARG_PLACEHOLDER_##val) #define ____is_defined(arg1_or_junk) __take_second_arg(arg1_or_junk 1, 0) @@ -41,13 +40,13 @@ * otherwise. For boolean options, this is equivalent to * IS_ENABLED(CONFIG_FOO). */ -#define IS_BUILTIN(option) config_enabled(option) +#define IS_BUILTIN(option) __is_defined(option) /* * IS_MODULE(CONFIG_FOO) evaluates to 1 if CONFIG_FOO is set to 'm', 0 * otherwise. */ -#define IS_MODULE(option) config_enabled(option##_MODULE) +#define IS_MODULE(option) __is_defined(option##_MODULE) /* * IS_REACHABLE(CONFIG_FOO) evaluates to 1 if the currently compiled diff --git a/include/linux/mlx4/device.h b/include/linux/mlx4/device.h index f6a164297358..3be7abd6e722 100644 --- a/include/linux/mlx4/device.h +++ b/include/linux/mlx4/device.h @@ -1399,7 +1399,8 @@ void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u32 *lkey, u32 *rkey); int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr); int mlx4_SYNC_TPT(struct mlx4_dev *dev); -int mlx4_test_interrupts(struct mlx4_dev *dev); +int mlx4_test_interrupt(struct mlx4_dev *dev, int vector); +int mlx4_test_async(struct mlx4_dev *dev); int mlx4_query_diag_counters(struct mlx4_dev *dev, u8 op_modifier, const u32 offset[], u32 value[], size_t array_len, u8 port); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 85c4786427e4..ecc451d89ccd 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -418,8 +418,12 @@ struct mlx5_core_health { u32 prev; int miss_counter; bool sick; + /* wq spinlock to synchronize draining */ + spinlock_t wq_lock; struct workqueue_struct *wq; + unsigned long flags; struct work_struct work; + struct delayed_work recover_work; }; struct mlx5_cq_table { @@ -626,10 +630,6 @@ struct mlx5_db { }; enum { - MLX5_DB_PER_PAGE = PAGE_SIZE / L1_CACHE_BYTES, -}; - -enum { MLX5_COMP_EQ_SIZE = 1024, }; @@ -638,13 +638,6 @@ enum { MLX5_PTYS_EN = 1 << 2, }; -struct mlx5_db_pgdir { - struct list_head list; - DECLARE_BITMAP(bitmap, MLX5_DB_PER_PAGE); - __be32 *db_page; - dma_addr_t db_dma; -}; - typedef void (*mlx5_cmd_cbk_t)(int status, void *context); struct mlx5_cmd_work_ent { @@ -789,6 +782,7 @@ void mlx5_health_cleanup(struct mlx5_core_dev *dev); int mlx5_health_init(struct mlx5_core_dev *dev); void mlx5_start_health_poll(struct mlx5_core_dev *dev); void mlx5_stop_health_poll(struct mlx5_core_dev *dev); +void mlx5_drain_health_wq(struct mlx5_core_dev *dev); int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf, int node); int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, struct mlx5_buf *buf); diff --git a/include/linux/mm.h b/include/linux/mm.h index 3a191853faaa..a92c8d73aeaf 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1271,10 +1271,6 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void * extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); -long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, - unsigned long start, unsigned long nr_pages, - unsigned int foll_flags, struct page **pages, - struct vm_area_struct **vmas, int *nonblocking); long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 7f2ae99e5daf..0f088f3a2fed 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -440,33 +440,7 @@ struct zone { seqlock_t span_seqlock; #endif - /* - * wait_table -- the array holding the hash table - * wait_table_hash_nr_entries -- the size of the hash table array - * wait_table_bits -- wait_table_size == (1 << wait_table_bits) - * - * The purpose of all these is to keep track of the people - * waiting for a page to become available and make them - * runnable again when possible. The trouble is that this - * consumes a lot of space, especially when so few things - * wait on pages at a given time. So instead of using - * per-page waitqueues, we use a waitqueue hash table. - * - * The bucket discipline is to sleep on the same queue when - * colliding and wake all in that wait queue when removing. - * When something wakes, it must check to be sure its page is - * truly available, a la thundering herd. The cost of a - * collision is great, but given the expected load of the - * table, they should be so rare as to be outweighed by the - * benefits from the saved space. - * - * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the - * primary users of these fields, and in mm/page_alloc.c - * free_area_init_core() performs the initialization of them. - */ - wait_queue_head_t *wait_table; - unsigned long wait_table_hash_nr_entries; - unsigned long wait_table_bits; + int initialized; /* Write-intensive fields used from the page allocator */ ZONE_PADDING(_pad1_) @@ -546,7 +520,7 @@ static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn) static inline bool zone_is_initialized(struct zone *zone) { - return !!zone->wait_table; + return zone->initialized; } static inline bool zone_is_empty(struct zone *zone) diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h index c5d3d5024fc8..d8905a229f34 100644 --- a/include/linux/mtd/nand.h +++ b/include/linux/mtd/nand.h @@ -1184,7 +1184,7 @@ int nand_read_oob_syndrome(struct mtd_info *mtd, struct nand_chip *chip, int page); /* Reset and initialize a NAND device */ -int nand_reset(struct nand_chip *chip); +int nand_reset(struct nand_chip *chip, int chipnr); /* Free resources held by the NAND device */ void nand_cleanup(struct nand_chip *chip); diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 136ae6bbe81e..91ee3643ccc8 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -2169,7 +2169,10 @@ struct napi_gro_cb { /* Used to determine if flush_id can be ignored */ u8 is_atomic:1; - /* 5 bit hole */ + /* Number of gro_receive callbacks this packet already went through */ + u8 recursion_counter:4; + + /* 1 bit hole */ /* used to support CHECKSUM_COMPLETE for tunneling protocols */ __wsum csum; @@ -2180,6 +2183,40 @@ struct napi_gro_cb { #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) +#define GRO_RECURSION_LIMIT 15 +static inline int gro_recursion_inc_test(struct sk_buff *skb) +{ + return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; +} + +typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *); +static inline struct sk_buff **call_gro_receive(gro_receive_t cb, + struct sk_buff **head, + struct sk_buff *skb) +{ + if (unlikely(gro_recursion_inc_test(skb))) { + NAPI_GRO_CB(skb)->flush |= 1; + return NULL; + } + + return cb(head, skb); +} + +typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **, + struct sk_buff *); +static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb, + struct sock *sk, + struct sk_buff **head, + struct sk_buff *skb) +{ + if (unlikely(gro_recursion_inc_test(skb))) { + NAPI_GRO_CB(skb)->flush |= 1; + return NULL; + } + + return cb(sk, head, skb); +} + struct packet_type { __be16 type; /* This is really htons(ether_type). */ struct net_device *dev; /* NULL is wildcarded here */ @@ -3877,7 +3914,7 @@ struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev, ldev = netdev_all_lower_get_next(dev, &(iter))) #define netdev_for_each_all_lower_dev_rcu(dev, ldev, iter) \ - for (iter = (dev)->all_adj_list.lower.next, \ + for (iter = &(dev)->all_adj_list.lower, \ ldev = netdev_all_lower_get_next_rcu(dev, &(iter)); \ ldev; \ ldev = netdev_all_lower_get_next_rcu(dev, &(iter))) diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 060d0ede88df..4741ecdb9817 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1257,6 +1257,7 @@ extern u64 perf_swevent_set_period(struct perf_event *event); extern void perf_event_enable(struct perf_event *event); extern void perf_event_disable(struct perf_event *event); extern void perf_event_disable_local(struct perf_event *event); +extern void perf_event_disable_inatomic(struct perf_event *event); extern void perf_event_task_tick(void); #else /* !CONFIG_PERF_EVENTS: */ static inline void * diff --git a/include/linux/qed/qed_if.h b/include/linux/qed/qed_if.h index f9ae903bbb84..8978a60371f4 100644 --- a/include/linux/qed/qed_if.h +++ b/include/linux/qed/qed_if.h @@ -146,6 +146,7 @@ enum qed_led_mode { #define DIRECT_REG_RD(reg_addr) readl((void __iomem *)(reg_addr)) #define QED_COALESCE_MAX 0xFF +#define QED_DEFAULT_RX_USECS 12 /* forward */ struct qed_dev; diff --git a/include/linux/qed/qede_roce.h b/include/linux/qed/qede_roce.h index 99fbe6d55acb..f48d64b0e2fb 100644 --- a/include/linux/qed/qede_roce.h +++ b/include/linux/qed/qede_roce.h @@ -68,7 +68,7 @@ void qede_roce_unregister_driver(struct qedr_driver *drv); bool qede_roce_supported(struct qede_dev *dev); -#if IS_ENABLED(CONFIG_INFINIBAND_QEDR) +#if IS_ENABLED(CONFIG_QED_RDMA) int qede_roce_dev_add(struct qede_dev *dev); void qede_roce_dev_event_open(struct qede_dev *dev); void qede_roce_dev_event_close(struct qede_dev *dev); diff --git a/include/linux/regmap.h b/include/linux/regmap.h index 9adc7b21903d..f6673132431d 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -15,6 +15,7 @@ #include <linux/list.h> #include <linux/rbtree.h> +#include <linux/delay.h> #include <linux/err.h> #include <linux/bug.h> #include <linux/lockdep.h> @@ -116,22 +117,22 @@ struct reg_sequence { #define regmap_read_poll_timeout(map, addr, val, cond, sleep_us, timeout_us) \ ({ \ ktime_t timeout = ktime_add_us(ktime_get(), timeout_us); \ - int ret; \ + int pollret; \ might_sleep_if(sleep_us); \ for (;;) { \ - ret = regmap_read((map), (addr), &(val)); \ - if (ret) \ + pollret = regmap_read((map), (addr), &(val)); \ + if (pollret) \ break; \ if (cond) \ break; \ if (timeout_us && ktime_compare(ktime_get(), timeout) > 0) { \ - ret = regmap_read((map), (addr), &(val)); \ + pollret = regmap_read((map), (addr), &(val)); \ break; \ } \ if (sleep_us) \ usleep_range((sleep_us >> 2) + 1, sleep_us); \ } \ - ret ?: ((cond) ? 0 : -ETIMEDOUT); \ + pollret ?: ((cond) ? 0 : -ETIMEDOUT); \ }) #ifdef CONFIG_REGMAP diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 601258f6e621..32810f279f8e 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -936,6 +936,7 @@ struct sk_buff_fclones { /** * skb_fclone_busy - check if fclone is busy + * @sk: socket * @skb: buffer * * Returns true if skb is a fast clone, and its clone is not freed. diff --git a/include/net/addrconf.h b/include/net/addrconf.h index f2d072787947..8f998afc1384 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -174,6 +174,7 @@ int ipv6_sock_mc_join(struct sock *sk, int ifindex, const struct in6_addr *addr); int ipv6_sock_mc_drop(struct sock *sk, int ifindex, const struct in6_addr *addr); +void __ipv6_sock_mc_close(struct sock *sk); void ipv6_sock_mc_close(struct sock *sk); bool inet6_mc_check(struct sock *sk, const struct in6_addr *mc_addr, const struct in6_addr *src_addr); diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index bd19faad0d96..14b51d739c3b 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -4047,14 +4047,29 @@ unsigned int ieee80211_get_mesh_hdrlen(struct ieee80211s_hdr *meshhdr); */ /** + * ieee80211_data_to_8023_exthdr - convert an 802.11 data frame to 802.3 + * @skb: the 802.11 data frame + * @ehdr: pointer to a &struct ethhdr that will get the header, instead + * of it being pushed into the SKB + * @addr: the device MAC address + * @iftype: the virtual interface type + * Return: 0 on success. Non-zero on error. + */ +int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, + const u8 *addr, enum nl80211_iftype iftype); + +/** * ieee80211_data_to_8023 - convert an 802.11 data frame to 802.3 * @skb: the 802.11 data frame * @addr: the device MAC address * @iftype: the virtual interface type * Return: 0 on success. Non-zero on error. */ -int ieee80211_data_to_8023(struct sk_buff *skb, const u8 *addr, - enum nl80211_iftype iftype); +static inline int ieee80211_data_to_8023(struct sk_buff *skb, const u8 *addr, + enum nl80211_iftype iftype) +{ + return ieee80211_data_to_8023_exthdr(skb, NULL, addr, iftype); +} /** * ieee80211_data_from_8023 - convert an 802.3 frame to 802.11 @@ -4072,22 +4087,23 @@ int ieee80211_data_from_8023(struct sk_buff *skb, const u8 *addr, /** * ieee80211_amsdu_to_8023s - decode an IEEE 802.11n A-MSDU frame * - * Decode an IEEE 802.11n A-MSDU frame and convert it to a list of - * 802.3 frames. The @list will be empty if the decode fails. The - * @skb is consumed after the function returns. + * Decode an IEEE 802.11 A-MSDU and convert it to a list of 802.3 frames. + * The @list will be empty if the decode fails. The @skb must be fully + * header-less before being passed in here; it is freed in this function. * - * @skb: The input IEEE 802.11n A-MSDU frame. + * @skb: The input A-MSDU frame without any headers. * @list: The output list of 802.3 frames. It must be allocated and * initialized by by the caller. * @addr: The device MAC address. * @iftype: The device interface type. * @extra_headroom: The hardware extra headroom for SKBs in the @list. - * @has_80211_header: Set it true if SKB is with IEEE 802.11 header. + * @check_da: DA to check in the inner ethernet header, or NULL + * @check_sa: SA to check in the inner ethernet header, or NULL */ void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list, const u8 *addr, enum nl80211_iftype iftype, const unsigned int extra_headroom, - bool has_80211_header); + const u8 *check_da, const u8 *check_sa); /** * cfg80211_classify8021d - determine the 802.1p/1d tag for a data frame diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index 515352c6280a..b0576cb2ab25 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -190,8 +190,8 @@ struct inet6_dev { __u32 if_flags; int dead; + u32 desync_factor; u8 rndid[8]; - struct timer_list regen_timer; struct list_head tempaddr_list; struct in6_addr token; diff --git a/include/net/ip.h b/include/net/ip.h index bc43c0fcae12..5413883ac47f 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -38,7 +38,7 @@ struct sock; struct inet_skb_parm { int iif; struct ip_options opt; /* Compiled IP options */ - unsigned char flags; + u16 flags; #define IPSKB_FORWARDED BIT(0) #define IPSKB_XFRM_TUNNEL_SIZE BIT(1) @@ -48,10 +48,16 @@ struct inet_skb_parm { #define IPSKB_DOREDIRECT BIT(5) #define IPSKB_FRAG_PMTU BIT(6) #define IPSKB_FRAG_SEGS BIT(7) +#define IPSKB_L3SLAVE BIT(8) u16 frag_max_size; }; +static inline bool ipv4_l3mdev_skb(u16 flags) +{ + return !!(flags & IPSKB_L3SLAVE); +} + static inline unsigned int ip_hdrlen(const struct sk_buff *skb) { return ip_hdr(skb)->ihl * 4; @@ -572,7 +578,7 @@ int ip_options_rcv_srr(struct sk_buff *skb); */ void ipv4_pktinfo_prepare(const struct sock *sk, struct sk_buff *skb); -void ip_cmsg_recv_offset(struct msghdr *msg, struct sk_buff *skb, int offset); +void ip_cmsg_recv_offset(struct msghdr *msg, struct sk_buff *skb, int tlen, int offset); int ip_cmsg_send(struct sock *sk, struct msghdr *msg, struct ipcm_cookie *ipc, bool allow_ipv6); int ip_setsockopt(struct sock *sk, int level, int optname, char __user *optval, @@ -594,7 +600,7 @@ void ip_local_error(struct sock *sk, int err, __be32 daddr, __be16 dport, static inline void ip_cmsg_recv(struct msghdr *msg, struct sk_buff *skb) { - ip_cmsg_recv_offset(msg, skb, 0); + ip_cmsg_recv_offset(msg, skb, 0, 0); } bool icmp_global_allow(void); diff --git a/include/net/ip6_fib.h b/include/net/ip6_fib.h index fb961a576abe..a74e2aa40ef4 100644 --- a/include/net/ip6_fib.h +++ b/include/net/ip6_fib.h @@ -230,6 +230,8 @@ struct fib6_table { rwlock_t tb6_lock; struct fib6_node tb6_root; struct inet_peer_base tb6_peers; + unsigned int flags; +#define RT6_TABLE_HAS_DFLT_ROUTER BIT(0) }; #define RT6_TABLE_UNSPEC RT_TABLE_UNSPEC diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index e0cd318d5103..f83e78d071a3 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -32,6 +32,7 @@ struct route_info { #define RT6_LOOKUP_F_SRCPREF_TMP 0x00000008 #define RT6_LOOKUP_F_SRCPREF_PUBLIC 0x00000010 #define RT6_LOOKUP_F_SRCPREF_COA 0x00000020 +#define RT6_LOOKUP_F_IGNORE_LINKSTATE 0x00000040 /* We do not (yet ?) support IPv6 jumbograms (RFC 2675) * Unlike IPv4, hdr->seg_len doesn't include the IPv6 header diff --git a/include/net/mac80211.h b/include/net/mac80211.h index a810dfcb83c2..e2dba93e374f 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -811,14 +811,18 @@ enum mac80211_rate_control_flags { * in the control information, and it will be filled by the rate * control algorithm according to what should be sent. For example, * if this array contains, in the format { <idx>, <count> } the - * information + * information:: + * * { 3, 2 }, { 2, 2 }, { 1, 4 }, { -1, 0 }, { -1, 0 } + * * then this means that the frame should be transmitted * up to twice at rate 3, up to twice at rate 2, and up to four * times at rate 1 if it doesn't get acknowledged. Say it gets * acknowledged by the peer after the fifth attempt, the status - * information should then contain + * information should then contain:: + * * { 3, 2 }, { 2, 2 }, { 1, 1 }, { -1, 0 } ... + * * since it was transmitted twice at rate 3, twice at rate 2 * and once at rate 1 after which we received an acknowledgement. */ @@ -1168,8 +1172,8 @@ enum mac80211_rx_vht_flags { * @rate_idx: index of data rate into band's supported rates or MCS index if * HT or VHT is used (%RX_FLAG_HT/%RX_FLAG_VHT) * @vht_nss: number of streams (VHT only) - * @flag: %RX_FLAG_* - * @vht_flag: %RX_VHT_FLAG_* + * @flag: %RX_FLAG_\* + * @vht_flag: %RX_VHT_FLAG_\* * @rx_flags: internal RX flags for mac80211 * @ampdu_reference: A-MPDU reference number, must be a different value for * each A-MPDU but the same for each subframe within one A-MPDU @@ -1432,7 +1436,7 @@ enum ieee80211_vif_flags { * @probe_req_reg: probe requests should be reported to mac80211 for this * interface. * @drv_priv: data area for driver use, will always be aligned to - * sizeof(void *). + * sizeof(void \*). * @txq: the multicast data TX queue (if driver uses the TXQ abstraction) */ struct ieee80211_vif { @@ -1743,7 +1747,7 @@ struct ieee80211_sta_rates { * @wme: indicates whether the STA supports QoS/WME (if local devices does, * otherwise always false) * @drv_priv: data area for driver use, will always be aligned to - * sizeof(void *), size is determined in hw information. + * sizeof(void \*), size is determined in hw information. * @uapsd_queues: bitmap of queues configured for uapsd. Only valid * if wme is supported. * @max_sp: max Service Period. Only valid if wme is supported. @@ -2146,12 +2150,12 @@ enum ieee80211_hw_flags { * * @radiotap_mcs_details: lists which MCS information can the HW * reports, by default it is set to _MCS, _GI and _BW but doesn't - * include _FMT. Use %IEEE80211_RADIOTAP_MCS_HAVE_* values, only + * include _FMT. Use %IEEE80211_RADIOTAP_MCS_HAVE_\* values, only * adding _BW is supported today. * * @radiotap_vht_details: lists which VHT MCS information the HW reports, * the default is _GI | _BANDWIDTH. - * Use the %IEEE80211_RADIOTAP_VHT_KNOWN_* values. + * Use the %IEEE80211_RADIOTAP_VHT_KNOWN_\* values. * * @radiotap_timestamp: Information for the radiotap timestamp field; if the * 'units_pos' member is set to a non-negative value it must be set to @@ -2486,6 +2490,7 @@ void ieee80211_free_txskb(struct ieee80211_hw *hw, struct sk_buff *skb); * in the software stack cares about, we will, in the future, have mac80211 * tell the driver which information elements are interesting in the sense * that we want to see changes in them. This will include + * * - a list of information element IDs * - a list of OUIs for the vendor information element * diff --git a/include/net/sock.h b/include/net/sock.h index ebf75db08e06..73c6b008f1b7 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -252,6 +252,7 @@ struct sock_common { * @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler) * @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE) * @sk_sndbuf: size of send buffer in bytes + * @sk_padding: unused element for alignment * @sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets * @sk_no_check_rx: allow zero checksum in RX packets * @sk_route_caps: route capabilities (e.g. %NETIF_F_TSO) @@ -302,7 +303,8 @@ struct sock_common { * @sk_backlog_rcv: callback to process the backlog * @sk_destruct: called at sock freeing time, i.e. when all refcnt == 0 * @sk_reuseport_cb: reuseport group container - */ + * @sk_rcu: used during RCU grace period + */ struct sock { /* * Now struct inet_timewait_sock also uses sock_common, so please just diff --git a/include/net/tcp.h b/include/net/tcp.h index f83b7f220a65..5b82d4d94834 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -794,12 +794,23 @@ struct tcp_skb_cb { */ static inline int tcp_v6_iif(const struct sk_buff *skb) { - bool l3_slave = skb_l3mdev_slave(TCP_SKB_CB(skb)->header.h6.flags); + bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif; } #endif +/* TCP_SKB_CB reference means this can not be used from early demux */ +static inline bool inet_exact_dif_match(struct net *net, struct sk_buff *skb) +{ +#if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) + if (!net->ipv4.sysctl_tcp_l3mdev_accept && + ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags)) + return true; +#endif + return false; +} + /* Due to TSO, an SKB can be composed of multiple actual * packets. To keep these tracked properly, we use this. */ diff --git a/include/net/udp.h b/include/net/udp.h index ea53a87d880f..4948790d393d 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -258,6 +258,7 @@ void udp_flush_pending_frames(struct sock *sk); void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst); int udp_rcv(struct sk_buff *skb); int udp_ioctl(struct sock *sk, int cmd, unsigned long arg); +int __udp_disconnect(struct sock *sk, int flags); int udp_disconnect(struct sock *sk, int flags); unsigned int udp_poll(struct file *file, struct socket *sock, poll_table *wait); struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb, diff --git a/include/net/vxlan.h b/include/net/vxlan.h index 0255613a54a4..308adc4154f4 100644 --- a/include/net/vxlan.h +++ b/include/net/vxlan.h @@ -225,9 +225,9 @@ struct vxlan_config { struct vxlan_dev { struct hlist_node hlist; /* vni hash table */ struct list_head next; /* vxlan's per namespace list */ - struct vxlan_sock *vn4_sock; /* listening socket for IPv4 */ + struct vxlan_sock __rcu *vn4_sock; /* listening socket for IPv4 */ #if IS_ENABLED(CONFIG_IPV6) - struct vxlan_sock *vn6_sock; /* listening socket for IPv6 */ + struct vxlan_sock __rcu *vn6_sock; /* listening socket for IPv6 */ #endif struct net_device *dev; struct net *net; /* netns for packet i/o */ diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index 099a4200732c..8e547231c1b7 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -119,8 +119,7 @@ struct ethtool_cmd { static inline void ethtool_cmd_speed_set(struct ethtool_cmd *ep, __u32 speed) { - - ep->speed = (__u16)speed; + ep->speed = (__u16)(speed & 0xFFFF); ep->speed_hi = (__u16)(speed >> 16); } diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h index 262f0379d83a..5a78be518101 100644 --- a/include/uapi/linux/rtnetlink.h +++ b/include/uapi/linux/rtnetlink.h @@ -350,7 +350,7 @@ struct rtnexthop { #define RTNH_F_OFFLOAD 8 /* offloaded route */ #define RTNH_F_LINKDOWN 16 /* carrier-down on nexthop */ -#define RTNH_COMPARE_MASK (RTNH_F_DEAD | RTNH_F_LINKDOWN) +#define RTNH_COMPARE_MASK (RTNH_F_DEAD | RTNH_F_LINKDOWN | RTNH_F_OFFLOAD) /* Macros to handle hexthops */ diff --git a/include/video/display_timing.h b/include/video/display_timing.h index 28d9d0d566ca..3d289e990aca 100644 --- a/include/video/display_timing.h +++ b/include/video/display_timing.h @@ -28,6 +28,10 @@ enum display_flags { DISPLAY_FLAGS_INTERLACED = BIT(8), DISPLAY_FLAGS_DOUBLESCAN = BIT(9), DISPLAY_FLAGS_DOUBLECLK = BIT(10), + /* drive sync on pos. edge */ + DISPLAY_FLAGS_SYNC_POSEDGE = BIT(11), + /* drive sync on neg. edge */ + DISPLAY_FLAGS_SYNC_NEGEDGE = BIT(12), }; /* diff --git a/ipc/msgutil.c b/ipc/msgutil.c index a521999de4f1..bf74eaa5c39f 100644 --- a/ipc/msgutil.c +++ b/ipc/msgutil.c @@ -53,7 +53,7 @@ static struct msg_msg *alloc_msg(size_t len) size_t alen; alen = min(len, DATALEN_MSG); - msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL); + msg = kmalloc(sizeof(*msg) + alen, GFP_KERNEL_ACCOUNT); if (msg == NULL) return NULL; @@ -65,7 +65,7 @@ static struct msg_msg *alloc_msg(size_t len) while (len > 0) { struct msg_msgseg *seg; alen = min(len, DATALEN_SEG); - seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL); + seg = kmalloc(sizeof(*seg) + alen, GFP_KERNEL_ACCOUNT); if (seg == NULL) goto out_err; *pseg = seg; diff --git a/kernel/events/core.c b/kernel/events/core.c index c6e47e97b33f..0e292132efac 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -1960,6 +1960,12 @@ void perf_event_disable(struct perf_event *event) } EXPORT_SYMBOL_GPL(perf_event_disable); +void perf_event_disable_inatomic(struct perf_event *event) +{ + event->pending_disable = 1; + irq_work_queue(&event->pending); +} + static void perf_set_shadow_time(struct perf_event *event, struct perf_event_context *ctx, u64 tstamp) @@ -7075,8 +7081,8 @@ static int __perf_event_overflow(struct perf_event *event, if (events && atomic_dec_and_test(&event->event_limit)) { ret = 1; event->pending_kill = POLL_HUP; - event->pending_disable = 1; - irq_work_queue(&event->pending); + + perf_event_disable_inatomic(event); } READ_ONCE(event->overflow_handler)(event, data, regs); @@ -8855,7 +8861,10 @@ EXPORT_SYMBOL_GPL(perf_pmu_register); void perf_pmu_unregister(struct pmu *pmu) { + int remove_device; + mutex_lock(&pmus_lock); + remove_device = pmu_bus_running; list_del_rcu(&pmu->entry); mutex_unlock(&pmus_lock); @@ -8869,10 +8878,12 @@ void perf_pmu_unregister(struct pmu *pmu) free_percpu(pmu->pmu_disable_count); if (pmu->type >= PERF_TYPE_MAX) idr_remove(&pmu_idr, pmu->type); - if (pmu->nr_addr_filters) - device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); - device_del(pmu->dev); - put_device(pmu->dev); + if (remove_device) { + if (pmu->nr_addr_filters) + device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); + device_del(pmu->dev); + put_device(pmu->dev); + } free_pmu_context(pmu); } EXPORT_SYMBOL_GPL(perf_pmu_unregister); diff --git a/kernel/fork.c b/kernel/fork.c index 623259fc794d..997ac1d584f7 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -315,6 +315,9 @@ static void account_kernel_stack(struct task_struct *tsk, int account) static void release_task_stack(struct task_struct *tsk) { + if (WARN_ON(tsk->state != TASK_DEAD)) + return; /* Better to leak the stack than to free prematurely */ + account_kernel_stack(tsk, -1); arch_release_thread_stack(tsk->stack); free_thread_stack(tsk); @@ -1862,6 +1865,7 @@ bad_fork_cleanup_count: atomic_dec(&p->cred->user->processes); exit_creds(p); bad_fork_free: + p->state = TASK_DEAD; put_task_stack(p); free_task(p); fork_out: diff --git a/kernel/kcov.c b/kernel/kcov.c index 8d44b3fea9d0..30e6d05aa5a9 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -53,8 +53,15 @@ void notrace __sanitizer_cov_trace_pc(void) /* * We are interested in code coverage as a function of a syscall inputs, * so we ignore code executed in interrupts. + * The checks for whether we are in an interrupt are open-coded, because + * 1. We can't use in_interrupt() here, since it also returns true + * when we are inside local_bh_disable() section. + * 2. We don't want to use (in_irq() | in_serving_softirq() | in_nmi()), + * since that leads to slower generated code (three separate tests, + * one for each of the flags). */ - if (!t || in_interrupt()) + if (!t || (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET + | NMI_MASK))) return; mode = READ_ONCE(t->kcov_mode); if (mode == KCOV_MODE_TRACE) { diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 1e7f5da648d9..6ccb08f57fcb 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -498,9 +498,9 @@ static int enter_state(suspend_state_t state) #ifndef CONFIG_SUSPEND_SKIP_SYNC trace_suspend_resume(TPS("sync_filesystems"), 0, true); - printk(KERN_INFO "PM: Syncing filesystems ... "); + pr_info("PM: Syncing filesystems ... "); sys_sync(); - printk("done.\n"); + pr_cont("done.\n"); trace_suspend_resume(TPS("sync_filesystems"), 0, false); #endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 94732d1ab00a..154fd689fe02 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5192,21 +5192,14 @@ void sched_show_task(struct task_struct *p) int ppid; unsigned long state = p->state; + if (!try_get_task_stack(p)) + return; if (state) state = __ffs(state) + 1; printk(KERN_INFO "%-15.15s %c", p->comm, state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); -#if BITS_PER_LONG == 32 - if (state == TASK_RUNNING) - printk(KERN_CONT " running "); - else - printk(KERN_CONT " %08lx ", thread_saved_pc(p)); -#else if (state == TASK_RUNNING) printk(KERN_CONT " running task "); - else - printk(KERN_CONT " %016lx ", thread_saved_pc(p)); -#endif #ifdef CONFIG_DEBUG_STACK_USAGE free = stack_not_used(p); #endif @@ -5221,6 +5214,7 @@ void sched_show_task(struct task_struct *p) print_worker_info(KERN_INFO, p); show_stack(p, NULL); + put_task_stack(p); } void show_state_filter(unsigned long state_filter) @@ -7515,11 +7509,27 @@ static struct kmem_cache *task_group_cache __read_mostly; DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); DECLARE_PER_CPU(cpumask_var_t, select_idle_mask); +#define WAIT_TABLE_BITS 8 +#define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS) +static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned; + +wait_queue_head_t *bit_waitqueue(void *word, int bit) +{ + const int shift = BITS_PER_LONG == 32 ? 5 : 6; + unsigned long val = (unsigned long)word << shift | bit; + + return bit_wait_table + hash_long(val, WAIT_TABLE_BITS); +} +EXPORT_SYMBOL(bit_waitqueue); + void __init sched_init(void) { int i, j; unsigned long alloc_size = 0, ptr; + for (i = 0; i < WAIT_TABLE_SIZE; i++) + init_waitqueue_head(bit_wait_table + i); + #ifdef CONFIG_FAIR_GROUP_SCHED alloc_size += 2 * nr_cpu_ids * sizeof(void **); #endif diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d941c97dfbc3..c242944f5cbd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8839,7 +8839,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) { struct sched_entity *se; struct cfs_rq *cfs_rq; - struct rq *rq; int i; tg->cfs_rq = kzalloc(sizeof(cfs_rq) * nr_cpu_ids, GFP_KERNEL); @@ -8854,8 +8853,6 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) init_cfs_bandwidth(tg_cfs_bandwidth(tg)); for_each_possible_cpu(i) { - rq = cpu_rq(i); - cfs_rq = kzalloc_node(sizeof(struct cfs_rq), GFP_KERNEL, cpu_to_node(i)); if (!cfs_rq) diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 4f7053579fe3..9453efe9b25a 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -480,16 +480,6 @@ void wake_up_bit(void *word, int bit) } EXPORT_SYMBOL(wake_up_bit); -wait_queue_head_t *bit_waitqueue(void *word, int bit) -{ - const int shift = BITS_PER_LONG == 32 ? 5 : 6; - const struct zone *zone = page_zone(virt_to_page(word)); - unsigned long val = (unsigned long)word << shift | bit; - - return &zone->wait_table[hash_long(val, zone->wait_table_bits)]; -} -EXPORT_SYMBOL(bit_waitqueue); - /* * Manipulate the atomic_t address to produce a better bit waitqueue table hash * index (we're keying off bit -1, but that would produce a horrible hash diff --git a/kernel/softirq.c b/kernel/softirq.c index 1bf81ef91375..744fa611cae0 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -58,7 +58,7 @@ static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp DEFINE_PER_CPU(struct task_struct *, ksoftirqd); const char * const softirq_to_name[NR_SOFTIRQS] = { - "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", + "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL", "TASKLET", "SCHED", "HRTIMER", "RCU" }; diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 2d47980a1bc4..c611c47de884 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -878,7 +878,7 @@ static inline struct timer_base *get_timer_base(u32 tflags) #ifdef CONFIG_NO_HZ_COMMON static inline struct timer_base * -__get_target_base(struct timer_base *base, unsigned tflags) +get_target_base(struct timer_base *base, unsigned tflags) { #ifdef CONFIG_SMP if ((tflags & TIMER_PINNED) || !base->migration_enabled) @@ -891,25 +891,27 @@ __get_target_base(struct timer_base *base, unsigned tflags) static inline void forward_timer_base(struct timer_base *base) { + unsigned long jnow = READ_ONCE(jiffies); + /* * We only forward the base when it's idle and we have a delta between * base clock and jiffies. */ - if (!base->is_idle || (long) (jiffies - base->clk) < 2) + if (!base->is_idle || (long) (jnow - base->clk) < 2) return; /* * If the next expiry value is > jiffies, then we fast forward to * jiffies otherwise we forward to the next expiry value. */ - if (time_after(base->next_expiry, jiffies)) - base->clk = jiffies; + if (time_after(base->next_expiry, jnow)) + base->clk = jnow; else base->clk = base->next_expiry; } #else static inline struct timer_base * -__get_target_base(struct timer_base *base, unsigned tflags) +get_target_base(struct timer_base *base, unsigned tflags) { return get_timer_this_cpu_base(tflags); } @@ -917,14 +919,6 @@ __get_target_base(struct timer_base *base, unsigned tflags) static inline void forward_timer_base(struct timer_base *base) { } #endif -static inline struct timer_base * -get_target_base(struct timer_base *base, unsigned tflags) -{ - struct timer_base *target = __get_target_base(base, tflags); - - forward_timer_base(target); - return target; -} /* * We are using hashed locking: Holding per_cpu(timer_bases[x]).lock means @@ -943,7 +937,14 @@ static struct timer_base *lock_timer_base(struct timer_list *timer, { for (;;) { struct timer_base *base; - u32 tf = timer->flags; + u32 tf; + + /* + * We need to use READ_ONCE() here, otherwise the compiler + * might re-read @tf between the check for TIMER_MIGRATING + * and spin_lock(). + */ + tf = READ_ONCE(timer->flags); if (!(tf & TIMER_MIGRATING)) { base = get_timer_base(tf); @@ -964,6 +965,8 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only) unsigned long clk = 0, flags; int ret = 0; + BUG_ON(!timer->function); + /* * This is a common optimization triggered by the networking code - if * the timer is re-modified to have the same timeout or ends up in the @@ -972,13 +975,16 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only) if (timer_pending(timer)) { if (timer->expires == expires) return 1; + /* - * Take the current timer_jiffies of base, but without holding - * the lock! + * We lock timer base and calculate the bucket index right + * here. If the timer ends up in the same bucket, then we + * just update the expiry time and avoid the whole + * dequeue/enqueue dance. */ - base = get_timer_base(timer->flags); - clk = base->clk; + base = lock_timer_base(timer, &flags); + clk = base->clk; idx = calc_wheel_index(expires, clk); /* @@ -988,14 +994,14 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only) */ if (idx == timer_get_idx(timer)) { timer->expires = expires; - return 1; + ret = 1; + goto out_unlock; } + } else { + base = lock_timer_base(timer, &flags); } timer_stats_timer_set_start_info(timer); - BUG_ON(!timer->function); - - base = lock_timer_base(timer, &flags); ret = detach_if_pending(timer, base, false); if (!ret && pending_only) @@ -1025,12 +1031,16 @@ __mod_timer(struct timer_list *timer, unsigned long expires, bool pending_only) } } + /* Try to forward a stale timer base clock */ + forward_timer_base(base); + timer->expires = expires; /* * If 'idx' was calculated above and the base time did not advance - * between calculating 'idx' and taking the lock, only enqueue_timer() - * and trigger_dyntick_cpu() is required. Otherwise we need to - * (re)calculate the wheel index via internal_add_timer(). + * between calculating 'idx' and possibly switching the base, only + * enqueue_timer() and trigger_dyntick_cpu() is required. Otherwise + * we need to (re)calculate the wheel index via + * internal_add_timer(). */ if (idx != UINT_MAX && clk == base->clk) { enqueue_timer(base, timer, idx); @@ -1510,12 +1520,16 @@ u64 get_next_timer_interrupt(unsigned long basej, u64 basem) is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA); base->next_expiry = nextevt; /* - * We have a fresh next event. Check whether we can forward the base: + * We have a fresh next event. Check whether we can forward the + * base. We can only do that when @basej is past base->clk + * otherwise we might rewind base->clk. */ - if (time_after(nextevt, jiffies)) - base->clk = jiffies; - else if (time_after(nextevt, base->clk)) - base->clk = nextevt; + if (time_after(basej, base->clk)) { + if (time_after(nextevt, basej)) + base->clk = basej; + else if (time_after(nextevt, base->clk)) + base->clk = nextevt; + } if (time_before_eq(nextevt, basej)) { expires = basem; diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 33bc56cf60d7..b01e547d4d04 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -198,6 +198,7 @@ config FRAME_WARN int "Warn for stack frames larger than (needs gcc 4.4)" range 0 8192 default 0 if KASAN + default 2048 if GCC_PLUGIN_LATENT_ENTROPY default 1024 if !64BIT default 2048 if 64BIT help diff --git a/lib/genalloc.c b/lib/genalloc.c index 0a1139644d32..144fe6b1a03e 100644 --- a/lib/genalloc.c +++ b/lib/genalloc.c @@ -292,7 +292,7 @@ unsigned long gen_pool_alloc_algo(struct gen_pool *pool, size_t size, struct gen_pool_chunk *chunk; unsigned long addr = 0; int order = pool->min_alloc_order; - int nbits, start_bit = 0, end_bit, remain; + int nbits, start_bit, end_bit, remain; #ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG BUG_ON(in_nmi()); @@ -307,6 +307,7 @@ unsigned long gen_pool_alloc_algo(struct gen_pool *pool, size_t size, if (size > atomic_read(&chunk->avail)) continue; + start_bit = 0; end_bit = chunk_size(chunk) >> order; retry: start_bit = algo(chunk->bits, end_bit, start_bit, diff --git a/lib/stackdepot.c b/lib/stackdepot.c index 60f77f1d470a..4d830e299989 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -50,7 +50,7 @@ STACK_ALLOC_ALIGN) #define STACK_ALLOC_INDEX_BITS (DEPOT_STACK_BITS - \ STACK_ALLOC_NULL_PROTECTION_BITS - STACK_ALLOC_OFFSET_BITS) -#define STACK_ALLOC_SLABS_CAP 1024 +#define STACK_ALLOC_SLABS_CAP 8192 #define STACK_ALLOC_MAX_SLABS \ (((1LL << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \ (1LL << (STACK_ALLOC_INDEX_BITS)) : STACK_ALLOC_SLABS_CAP) diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 94346b4d8984..0362da0b66c3 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -4831,7 +4831,7 @@ static struct bpf_test tests[] = { { }, INTERNAL, { 0x34 }, - { { 1, 0xbef } }, + { { ETH_HLEN, 0xbef } }, .fill_helper = bpf_fill_ld_abs_vlan_push_pop, }, /* diff --git a/mm/Kconfig b/mm/Kconfig index be0ee11fa0d9..86e3e0e74d20 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -187,7 +187,7 @@ config MEMORY_HOTPLUG bool "Allow for memory hot-add" depends on SPARSEMEM || X86_64_ACPI_NUMA depends on ARCH_ENABLE_MEMORY_HOTPLUG - depends on !KASAN + depends on COMPILE_TEST || !KASAN config MEMORY_HOTPLUG_SPARSE def_bool y diff --git a/mm/filemap.c b/mm/filemap.c index 849f459ad078..c7fe2f16503f 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -790,9 +790,7 @@ EXPORT_SYMBOL(__page_cache_alloc); */ wait_queue_head_t *page_waitqueue(struct page *page) { - const struct zone *zone = page_zone(page); - - return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)]; + return bit_waitqueue(page, 0); } EXPORT_SYMBOL(page_waitqueue); @@ -526,7 +526,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) * instead of __get_user_pages. __get_user_pages should be used only if * you need some special @gup_flags. */ -long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, +static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int gup_flags, struct page **pages, struct vm_area_struct **vmas, int *nonblocking) @@ -631,7 +631,6 @@ next_page: } while (nr_pages); return i; } -EXPORT_SYMBOL(__get_user_pages); bool vma_permits_fault(struct vm_area_struct *vma, unsigned int fault_flags) { diff --git a/mm/kmemleak.c b/mm/kmemleak.c index a5e453cf05c4..e5355a5b423f 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1453,8 +1453,11 @@ static void kmemleak_scan(void) read_lock(&tasklist_lock); do_each_thread(g, p) { - scan_block(task_stack_page(p), task_stack_page(p) + - THREAD_SIZE, NULL); + void *stack = try_get_task_stack(p); + if (stack) { + scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } } while_each_thread(g, p); read_unlock(&tasklist_lock); } diff --git a/mm/list_lru.c b/mm/list_lru.c index 1d05cb9d363d..234676e31edd 100644 --- a/mm/list_lru.c +++ b/mm/list_lru.c @@ -554,6 +554,8 @@ int __list_lru_init(struct list_lru *lru, bool memcg_aware, err = memcg_init_list_lru(lru, memcg_aware); if (err) { kfree(lru->node); + /* Do this so a list_lru_destroy() doesn't crash: */ + lru->node = NULL; goto out; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index ae052b5e3315..0f870ba43942 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1917,6 +1917,15 @@ retry: current->flags & PF_EXITING)) goto force; + /* + * Prevent unbounded recursion when reclaim operations need to + * allocate memory. This might exceed the limits temporarily, + * but we prefer facilitating memory reclaim and getting back + * under the limit over triggering OOM kills in these cases. + */ + if (unlikely(current->flags & PF_MEMALLOC)) + goto force; + if (unlikely(task_in_memcg_oom(current))) goto nomem; diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 962927309b6e..cad4b9125695 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -268,7 +268,6 @@ void __init register_page_bootmem_info_node(struct pglist_data *pgdat) unsigned long i, pfn, end_pfn, nr_pages; int node = pgdat->node_id; struct page *page; - struct zone *zone; nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; page = virt_to_page(pgdat); @@ -276,19 +275,6 @@ void __init register_page_bootmem_info_node(struct pglist_data *pgdat) for (i = 0; i < nr_pages; i++, page++) get_page_bootmem(node, page, NODE_INFO); - zone = &pgdat->node_zones[0]; - for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { - if (zone_is_initialized(zone)) { - nr_pages = zone->wait_table_hash_nr_entries - * sizeof(wait_queue_head_t); - nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; - page = virt_to_page(zone->wait_table); - - for (i = 0; i < nr_pages; i++, page++) - get_page_bootmem(node, page, NODE_INFO); - } - } - pfn = pgdat->node_start_pfn; end_pfn = pgdat_end_pfn(pgdat); @@ -2131,7 +2117,6 @@ void try_offline_node(int nid) unsigned long start_pfn = pgdat->node_start_pfn; unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages; unsigned long pfn; - int i; for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { unsigned long section_nr = pfn_to_section_nr(pfn); @@ -2158,20 +2143,6 @@ void try_offline_node(int nid) */ node_set_offline(nid); unregister_one_node(nid); - - /* free waittable in each zone */ - for (i = 0; i < MAX_NR_ZONES; i++) { - struct zone *zone = pgdat->node_zones + i; - - /* - * wait_table may be allocated from boot memory, - * here only free if it's allocated by vmalloc. - */ - if (is_vmalloc_addr(zone->wait_table)) { - vfree(zone->wait_table); - zone->wait_table = NULL; - } - } } EXPORT_SYMBOL(try_offline_node); diff --git a/mm/nommu.c b/mm/nommu.c index db5fd1795298..8b8faaf2a9e9 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -109,7 +109,7 @@ unsigned int kobjsize(const void *objp) return PAGE_SIZE << compound_order(page); } -long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, +static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, unsigned long nr_pages, unsigned int foll_flags, struct page **pages, struct vm_area_struct **vmas, int *nonblocking) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 2b3bf6767d54..072d791dce2d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -92,7 +92,7 @@ int _node_numa_mem_[MAX_NUMNODES]; #endif #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY -volatile u64 latent_entropy __latent_entropy; +volatile unsigned long latent_entropy __latent_entropy; EXPORT_SYMBOL(latent_entropy); #endif @@ -4224,7 +4224,7 @@ static void show_migration_types(unsigned char type) } *p = '\0'; - printk("(%s) ", tmp); + printk(KERN_CONT "(%s) ", tmp); } /* @@ -4335,7 +4335,8 @@ void show_free_areas(unsigned int filter) free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; show_node(zone); - printk("%s" + printk(KERN_CONT + "%s" " free:%lukB" " min:%lukB" " low:%lukB" @@ -4382,8 +4383,8 @@ void show_free_areas(unsigned int filter) K(zone_page_state(zone, NR_FREE_CMA_PAGES))); printk("lowmem_reserve[]:"); for (i = 0; i < MAX_NR_ZONES; i++) - printk(" %ld", zone->lowmem_reserve[i]); - printk("\n"); + printk(KERN_CONT " %ld", zone->lowmem_reserve[i]); + printk(KERN_CONT "\n"); } for_each_populated_zone(zone) { @@ -4394,7 +4395,7 @@ void show_free_areas(unsigned int filter) if (skip_free_areas_node(filter, zone_to_nid(zone))) continue; show_node(zone); - printk("%s: ", zone->name); + printk(KERN_CONT "%s: ", zone->name); spin_lock_irqsave(&zone->lock, flags); for (order = 0; order < MAX_ORDER; order++) { @@ -4412,11 +4413,12 @@ void show_free_areas(unsigned int filter) } spin_unlock_irqrestore(&zone->lock, flags); for (order = 0; order < MAX_ORDER; order++) { - printk("%lu*%lukB ", nr[order], K(1UL) << order); + printk(KERN_CONT "%lu*%lukB ", + nr[order], K(1UL) << order); if (nr[order]) show_migration_types(types[order]); } - printk("= %lukB\n", K(total)); + printk(KERN_CONT "= %lukB\n", K(total)); } hugetlb_show_meminfo(); @@ -4977,72 +4979,6 @@ void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) } /* - * Helper functions to size the waitqueue hash table. - * Essentially these want to choose hash table sizes sufficiently - * large so that collisions trying to wait on pages are rare. - * But in fact, the number of active page waitqueues on typical - * systems is ridiculously low, less than 200. So this is even - * conservative, even though it seems large. - * - * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to - * waitqueues, i.e. the size of the waitq table given the number of pages. - */ -#define PAGES_PER_WAITQUEUE 256 - -#ifndef CONFIG_MEMORY_HOTPLUG -static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) -{ - unsigned long size = 1; - - pages /= PAGES_PER_WAITQUEUE; - - while (size < pages) - size <<= 1; - - /* - * Once we have dozens or even hundreds of threads sleeping - * on IO we've got bigger problems than wait queue collision. - * Limit the size of the wait table to a reasonable size. - */ - size = min(size, 4096UL); - - return max(size, 4UL); -} -#else -/* - * A zone's size might be changed by hot-add, so it is not possible to determine - * a suitable size for its wait_table. So we use the maximum size now. - * - * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: - * - * i386 (preemption config) : 4096 x 16 = 64Kbyte. - * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. - * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. - * - * The maximum entries are prepared when a zone's memory is (512K + 256) pages - * or more by the traditional way. (See above). It equals: - * - * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. - * ia64(16K page size) : = ( 8G + 4M)byte. - * powerpc (64K page size) : = (32G +16M)byte. - */ -static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) -{ - return 4096UL; -} -#endif - -/* - * This is an integer logarithm so that shifts can be used later - * to extract the more random high bits from the multiplicative - * hash function before the remainder is taken. - */ -static inline unsigned long wait_table_bits(unsigned long size) -{ - return ffz(~size); -} - -/* * Initially all pages are reserved - free ones are freed * up by free_all_bootmem() once the early boot process is * done. Non-atomic initialization, single-pass. @@ -5304,49 +5240,6 @@ void __init setup_per_cpu_pageset(void) alloc_percpu(struct per_cpu_nodestat); } -static noinline __ref -int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) -{ - int i; - size_t alloc_size; - - /* - * The per-page waitqueue mechanism uses hashed waitqueues - * per zone. - */ - zone->wait_table_hash_nr_entries = - wait_table_hash_nr_entries(zone_size_pages); - zone->wait_table_bits = - wait_table_bits(zone->wait_table_hash_nr_entries); - alloc_size = zone->wait_table_hash_nr_entries - * sizeof(wait_queue_head_t); - - if (!slab_is_available()) { - zone->wait_table = (wait_queue_head_t *) - memblock_virt_alloc_node_nopanic( - alloc_size, zone->zone_pgdat->node_id); - } else { - /* - * This case means that a zone whose size was 0 gets new memory - * via memory hot-add. - * But it may be the case that a new node was hot-added. In - * this case vmalloc() will not be able to use this new node's - * memory - this wait_table must be initialized to use this new - * node itself as well. - * To use this new node's memory, further consideration will be - * necessary. - */ - zone->wait_table = vmalloc(alloc_size); - } - if (!zone->wait_table) - return -ENOMEM; - - for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) - init_waitqueue_head(zone->wait_table + i); - - return 0; -} - static __meminit void zone_pcp_init(struct zone *zone) { /* @@ -5367,10 +5260,7 @@ int __meminit init_currently_empty_zone(struct zone *zone, unsigned long size) { struct pglist_data *pgdat = zone->zone_pgdat; - int ret; - ret = zone_wait_table_init(zone, size); - if (ret) - return ret; + pgdat->nr_zones = zone_idx(zone) + 1; zone->zone_start_pfn = zone_start_pfn; @@ -5382,6 +5272,7 @@ int __meminit init_currently_empty_zone(struct zone *zone, zone_start_pfn, (zone_start_pfn + size)); zone_init_free_lists(zone); + zone->initialized = 1; return 0; } diff --git a/mm/slab.c b/mm/slab.c index 090fb26b3a39..0b0550ca85b4 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -233,6 +233,7 @@ static void kmem_cache_node_init(struct kmem_cache_node *parent) spin_lock_init(&parent->list_lock); parent->free_objects = 0; parent->free_touched = 0; + parent->num_slabs = 0; } #define MAKE_LIST(cachep, listp, slab, nodeid) \ @@ -966,7 +967,7 @@ static int setup_kmem_cache_node(struct kmem_cache *cachep, * guaranteed to be valid until irq is re-enabled, because it will be * freed after synchronize_sched(). */ - if (force_change) + if (old_shared && force_change) synchronize_sched(); fail: @@ -1382,24 +1383,27 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) for_each_kmem_cache_node(cachep, node, n) { unsigned long active_objs = 0, num_objs = 0, free_objects = 0; unsigned long active_slabs = 0, num_slabs = 0; + unsigned long num_slabs_partial = 0, num_slabs_free = 0; + unsigned long num_slabs_full; spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->slabs_full, lru) { - active_objs += cachep->num; - active_slabs++; - } + num_slabs = n->num_slabs; list_for_each_entry(page, &n->slabs_partial, lru) { active_objs += page->active; - active_slabs++; + num_slabs_partial++; } list_for_each_entry(page, &n->slabs_free, lru) - num_slabs++; + num_slabs_free++; free_objects += n->free_objects; spin_unlock_irqrestore(&n->list_lock, flags); - num_slabs += active_slabs; num_objs = num_slabs * cachep->num; + active_slabs = num_slabs - num_slabs_free; + num_slabs_full = num_slabs - + (num_slabs_partial + num_slabs_free); + active_objs += (num_slabs_full * cachep->num); + pr_warn(" node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n", node, active_slabs, num_slabs, active_objs, num_objs, free_objects); @@ -2314,6 +2318,7 @@ static int drain_freelist(struct kmem_cache *cache, page = list_entry(p, struct page, lru); list_del(&page->lru); + n->num_slabs--; /* * Safe to drop the lock. The slab is no longer linked * to the cache. @@ -2752,6 +2757,8 @@ static void cache_grow_end(struct kmem_cache *cachep, struct page *page) list_add_tail(&page->lru, &(n->slabs_free)); else fixup_slab_list(cachep, n, page, &list); + + n->num_slabs++; STATS_INC_GROWN(cachep); n->free_objects += cachep->num - page->active; spin_unlock(&n->list_lock); @@ -3443,6 +3450,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, page = list_last_entry(&n->slabs_free, struct page, lru); list_move(&page->lru, list); + n->num_slabs--; } } @@ -4099,6 +4107,8 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) unsigned long num_objs; unsigned long active_slabs = 0; unsigned long num_slabs, free_objects = 0, shared_avail = 0; + unsigned long num_slabs_partial = 0, num_slabs_free = 0; + unsigned long num_slabs_full = 0; const char *name; char *error = NULL; int node; @@ -4111,33 +4121,34 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) check_irq_on(); spin_lock_irq(&n->list_lock); - list_for_each_entry(page, &n->slabs_full, lru) { - if (page->active != cachep->num && !error) - error = "slabs_full accounting error"; - active_objs += cachep->num; - active_slabs++; - } + num_slabs += n->num_slabs; + list_for_each_entry(page, &n->slabs_partial, lru) { if (page->active == cachep->num && !error) error = "slabs_partial accounting error"; if (!page->active && !error) error = "slabs_partial accounting error"; active_objs += page->active; - active_slabs++; + num_slabs_partial++; } + list_for_each_entry(page, &n->slabs_free, lru) { if (page->active && !error) error = "slabs_free accounting error"; - num_slabs++; + num_slabs_free++; } + free_objects += n->free_objects; if (n->shared) shared_avail += n->shared->avail; spin_unlock_irq(&n->list_lock); } - num_slabs += active_slabs; num_objs = num_slabs * cachep->num; + active_slabs = num_slabs - num_slabs_free; + num_slabs_full = num_slabs - (num_slabs_partial + num_slabs_free); + active_objs += (num_slabs_full * cachep->num); + if (num_objs - active_objs != free_objects && !error) error = "free_objects accounting error"; diff --git a/mm/slab.h b/mm/slab.h index 9653f2e2591a..bc05fdc3edce 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -432,6 +432,7 @@ struct kmem_cache_node { struct list_head slabs_partial; /* partial list first, better asm code */ struct list_head slabs_full; struct list_head slabs_free; + unsigned long num_slabs; unsigned long free_objects; unsigned int free_limit; unsigned int colour_next; /* Per-node cache coloring */ diff --git a/mm/vmscan.c b/mm/vmscan.c index 744f926af442..76fda2268148 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3043,7 +3043,9 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, sc.gfp_mask, sc.reclaim_idx); + current->flags |= PF_MEMALLOC; nr_reclaimed = do_try_to_free_pages(zonelist, &sc); + current->flags &= ~PF_MEMALLOC; trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 8de138d3306b..f2531ad66b68 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -664,7 +664,7 @@ static struct sk_buff **vlan_gro_receive(struct sk_buff **head, skb_gro_pull(skb, sizeof(*vhdr)); skb_gro_postpull_rcsum(skb, vhdr, sizeof(*vhdr)); - pp = ptype->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); out_unlock: rcu_read_unlock(); diff --git a/net/batman-adv/hard-interface.c b/net/batman-adv/hard-interface.c index 08ce36147c4c..e034afbd1bb0 100644 --- a/net/batman-adv/hard-interface.c +++ b/net/batman-adv/hard-interface.c @@ -652,7 +652,6 @@ void batadv_hardif_disable_interface(struct batadv_hard_iface *hard_iface, batadv_softif_destroy_sysfs(hard_iface->soft_iface); } - hard_iface->soft_iface = NULL; batadv_hardif_put(hard_iface); out: diff --git a/net/batman-adv/log.h b/net/batman-adv/log.h index e0e1a88c3e58..d2905a855d1b 100644 --- a/net/batman-adv/log.h +++ b/net/batman-adv/log.h @@ -63,7 +63,7 @@ enum batadv_dbg_level { BATADV_DBG_NC = BIT(5), BATADV_DBG_MCAST = BIT(6), BATADV_DBG_TP_METER = BIT(7), - BATADV_DBG_ALL = 127, + BATADV_DBG_ALL = 255, }; #ifdef CONFIG_BATMAN_ADV_DEBUG diff --git a/net/batman-adv/originator.c b/net/batman-adv/originator.c index 5f3bfc41aeb1..7c8d16086f0f 100644 --- a/net/batman-adv/originator.c +++ b/net/batman-adv/originator.c @@ -544,7 +544,7 @@ batadv_hardif_neigh_create(struct batadv_hard_iface *hard_iface, if (bat_priv->algo_ops->neigh.hardif_init) bat_priv->algo_ops->neigh.hardif_init(hardif_neigh); - hlist_add_head(&hardif_neigh->list, &hard_iface->neigh_list); + hlist_add_head_rcu(&hardif_neigh->list, &hard_iface->neigh_list); out: spin_unlock_bh(&hard_iface->neigh_list_lock); diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index e2288421fe6b..1015d9c8d97d 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -969,41 +969,38 @@ void __hci_req_enable_advertising(struct hci_request *req) hci_req_add(req, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable); } -static u8 append_local_name(struct hci_dev *hdev, u8 *ptr, u8 ad_len) +u8 append_local_name(struct hci_dev *hdev, u8 *ptr, u8 ad_len) { - size_t complete_len; size_t short_len; - int max_len; - - max_len = HCI_MAX_AD_LENGTH - ad_len - 2; - complete_len = strlen(hdev->dev_name); - short_len = strlen(hdev->short_name); - - /* no space left for name */ - if (max_len < 1) - return ad_len; + size_t complete_len; - /* no name set */ - if (!complete_len) + /* no space left for name (+ NULL + type + len) */ + if ((HCI_MAX_AD_LENGTH - ad_len) < HCI_MAX_SHORT_NAME_LENGTH + 3) return ad_len; - /* complete name fits and is eq to max short name len or smaller */ - if (complete_len <= max_len && - complete_len <= HCI_MAX_SHORT_NAME_LENGTH) { + /* use complete name if present and fits */ + complete_len = strlen(hdev->dev_name); + if (complete_len && complete_len <= HCI_MAX_SHORT_NAME_LENGTH) return eir_append_data(ptr, ad_len, EIR_NAME_COMPLETE, - hdev->dev_name, complete_len); - } + hdev->dev_name, complete_len + 1); - /* short name set and fits */ - if (short_len && short_len <= max_len) { + /* use short name if present */ + short_len = strlen(hdev->short_name); + if (short_len) return eir_append_data(ptr, ad_len, EIR_NAME_SHORT, - hdev->short_name, short_len); - } + hdev->short_name, short_len + 1); - /* no short name set so shorten complete name */ - if (!short_len) { - return eir_append_data(ptr, ad_len, EIR_NAME_SHORT, - hdev->dev_name, max_len); + /* use shortened full name if present, we already know that name + * is longer then HCI_MAX_SHORT_NAME_LENGTH + */ + if (complete_len) { + u8 name[HCI_MAX_SHORT_NAME_LENGTH + 1]; + + memcpy(name, hdev->dev_name, HCI_MAX_SHORT_NAME_LENGTH); + name[HCI_MAX_SHORT_NAME_LENGTH] = '\0'; + + return eir_append_data(ptr, ad_len, EIR_NAME_SHORT, name, + sizeof(name)); } return ad_len; diff --git a/net/bluetooth/hci_request.h b/net/bluetooth/hci_request.h index 6b06629245a8..dde77bd59f91 100644 --- a/net/bluetooth/hci_request.h +++ b/net/bluetooth/hci_request.h @@ -106,6 +106,8 @@ static inline void hci_update_background_scan(struct hci_dev *hdev) void hci_request_setup(struct hci_dev *hdev); void hci_request_cancel_all(struct hci_dev *hdev); +u8 append_local_name(struct hci_dev *hdev, u8 *ptr, u8 ad_len); + static inline u16 eir_append_data(u8 *eir, u16 eir_len, u8 type, u8 *data, u8 data_len) { diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 736038085feb..1fba2a03f8ae 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -6017,7 +6017,15 @@ static int read_adv_features(struct sock *sk, struct hci_dev *hdev, return err; } -static u8 tlv_data_max_len(u32 adv_flags, bool is_adv_data) +static u8 calculate_name_len(struct hci_dev *hdev) +{ + u8 buf[HCI_MAX_SHORT_NAME_LENGTH + 3]; + + return append_local_name(hdev, buf, 0); +} + +static u8 tlv_data_max_len(struct hci_dev *hdev, u32 adv_flags, + bool is_adv_data) { u8 max_len = HCI_MAX_AD_LENGTH; @@ -6030,9 +6038,8 @@ static u8 tlv_data_max_len(u32 adv_flags, bool is_adv_data) if (adv_flags & MGMT_ADV_FLAG_TX_POWER) max_len -= 3; } else { - /* at least 1 byte of name should fit in */ if (adv_flags & MGMT_ADV_FLAG_LOCAL_NAME) - max_len -= 3; + max_len -= calculate_name_len(hdev); if (adv_flags & (MGMT_ADV_FLAG_APPEARANCE)) max_len -= 4; @@ -6063,12 +6070,13 @@ static bool appearance_managed(u32 adv_flags) return adv_flags & MGMT_ADV_FLAG_APPEARANCE; } -static bool tlv_data_is_valid(u32 adv_flags, u8 *data, u8 len, bool is_adv_data) +static bool tlv_data_is_valid(struct hci_dev *hdev, u32 adv_flags, u8 *data, + u8 len, bool is_adv_data) { int i, cur_len; u8 max_len; - max_len = tlv_data_max_len(adv_flags, is_adv_data); + max_len = tlv_data_max_len(hdev, adv_flags, is_adv_data); if (len > max_len) return false; @@ -6215,8 +6223,8 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, goto unlock; } - if (!tlv_data_is_valid(flags, cp->data, cp->adv_data_len, true) || - !tlv_data_is_valid(flags, cp->data + cp->adv_data_len, + if (!tlv_data_is_valid(hdev, flags, cp->data, cp->adv_data_len, true) || + !tlv_data_is_valid(hdev, flags, cp->data + cp->adv_data_len, cp->scan_rsp_len, false)) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, MGMT_STATUS_INVALID_PARAMS); @@ -6429,8 +6437,8 @@ static int get_adv_size_info(struct sock *sk, struct hci_dev *hdev, rp.instance = cp->instance; rp.flags = cp->flags; - rp.max_adv_data_len = tlv_data_max_len(flags, true); - rp.max_scan_rsp_len = tlv_data_max_len(flags, false); + rp.max_adv_data_len = tlv_data_max_len(hdev, flags, true); + rp.max_scan_rsp_len = tlv_data_max_len(hdev, flags, false); err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_GET_ADV_SIZE_INFO, MGMT_STATUS_SUCCESS, &rp, sizeof(rp)); diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index c5fea9393946..2136e45f5277 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -972,13 +972,12 @@ static void br_multicast_enable(struct bridge_mcast_own_query *query) mod_timer(&query->timer, jiffies); } -void br_multicast_enable_port(struct net_bridge_port *port) +static void __br_multicast_enable_port(struct net_bridge_port *port) { struct net_bridge *br = port->br; - spin_lock(&br->multicast_lock); if (br->multicast_disabled || !netif_running(br->dev)) - goto out; + return; br_multicast_enable(&port->ip4_own_query); #if IS_ENABLED(CONFIG_IPV6) @@ -987,8 +986,14 @@ void br_multicast_enable_port(struct net_bridge_port *port) if (port->multicast_router == MDB_RTR_TYPE_PERM && hlist_unhashed(&port->rlist)) br_multicast_add_router(br, port); +} -out: +void br_multicast_enable_port(struct net_bridge_port *port) +{ + struct net_bridge *br = port->br; + + spin_lock(&br->multicast_lock); + __br_multicast_enable_port(port); spin_unlock(&br->multicast_lock); } @@ -1994,8 +1999,9 @@ static void br_multicast_start_querier(struct net_bridge *br, int br_multicast_toggle(struct net_bridge *br, unsigned long val) { - int err = 0; struct net_bridge_mdb_htable *mdb; + struct net_bridge_port *port; + int err = 0; spin_lock_bh(&br->multicast_lock); if (br->multicast_disabled == !val) @@ -2023,10 +2029,9 @@ rollback: goto rollback; } - br_multicast_start_querier(br, &br->ip4_own_query); -#if IS_ENABLED(CONFIG_IPV6) - br_multicast_start_querier(br, &br->ip6_own_query); -#endif + br_multicast_open(br); + list_for_each_entry(port, &br->port_list, list) + __br_multicast_enable_port(port); unlock: spin_unlock_bh(&br->multicast_lock); diff --git a/net/core/dev.c b/net/core/dev.c index 4bc19a164ba5..820bac239738 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -3035,6 +3035,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d } return head; } +EXPORT_SYMBOL_GPL(validate_xmit_skb_list); static void qdisc_pkt_len_init(struct sk_buff *skb) { @@ -4511,6 +4512,7 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff NAPI_GRO_CB(skb)->flush = 0; NAPI_GRO_CB(skb)->free = 0; NAPI_GRO_CB(skb)->encap_mark = 0; + NAPI_GRO_CB(skb)->recursion_counter = 0; NAPI_GRO_CB(skb)->is_fou = 0; NAPI_GRO_CB(skb)->is_atomic = 1; NAPI_GRO_CB(skb)->gro_remcsum_start = 0; @@ -5511,10 +5513,14 @@ struct net_device *netdev_all_lower_get_next_rcu(struct net_device *dev, { struct netdev_adjacent *lower; - lower = list_first_or_null_rcu(&dev->all_adj_list.lower, - struct netdev_adjacent, list); + lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); + + if (&lower->list == &dev->all_adj_list.lower) + return NULL; + + *iter = &lower->list; - return lower ? lower->dev : NULL; + return lower->dev; } EXPORT_SYMBOL(netdev_all_lower_get_next_rcu); diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 1a7b80f73376..ab193e5def07 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -246,15 +246,13 @@ ipv6: case htons(ETH_P_8021AD): case htons(ETH_P_8021Q): { const struct vlan_hdr *vlan; + struct vlan_hdr _vlan; + bool vlan_tag_present = skb && skb_vlan_tag_present(skb); - if (skb_vlan_tag_present(skb)) + if (vlan_tag_present) proto = skb->protocol; - if (!skb_vlan_tag_present(skb) || - proto == cpu_to_be16(ETH_P_8021Q) || - proto == cpu_to_be16(ETH_P_8021AD)) { - struct vlan_hdr _vlan; - + if (!vlan_tag_present || eth_type_vlan(skb->protocol)) { vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan); if (!vlan) @@ -272,7 +270,7 @@ ipv6: FLOW_DISSECTOR_KEY_VLAN, target_container); - if (skb_vlan_tag_present(skb)) { + if (vlan_tag_present) { key_vlan->vlan_id = skb_vlan_tag_get_id(skb); key_vlan->vlan_priority = (skb_vlan_tag_get_prio(skb) >> VLAN_PRIO_SHIFT); diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 989434f36f96..f61c0e02a413 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -215,13 +215,14 @@ static void rtnl_net_notifyid(struct net *net, int cmd, int id); */ int peernet2id_alloc(struct net *net, struct net *peer) { + unsigned long flags; bool alloc; int id; - spin_lock_bh(&net->nsid_lock); + spin_lock_irqsave(&net->nsid_lock, flags); alloc = atomic_read(&peer->count) == 0 ? false : true; id = __peernet2id_alloc(net, peer, &alloc); - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irqrestore(&net->nsid_lock, flags); if (alloc && id >= 0) rtnl_net_notifyid(net, RTM_NEWNSID, id); return id; @@ -230,11 +231,12 @@ int peernet2id_alloc(struct net *net, struct net *peer) /* This function returns, if assigned, the id of a peer netns. */ int peernet2id(struct net *net, struct net *peer) { + unsigned long flags; int id; - spin_lock_bh(&net->nsid_lock); + spin_lock_irqsave(&net->nsid_lock, flags); id = __peernet2id(net, peer); - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irqrestore(&net->nsid_lock, flags); return id; } EXPORT_SYMBOL(peernet2id); @@ -249,17 +251,18 @@ bool peernet_has_id(struct net *net, struct net *peer) struct net *get_net_ns_by_id(struct net *net, int id) { + unsigned long flags; struct net *peer; if (id < 0) return NULL; rcu_read_lock(); - spin_lock_bh(&net->nsid_lock); + spin_lock_irqsave(&net->nsid_lock, flags); peer = idr_find(&net->netns_ids, id); if (peer) get_net(peer); - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irqrestore(&net->nsid_lock, flags); rcu_read_unlock(); return peer; @@ -422,17 +425,17 @@ static void cleanup_net(struct work_struct *work) for_each_net(tmp) { int id; - spin_lock_bh(&tmp->nsid_lock); + spin_lock_irq(&tmp->nsid_lock); id = __peernet2id(tmp, net); if (id >= 0) idr_remove(&tmp->netns_ids, id); - spin_unlock_bh(&tmp->nsid_lock); + spin_unlock_irq(&tmp->nsid_lock); if (id >= 0) rtnl_net_notifyid(tmp, RTM_DELNSID, id); } - spin_lock_bh(&net->nsid_lock); + spin_lock_irq(&net->nsid_lock); idr_destroy(&net->netns_ids); - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irq(&net->nsid_lock); } rtnl_unlock(); @@ -561,6 +564,7 @@ static int rtnl_net_newid(struct sk_buff *skb, struct nlmsghdr *nlh) { struct net *net = sock_net(skb->sk); struct nlattr *tb[NETNSA_MAX + 1]; + unsigned long flags; struct net *peer; int nsid, err; @@ -581,15 +585,15 @@ static int rtnl_net_newid(struct sk_buff *skb, struct nlmsghdr *nlh) if (IS_ERR(peer)) return PTR_ERR(peer); - spin_lock_bh(&net->nsid_lock); + spin_lock_irqsave(&net->nsid_lock, flags); if (__peernet2id(net, peer) >= 0) { - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irqrestore(&net->nsid_lock, flags); err = -EEXIST; goto out; } err = alloc_netid(net, peer, nsid); - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irqrestore(&net->nsid_lock, flags); if (err >= 0) { rtnl_net_notifyid(net, RTM_NEWNSID, err); err = 0; @@ -711,10 +715,11 @@ static int rtnl_net_dumpid(struct sk_buff *skb, struct netlink_callback *cb) .idx = 0, .s_idx = cb->args[0], }; + unsigned long flags; - spin_lock_bh(&net->nsid_lock); + spin_lock_irqsave(&net->nsid_lock, flags); idr_for_each(&net->netns_ids, rtnl_net_dumpid_one, &net_cb); - spin_unlock_bh(&net->nsid_lock); + spin_unlock_irqrestore(&net->nsid_lock, flags); cb->args[0] = net_cb.idx; return skb->len; diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 5219a9e2127a..306b8f0e03c1 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -216,8 +216,8 @@ #define M_QUEUE_XMIT 2 /* Inject packet into qdisc */ /* If lock -- protects updating of if_list */ -#define if_lock(t) spin_lock(&(t->if_lock)); -#define if_unlock(t) spin_unlock(&(t->if_lock)); +#define if_lock(t) mutex_lock(&(t->if_lock)); +#define if_unlock(t) mutex_unlock(&(t->if_lock)); /* Used to help with determining the pkts on receive */ #define PKTGEN_MAGIC 0xbe9be955 @@ -423,7 +423,7 @@ struct pktgen_net { }; struct pktgen_thread { - spinlock_t if_lock; /* for list of devices */ + struct mutex if_lock; /* for list of devices */ struct list_head if_list; /* All device here */ struct list_head th_list; struct task_struct *tsk; @@ -2010,11 +2010,13 @@ static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *d { struct pktgen_thread *t; + mutex_lock(&pktgen_thread_lock); + list_for_each_entry(t, &pn->pktgen_threads, th_list) { struct pktgen_dev *pkt_dev; - rcu_read_lock(); - list_for_each_entry_rcu(pkt_dev, &t->if_list, list) { + if_lock(t); + list_for_each_entry(pkt_dev, &t->if_list, list) { if (pkt_dev->odev != dev) continue; @@ -2029,8 +2031,9 @@ static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *d dev->name); break; } - rcu_read_unlock(); + if_unlock(t); } + mutex_unlock(&pktgen_thread_lock); } static int pktgen_device_event(struct notifier_block *unused, @@ -3762,7 +3765,7 @@ static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn) return -ENOMEM; } - spin_lock_init(&t->if_lock); + mutex_init(&t->if_lock); t->cpu = cpu; INIT_LIST_HEAD(&t->if_list); diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c index e92b759d906c..9a1a352fd1eb 100644 --- a/net/core/sock_reuseport.c +++ b/net/core/sock_reuseport.c @@ -129,7 +129,6 @@ int reuseport_add_sock(struct sock *sk, struct sock *sk2) return 0; } -EXPORT_SYMBOL(reuseport_add_sock); static void reuseport_free_rcu(struct rcu_head *head) { diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index 66dff5e3d772..02acfff36028 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -439,7 +439,7 @@ struct sk_buff **eth_gro_receive(struct sk_buff **head, skb_gro_pull(skb, sizeof(*eh)); skb_gro_postpull_rcsum(skb, eh, sizeof(*eh)); - pp = ptype->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); out_unlock: rcu_read_unlock(); diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c index 5ee1d43f1310..4ebe2aa3e7d3 100644 --- a/net/hsr/hsr_forward.c +++ b/net/hsr/hsr_forward.c @@ -300,10 +300,6 @@ static void hsr_forward_do(struct hsr_frame_info *frame) static void check_local_dest(struct hsr_priv *hsr, struct sk_buff *skb, struct hsr_frame_info *frame) { - struct net_device *master_dev; - - master_dev = hsr_port_get_hsr(hsr, HSR_PT_MASTER)->dev; - if (hsr_addr_is_self(hsr, eth_hdr(skb)->h_dest)) { frame->is_local_exclusive = true; skb->pkt_type = PACKET_HOST; diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 1effc986739e..9648c97e541f 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1391,7 +1391,7 @@ struct sk_buff **inet_gro_receive(struct sk_buff **head, struct sk_buff *skb) skb_gro_pull(skb, sizeof(*iph)); skb_set_transport_header(skb, skb_gro_offset(skb)); - pp = ops->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); out_unlock: rcu_read_unlock(); diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c index cf50f7e2b012..030d1531e897 100644 --- a/net/ipv4/fou.c +++ b/net/ipv4/fou.c @@ -249,7 +249,7 @@ static struct sk_buff **fou_gro_receive(struct sock *sk, if (!ops || !ops->callbacks.gro_receive) goto out_unlock; - pp = ops->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); out_unlock: rcu_read_unlock(); @@ -441,7 +441,7 @@ next_proto: if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive)) goto out_unlock; - pp = ops->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); flush = 0; out_unlock: diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c index 96e0efecefa6..d5cac99170b1 100644 --- a/net/ipv4/gre_offload.c +++ b/net/ipv4/gre_offload.c @@ -229,7 +229,7 @@ static struct sk_buff **gre_gro_receive(struct sk_buff **head, /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/ skb_gro_postpull_rcsum(skb, greh, grehlen); - pp = ptype->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); flush = 0; out_unlock: diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 77c20a489218..ca97835bfec4 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -25,6 +25,7 @@ #include <net/inet_hashtables.h> #include <net/secure_seq.h> #include <net/ip.h> +#include <net/tcp.h> #include <net/sock_reuseport.h> static u32 inet_ehashfn(const struct net *net, const __be32 laddr, @@ -172,7 +173,7 @@ EXPORT_SYMBOL_GPL(__inet_inherit_port); static inline int compute_score(struct sock *sk, struct net *net, const unsigned short hnum, const __be32 daddr, - const int dif) + const int dif, bool exact_dif) { int score = -1; struct inet_sock *inet = inet_sk(sk); @@ -186,7 +187,7 @@ static inline int compute_score(struct sock *sk, struct net *net, return -1; score += 4; } - if (sk->sk_bound_dev_if) { + if (sk->sk_bound_dev_if || exact_dif) { if (sk->sk_bound_dev_if != dif) return -1; score += 4; @@ -215,11 +216,12 @@ struct sock *__inet_lookup_listener(struct net *net, unsigned int hash = inet_lhashfn(net, hnum); struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; int score, hiscore = 0, matches = 0, reuseport = 0; + bool exact_dif = inet_exact_dif_match(net, skb); struct sock *sk, *result = NULL; u32 phash = 0; sk_for_each_rcu(sk, &ilb->head) { - score = compute_score(sk, net, hnum, daddr, dif); + score = compute_score(sk, net, hnum, daddr, dif, exact_dif); if (score > hiscore) { reuseport = sk->sk_reuseport; if (reuseport) { diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 05d105832bdb..03e7f7310423 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -538,7 +538,6 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, { struct iphdr *iph; int ptr; - struct net_device *dev; struct sk_buff *skb2; unsigned int mtu, hlen, left, len, ll_rs; int offset; @@ -546,8 +545,6 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, struct rtable *rt = skb_rtable(skb); int err = 0; - dev = rt->dst.dev; - /* for offloaded checksums cleanup checksum before fragmentation */ if (skb->ip_summed == CHECKSUM_PARTIAL && (err = skb_checksum_help(skb))) diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index af4919792b6a..b8a2d63d1fb8 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -98,7 +98,7 @@ static void ip_cmsg_recv_retopts(struct msghdr *msg, struct sk_buff *skb) } static void ip_cmsg_recv_checksum(struct msghdr *msg, struct sk_buff *skb, - int offset) + int tlen, int offset) { __wsum csum = skb->csum; @@ -106,8 +106,9 @@ static void ip_cmsg_recv_checksum(struct msghdr *msg, struct sk_buff *skb, return; if (offset != 0) - csum = csum_sub(csum, csum_partial(skb_transport_header(skb), - offset, 0)); + csum = csum_sub(csum, + csum_partial(skb_transport_header(skb) + tlen, + offset, 0)); put_cmsg(msg, SOL_IP, IP_CHECKSUM, sizeof(__wsum), &csum); } @@ -153,7 +154,7 @@ static void ip_cmsg_recv_dstaddr(struct msghdr *msg, struct sk_buff *skb) } void ip_cmsg_recv_offset(struct msghdr *msg, struct sk_buff *skb, - int offset) + int tlen, int offset) { struct inet_sock *inet = inet_sk(skb->sk); unsigned int flags = inet->cmsg_flags; @@ -216,7 +217,7 @@ void ip_cmsg_recv_offset(struct msghdr *msg, struct sk_buff *skb, } if (flags & IP_CMSG_CHECKSUM) - ip_cmsg_recv_checksum(msg, skb, offset); + ip_cmsg_recv_checksum(msg, skb, tlen, offset); } EXPORT_SYMBOL(ip_cmsg_recv_offset); diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 7cf7d6e380c2..205e2000d395 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -994,7 +994,7 @@ struct proto ping_prot = { .init = ping_init_sock, .close = ping_close, .connect = ip4_datagram_connect, - .disconnect = udp_disconnect, + .disconnect = __udp_disconnect, .setsockopt = ip_setsockopt, .getsockopt = ip_getsockopt, .sendmsg = ping_v4_sendmsg, diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index 90a85c955872..ecbe5a7c2d6d 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -918,7 +918,7 @@ struct proto raw_prot = { .close = raw_close, .destroy = raw_destroy, .connect = ip4_datagram_connect, - .disconnect = udp_disconnect, + .disconnect = __udp_disconnect, .ioctl = raw_ioctl, .init = raw_init, .setsockopt = raw_setsockopt, diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c index 1cb67de106fe..80bc36b25de2 100644 --- a/net/ipv4/sysctl_net_ipv4.c +++ b/net/ipv4/sysctl_net_ipv4.c @@ -96,11 +96,11 @@ static void inet_get_ping_group_range_table(struct ctl_table *table, kgid_t *low container_of(table->data, struct net, ipv4.ping_group_range.range); unsigned int seq; do { - seq = read_seqbegin(&net->ipv4.ip_local_ports.lock); + seq = read_seqbegin(&net->ipv4.ping_group_range.lock); *low = data[0]; *high = data[1]; - } while (read_seqretry(&net->ipv4.ip_local_ports.lock, seq)); + } while (read_seqretry(&net->ipv4.ping_group_range.lock, seq)); } /* Update system visible IP port range */ @@ -109,10 +109,10 @@ static void set_ping_group_range(struct ctl_table *table, kgid_t low, kgid_t hig kgid_t *data = table->data; struct net *net = container_of(table->data, struct net, ipv4.ping_group_range.range); - write_seqlock(&net->ipv4.ip_local_ports.lock); + write_seqlock(&net->ipv4.ping_group_range.lock); data[0] = low; data[1] = high; - write_sequnlock(&net->ipv4.ip_local_ports.lock); + write_sequnlock(&net->ipv4.ping_group_range.lock); } /* Validate changes from /proc interface. */ diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index bd5e8d10893f..61b7be303eec 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -86,7 +86,6 @@ int sysctl_tcp_tw_reuse __read_mostly; int sysctl_tcp_low_latency __read_mostly; -EXPORT_SYMBOL(sysctl_tcp_low_latency); #ifdef CONFIG_TCP_MD5SIG static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key, @@ -1887,7 +1886,6 @@ static void *listening_get_next(struct seq_file *seq, void *cur) struct tcp_iter_state *st = seq->private; struct net *net = seq_file_net(seq); struct inet_listen_hashbucket *ilb; - struct inet_connection_sock *icsk; struct sock *sk = cur; if (!sk) { @@ -1909,7 +1907,6 @@ get_sk: continue; if (sk->sk_family == st->family) return sk; - icsk = inet_csk(sk); } spin_unlock_bh(&ilb->lock); st->offset = 0; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 7d96dc2d3d08..d123d68f4d1d 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1322,7 +1322,7 @@ try_again: *addr_len = sizeof(*sin); } if (inet->cmsg_flags) - ip_cmsg_recv_offset(msg, skb, sizeof(struct udphdr) + off); + ip_cmsg_recv_offset(msg, skb, sizeof(struct udphdr), off); err = copied; if (flags & MSG_TRUNC) @@ -1345,7 +1345,7 @@ csum_copy_err: goto try_again; } -int udp_disconnect(struct sock *sk, int flags) +int __udp_disconnect(struct sock *sk, int flags) { struct inet_sock *inet = inet_sk(sk); /* @@ -1367,6 +1367,15 @@ int udp_disconnect(struct sock *sk, int flags) sk_dst_reset(sk); return 0; } +EXPORT_SYMBOL(__udp_disconnect); + +int udp_disconnect(struct sock *sk, int flags) +{ + lock_sock(sk); + __udp_disconnect(sk, flags); + release_sock(sk); + return 0; +} EXPORT_SYMBOL(udp_disconnect); void udp_lib_unhash(struct sock *sk) @@ -2193,7 +2202,7 @@ int udp_abort(struct sock *sk, int err) sk->sk_err = err; sk->sk_error_report(sk); - udp_disconnect(sk, 0); + __udp_disconnect(sk, 0); release_sock(sk); diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index f9333c963607..b2be1d9757ef 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -295,7 +295,7 @@ unflush: skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */ skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr)); - pp = udp_sk(sk)->gro_receive(sk, head, skb); + pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb); out_unlock: rcu_read_unlock(); diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index d8983e15f859..060dd9922018 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -147,9 +147,8 @@ static inline void addrconf_sysctl_unregister(struct inet6_dev *idev) } #endif -static void __ipv6_regen_rndid(struct inet6_dev *idev); -static void __ipv6_try_regen_rndid(struct inet6_dev *idev, struct in6_addr *tmpaddr); -static void ipv6_regen_rndid(unsigned long data); +static void ipv6_regen_rndid(struct inet6_dev *idev); +static void ipv6_try_regen_rndid(struct inet6_dev *idev, struct in6_addr *tmpaddr); static int ipv6_generate_eui64(u8 *eui, struct net_device *dev); static int ipv6_count_addresses(struct inet6_dev *idev); @@ -409,9 +408,7 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev) goto err_release; } - /* One reference from device. We must do this before - * we invoke __ipv6_regen_rndid(). - */ + /* One reference from device. */ in6_dev_hold(ndev); if (dev->flags & (IFF_NOARP | IFF_LOOPBACK)) @@ -425,17 +422,15 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev) #endif INIT_LIST_HEAD(&ndev->tempaddr_list); - setup_timer(&ndev->regen_timer, ipv6_regen_rndid, (unsigned long)ndev); + ndev->desync_factor = U32_MAX; if ((dev->flags&IFF_LOOPBACK) || dev->type == ARPHRD_TUNNEL || dev->type == ARPHRD_TUNNEL6 || dev->type == ARPHRD_SIT || dev->type == ARPHRD_NONE) { ndev->cnf.use_tempaddr = -1; - } else { - in6_dev_hold(ndev); - ipv6_regen_rndid((unsigned long) ndev); - } + } else + ipv6_regen_rndid(ndev); ndev->token = in6addr_any; @@ -447,7 +442,6 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev) err = addrconf_sysctl_register(ndev); if (err) { ipv6_mc_destroy_dev(ndev); - del_timer(&ndev->regen_timer); snmp6_unregister_dev(ndev); goto err_release; } @@ -1190,6 +1184,8 @@ static int ipv6_create_tempaddr(struct inet6_ifaddr *ifp, struct inet6_ifaddr *i int ret = 0; u32 addr_flags; unsigned long now = jiffies; + long max_desync_factor; + s32 cnf_temp_preferred_lft; write_lock_bh(&idev->lock); if (ift) { @@ -1222,23 +1218,42 @@ retry: } in6_ifa_hold(ifp); memcpy(addr.s6_addr, ifp->addr.s6_addr, 8); - __ipv6_try_regen_rndid(idev, tmpaddr); + ipv6_try_regen_rndid(idev, tmpaddr); memcpy(&addr.s6_addr[8], idev->rndid, 8); age = (now - ifp->tstamp) / HZ; + + regen_advance = idev->cnf.regen_max_retry * + idev->cnf.dad_transmits * + NEIGH_VAR(idev->nd_parms, RETRANS_TIME) / HZ; + + /* recalculate max_desync_factor each time and update + * idev->desync_factor if it's larger + */ + cnf_temp_preferred_lft = READ_ONCE(idev->cnf.temp_prefered_lft); + max_desync_factor = min_t(__u32, + idev->cnf.max_desync_factor, + cnf_temp_preferred_lft - regen_advance); + + if (unlikely(idev->desync_factor > max_desync_factor)) { + if (max_desync_factor > 0) { + get_random_bytes(&idev->desync_factor, + sizeof(idev->desync_factor)); + idev->desync_factor %= max_desync_factor; + } else { + idev->desync_factor = 0; + } + } + tmp_valid_lft = min_t(__u32, ifp->valid_lft, idev->cnf.temp_valid_lft + age); - tmp_prefered_lft = min_t(__u32, - ifp->prefered_lft, - idev->cnf.temp_prefered_lft + age - - idev->cnf.max_desync_factor); + tmp_prefered_lft = cnf_temp_preferred_lft + age - + idev->desync_factor; + tmp_prefered_lft = min_t(__u32, ifp->prefered_lft, tmp_prefered_lft); tmp_plen = ifp->prefix_len; tmp_tstamp = ifp->tstamp; spin_unlock_bh(&ifp->lock); - regen_advance = idev->cnf.regen_max_retry * - idev->cnf.dad_transmits * - NEIGH_VAR(idev->nd_parms, RETRANS_TIME) / HZ; write_unlock_bh(&idev->lock); /* A temporary address is created only if this calculated Preferred @@ -2150,7 +2165,7 @@ static int ipv6_inherit_eui64(u8 *eui, struct inet6_dev *idev) } /* (re)generation of randomized interface identifier (RFC 3041 3.2, 3.5) */ -static void __ipv6_regen_rndid(struct inet6_dev *idev) +static void ipv6_regen_rndid(struct inet6_dev *idev) { regen: get_random_bytes(idev->rndid, sizeof(idev->rndid)); @@ -2179,43 +2194,10 @@ regen: } } -static void ipv6_regen_rndid(unsigned long data) -{ - struct inet6_dev *idev = (struct inet6_dev *) data; - unsigned long expires; - - rcu_read_lock_bh(); - write_lock_bh(&idev->lock); - - if (idev->dead) - goto out; - - __ipv6_regen_rndid(idev); - - expires = jiffies + - idev->cnf.temp_prefered_lft * HZ - - idev->cnf.regen_max_retry * idev->cnf.dad_transmits * - NEIGH_VAR(idev->nd_parms, RETRANS_TIME) - - idev->cnf.max_desync_factor * HZ; - if (time_before(expires, jiffies)) { - pr_warn("%s: too short regeneration interval; timer disabled for %s\n", - __func__, idev->dev->name); - goto out; - } - - if (!mod_timer(&idev->regen_timer, expires)) - in6_dev_hold(idev); - -out: - write_unlock_bh(&idev->lock); - rcu_read_unlock_bh(); - in6_dev_put(idev); -} - -static void __ipv6_try_regen_rndid(struct inet6_dev *idev, struct in6_addr *tmpaddr) +static void ipv6_try_regen_rndid(struct inet6_dev *idev, struct in6_addr *tmpaddr) { if (tmpaddr && memcmp(idev->rndid, &tmpaddr->s6_addr[8], 8) == 0) - __ipv6_regen_rndid(idev); + ipv6_regen_rndid(idev); } /* @@ -2356,7 +2338,7 @@ static void manage_tempaddrs(struct inet6_dev *idev, max_valid = 0; max_prefered = idev->cnf.temp_prefered_lft - - idev->cnf.max_desync_factor - age; + idev->desync_factor - age; if (max_prefered < 0) max_prefered = 0; @@ -3018,7 +3000,7 @@ static void init_loopback(struct net_device *dev) * lo device down, release this obsolete dst and * reallocate a new router for ifa. */ - if (sp_ifa->rt->dst.obsolete > 0) { + if (!atomic_read(&sp_ifa->rt->rt6i_ref)) { ip6_rt_put(sp_ifa->rt); sp_ifa->rt = NULL; } else { @@ -3594,9 +3576,6 @@ restart: if (!how) idev->if_flags &= ~(IF_RS_SENT|IF_RA_RCVD|IF_READY); - if (how && del_timer(&idev->regen_timer)) - in6_dev_put(idev); - /* Step 3: clear tempaddr list */ while (!list_empty(&idev->tempaddr_list)) { ifa = list_first_entry(&idev->tempaddr_list, diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 00cf28ad4565..02761c9fe43e 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -96,7 +96,7 @@ EXPORT_SYMBOL(__inet6_lookup_established); static inline int compute_score(struct sock *sk, struct net *net, const unsigned short hnum, const struct in6_addr *daddr, - const int dif) + const int dif, bool exact_dif) { int score = -1; @@ -109,7 +109,7 @@ static inline int compute_score(struct sock *sk, struct net *net, return -1; score++; } - if (sk->sk_bound_dev_if) { + if (sk->sk_bound_dev_if || exact_dif) { if (sk->sk_bound_dev_if != dif) return -1; score++; @@ -131,11 +131,12 @@ struct sock *inet6_lookup_listener(struct net *net, unsigned int hash = inet_lhashfn(net, hnum); struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; int score, hiscore = 0, matches = 0, reuseport = 0; + bool exact_dif = inet6_exact_dif_match(net, skb); struct sock *sk, *result = NULL; u32 phash = 0; sk_for_each(sk, &ilb->head) { - score = compute_score(sk, net, hnum, daddr, dif); + score = compute_score(sk, net, hnum, daddr, dif, exact_dif); if (score > hiscore) { reuseport = sk->sk_reuseport; if (reuseport) { @@ -263,13 +264,15 @@ EXPORT_SYMBOL_GPL(inet6_hash_connect); int inet6_hash(struct sock *sk) { + int err = 0; + if (sk->sk_state != TCP_CLOSE) { local_bh_disable(); - __inet_hash(sk, NULL, ipv6_rcv_saddr_equal); + err = __inet_hash(sk, NULL, ipv6_rcv_saddr_equal); local_bh_enable(); } - return 0; + return err; } EXPORT_SYMBOL_GPL(inet6_hash); diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index e7bfd55899a3..1fcf61f1cbc3 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -246,7 +246,7 @@ static struct sk_buff **ipv6_gro_receive(struct sk_buff **head, skb_gro_postpull_rcsum(skb, iph, nlen); - pp = ops->callbacks.gro_receive(head, skb); + pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); out_unlock: rcu_read_unlock(); diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 6a66adba0c22..87784560dc46 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -157,6 +157,7 @@ ip6_tnl_lookup(struct net *net, const struct in6_addr *remote, const struct in6_ hash = HASH(&any, local); for_each_ip6_tunnel_rcu(ip6n->tnls_r_l[hash]) { if (ipv6_addr_equal(local, &t->parms.laddr) && + ipv6_addr_any(&t->parms.raddr) && (t->dev->flags & IFF_UP)) return t; } @@ -164,6 +165,7 @@ ip6_tnl_lookup(struct net *net, const struct in6_addr *remote, const struct in6_ hash = HASH(remote, &any); for_each_ip6_tunnel_rcu(ip6n->tnls_r_l[hash]) { if (ipv6_addr_equal(remote, &t->parms.raddr) && + ipv6_addr_any(&t->parms.laddr) && (t->dev->flags & IFF_UP)) return t; } @@ -1170,6 +1172,7 @@ route_lookup: if (err) return err; + skb->protocol = htons(ETH_P_IPV6); skb_push(skb, sizeof(struct ipv6hdr)); skb_reset_network_header(skb); ipv6h = ipv6_hdr(skb); diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index 5330262ab673..636ec56f5f50 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -120,6 +120,7 @@ struct ipv6_txoptions *ipv6_update_options(struct sock *sk, static bool setsockopt_needs_rtnl(int optname) { switch (optname) { + case IPV6_ADDRFORM: case IPV6_ADD_MEMBERSHIP: case IPV6_DROP_MEMBERSHIP: case IPV6_JOIN_ANYCAST: @@ -198,7 +199,7 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, } fl6_free_socklist(sk); - ipv6_sock_mc_close(sk); + __ipv6_sock_mc_close(sk); /* * Sock is moving from IPv6 to IPv4 (sk_prot), so diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index 75c1fc54f188..14a3903f1c82 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -276,16 +276,14 @@ static struct inet6_dev *ip6_mc_find_dev_rcu(struct net *net, return idev; } -void ipv6_sock_mc_close(struct sock *sk) +void __ipv6_sock_mc_close(struct sock *sk) { struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_mc_socklist *mc_lst; struct net *net = sock_net(sk); - if (!rcu_access_pointer(np->ipv6_mc_list)) - return; + ASSERT_RTNL(); - rtnl_lock(); while ((mc_lst = rtnl_dereference(np->ipv6_mc_list)) != NULL) { struct net_device *dev; @@ -303,8 +301,17 @@ void ipv6_sock_mc_close(struct sock *sk) atomic_sub(sizeof(*mc_lst), &sk->sk_omem_alloc); kfree_rcu(mc_lst, rcu); - } +} + +void ipv6_sock_mc_close(struct sock *sk) +{ + struct ipv6_pinfo *np = inet6_sk(sk); + + if (!rcu_access_pointer(np->ipv6_mc_list)) + return; + rtnl_lock(); + __ipv6_sock_mc_close(sk); rtnl_unlock(); } diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c index 0e983b694ee8..66e2d9dfc43a 100644 --- a/net/ipv6/ping.c +++ b/net/ipv6/ping.c @@ -180,7 +180,7 @@ struct proto pingv6_prot = { .init = ping_init_sock, .close = ping_close, .connect = ip6_datagram_connect_v6_only, - .disconnect = udp_disconnect, + .disconnect = __udp_disconnect, .setsockopt = ipv6_setsockopt, .getsockopt = ipv6_getsockopt, .sendmsg = ping_v6_sendmsg, diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index 54404f08efcc..054a1d84fc5e 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -1241,7 +1241,7 @@ struct proto rawv6_prot = { .close = rawv6_close, .destroy = raw6_destroy, .connect = ip6_datagram_connect_v6_only, - .disconnect = udp_disconnect, + .disconnect = __udp_disconnect, .ioctl = rawv6_ioctl, .init = rawv6_init_sk, .setsockopt = rawv6_setsockopt, diff --git a/net/ipv6/reassembly.c b/net/ipv6/reassembly.c index 2160d5d009cb..3815e8505ed2 100644 --- a/net/ipv6/reassembly.c +++ b/net/ipv6/reassembly.c @@ -456,7 +456,8 @@ static int ip6_frag_reasm(struct frag_queue *fq, struct sk_buff *prev, skb_network_header(head)[nhoff] = skb_transport_header(head)[0]; memmove(head->head + sizeof(struct frag_hdr), head->head, (head->data - head->head) - sizeof(struct frag_hdr)); - head->mac_header += sizeof(struct frag_hdr); + if (skb_mac_header_was_set(head)) + head->mac_header += sizeof(struct frag_hdr); head->network_header += sizeof(struct frag_hdr); skb_reset_transport_header(head); diff --git a/net/ipv6/route.c b/net/ipv6/route.c index bdbc38e8bf29..947ed1ded026 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -102,11 +102,13 @@ static int rt6_score_route(struct rt6_info *rt, int oif, int strict); #ifdef CONFIG_IPV6_ROUTE_INFO static struct rt6_info *rt6_add_route_info(struct net *net, const struct in6_addr *prefix, int prefixlen, - const struct in6_addr *gwaddr, int ifindex, + const struct in6_addr *gwaddr, + struct net_device *dev, unsigned int pref); static struct rt6_info *rt6_get_route_info(struct net *net, const struct in6_addr *prefix, int prefixlen, - const struct in6_addr *gwaddr, int ifindex); + const struct in6_addr *gwaddr, + struct net_device *dev); #endif struct uncached_list { @@ -656,7 +658,8 @@ static struct rt6_info *find_match(struct rt6_info *rt, int oif, int strict, struct net_device *dev = rt->dst.dev; if (dev && !netif_carrier_ok(dev) && - idev->cnf.ignore_routes_with_linkdown) + idev->cnf.ignore_routes_with_linkdown && + !(strict & RT6_LOOKUP_F_IGNORE_LINKSTATE)) goto out; if (rt6_check_expired(rt)) @@ -803,7 +806,7 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, rt = rt6_get_dflt_router(gwaddr, dev); else rt = rt6_get_route_info(net, prefix, rinfo->prefix_len, - gwaddr, dev->ifindex); + gwaddr, dev); if (rt && !lifetime) { ip6_del_rt(rt); @@ -811,8 +814,8 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, } if (!rt && lifetime) - rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, dev->ifindex, - pref); + rt = rt6_add_route_info(net, prefix, rinfo->prefix_len, gwaddr, + dev, pref); else if (rt) rt->rt6i_flags = RTF_ROUTEINFO | (rt->rt6i_flags & ~RTF_PREF_MASK) | RTF_PREF(pref); @@ -1050,6 +1053,7 @@ struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, int strict = 0; strict |= flags & RT6_LOOKUP_F_IFACE; + strict |= flags & RT6_LOOKUP_F_IGNORE_LINKSTATE; if (net->ipv6.devconf_all->forwarding == 0) strict |= RT6_LOOKUP_F_REACHABLE; @@ -1789,7 +1793,7 @@ static struct rt6_info *ip6_nh_lookup_table(struct net *net, }; struct fib6_table *table; struct rt6_info *rt; - int flags = RT6_LOOKUP_F_IFACE; + int flags = RT6_LOOKUP_F_IFACE | RT6_LOOKUP_F_IGNORE_LINKSTATE; table = fib6_get_table(net, cfg->fc_table); if (!table) @@ -2325,13 +2329,16 @@ static void ip6_rt_copy_init(struct rt6_info *rt, struct rt6_info *ort) #ifdef CONFIG_IPV6_ROUTE_INFO static struct rt6_info *rt6_get_route_info(struct net *net, const struct in6_addr *prefix, int prefixlen, - const struct in6_addr *gwaddr, int ifindex) + const struct in6_addr *gwaddr, + struct net_device *dev) { + u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO; + int ifindex = dev->ifindex; struct fib6_node *fn; struct rt6_info *rt = NULL; struct fib6_table *table; - table = fib6_get_table(net, RT6_TABLE_INFO); + table = fib6_get_table(net, tb_id); if (!table) return NULL; @@ -2357,12 +2364,13 @@ out: static struct rt6_info *rt6_add_route_info(struct net *net, const struct in6_addr *prefix, int prefixlen, - const struct in6_addr *gwaddr, int ifindex, + const struct in6_addr *gwaddr, + struct net_device *dev, unsigned int pref) { struct fib6_config cfg = { .fc_metric = IP6_RT_PRIO_USER, - .fc_ifindex = ifindex, + .fc_ifindex = dev->ifindex, .fc_dst_len = prefixlen, .fc_flags = RTF_GATEWAY | RTF_ADDRCONF | RTF_ROUTEINFO | RTF_UP | RTF_PREF(pref), @@ -2371,7 +2379,7 @@ static struct rt6_info *rt6_add_route_info(struct net *net, .fc_nlinfo.nl_net = net, }; - cfg.fc_table = l3mdev_fib_table_by_index(net, ifindex) ? : RT6_TABLE_INFO; + cfg.fc_table = l3mdev_fib_table(dev) ? : RT6_TABLE_INFO, cfg.fc_dst = *prefix; cfg.fc_gateway = *gwaddr; @@ -2381,16 +2389,17 @@ static struct rt6_info *rt6_add_route_info(struct net *net, ip6_route_add(&cfg); - return rt6_get_route_info(net, prefix, prefixlen, gwaddr, ifindex); + return rt6_get_route_info(net, prefix, prefixlen, gwaddr, dev); } #endif struct rt6_info *rt6_get_dflt_router(const struct in6_addr *addr, struct net_device *dev) { + u32 tb_id = l3mdev_fib_table(dev) ? : RT6_TABLE_DFLT; struct rt6_info *rt; struct fib6_table *table; - table = fib6_get_table(dev_net(dev), RT6_TABLE_DFLT); + table = fib6_get_table(dev_net(dev), tb_id); if (!table) return NULL; @@ -2424,20 +2433,20 @@ struct rt6_info *rt6_add_dflt_router(const struct in6_addr *gwaddr, cfg.fc_gateway = *gwaddr; - ip6_route_add(&cfg); + if (!ip6_route_add(&cfg)) { + struct fib6_table *table; + + table = fib6_get_table(dev_net(dev), cfg.fc_table); + if (table) + table->flags |= RT6_TABLE_HAS_DFLT_ROUTER; + } return rt6_get_dflt_router(gwaddr, dev); } -void rt6_purge_dflt_routers(struct net *net) +static void __rt6_purge_dflt_routers(struct fib6_table *table) { struct rt6_info *rt; - struct fib6_table *table; - - /* NOTE: Keep consistent with rt6_get_dflt_router */ - table = fib6_get_table(net, RT6_TABLE_DFLT); - if (!table) - return; restart: read_lock_bh(&table->tb6_lock); @@ -2451,6 +2460,27 @@ restart: } } read_unlock_bh(&table->tb6_lock); + + table->flags &= ~RT6_TABLE_HAS_DFLT_ROUTER; +} + +void rt6_purge_dflt_routers(struct net *net) +{ + struct fib6_table *table; + struct hlist_head *head; + unsigned int h; + + rcu_read_lock(); + + for (h = 0; h < FIB6_TABLE_HASHSZ; h++) { + head = &net->ipv6.fib_table_hash[h]; + hlist_for_each_entry_rcu(table, head, tb6_hlist) { + if (table->flags & RT6_TABLE_HAS_DFLT_ROUTER) + __rt6_purge_dflt_routers(table); + } + } + + rcu_read_unlock(); } static void rtmsg_to_fib6_config(struct net *net, diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 9aa7c1c7a9ce..b2ef061e6836 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -427,7 +427,8 @@ try_again: if (is_udp4) { if (inet->cmsg_flags) - ip_cmsg_recv(msg, skb); + ip_cmsg_recv_offset(msg, skb, + sizeof(struct udphdr), off); } else { if (np->rxopt.all) ip6_datagram_recv_specific_ctl(sk, msg, skb); diff --git a/net/l2tp/l2tp_ip.c b/net/l2tp/l2tp_ip.c index 42de4ccd159f..fce25afb652a 100644 --- a/net/l2tp/l2tp_ip.c +++ b/net/l2tp/l2tp_ip.c @@ -338,7 +338,7 @@ static int l2tp_ip_disconnect(struct sock *sk, int flags) if (sock_flag(sk, SOCK_ZAPPED)) return 0; - return udp_disconnect(sk, flags); + return __udp_disconnect(sk, flags); } static int l2tp_ip_getname(struct socket *sock, struct sockaddr *uaddr, diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c index ea2ae6664cc8..ad3468c32b53 100644 --- a/net/l2tp/l2tp_ip6.c +++ b/net/l2tp/l2tp_ip6.c @@ -410,7 +410,7 @@ static int l2tp_ip6_disconnect(struct sock *sk, int flags) if (sock_flag(sk, SOCK_ZAPPED)) return 0; - return udp_disconnect(sk, flags); + return __udp_disconnect(sk, flags); } static int l2tp_ip6_getname(struct socket *sock, struct sockaddr *uaddr, diff --git a/net/mac80211/aes_ccm.c b/net/mac80211/aes_ccm.c index 7663c28ba353..a4e0d59a40dd 100644 --- a/net/mac80211/aes_ccm.c +++ b/net/mac80211/aes_ccm.c @@ -18,21 +18,24 @@ #include "key.h" #include "aes_ccm.h" -void ieee80211_aes_ccm_encrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, - u8 *data, size_t data_len, u8 *mic, - size_t mic_len) +int ieee80211_aes_ccm_encrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, + u8 *data, size_t data_len, u8 *mic, + size_t mic_len) { struct scatterlist sg[3]; + struct aead_request *aead_req; + int reqsize = sizeof(*aead_req) + crypto_aead_reqsize(tfm); + u8 *__aad; - char aead_req_data[sizeof(struct aead_request) + - crypto_aead_reqsize(tfm)] - __aligned(__alignof__(struct aead_request)); - struct aead_request *aead_req = (void *) aead_req_data; + aead_req = kzalloc(reqsize + CCM_AAD_LEN, GFP_ATOMIC); + if (!aead_req) + return -ENOMEM; - memset(aead_req, 0, sizeof(aead_req_data)); + __aad = (u8 *)aead_req + reqsize; + memcpy(__aad, aad, CCM_AAD_LEN); sg_init_table(sg, 3); - sg_set_buf(&sg[0], &aad[2], be16_to_cpup((__be16 *)aad)); + sg_set_buf(&sg[0], &__aad[2], be16_to_cpup((__be16 *)__aad)); sg_set_buf(&sg[1], data, data_len); sg_set_buf(&sg[2], mic, mic_len); @@ -41,6 +44,9 @@ void ieee80211_aes_ccm_encrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, aead_request_set_ad(aead_req, sg[0].length); crypto_aead_encrypt(aead_req); + kzfree(aead_req); + + return 0; } int ieee80211_aes_ccm_decrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, @@ -48,18 +54,23 @@ int ieee80211_aes_ccm_decrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, size_t mic_len) { struct scatterlist sg[3]; - char aead_req_data[sizeof(struct aead_request) + - crypto_aead_reqsize(tfm)] - __aligned(__alignof__(struct aead_request)); - struct aead_request *aead_req = (void *) aead_req_data; + struct aead_request *aead_req; + int reqsize = sizeof(*aead_req) + crypto_aead_reqsize(tfm); + u8 *__aad; + int err; if (data_len == 0) return -EINVAL; - memset(aead_req, 0, sizeof(aead_req_data)); + aead_req = kzalloc(reqsize + CCM_AAD_LEN, GFP_ATOMIC); + if (!aead_req) + return -ENOMEM; + + __aad = (u8 *)aead_req + reqsize; + memcpy(__aad, aad, CCM_AAD_LEN); sg_init_table(sg, 3); - sg_set_buf(&sg[0], &aad[2], be16_to_cpup((__be16 *)aad)); + sg_set_buf(&sg[0], &__aad[2], be16_to_cpup((__be16 *)__aad)); sg_set_buf(&sg[1], data, data_len); sg_set_buf(&sg[2], mic, mic_len); @@ -67,7 +78,10 @@ int ieee80211_aes_ccm_decrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, aead_request_set_crypt(aead_req, sg, sg, data_len + mic_len, b_0); aead_request_set_ad(aead_req, sg[0].length); - return crypto_aead_decrypt(aead_req); + err = crypto_aead_decrypt(aead_req); + kzfree(aead_req); + + return err; } struct crypto_aead *ieee80211_aes_key_setup_encrypt(const u8 key[], diff --git a/net/mac80211/aes_ccm.h b/net/mac80211/aes_ccm.h index 6a73d1e4d186..fcd3254c5cf0 100644 --- a/net/mac80211/aes_ccm.h +++ b/net/mac80211/aes_ccm.h @@ -12,12 +12,14 @@ #include <linux/crypto.h> +#define CCM_AAD_LEN 32 + struct crypto_aead *ieee80211_aes_key_setup_encrypt(const u8 key[], size_t key_len, size_t mic_len); -void ieee80211_aes_ccm_encrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, - u8 *data, size_t data_len, u8 *mic, - size_t mic_len); +int ieee80211_aes_ccm_encrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, + u8 *data, size_t data_len, u8 *mic, + size_t mic_len); int ieee80211_aes_ccm_decrypt(struct crypto_aead *tfm, u8 *b_0, u8 *aad, u8 *data, size_t data_len, u8 *mic, size_t mic_len); diff --git a/net/mac80211/aes_gcm.c b/net/mac80211/aes_gcm.c index 3afe361fd27c..8a4397cc1b08 100644 --- a/net/mac80211/aes_gcm.c +++ b/net/mac80211/aes_gcm.c @@ -15,20 +15,23 @@ #include "key.h" #include "aes_gcm.h" -void ieee80211_aes_gcm_encrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, - u8 *data, size_t data_len, u8 *mic) +int ieee80211_aes_gcm_encrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, + u8 *data, size_t data_len, u8 *mic) { struct scatterlist sg[3]; + struct aead_request *aead_req; + int reqsize = sizeof(*aead_req) + crypto_aead_reqsize(tfm); + u8 *__aad; - char aead_req_data[sizeof(struct aead_request) + - crypto_aead_reqsize(tfm)] - __aligned(__alignof__(struct aead_request)); - struct aead_request *aead_req = (void *)aead_req_data; + aead_req = kzalloc(reqsize + GCM_AAD_LEN, GFP_ATOMIC); + if (!aead_req) + return -ENOMEM; - memset(aead_req, 0, sizeof(aead_req_data)); + __aad = (u8 *)aead_req + reqsize; + memcpy(__aad, aad, GCM_AAD_LEN); sg_init_table(sg, 3); - sg_set_buf(&sg[0], &aad[2], be16_to_cpup((__be16 *)aad)); + sg_set_buf(&sg[0], &__aad[2], be16_to_cpup((__be16 *)__aad)); sg_set_buf(&sg[1], data, data_len); sg_set_buf(&sg[2], mic, IEEE80211_GCMP_MIC_LEN); @@ -37,24 +40,31 @@ void ieee80211_aes_gcm_encrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, aead_request_set_ad(aead_req, sg[0].length); crypto_aead_encrypt(aead_req); + kzfree(aead_req); + return 0; } int ieee80211_aes_gcm_decrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, u8 *data, size_t data_len, u8 *mic) { struct scatterlist sg[3]; - char aead_req_data[sizeof(struct aead_request) + - crypto_aead_reqsize(tfm)] - __aligned(__alignof__(struct aead_request)); - struct aead_request *aead_req = (void *)aead_req_data; + struct aead_request *aead_req; + int reqsize = sizeof(*aead_req) + crypto_aead_reqsize(tfm); + u8 *__aad; + int err; if (data_len == 0) return -EINVAL; - memset(aead_req, 0, sizeof(aead_req_data)); + aead_req = kzalloc(reqsize + GCM_AAD_LEN, GFP_ATOMIC); + if (!aead_req) + return -ENOMEM; + + __aad = (u8 *)aead_req + reqsize; + memcpy(__aad, aad, GCM_AAD_LEN); sg_init_table(sg, 3); - sg_set_buf(&sg[0], &aad[2], be16_to_cpup((__be16 *)aad)); + sg_set_buf(&sg[0], &__aad[2], be16_to_cpup((__be16 *)__aad)); sg_set_buf(&sg[1], data, data_len); sg_set_buf(&sg[2], mic, IEEE80211_GCMP_MIC_LEN); @@ -63,7 +73,10 @@ int ieee80211_aes_gcm_decrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, data_len + IEEE80211_GCMP_MIC_LEN, j_0); aead_request_set_ad(aead_req, sg[0].length); - return crypto_aead_decrypt(aead_req); + err = crypto_aead_decrypt(aead_req); + kzfree(aead_req); + + return err; } struct crypto_aead *ieee80211_aes_gcm_key_setup_encrypt(const u8 key[], diff --git a/net/mac80211/aes_gcm.h b/net/mac80211/aes_gcm.h index 1347fda6b76a..55aed5352494 100644 --- a/net/mac80211/aes_gcm.h +++ b/net/mac80211/aes_gcm.h @@ -11,8 +11,10 @@ #include <linux/crypto.h> -void ieee80211_aes_gcm_encrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, - u8 *data, size_t data_len, u8 *mic); +#define GCM_AAD_LEN 32 + +int ieee80211_aes_gcm_encrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, + u8 *data, size_t data_len, u8 *mic); int ieee80211_aes_gcm_decrypt(struct crypto_aead *tfm, u8 *j_0, u8 *aad, u8 *data, size_t data_len, u8 *mic); struct crypto_aead *ieee80211_aes_gcm_key_setup_encrypt(const u8 key[], diff --git a/net/mac80211/aes_gmac.c b/net/mac80211/aes_gmac.c index 3ddd927aaf30..bd72a862ddb7 100644 --- a/net/mac80211/aes_gmac.c +++ b/net/mac80211/aes_gmac.c @@ -17,28 +17,27 @@ #include "key.h" #include "aes_gmac.h" -#define GMAC_MIC_LEN 16 -#define GMAC_NONCE_LEN 12 -#define AAD_LEN 20 - int ieee80211_aes_gmac(struct crypto_aead *tfm, const u8 *aad, u8 *nonce, const u8 *data, size_t data_len, u8 *mic) { struct scatterlist sg[4]; - char aead_req_data[sizeof(struct aead_request) + - crypto_aead_reqsize(tfm)] - __aligned(__alignof__(struct aead_request)); - struct aead_request *aead_req = (void *)aead_req_data; - u8 zero[GMAC_MIC_LEN], iv[AES_BLOCK_SIZE]; + u8 *zero, *__aad, iv[AES_BLOCK_SIZE]; + struct aead_request *aead_req; + int reqsize = sizeof(*aead_req) + crypto_aead_reqsize(tfm); if (data_len < GMAC_MIC_LEN) return -EINVAL; - memset(aead_req, 0, sizeof(aead_req_data)); + aead_req = kzalloc(reqsize + GMAC_MIC_LEN + GMAC_AAD_LEN, GFP_ATOMIC); + if (!aead_req) + return -ENOMEM; + + zero = (u8 *)aead_req + reqsize; + __aad = zero + GMAC_MIC_LEN; + memcpy(__aad, aad, GMAC_AAD_LEN); - memset(zero, 0, GMAC_MIC_LEN); sg_init_table(sg, 4); - sg_set_buf(&sg[0], aad, AAD_LEN); + sg_set_buf(&sg[0], __aad, GMAC_AAD_LEN); sg_set_buf(&sg[1], data, data_len - GMAC_MIC_LEN); sg_set_buf(&sg[2], zero, GMAC_MIC_LEN); sg_set_buf(&sg[3], mic, GMAC_MIC_LEN); @@ -49,9 +48,10 @@ int ieee80211_aes_gmac(struct crypto_aead *tfm, const u8 *aad, u8 *nonce, aead_request_set_tfm(aead_req, tfm); aead_request_set_crypt(aead_req, sg, sg, 0, iv); - aead_request_set_ad(aead_req, AAD_LEN + data_len); + aead_request_set_ad(aead_req, GMAC_AAD_LEN + data_len); crypto_aead_encrypt(aead_req); + kzfree(aead_req); return 0; } diff --git a/net/mac80211/aes_gmac.h b/net/mac80211/aes_gmac.h index d328204d73a8..32e6442c95be 100644 --- a/net/mac80211/aes_gmac.h +++ b/net/mac80211/aes_gmac.h @@ -11,6 +11,10 @@ #include <linux/crypto.h> +#define GMAC_AAD_LEN 20 +#define GMAC_MIC_LEN 16 +#define GMAC_NONCE_LEN 12 + struct crypto_aead *ieee80211_aes_gmac_key_setup(const u8 key[], size_t key_len); int ieee80211_aes_gmac(struct crypto_aead *tfm, const u8 *aad, u8 *nonce, diff --git a/net/mac80211/offchannel.c b/net/mac80211/offchannel.c index c3f610bba3fe..eede5c6db8d5 100644 --- a/net/mac80211/offchannel.c +++ b/net/mac80211/offchannel.c @@ -820,7 +820,7 @@ int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev, mgmt->u.action.category == WLAN_CATEGORY_SPECTRUM_MGMT) break; rcu_read_lock(); - sta = sta_info_get(sdata, mgmt->da); + sta = sta_info_get_bss(sdata, mgmt->da); rcu_read_unlock(); if (!sta) return -ENOLINK; diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 6175db385ba7..a47bbc973f2d 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -2298,6 +2298,8 @@ ieee80211_rx_h_amsdu(struct ieee80211_rx_data *rx) __le16 fc = hdr->frame_control; struct sk_buff_head frame_list; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb); + struct ethhdr ethhdr; + const u8 *check_da = ethhdr.h_dest, *check_sa = ethhdr.h_source; if (unlikely(!ieee80211_is_data(fc))) return RX_CONTINUE; @@ -2308,24 +2310,53 @@ ieee80211_rx_h_amsdu(struct ieee80211_rx_data *rx) if (!(status->rx_flags & IEEE80211_RX_AMSDU)) return RX_CONTINUE; - if (ieee80211_has_a4(hdr->frame_control) && - rx->sdata->vif.type == NL80211_IFTYPE_AP_VLAN && - !rx->sdata->u.vlan.sta) - return RX_DROP_UNUSABLE; + if (unlikely(ieee80211_has_a4(hdr->frame_control))) { + switch (rx->sdata->vif.type) { + case NL80211_IFTYPE_AP_VLAN: + if (!rx->sdata->u.vlan.sta) + return RX_DROP_UNUSABLE; + break; + case NL80211_IFTYPE_STATION: + if (!rx->sdata->u.mgd.use_4addr) + return RX_DROP_UNUSABLE; + break; + default: + return RX_DROP_UNUSABLE; + } + check_da = NULL; + check_sa = NULL; + } else switch (rx->sdata->vif.type) { + case NL80211_IFTYPE_AP: + case NL80211_IFTYPE_AP_VLAN: + check_da = NULL; + break; + case NL80211_IFTYPE_STATION: + if (!rx->sta || + !test_sta_flag(rx->sta, WLAN_STA_TDLS_PEER)) + check_sa = NULL; + break; + case NL80211_IFTYPE_MESH_POINT: + check_sa = NULL; + break; + default: + break; + } - if (is_multicast_ether_addr(hdr->addr1) && - ((rx->sdata->vif.type == NL80211_IFTYPE_AP_VLAN && - rx->sdata->u.vlan.sta) || - (rx->sdata->vif.type == NL80211_IFTYPE_STATION && - rx->sdata->u.mgd.use_4addr))) + if (is_multicast_ether_addr(hdr->addr1)) return RX_DROP_UNUSABLE; skb->dev = dev; __skb_queue_head_init(&frame_list); + if (ieee80211_data_to_8023_exthdr(skb, ðhdr, + rx->sdata->vif.addr, + rx->sdata->vif.type)) + return RX_DROP_UNUSABLE; + ieee80211_amsdu_to_8023s(skb, &frame_list, dev->dev_addr, rx->sdata->vif.type, - rx->local->hw.extra_tx_headroom, true); + rx->local->hw.extra_tx_headroom, + check_da, check_sa); while (!skb_queue_empty(&frame_list)) { rx->skb = __skb_dequeue(&frame_list); diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index b48c1e13e281..42ce9bd4426f 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -405,7 +405,7 @@ static int ccmp_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb, u8 *pos; u8 pn[6]; u64 pn64; - u8 aad[2 * AES_BLOCK_SIZE]; + u8 aad[CCM_AAD_LEN]; u8 b_0[AES_BLOCK_SIZE]; if (info->control.hw_key && @@ -461,10 +461,8 @@ static int ccmp_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb, pos += IEEE80211_CCMP_HDR_LEN; ccmp_special_blocks(skb, pn, b_0, aad); - ieee80211_aes_ccm_encrypt(key->u.ccmp.tfm, b_0, aad, pos, len, - skb_put(skb, mic_len), mic_len); - - return 0; + return ieee80211_aes_ccm_encrypt(key->u.ccmp.tfm, b_0, aad, pos, len, + skb_put(skb, mic_len), mic_len); } @@ -639,7 +637,7 @@ static int gcmp_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb) u8 *pos; u8 pn[6]; u64 pn64; - u8 aad[2 * AES_BLOCK_SIZE]; + u8 aad[GCM_AAD_LEN]; u8 j_0[AES_BLOCK_SIZE]; if (info->control.hw_key && @@ -696,10 +694,8 @@ static int gcmp_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb) pos += IEEE80211_GCMP_HDR_LEN; gcmp_special_blocks(skb, pn, j_0, aad); - ieee80211_aes_gcm_encrypt(key->u.gcmp.tfm, j_0, aad, pos, len, - skb_put(skb, IEEE80211_GCMP_MIC_LEN)); - - return 0; + return ieee80211_aes_gcm_encrypt(key->u.gcmp.tfm, j_0, aad, pos, len, + skb_put(skb, IEEE80211_GCMP_MIC_LEN)); } ieee80211_tx_result @@ -1123,9 +1119,9 @@ ieee80211_crypto_aes_gmac_encrypt(struct ieee80211_tx_data *tx) struct ieee80211_key *key = tx->key; struct ieee80211_mmie_16 *mmie; struct ieee80211_hdr *hdr; - u8 aad[20]; + u8 aad[GMAC_AAD_LEN]; u64 pn64; - u8 nonce[12]; + u8 nonce[GMAC_NONCE_LEN]; if (WARN_ON(skb_queue_len(&tx->skbs) != 1)) return TX_DROP; @@ -1171,7 +1167,7 @@ ieee80211_crypto_aes_gmac_decrypt(struct ieee80211_rx_data *rx) struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); struct ieee80211_key *key = rx->key; struct ieee80211_mmie_16 *mmie; - u8 aad[20], mic[16], ipn[6], nonce[12]; + u8 aad[GMAC_AAD_LEN], mic[GMAC_MIC_LEN], ipn[6], nonce[GMAC_NONCE_LEN]; struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; if (!ieee80211_is_mgmt(hdr->frame_control)) diff --git a/net/ncsi/internal.h b/net/ncsi/internal.h index 13290a70fa71..1308a56f2591 100644 --- a/net/ncsi/internal.h +++ b/net/ncsi/internal.h @@ -246,6 +246,7 @@ enum { ncsi_dev_state_config_gls, ncsi_dev_state_config_done, ncsi_dev_state_suspend_select = 0x0401, + ncsi_dev_state_suspend_gls, ncsi_dev_state_suspend_dcnt, ncsi_dev_state_suspend_dc, ncsi_dev_state_suspend_deselect, @@ -264,6 +265,7 @@ struct ncsi_dev_priv { #endif unsigned int package_num; /* Number of packages */ struct list_head packages; /* List of packages */ + struct ncsi_channel *hot_channel; /* Channel was ever active */ struct ncsi_request requests[256]; /* Request table */ unsigned int request_id; /* Last used request ID */ #define NCSI_REQ_START_IDX 1 diff --git a/net/ncsi/ncsi-aen.c b/net/ncsi/ncsi-aen.c index b41a6617d498..6898e7229285 100644 --- a/net/ncsi/ncsi-aen.c +++ b/net/ncsi/ncsi-aen.c @@ -141,23 +141,35 @@ static int ncsi_aen_handler_hncdsc(struct ncsi_dev_priv *ndp, return -ENODEV; /* If the channel is active one, we need reconfigure it */ + spin_lock_irqsave(&nc->lock, flags); ncm = &nc->modes[NCSI_MODE_LINK]; hncdsc = (struct ncsi_aen_hncdsc_pkt *)h; ncm->data[3] = ntohl(hncdsc->status); if (!list_empty(&nc->link) || - nc->state != NCSI_CHANNEL_ACTIVE || - (ncm->data[3] & 0x1)) + nc->state != NCSI_CHANNEL_ACTIVE) { + spin_unlock_irqrestore(&nc->lock, flags); return 0; + } - if (ndp->flags & NCSI_DEV_HWA) + spin_unlock_irqrestore(&nc->lock, flags); + if (!(ndp->flags & NCSI_DEV_HWA) && !(ncm->data[3] & 0x1)) ndp->flags |= NCSI_DEV_RESHUFFLE; /* If this channel is the active one and the link doesn't * work, we have to choose another channel to be active one. * The logic here is exactly similar to what we do when link * is down on the active channel. + * + * On the other hand, we need configure it when host driver + * state on the active channel becomes ready. */ ncsi_stop_channel_monitor(nc); + + spin_lock_irqsave(&nc->lock, flags); + nc->state = (ncm->data[3] & 0x1) ? NCSI_CHANNEL_INACTIVE : + NCSI_CHANNEL_ACTIVE; + spin_unlock_irqrestore(&nc->lock, flags); + spin_lock_irqsave(&ndp->lock, flags); list_add_tail_rcu(&nc->link, &ndp->channel_queue); spin_unlock_irqrestore(&ndp->lock, flags); diff --git a/net/ncsi/ncsi-manage.c b/net/ncsi/ncsi-manage.c index 5e509e547c2d..a3bd5fa8ad09 100644 --- a/net/ncsi/ncsi-manage.c +++ b/net/ncsi/ncsi-manage.c @@ -540,42 +540,86 @@ static void ncsi_suspend_channel(struct ncsi_dev_priv *ndp) nd->state = ncsi_dev_state_suspend_select; /* Fall through */ case ncsi_dev_state_suspend_select: - case ncsi_dev_state_suspend_dcnt: - case ncsi_dev_state_suspend_dc: - case ncsi_dev_state_suspend_deselect: ndp->pending_req_num = 1; - np = ndp->active_package; - nc = ndp->active_channel; + nca.type = NCSI_PKT_CMD_SP; nca.package = np->id; - if (nd->state == ncsi_dev_state_suspend_select) { - nca.type = NCSI_PKT_CMD_SP; - nca.channel = NCSI_RESERVED_CHANNEL; - if (ndp->flags & NCSI_DEV_HWA) - nca.bytes[0] = 0; - else - nca.bytes[0] = 1; + nca.channel = NCSI_RESERVED_CHANNEL; + if (ndp->flags & NCSI_DEV_HWA) + nca.bytes[0] = 0; + else + nca.bytes[0] = 1; + + /* To retrieve the last link states of channels in current + * package when current active channel needs fail over to + * another one. It means we will possibly select another + * channel as next active one. The link states of channels + * are most important factor of the selection. So we need + * accurate link states. Unfortunately, the link states on + * inactive channels can't be updated with LSC AEN in time. + */ + if (ndp->flags & NCSI_DEV_RESHUFFLE) + nd->state = ncsi_dev_state_suspend_gls; + else nd->state = ncsi_dev_state_suspend_dcnt; - } else if (nd->state == ncsi_dev_state_suspend_dcnt) { - nca.type = NCSI_PKT_CMD_DCNT; - nca.channel = nc->id; - nd->state = ncsi_dev_state_suspend_dc; - } else if (nd->state == ncsi_dev_state_suspend_dc) { - nca.type = NCSI_PKT_CMD_DC; + ret = ncsi_xmit_cmd(&nca); + if (ret) + goto error; + + break; + case ncsi_dev_state_suspend_gls: + ndp->pending_req_num = np->channel_num; + + nca.type = NCSI_PKT_CMD_GLS; + nca.package = np->id; + + nd->state = ncsi_dev_state_suspend_dcnt; + NCSI_FOR_EACH_CHANNEL(np, nc) { nca.channel = nc->id; - nca.bytes[0] = 1; - nd->state = ncsi_dev_state_suspend_deselect; - } else if (nd->state == ncsi_dev_state_suspend_deselect) { - nca.type = NCSI_PKT_CMD_DP; - nca.channel = NCSI_RESERVED_CHANNEL; - nd->state = ncsi_dev_state_suspend_done; + ret = ncsi_xmit_cmd(&nca); + if (ret) + goto error; } + break; + case ncsi_dev_state_suspend_dcnt: + ndp->pending_req_num = 1; + + nca.type = NCSI_PKT_CMD_DCNT; + nca.package = np->id; + nca.channel = nc->id; + + nd->state = ncsi_dev_state_suspend_dc; ret = ncsi_xmit_cmd(&nca); - if (ret) { - nd->state = ncsi_dev_state_functional; - return; - } + if (ret) + goto error; + + break; + case ncsi_dev_state_suspend_dc: + ndp->pending_req_num = 1; + + nca.type = NCSI_PKT_CMD_DC; + nca.package = np->id; + nca.channel = nc->id; + nca.bytes[0] = 1; + + nd->state = ncsi_dev_state_suspend_deselect; + ret = ncsi_xmit_cmd(&nca); + if (ret) + goto error; + + break; + case ncsi_dev_state_suspend_deselect: + ndp->pending_req_num = 1; + + nca.type = NCSI_PKT_CMD_DP; + nca.package = np->id; + nca.channel = NCSI_RESERVED_CHANNEL; + + nd->state = ncsi_dev_state_suspend_done; + ret = ncsi_xmit_cmd(&nca); + if (ret) + goto error; break; case ncsi_dev_state_suspend_done: @@ -589,6 +633,10 @@ static void ncsi_suspend_channel(struct ncsi_dev_priv *ndp) netdev_warn(nd->dev, "Wrong NCSI state 0x%x in suspend\n", nd->state); } + + return; +error: + nd->state = ncsi_dev_state_functional; } static void ncsi_configure_channel(struct ncsi_dev_priv *ndp) @@ -597,6 +645,7 @@ static void ncsi_configure_channel(struct ncsi_dev_priv *ndp) struct net_device *dev = nd->dev; struct ncsi_package *np = ndp->active_package; struct ncsi_channel *nc = ndp->active_channel; + struct ncsi_channel *hot_nc = NULL; struct ncsi_cmd_arg nca; unsigned char index; unsigned long flags; @@ -702,12 +751,20 @@ static void ncsi_configure_channel(struct ncsi_dev_priv *ndp) break; case ncsi_dev_state_config_done: spin_lock_irqsave(&nc->lock, flags); - if (nc->modes[NCSI_MODE_LINK].data[2] & 0x1) + if (nc->modes[NCSI_MODE_LINK].data[2] & 0x1) { + hot_nc = nc; nc->state = NCSI_CHANNEL_ACTIVE; - else + } else { + hot_nc = NULL; nc->state = NCSI_CHANNEL_INACTIVE; + } spin_unlock_irqrestore(&nc->lock, flags); + /* Update the hot channel */ + spin_lock_irqsave(&ndp->lock, flags); + ndp->hot_channel = hot_nc; + spin_unlock_irqrestore(&ndp->lock, flags); + ncsi_start_channel_monitor(nc); ncsi_process_next_channel(ndp); break; @@ -725,10 +782,14 @@ error: static int ncsi_choose_active_channel(struct ncsi_dev_priv *ndp) { struct ncsi_package *np; - struct ncsi_channel *nc, *found; + struct ncsi_channel *nc, *found, *hot_nc; struct ncsi_channel_mode *ncm; unsigned long flags; + spin_lock_irqsave(&ndp->lock, flags); + hot_nc = ndp->hot_channel; + spin_unlock_irqrestore(&ndp->lock, flags); + /* The search is done once an inactive channel with up * link is found. */ @@ -746,6 +807,9 @@ static int ncsi_choose_active_channel(struct ncsi_dev_priv *ndp) if (!found) found = nc; + if (nc == hot_nc) + found = nc; + ncm = &nc->modes[NCSI_MODE_LINK]; if (ncm->data[2] & 0x1) { spin_unlock_irqrestore(&nc->lock, flags); diff --git a/net/netfilter/core.c b/net/netfilter/core.c index fcb5d1df11e9..004af030ef1a 100644 --- a/net/netfilter/core.c +++ b/net/netfilter/core.c @@ -361,16 +361,9 @@ next_hook: if (ret == 0) ret = -EPERM; } else if ((verdict & NF_VERDICT_MASK) == NF_QUEUE) { - int err; - - RCU_INIT_POINTER(state->hook_entries, entry); - err = nf_queue(skb, state, verdict >> NF_VERDICT_QBITS); - if (err < 0) { - if (err == -ESRCH && - (verdict & NF_VERDICT_FLAG_QUEUE_BYPASS)) - goto next_hook; - kfree_skb(skb); - } + ret = nf_queue(skb, state, &entry, verdict); + if (ret == 1 && entry) + goto next_hook; } return ret; } diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index ba6a1d421222..df2f5a3901df 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -983,7 +983,7 @@ static void gc_worker(struct work_struct *work) return; ratio = scanned ? expired_count * 100 / scanned : 0; - if (ratio >= 90) + if (ratio >= 90 || expired_count == GC_MAX_EVICTS) next_run = 0; gc_work->last_bucket = i; diff --git a/net/netfilter/nf_internals.h b/net/netfilter/nf_internals.h index e0adb5959342..9fdb655f85bc 100644 --- a/net/netfilter/nf_internals.h +++ b/net/netfilter/nf_internals.h @@ -18,7 +18,7 @@ unsigned int nf_iterate(struct sk_buff *skb, struct nf_hook_state *state, /* nf_queue.c */ int nf_queue(struct sk_buff *skb, struct nf_hook_state *state, - unsigned int queuenum); + struct nf_hook_entry **entryp, unsigned int verdict); void nf_queue_nf_hook_drop(struct net *net, const struct nf_hook_entry *entry); int __init netfilter_queue_init(void); diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c index 96964a0070e1..8f08d759844a 100644 --- a/net/netfilter/nf_queue.c +++ b/net/netfilter/nf_queue.c @@ -107,13 +107,8 @@ void nf_queue_nf_hook_drop(struct net *net, const struct nf_hook_entry *entry) rcu_read_unlock(); } -/* - * Any packet that leaves via this function must come back - * through nf_reinject(). - */ -int nf_queue(struct sk_buff *skb, - struct nf_hook_state *state, - unsigned int queuenum) +static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state, + unsigned int queuenum) { int status = -ENOENT; struct nf_queue_entry *entry = NULL; @@ -161,6 +156,27 @@ err: return status; } +/* Packets leaving via this function must come back through nf_reinject(). */ +int nf_queue(struct sk_buff *skb, struct nf_hook_state *state, + struct nf_hook_entry **entryp, unsigned int verdict) +{ + struct nf_hook_entry *entry = *entryp; + int ret; + + RCU_INIT_POINTER(state->hook_entries, entry); + ret = __nf_queue(skb, state, verdict >> NF_VERDICT_QBITS); + if (ret < 0) { + if (ret == -ESRCH && + (verdict & NF_VERDICT_FLAG_QUEUE_BYPASS)) { + *entryp = rcu_dereference(entry->next); + return 1; + } + kfree_skb(skb); + } + + return 0; +} + void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) { struct nf_hook_entry *hook_entry; @@ -187,26 +203,26 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict) entry->state.thresh = INT_MIN; if (verdict == NF_ACCEPT) { - next_hook: - verdict = nf_iterate(skb, &entry->state, &hook_entry); + hook_entry = rcu_dereference(hook_entry->next); + if (hook_entry) +next_hook: + verdict = nf_iterate(skb, &entry->state, &hook_entry); } switch (verdict & NF_VERDICT_MASK) { case NF_ACCEPT: case NF_STOP: +okfn: local_bh_disable(); entry->state.okfn(entry->state.net, entry->state.sk, skb); local_bh_enable(); break; case NF_QUEUE: - RCU_INIT_POINTER(entry->state.hook_entries, hook_entry); - err = nf_queue(skb, &entry->state, - verdict >> NF_VERDICT_QBITS); - if (err < 0) { - if (err == -ESRCH && - (verdict & NF_VERDICT_FLAG_QUEUE_BYPASS)) + err = nf_queue(skb, &entry->state, &hook_entry, verdict); + if (err == 1) { + if (hook_entry) goto next_hook; - kfree_skb(skb); + goto okfn; } break; case NF_STOLEN: diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index b70d3ea1430e..24db22257586 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -4423,7 +4423,7 @@ static int nf_tables_check_loops(const struct nft_ctx *ctx, */ unsigned int nft_parse_u32_check(const struct nlattr *attr, int max, u32 *dest) { - int val; + u32 val; val = ntohl(nla_get_be32(attr)); if (val > max) diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index e3b83c31da2e..517f08767a3c 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -158,7 +158,8 @@ static int nft_dynset_init(const struct nft_ctx *ctx, if (tb[NFTA_DYNSET_TIMEOUT] != NULL) { if (!(set->flags & NFT_SET_TIMEOUT)) return -EINVAL; - timeout = be64_to_cpu(nla_get_be64(tb[NFTA_DYNSET_TIMEOUT])); + timeout = msecs_to_jiffies(be64_to_cpu(nla_get_be64( + tb[NFTA_DYNSET_TIMEOUT]))); } priv->sreg_key = nft_parse_register(tb[NFTA_DYNSET_SREG_KEY]); @@ -246,7 +247,8 @@ static int nft_dynset_dump(struct sk_buff *skb, const struct nft_expr *expr) goto nla_put_failure; if (nla_put_string(skb, NFTA_DYNSET_SET_NAME, priv->set->name)) goto nla_put_failure; - if (nla_put_be64(skb, NFTA_DYNSET_TIMEOUT, cpu_to_be64(priv->timeout), + if (nla_put_be64(skb, NFTA_DYNSET_TIMEOUT, + cpu_to_be64(jiffies_to_msecs(priv->timeout)), NFTA_DYNSET_PAD)) goto nla_put_failure; if (priv->expr && nft_expr_dump(skb, NFTA_DYNSET_EXPR, priv->expr)) diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c index a84cf3d66056..47beb3abcc9d 100644 --- a/net/netfilter/nft_exthdr.c +++ b/net/netfilter/nft_exthdr.c @@ -59,7 +59,8 @@ static int nft_exthdr_init(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { struct nft_exthdr *priv = nft_expr_priv(expr); - u32 offset, len, err; + u32 offset, len; + int err; if (tb[NFTA_EXTHDR_DREG] == NULL || tb[NFTA_EXTHDR_TYPE] == NULL || diff --git a/net/netfilter/nft_hash.c b/net/netfilter/nft_hash.c index 09473b415b95..baf694de3935 100644 --- a/net/netfilter/nft_hash.c +++ b/net/netfilter/nft_hash.c @@ -44,6 +44,7 @@ static const struct nla_policy nft_hash_policy[NFTA_HASH_MAX + 1] = { [NFTA_HASH_LEN] = { .type = NLA_U32 }, [NFTA_HASH_MODULUS] = { .type = NLA_U32 }, [NFTA_HASH_SEED] = { .type = NLA_U32 }, + [NFTA_HASH_OFFSET] = { .type = NLA_U32 }, }; static int nft_hash_init(const struct nft_ctx *ctx, diff --git a/net/netfilter/nft_range.c b/net/netfilter/nft_range.c index c6d5358482d1..fbc88009ca2e 100644 --- a/net/netfilter/nft_range.c +++ b/net/netfilter/nft_range.c @@ -28,22 +28,20 @@ static void nft_range_eval(const struct nft_expr *expr, const struct nft_pktinfo *pkt) { const struct nft_range_expr *priv = nft_expr_priv(expr); - bool mismatch; int d1, d2; d1 = memcmp(®s->data[priv->sreg], &priv->data_from, priv->len); d2 = memcmp(®s->data[priv->sreg], &priv->data_to, priv->len); switch (priv->op) { case NFT_RANGE_EQ: - mismatch = (d1 < 0 || d2 > 0); + if (d1 < 0 || d2 > 0) + regs->verdict.code = NFT_BREAK; break; case NFT_RANGE_NEQ: - mismatch = (d1 >= 0 && d2 <= 0); + if (d1 >= 0 && d2 <= 0) + regs->verdict.code = NFT_BREAK; break; } - - if (mismatch) - regs->verdict.code = NFT_BREAK; } static const struct nla_policy nft_range_policy[NFTA_RANGE_MAX + 1] = { @@ -59,6 +57,7 @@ static int nft_range_init(const struct nft_ctx *ctx, const struct nft_expr *expr struct nft_range_expr *priv = nft_expr_priv(expr); struct nft_data_desc desc_from, desc_to; int err; + u32 op; err = nft_data_init(NULL, &priv->data_from, sizeof(priv->data_from), &desc_from, tb[NFTA_RANGE_FROM_DATA]); @@ -80,7 +79,20 @@ static int nft_range_init(const struct nft_ctx *ctx, const struct nft_expr *expr if (err < 0) goto err2; - priv->op = ntohl(nla_get_be32(tb[NFTA_RANGE_OP])); + err = nft_parse_u32_check(tb[NFTA_RANGE_OP], U8_MAX, &op); + if (err < 0) + goto err2; + + switch (op) { + case NFT_RANGE_EQ: + case NFT_RANGE_NEQ: + break; + default: + err = -EINVAL; + goto err2; + } + + priv->op = op; priv->len = desc_from.len; return 0; err2: diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index e0aa7c1d0224..fc4977456c30 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -1513,7 +1513,7 @@ xt_hook_ops_alloc(const struct xt_table *table, nf_hookfn *fn) if (!num_hooks) return ERR_PTR(-EINVAL); - ops = kmalloc(sizeof(*ops) * num_hooks, GFP_KERNEL); + ops = kcalloc(num_hooks, sizeof(*ops), GFP_KERNEL); if (ops == NULL) return ERR_PTR(-ENOMEM); diff --git a/net/netfilter/xt_NFLOG.c b/net/netfilter/xt_NFLOG.c index 018eed7e1ff1..8668a5c18dc3 100644 --- a/net/netfilter/xt_NFLOG.c +++ b/net/netfilter/xt_NFLOG.c @@ -32,6 +32,7 @@ nflog_tg(struct sk_buff *skb, const struct xt_action_param *par) li.u.ulog.copy_len = info->len; li.u.ulog.group = info->group; li.u.ulog.qthreshold = info->threshold; + li.u.ulog.flags = 0; if (info->flags & XT_NFLOG_F_COPY_LEN) li.u.ulog.flags |= NF_LOG_F_COPY_LEN; diff --git a/net/netfilter/xt_hashlimit.c b/net/netfilter/xt_hashlimit.c index 2fab0c65aa94..b89b688e9d01 100644 --- a/net/netfilter/xt_hashlimit.c +++ b/net/netfilter/xt_hashlimit.c @@ -431,7 +431,7 @@ static void htable_put(struct xt_hashlimit_htable *hinfo) CREDITS_PER_JIFFY*HZ*60*60*24 < 2^32 ie. */ #define MAX_CPJ_v1 (0xFFFFFFFF / (HZ*60*60*24)) -#define MAX_CPJ (0xFFFFFFFFFFFFFFFF / (HZ*60*60*24)) +#define MAX_CPJ (0xFFFFFFFFFFFFFFFFULL / (HZ*60*60*24)) /* Repeated shift and or gives us all 1s, final shift and add 1 gives * us the power of 2 below the theoretical max, so GCC simply does a @@ -473,7 +473,7 @@ static u64 user2credits(u64 user, int revision) return div64_u64(user * HZ * CREDITS_PER_JIFFY_v1, XT_HASHLIMIT_SCALE); } else { - if (user > 0xFFFFFFFFFFFFFFFF / (HZ*CREDITS_PER_JIFFY)) + if (user > 0xFFFFFFFFFFFFFFFFULL / (HZ*CREDITS_PER_JIFFY)) return div64_u64(user, XT_HASHLIMIT_SCALE_v2) * HZ * CREDITS_PER_JIFFY; diff --git a/net/netfilter/xt_ipcomp.c b/net/netfilter/xt_ipcomp.c index 89d53104c6b3..000e70377f85 100644 --- a/net/netfilter/xt_ipcomp.c +++ b/net/netfilter/xt_ipcomp.c @@ -26,6 +26,8 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Fan Du <fan.du@windriver.com>"); MODULE_DESCRIPTION("Xtables: IPv4/6 IPsec-IPComp SPI match"); +MODULE_ALIAS("ipt_ipcomp"); +MODULE_ALIAS("ip6t_ipcomp"); /* Returns 1 if the spi is matched by the range, 0 otherwise */ static inline bool diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 11db0d619c00..d2238b204691 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -250,7 +250,7 @@ static void __fanout_link(struct sock *sk, struct packet_sock *po); static int packet_direct_xmit(struct sk_buff *skb) { struct net_device *dev = skb->dev; - netdev_features_t features; + struct sk_buff *orig_skb = skb; struct netdev_queue *txq; int ret = NETDEV_TX_BUSY; @@ -258,9 +258,8 @@ static int packet_direct_xmit(struct sk_buff *skb) !netif_carrier_ok(dev))) goto drop; - features = netif_skb_features(skb); - if (skb_needs_linearize(skb, features) && - __skb_linearize(skb)) + skb = validate_xmit_skb_list(skb, dev); + if (skb != orig_skb) goto drop; txq = skb_get_tx_queue(dev, skb); @@ -280,7 +279,7 @@ static int packet_direct_xmit(struct sk_buff *skb) return ret; drop: atomic_long_inc(&dev->tx_dropped); - kfree_skb(skb); + kfree_skb_list(skb); return NET_XMIT_DROP; } diff --git a/net/rds/Makefile b/net/rds/Makefile index 0e72bec1529f..56c7d27eefee 100644 --- a/net/rds/Makefile +++ b/net/rds/Makefile @@ -13,5 +13,5 @@ obj-$(CONFIG_RDS_TCP) += rds_tcp.o rds_tcp-y := tcp.o tcp_connect.o tcp_listen.o tcp_recv.o \ tcp_send.o tcp_stats.o -ccflags-$(CONFIG_RDS_DEBUG) := -DDEBUG +ccflags-$(CONFIG_RDS_DEBUG) := -DRDS_DEBUG diff --git a/net/rds/rds.h b/net/rds/rds.h index fd0bccb2f9f9..67ba67c058b1 100644 --- a/net/rds/rds.h +++ b/net/rds/rds.h @@ -33,7 +33,7 @@ #define KERNEL_HAS_ATOMIC64 #endif -#ifdef DEBUG +#ifdef RDS_DEBUG #define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args) #else /* sigh, pr_debug() causes unused variable warnings */ diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c index 4353a29f3b57..1ed18d8c9c9f 100644 --- a/net/rxrpc/call_object.c +++ b/net/rxrpc/call_object.c @@ -276,7 +276,7 @@ struct rxrpc_call *rxrpc_new_client_call(struct rxrpc_sock *rx, goto error; trace_rxrpc_call(call, rxrpc_call_connected, atomic_read(&call->usage), - here, ERR_PTR(ret)); + here, NULL); spin_lock_bh(&call->conn->params.peer->lock); hlist_add_head(&call->error_link, diff --git a/net/rxrpc/peer_object.c b/net/rxrpc/peer_object.c index 941b724d523b..862eea6b266c 100644 --- a/net/rxrpc/peer_object.c +++ b/net/rxrpc/peer_object.c @@ -193,8 +193,8 @@ static void rxrpc_assess_MTU_size(struct rxrpc_peer *peer) fl6->fl6_dport = htons(7001); fl6->fl6_sport = htons(7000); dst = ip6_route_output(&init_net, NULL, fl6); - if (IS_ERR(dst)) { - _leave(" [route err %ld]", PTR_ERR(dst)); + if (dst->error) { + _leave(" [route err %d]", dst->error); return; } break; diff --git a/net/sched/act_api.c b/net/sched/act_api.c index a512b18c0088..f893d180da1c 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -1028,8 +1028,7 @@ static struct nlattr *find_dump_kind(const struct nlmsghdr *n) if (tb[1] == NULL) return NULL; - if (nla_parse(tb2, TCA_ACT_MAX, nla_data(tb[1]), - nla_len(tb[1]), NULL) < 0) + if (nla_parse_nested(tb2, TCA_ACT_MAX, tb[1], NULL) < 0) return NULL; kind = tb2[TCA_ACT_KIND]; diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index 667dc382df82..6b07fba5770b 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -207,8 +207,11 @@ out: static void tcf_stats_update(struct tc_action *a, u64 bytes, u32 packets, u64 lastuse) { - tcf_lastuse_update(&a->tcfa_tm); + struct tcf_mirred *m = to_mirred(a); + struct tcf_t *tm = &m->tcf_tm; + _bstats_cpu_update(this_cpu_ptr(a->cpu_bstats), bytes, packets); + tm->lastuse = lastuse; } static int tcf_mirred_dump(struct sk_buff *skb, struct tc_action *a, int bind, diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 2ee29a3375f6..2b2a7974e4bb 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -345,7 +345,8 @@ replay: if (err == 0) { struct tcf_proto *next = rtnl_dereference(tp->next); - tfilter_notify(net, skb, n, tp, fh, + tfilter_notify(net, skb, n, tp, + t->tcm_handle, RTM_DELTFILTER, false); if (tcf_destroy(tp, false)) RCU_INIT_POINTER(*back, next); diff --git a/net/sctp/output.c b/net/sctp/output.c index 2a5c1896d18f..6cb0df859195 100644 --- a/net/sctp/output.c +++ b/net/sctp/output.c @@ -418,6 +418,7 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp) __u8 has_data = 0; int gso = 0; int pktcount = 0; + int auth_len = 0; struct dst_entry *dst; unsigned char *auth = NULL; /* pointer to auth in skb data */ @@ -510,7 +511,12 @@ int sctp_packet_transmit(struct sctp_packet *packet, gfp_t gfp) list_for_each_entry(chunk, &packet->chunk_list, list) { int padded = SCTP_PAD4(chunk->skb->len); - if (pkt_size + padded > tp->pathmtu) + if (chunk == packet->auth) + auth_len = padded; + else if (auth_len + padded + packet->overhead > + tp->pathmtu) + goto nomem; + else if (pkt_size + padded > tp->pathmtu) break; pkt_size += padded; } diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 026e3bca4a94..8ec20a64a3f8 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -3422,6 +3422,12 @@ sctp_disposition_t sctp_sf_ootb(struct net *net, return sctp_sf_violation_chunklen(net, ep, asoc, type, arg, commands); + /* Report violation if chunk len overflows */ + ch_end = ((__u8 *)ch) + SCTP_PAD4(ntohs(ch->length)); + if (ch_end > skb_tail_pointer(skb)) + return sctp_sf_violation_chunklen(net, ep, asoc, type, arg, + commands); + /* Now that we know we at least have a chunk header, * do things that are type appropriate. */ @@ -3453,12 +3459,6 @@ sctp_disposition_t sctp_sf_ootb(struct net *net, } } - /* Report violation if chunk len overflows */ - ch_end = ((__u8 *)ch) + SCTP_PAD4(ntohs(ch->length)); - if (ch_end > skb_tail_pointer(skb)) - return sctp_sf_violation_chunklen(net, ep, asoc, type, arg, - commands); - ch = (sctp_chunkhdr_t *) ch_end; } while (ch_end < skb_tail_pointer(skb)); diff --git a/net/sctp/socket.c b/net/sctp/socket.c index fb02c7033307..9fbb6feb8c27 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -4687,7 +4687,7 @@ static int sctp_getsockopt_disable_fragments(struct sock *sk, int len, static int sctp_getsockopt_events(struct sock *sk, int len, char __user *optval, int __user *optlen) { - if (len <= 0) + if (len == 0) return -EINVAL; if (len > sizeof(struct sctp_event_subscribe)) len = sizeof(struct sctp_event_subscribe); @@ -6430,6 +6430,9 @@ static int sctp_getsockopt(struct sock *sk, int level, int optname, if (get_user(len, optlen)) return -EFAULT; + if (len < 0) + return -EINVAL; + lock_sock(sk); switch (optname) { diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index d8bd97a5a7c9..3dfd769dc5b5 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -1616,7 +1616,7 @@ gss_validate(struct rpc_task *task, __be32 *p) { struct rpc_cred *cred = task->tk_rqstp->rq_cred; struct gss_cl_ctx *ctx = gss_cred_get_ctx(cred); - __be32 seq; + __be32 *seq = NULL; struct kvec iov; struct xdr_buf verf_buf; struct xdr_netobj mic; @@ -1631,9 +1631,12 @@ gss_validate(struct rpc_task *task, __be32 *p) goto out_bad; if (flav != RPC_AUTH_GSS) goto out_bad; - seq = htonl(task->tk_rqstp->rq_seqno); - iov.iov_base = &seq; - iov.iov_len = sizeof(seq); + seq = kmalloc(4, GFP_NOFS); + if (!seq) + goto out_bad; + *seq = htonl(task->tk_rqstp->rq_seqno); + iov.iov_base = seq; + iov.iov_len = 4; xdr_buf_from_iov(&iov, &verf_buf); mic.data = (u8 *)p; mic.len = len; @@ -1653,11 +1656,13 @@ gss_validate(struct rpc_task *task, __be32 *p) gss_put_ctx(ctx); dprintk("RPC: %5u %s: gss_verify_mic succeeded.\n", task->tk_pid, __func__); + kfree(seq); return p + XDR_QUADLEN(len); out_bad: gss_put_ctx(ctx); dprintk("RPC: %5u %s failed ret %ld.\n", task->tk_pid, __func__, PTR_ERR(ret)); + kfree(seq); return ret; } diff --git a/net/sunrpc/auth_gss/gss_krb5_crypto.c b/net/sunrpc/auth_gss/gss_krb5_crypto.c index 244245bcbbd2..90115ceefd49 100644 --- a/net/sunrpc/auth_gss/gss_krb5_crypto.c +++ b/net/sunrpc/auth_gss/gss_krb5_crypto.c @@ -166,8 +166,8 @@ make_checksum_hmac_md5(struct krb5_ctx *kctx, char *header, int hdrlen, unsigned int usage, struct xdr_netobj *cksumout) { struct scatterlist sg[1]; - int err; - u8 checksumdata[GSS_KRB5_MAX_CKSUM_LEN]; + int err = -1; + u8 *checksumdata; u8 rc4salt[4]; struct crypto_ahash *md5; struct crypto_ahash *hmac_md5; @@ -187,23 +187,22 @@ make_checksum_hmac_md5(struct krb5_ctx *kctx, char *header, int hdrlen, return GSS_S_FAILURE; } + checksumdata = kmalloc(GSS_KRB5_MAX_CKSUM_LEN, GFP_NOFS); + if (!checksumdata) + return GSS_S_FAILURE; + md5 = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); if (IS_ERR(md5)) - return GSS_S_FAILURE; + goto out_free_cksum; hmac_md5 = crypto_alloc_ahash(kctx->gk5e->cksum_name, 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(hmac_md5)) { - crypto_free_ahash(md5); - return GSS_S_FAILURE; - } + if (IS_ERR(hmac_md5)) + goto out_free_md5; req = ahash_request_alloc(md5, GFP_KERNEL); - if (!req) { - crypto_free_ahash(hmac_md5); - crypto_free_ahash(md5); - return GSS_S_FAILURE; - } + if (!req) + goto out_free_hmac_md5; ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); @@ -232,11 +231,8 @@ make_checksum_hmac_md5(struct krb5_ctx *kctx, char *header, int hdrlen, ahash_request_free(req); req = ahash_request_alloc(hmac_md5, GFP_KERNEL); - if (!req) { - crypto_free_ahash(hmac_md5); - crypto_free_ahash(md5); - return GSS_S_FAILURE; - } + if (!req) + goto out_free_hmac_md5; ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); @@ -258,8 +254,12 @@ make_checksum_hmac_md5(struct krb5_ctx *kctx, char *header, int hdrlen, cksumout->len = kctx->gk5e->cksumlength; out: ahash_request_free(req); - crypto_free_ahash(md5); +out_free_hmac_md5: crypto_free_ahash(hmac_md5); +out_free_md5: + crypto_free_ahash(md5); +out_free_cksum: + kfree(checksumdata); return err ? GSS_S_FAILURE : 0; } @@ -276,8 +276,8 @@ make_checksum(struct krb5_ctx *kctx, char *header, int hdrlen, struct crypto_ahash *tfm; struct ahash_request *req; struct scatterlist sg[1]; - int err; - u8 checksumdata[GSS_KRB5_MAX_CKSUM_LEN]; + int err = -1; + u8 *checksumdata; unsigned int checksumlen; if (kctx->gk5e->ctype == CKSUMTYPE_HMAC_MD5_ARCFOUR) @@ -291,15 +291,17 @@ make_checksum(struct krb5_ctx *kctx, char *header, int hdrlen, return GSS_S_FAILURE; } + checksumdata = kmalloc(GSS_KRB5_MAX_CKSUM_LEN, GFP_NOFS); + if (checksumdata == NULL) + return GSS_S_FAILURE; + tfm = crypto_alloc_ahash(kctx->gk5e->cksum_name, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) - return GSS_S_FAILURE; + goto out_free_cksum; req = ahash_request_alloc(tfm, GFP_KERNEL); - if (!req) { - crypto_free_ahash(tfm); - return GSS_S_FAILURE; - } + if (!req) + goto out_free_ahash; ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); @@ -349,7 +351,10 @@ make_checksum(struct krb5_ctx *kctx, char *header, int hdrlen, cksumout->len = kctx->gk5e->cksumlength; out: ahash_request_free(req); +out_free_ahash: crypto_free_ahash(tfm); +out_free_cksum: + kfree(checksumdata); return err ? GSS_S_FAILURE : 0; } @@ -368,8 +373,8 @@ make_checksum_v2(struct krb5_ctx *kctx, char *header, int hdrlen, struct crypto_ahash *tfm; struct ahash_request *req; struct scatterlist sg[1]; - int err; - u8 checksumdata[GSS_KRB5_MAX_CKSUM_LEN]; + int err = -1; + u8 *checksumdata; unsigned int checksumlen; if (kctx->gk5e->keyed_cksum == 0) { @@ -383,16 +388,18 @@ make_checksum_v2(struct krb5_ctx *kctx, char *header, int hdrlen, return GSS_S_FAILURE; } + checksumdata = kmalloc(GSS_KRB5_MAX_CKSUM_LEN, GFP_NOFS); + if (!checksumdata) + return GSS_S_FAILURE; + tfm = crypto_alloc_ahash(kctx->gk5e->cksum_name, 0, CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) - return GSS_S_FAILURE; + goto out_free_cksum; checksumlen = crypto_ahash_digestsize(tfm); req = ahash_request_alloc(tfm, GFP_KERNEL); - if (!req) { - crypto_free_ahash(tfm); - return GSS_S_FAILURE; - } + if (!req) + goto out_free_ahash; ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); @@ -433,7 +440,10 @@ make_checksum_v2(struct krb5_ctx *kctx, char *header, int hdrlen, } out: ahash_request_free(req); +out_free_ahash: crypto_free_ahash(tfm); +out_free_cksum: + kfree(checksumdata); return err ? GSS_S_FAILURE : 0; } @@ -666,14 +676,17 @@ gss_krb5_cts_crypt(struct crypto_skcipher *cipher, struct xdr_buf *buf, u32 ret; struct scatterlist sg[1]; SKCIPHER_REQUEST_ON_STACK(req, cipher); - u8 data[GSS_KRB5_MAX_BLOCKSIZE * 2]; + u8 *data; struct page **save_pages; u32 len = buf->len - offset; - if (len > ARRAY_SIZE(data)) { + if (len > GSS_KRB5_MAX_BLOCKSIZE * 2) { WARN_ON(0); return -ENOMEM; } + data = kmalloc(GSS_KRB5_MAX_BLOCKSIZE * 2, GFP_NOFS); + if (!data) + return -ENOMEM; /* * For encryption, we want to read from the cleartext @@ -708,6 +721,7 @@ gss_krb5_cts_crypt(struct crypto_skcipher *cipher, struct xdr_buf *buf, ret = write_bytes_to_xdr_buf(buf, offset, data, len); out: + kfree(data); return ret; } diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index d67f7e1bc82d..45662d7f0943 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -718,30 +718,37 @@ gss_write_null_verf(struct svc_rqst *rqstp) static int gss_write_verf(struct svc_rqst *rqstp, struct gss_ctx *ctx_id, u32 seq) { - __be32 xdr_seq; + __be32 *xdr_seq; u32 maj_stat; struct xdr_buf verf_data; struct xdr_netobj mic; __be32 *p; struct kvec iov; + int err = -1; svc_putnl(rqstp->rq_res.head, RPC_AUTH_GSS); - xdr_seq = htonl(seq); + xdr_seq = kmalloc(4, GFP_KERNEL); + if (!xdr_seq) + return -1; + *xdr_seq = htonl(seq); - iov.iov_base = &xdr_seq; - iov.iov_len = sizeof(xdr_seq); + iov.iov_base = xdr_seq; + iov.iov_len = 4; xdr_buf_from_iov(&iov, &verf_data); p = rqstp->rq_res.head->iov_base + rqstp->rq_res.head->iov_len; mic.data = (u8 *)(p + 1); maj_stat = gss_get_mic(ctx_id, &verf_data, &mic); if (maj_stat != GSS_S_COMPLETE) - return -1; + goto out; *p++ = htonl(mic.len); memset((u8 *)p + mic.len, 0, round_up_to_quad(mic.len) - mic.len); p += XDR_QUADLEN(mic.len); if (!xdr_ressize_check(rqstp, p)) - return -1; - return 0; + goto out; + err = 0; +out: + kfree(xdr_seq); + return err; } struct gss_domain { diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c index 2d8545c34095..20027f8de129 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c +++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c @@ -177,18 +177,26 @@ xprt_rdma_bc_allocate(struct rpc_task *task) return -EINVAL; } + /* svc_rdma_sendto releases this page */ page = alloc_page(RPCRDMA_DEF_GFP); if (!page) return -ENOMEM; - rqst->rq_buffer = page_address(page); + + rqst->rq_rbuffer = kmalloc(rqst->rq_rcvsize, RPCRDMA_DEF_GFP); + if (!rqst->rq_rbuffer) { + put_page(page); + return -ENOMEM; + } return 0; } static void xprt_rdma_bc_free(struct rpc_task *task) { - /* No-op: ctxt and page have already been freed. */ + struct rpc_rqst *rqst = task->tk_rqstp; + + kfree(rqst->rq_rbuffer); } static int diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 0137af1c0916..e01c825bc683 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -2563,6 +2563,7 @@ static int bc_malloc(struct rpc_task *task) buf->len = PAGE_SIZE; rqst->rq_buffer = buf->data; + rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; return 0; } diff --git a/net/switchdev/switchdev.c b/net/switchdev/switchdev.c index 02beb35f577f..3b95fe980fa2 100644 --- a/net/switchdev/switchdev.c +++ b/net/switchdev/switchdev.c @@ -771,6 +771,9 @@ int switchdev_port_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, u32 mask = BR_LEARNING | BR_LEARNING_SYNC | BR_FLOOD; int err; + if (!netif_is_bridge_port(dev)) + return -EOPNOTSUPP; + err = switchdev_port_attr_get(dev, &attr); if (err && err != -EOPNOTSUPP) return err; @@ -926,6 +929,9 @@ int switchdev_port_bridge_setlink(struct net_device *dev, struct nlattr *afspec; int err = 0; + if (!netif_is_bridge_port(dev)) + return -EOPNOTSUPP; + protinfo = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_PROTINFO); if (protinfo) { @@ -959,6 +965,9 @@ int switchdev_port_bridge_dellink(struct net_device *dev, { struct nlattr *afspec; + if (!netif_is_bridge_port(dev)) + return -EOPNOTSUPP; + afspec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); if (afspec) diff --git a/net/tipc/bcast.c b/net/tipc/bcast.c index 753f774cb46f..aa1babbea385 100644 --- a/net/tipc/bcast.c +++ b/net/tipc/bcast.c @@ -247,11 +247,17 @@ int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb) * * RCU is locked, no other locks set */ -void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, u32 acked) +void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, + struct tipc_msg *hdr) { struct sk_buff_head *inputq = &tipc_bc_base(net)->inputq; + u16 acked = msg_bcast_ack(hdr); struct sk_buff_head xmitq; + /* Ignore bc acks sent by peer before bcast synch point was received */ + if (msg_bc_ack_invalid(hdr)) + return; + __skb_queue_head_init(&xmitq); tipc_bcast_lock(net); @@ -279,11 +285,11 @@ int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, __skb_queue_head_init(&xmitq); tipc_bcast_lock(net); - if (msg_type(hdr) == STATE_MSG) { + if (msg_type(hdr) != STATE_MSG) { + tipc_link_bc_init_rcv(l, hdr); + } else if (!msg_bc_ack_invalid(hdr)) { tipc_link_bc_ack_rcv(l, msg_bcast_ack(hdr), &xmitq); rc = tipc_link_bc_sync_rcv(l, hdr, &xmitq); - } else { - tipc_link_bc_init_rcv(l, hdr); } tipc_bcast_unlock(net); diff --git a/net/tipc/bcast.h b/net/tipc/bcast.h index 5ffe34472ccd..855d53c64ab3 100644 --- a/net/tipc/bcast.h +++ b/net/tipc/bcast.h @@ -55,7 +55,8 @@ void tipc_bcast_dec_bearer_dst_cnt(struct net *net, int bearer_id); int tipc_bcast_get_mtu(struct net *net); int tipc_bcast_xmit(struct net *net, struct sk_buff_head *list); int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb); -void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, u32 acked); +void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, + struct tipc_msg *hdr); int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr); int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg); diff --git a/net/tipc/link.c b/net/tipc/link.c index b36e16cdc945..1055164c6232 100644 --- a/net/tipc/link.c +++ b/net/tipc/link.c @@ -1312,6 +1312,7 @@ static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe, msg_set_next_sent(hdr, l->snd_nxt); msg_set_ack(hdr, l->rcv_nxt - 1); msg_set_bcast_ack(hdr, bcl->rcv_nxt - 1); + msg_set_bc_ack_invalid(hdr, !node_up); msg_set_last_bcast(hdr, l->bc_sndlink->snd_nxt - 1); msg_set_link_tolerance(hdr, tolerance); msg_set_linkprio(hdr, priority); @@ -1574,6 +1575,7 @@ static void tipc_link_build_bc_init_msg(struct tipc_link *l, __skb_queue_head_init(&list); if (!tipc_link_build_bc_proto_msg(l->bc_rcvlink, false, 0, &list)) return; + msg_set_bc_ack_invalid(buf_msg(skb_peek(&list)), true); tipc_link_xmit(l, &list, xmitq); } diff --git a/net/tipc/msg.h b/net/tipc/msg.h index c3832cdf2278..50a739860d37 100644 --- a/net/tipc/msg.h +++ b/net/tipc/msg.h @@ -714,6 +714,23 @@ static inline void msg_set_peer_stopping(struct tipc_msg *m, u32 s) msg_set_bits(m, 5, 13, 0x1, s); } +static inline bool msg_bc_ack_invalid(struct tipc_msg *m) +{ + switch (msg_user(m)) { + case BCAST_PROTOCOL: + case NAME_DISTRIBUTOR: + case LINK_PROTOCOL: + return msg_bits(m, 5, 14, 0x1); + default: + return false; + } +} + +static inline void msg_set_bc_ack_invalid(struct tipc_msg *m, bool invalid) +{ + msg_set_bits(m, 5, 14, 0x1, invalid); +} + static inline char *msg_media_addr(struct tipc_msg *m) { return (char *)&m->hdr[TIPC_MEDIA_INFO_OFFSET]; diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c index a04fe9be1c60..c1cfd92de17a 100644 --- a/net/tipc/name_distr.c +++ b/net/tipc/name_distr.c @@ -156,6 +156,7 @@ static void named_distribute(struct net *net, struct sk_buff_head *list, pr_warn("Bulk publication failure\n"); return; } + msg_set_bc_ack_invalid(buf_msg(skb), true); item = (struct distr_item *)msg_data(buf_msg(skb)); } diff --git a/net/tipc/node.c b/net/tipc/node.c index 7ef14e2d2356..9d2f4c2b08ab 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -1535,7 +1535,7 @@ void tipc_rcv(struct net *net, struct sk_buff *skb, struct tipc_bearer *b) if (unlikely(usr == LINK_PROTOCOL)) tipc_node_bc_sync_rcv(n, hdr, bearer_id, &xmitq); else if (unlikely(tipc_link_acked(n->bc_entry.link) != bc_ack)) - tipc_bcast_ack_rcv(net, n->bc_entry.link, bc_ack); + tipc_bcast_ack_rcv(net, n->bc_entry.link, hdr); /* Receive packet directly if conditions permit */ tipc_node_read_lock(n); diff --git a/net/wireless/sysfs.c b/net/wireless/sysfs.c index 0082f4b01795..14b3f007826d 100644 --- a/net/wireless/sysfs.c +++ b/net/wireless/sysfs.c @@ -104,13 +104,16 @@ static int wiphy_suspend(struct device *dev) rtnl_lock(); if (rdev->wiphy.registered) { - if (!rdev->wiphy.wowlan_config) + if (!rdev->wiphy.wowlan_config) { cfg80211_leave_all(rdev); + cfg80211_process_rdev_events(rdev); + } if (rdev->ops->suspend) ret = rdev_suspend(rdev, rdev->wiphy.wowlan_config); if (ret == 1) { /* Driver refuse to configure wowlan */ cfg80211_leave_all(rdev); + cfg80211_process_rdev_events(rdev); ret = rdev_suspend(rdev, NULL); } } diff --git a/net/wireless/util.c b/net/wireless/util.c index 8edce22d1b93..5ea12afc7706 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -420,8 +420,8 @@ unsigned int ieee80211_get_mesh_hdrlen(struct ieee80211s_hdr *meshhdr) } EXPORT_SYMBOL(ieee80211_get_mesh_hdrlen); -static int __ieee80211_data_to_8023(struct sk_buff *skb, struct ethhdr *ehdr, - const u8 *addr, enum nl80211_iftype iftype) +int ieee80211_data_to_8023_exthdr(struct sk_buff *skb, struct ethhdr *ehdr, + const u8 *addr, enum nl80211_iftype iftype) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data; struct { @@ -525,13 +525,7 @@ static int __ieee80211_data_to_8023(struct sk_buff *skb, struct ethhdr *ehdr, return 0; } - -int ieee80211_data_to_8023(struct sk_buff *skb, const u8 *addr, - enum nl80211_iftype iftype) -{ - return __ieee80211_data_to_8023(skb, NULL, addr, iftype); -} -EXPORT_SYMBOL(ieee80211_data_to_8023); +EXPORT_SYMBOL(ieee80211_data_to_8023_exthdr); int ieee80211_data_from_8023(struct sk_buff *skb, const u8 *addr, enum nl80211_iftype iftype, @@ -746,24 +740,18 @@ __ieee80211_amsdu_copy(struct sk_buff *skb, unsigned int hlen, void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list, const u8 *addr, enum nl80211_iftype iftype, const unsigned int extra_headroom, - bool has_80211_header) + const u8 *check_da, const u8 *check_sa) { unsigned int hlen = ALIGN(extra_headroom, 4); struct sk_buff *frame = NULL; u16 ethertype; u8 *payload; - int offset = 0, remaining, err; + int offset = 0, remaining; struct ethhdr eth; bool reuse_frag = skb->head_frag && !skb_has_frag_list(skb); bool reuse_skb = false; bool last = false; - if (has_80211_header) { - err = __ieee80211_data_to_8023(skb, ð, addr, iftype); - if (err) - goto out; - } - while (!last) { unsigned int subframe_len; int len; @@ -780,8 +768,17 @@ void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list, goto purge; offset += sizeof(struct ethhdr); - /* reuse skb for the last subframe */ last = remaining <= subframe_len + padding; + + /* FIXME: should we really accept multicast DA? */ + if ((check_da && !is_multicast_ether_addr(eth.h_dest) && + !ether_addr_equal(check_da, eth.h_dest)) || + (check_sa && !ether_addr_equal(check_sa, eth.h_source))) { + offset += len + padding; + continue; + } + + /* reuse skb for the last subframe */ if (!skb_is_nonlinear(skb) && !reuse_frag && last) { skb_pull(skb, offset); frame = skb; @@ -819,7 +816,6 @@ void ieee80211_amsdu_to_8023s(struct sk_buff *skb, struct sk_buff_head *list, purge: __skb_queue_purge(list); - out: dev_kfree_skb(skb); } EXPORT_SYMBOL(ieee80211_amsdu_to_8023s); diff --git a/samples/bpf/parse_ldabs.c b/samples/bpf/parse_ldabs.c index d17550198d06..6db6b21fdc6d 100644 --- a/samples/bpf/parse_ldabs.c +++ b/samples/bpf/parse_ldabs.c @@ -4,6 +4,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define KBUILD_MODNAME "foo" #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/in.h> diff --git a/samples/bpf/parse_simple.c b/samples/bpf/parse_simple.c index cf2511c33905..10af53d33cc2 100644 --- a/samples/bpf/parse_simple.c +++ b/samples/bpf/parse_simple.c @@ -4,6 +4,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define KBUILD_MODNAME "foo" #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/in.h> diff --git a/samples/bpf/parse_varlen.c b/samples/bpf/parse_varlen.c index edab34dce79b..95c16324760c 100644 --- a/samples/bpf/parse_varlen.c +++ b/samples/bpf/parse_varlen.c @@ -4,6 +4,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define KBUILD_MODNAME "foo" #include <linux/if_ether.h> #include <linux/ip.h> #include <linux/ipv6.h> diff --git a/samples/bpf/tcbpf1_kern.c b/samples/bpf/tcbpf1_kern.c index fa051b3d53ee..274c884c87fe 100644 --- a/samples/bpf/tcbpf1_kern.c +++ b/samples/bpf/tcbpf1_kern.c @@ -1,3 +1,4 @@ +#define KBUILD_MODNAME "foo" #include <uapi/linux/bpf.h> #include <uapi/linux/if_ether.h> #include <uapi/linux/if_packet.h> diff --git a/samples/bpf/tcbpf2_kern.c b/samples/bpf/tcbpf2_kern.c index 3303bb85593b..9c823a609e75 100644 --- a/samples/bpf/tcbpf2_kern.c +++ b/samples/bpf/tcbpf2_kern.c @@ -5,6 +5,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define KBUILD_MODNAME "foo" #include <uapi/linux/bpf.h> #include <uapi/linux/if_ether.h> #include <uapi/linux/if_packet.h> diff --git a/samples/bpf/test_cgrp2_tc_kern.c b/samples/bpf/test_cgrp2_tc_kern.c index 10ff73404e3a..1547b36a7b7b 100644 --- a/samples/bpf/test_cgrp2_tc_kern.c +++ b/samples/bpf/test_cgrp2_tc_kern.c @@ -4,6 +4,7 @@ * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ +#define KBUILD_MODNAME "foo" #include <uapi/linux/if_ether.h> #include <uapi/linux/in6.h> #include <uapi/linux/ipv6.h> diff --git a/scripts/gcc-plugins/cyc_complexity_plugin.c b/scripts/gcc-plugins/cyc_complexity_plugin.c index 34df974c6ba3..8af7db06122d 100644 --- a/scripts/gcc-plugins/cyc_complexity_plugin.c +++ b/scripts/gcc-plugins/cyc_complexity_plugin.c @@ -20,7 +20,7 @@ #include "gcc-common.h" -int plugin_is_GPL_compatible; +__visible int plugin_is_GPL_compatible; static struct plugin_info cyc_complexity_plugin_info = { .version = "20160225", @@ -49,7 +49,7 @@ static unsigned int cyc_complexity_execute(void) #include "gcc-generate-gimple-pass.h" -int plugin_init(struct plugin_name_args *plugin_info, struct plugin_gcc_version *version) +__visible int plugin_init(struct plugin_name_args *plugin_info, struct plugin_gcc_version *version) { const char * const plugin_name = plugin_info->base_name; struct register_pass_info cyc_complexity_pass_info; diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 172850bcd0d9..950fd2e64bb7 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -130,6 +130,7 @@ extern void dump_gimple_stmt(pretty_printer *, gimple, int, int); #endif #define __unused __attribute__((__unused__)) +#define __visible __attribute__((visibility("default"))) #define DECL_NAME_POINTER(node) IDENTIFIER_POINTER(DECL_NAME(node)) #define DECL_NAME_LENGTH(node) IDENTIFIER_LENGTH(DECL_NAME(node)) diff --git a/scripts/gcc-plugins/latent_entropy_plugin.c b/scripts/gcc-plugins/latent_entropy_plugin.c index ff1939b804ae..8160f1c1b56e 100644 --- a/scripts/gcc-plugins/latent_entropy_plugin.c +++ b/scripts/gcc-plugins/latent_entropy_plugin.c @@ -77,7 +77,7 @@ #include "gcc-common.h" -int plugin_is_GPL_compatible; +__visible int plugin_is_GPL_compatible; static GTY(()) tree latent_entropy_decl; @@ -340,7 +340,7 @@ static enum tree_code get_op(tree *rhs) break; } if (rhs) - *rhs = build_int_cstu(unsigned_intDI_type_node, random_const); + *rhs = build_int_cstu(long_unsigned_type_node, random_const); return op; } @@ -372,7 +372,7 @@ static void __perturb_latent_entropy(gimple_stmt_iterator *gsi, enum tree_code op; /* 1. create temporary copy of latent_entropy */ - temp = create_var(unsigned_intDI_type_node, "tmp_latent_entropy"); + temp = create_var(long_unsigned_type_node, "temp_latent_entropy"); /* 2. read... */ add_referenced_var(latent_entropy_decl); @@ -459,13 +459,13 @@ static void init_local_entropy(basic_block bb, tree local_entropy) gsi_insert_before(&gsi, call, GSI_NEW_STMT); update_stmt(call); - udi_frame_addr = fold_convert(unsigned_intDI_type_node, frame_addr); + udi_frame_addr = fold_convert(long_unsigned_type_node, frame_addr); assign = gimple_build_assign(local_entropy, udi_frame_addr); gsi_insert_after(&gsi, assign, GSI_NEW_STMT); update_stmt(assign); /* 3. create temporary copy of latent_entropy */ - tmp = create_var(unsigned_intDI_type_node, "tmp_latent_entropy"); + tmp = create_var(long_unsigned_type_node, "temp_latent_entropy"); /* 4. read the global entropy variable into local entropy */ add_referenced_var(latent_entropy_decl); @@ -480,7 +480,7 @@ static void init_local_entropy(basic_block bb, tree local_entropy) update_stmt(assign); rand_cst = get_random_const(); - rand_const = build_int_cstu(unsigned_intDI_type_node, rand_cst); + rand_const = build_int_cstu(long_unsigned_type_node, rand_cst); op = get_op(NULL); assign = create_assign(op, local_entropy, local_entropy, rand_const); gsi_insert_after(&gsi, assign, GSI_NEW_STMT); @@ -529,7 +529,7 @@ static unsigned int latent_entropy_execute(void) } /* 1. create the local entropy variable */ - local_entropy = create_var(unsigned_intDI_type_node, "local_entropy"); + local_entropy = create_var(long_unsigned_type_node, "local_entropy"); /* 2. initialize the local entropy variable */ init_local_entropy(bb, local_entropy); @@ -561,10 +561,9 @@ static void latent_entropy_start_unit(void *gcc_data __unused, if (in_lto_p) return; - /* extern volatile u64 latent_entropy */ - gcc_assert(TYPE_PRECISION(long_long_unsigned_type_node) == 64); - quals = TYPE_QUALS(long_long_unsigned_type_node) | TYPE_QUAL_VOLATILE; - type = build_qualified_type(long_long_unsigned_type_node, quals); + /* extern volatile unsigned long latent_entropy */ + quals = TYPE_QUALS(long_unsigned_type_node) | TYPE_QUAL_VOLATILE; + type = build_qualified_type(long_unsigned_type_node, quals); id = get_identifier("latent_entropy"); latent_entropy_decl = build_decl(UNKNOWN_LOCATION, VAR_DECL, id, type); @@ -584,8 +583,8 @@ static void latent_entropy_start_unit(void *gcc_data __unused, | TODO_update_ssa #include "gcc-generate-gimple-pass.h" -int plugin_init(struct plugin_name_args *plugin_info, - struct plugin_gcc_version *version) +__visible int plugin_init(struct plugin_name_args *plugin_info, + struct plugin_gcc_version *version) { bool enabled = true; const char * const plugin_name = plugin_info->base_name; diff --git a/scripts/gcc-plugins/sancov_plugin.c b/scripts/gcc-plugins/sancov_plugin.c index aedd6113cb73..7ea0b3f50739 100644 --- a/scripts/gcc-plugins/sancov_plugin.c +++ b/scripts/gcc-plugins/sancov_plugin.c @@ -21,7 +21,7 @@ #include "gcc-common.h" -int plugin_is_GPL_compatible; +__visible int plugin_is_GPL_compatible; tree sancov_fndecl; @@ -86,7 +86,7 @@ static void sancov_start_unit(void __unused *gcc_data, void __unused *user_data) #endif } -int plugin_init(struct plugin_name_args *plugin_info, struct plugin_gcc_version *version) +__visible int plugin_init(struct plugin_name_args *plugin_info, struct plugin_gcc_version *version) { int i; struct register_pass_info sancov_plugin_pass_info; diff --git a/security/keys/Kconfig b/security/keys/Kconfig index f826e8739023..d942c7c2bc0a 100644 --- a/security/keys/Kconfig +++ b/security/keys/Kconfig @@ -41,7 +41,7 @@ config BIG_KEYS bool "Large payload keys" depends on KEYS depends on TMPFS - select CRYPTO + depends on (CRYPTO_ANSI_CPRNG = y || CRYPTO_DRBG = y) select CRYPTO_AES select CRYPTO_ECB select CRYPTO_RNG diff --git a/security/keys/big_key.c b/security/keys/big_key.c index c0b3030b5634..835c1ab30d01 100644 --- a/security/keys/big_key.c +++ b/security/keys/big_key.c @@ -9,6 +9,7 @@ * 2 of the Licence, or (at your option) any later version. */ +#define pr_fmt(fmt) "big_key: "fmt #include <linux/init.h> #include <linux/seq_file.h> #include <linux/file.h> @@ -341,44 +342,48 @@ error: */ static int __init big_key_init(void) { - return register_key_type(&key_type_big_key); -} - -/* - * Initialize big_key crypto and RNG algorithms - */ -static int __init big_key_crypto_init(void) -{ - int ret = -EINVAL; + struct crypto_skcipher *cipher; + struct crypto_rng *rng; + int ret; - /* init RNG */ - big_key_rng = crypto_alloc_rng(big_key_rng_name, 0, 0); - if (IS_ERR(big_key_rng)) { - big_key_rng = NULL; - return -EFAULT; + rng = crypto_alloc_rng(big_key_rng_name, 0, 0); + if (IS_ERR(rng)) { + pr_err("Can't alloc rng: %ld\n", PTR_ERR(rng)); + return PTR_ERR(rng); } + big_key_rng = rng; + /* seed RNG */ - ret = crypto_rng_reset(big_key_rng, NULL, crypto_rng_seedsize(big_key_rng)); - if (ret) - goto error; + ret = crypto_rng_reset(rng, NULL, crypto_rng_seedsize(rng)); + if (ret) { + pr_err("Can't reset rng: %d\n", ret); + goto error_rng; + } /* init block cipher */ - big_key_skcipher = crypto_alloc_skcipher(big_key_alg_name, - 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(big_key_skcipher)) { - big_key_skcipher = NULL; - ret = -EFAULT; - goto error; + cipher = crypto_alloc_skcipher(big_key_alg_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(cipher)) { + ret = PTR_ERR(cipher); + pr_err("Can't alloc crypto: %d\n", ret); + goto error_rng; + } + + big_key_skcipher = cipher; + + ret = register_key_type(&key_type_big_key); + if (ret < 0) { + pr_err("Can't register type: %d\n", ret); + goto error_cipher; } return 0; -error: +error_cipher: + crypto_free_skcipher(big_key_skcipher); +error_rng: crypto_free_rng(big_key_rng); - big_key_rng = NULL; return ret; } -device_initcall(big_key_init); -late_initcall(big_key_crypto_init); +late_initcall(big_key_init); diff --git a/security/keys/proc.c b/security/keys/proc.c index f0611a6368cd..b9f531c9e4fa 100644 --- a/security/keys/proc.c +++ b/security/keys/proc.c @@ -181,7 +181,7 @@ static int proc_keys_show(struct seq_file *m, void *v) struct timespec now; unsigned long timo; key_ref_t key_ref, skey_ref; - char xbuf[12]; + char xbuf[16]; int rc; struct keyring_search_context ctx = { diff --git a/sound/core/seq/seq_timer.c b/sound/core/seq/seq_timer.c index dcc102813aef..37d9cfbc29f9 100644 --- a/sound/core/seq/seq_timer.c +++ b/sound/core/seq/seq_timer.c @@ -448,8 +448,8 @@ snd_seq_real_time_t snd_seq_timer_get_cur_time(struct snd_seq_timer *tmr) ktime_get_ts64(&tm); tm = timespec64_sub(tm, tmr->last_update); - cur_time.tv_nsec = tm.tv_nsec; - cur_time.tv_sec = tm.tv_sec; + cur_time.tv_nsec += tm.tv_nsec; + cur_time.tv_sec += tm.tv_sec; snd_seq_sanity_real_time(&cur_time); } spin_unlock_irqrestore(&tmr->lock, flags); diff --git a/sound/pci/asihpi/hpioctl.c b/sound/pci/asihpi/hpioctl.c index d17937b92331..7e3aa50b21f9 100644 --- a/sound/pci/asihpi/hpioctl.c +++ b/sound/pci/asihpi/hpioctl.c @@ -111,7 +111,7 @@ long asihpi_hpi_ioctl(struct file *file, unsigned int cmd, unsigned long arg) return -EINVAL; hm = kmalloc(sizeof(*hm), GFP_KERNEL); - hr = kmalloc(sizeof(*hr), GFP_KERNEL); + hr = kzalloc(sizeof(*hr), GFP_KERNEL); if (!hm || !hr) { err = -ENOMEM; goto out; diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index c3469f756ec2..c64d986009a9 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -341,8 +341,7 @@ enum { /* quirks for Nvidia */ #define AZX_DCAPS_PRESET_NVIDIA \ - (AZX_DCAPS_NO_MSI | /*AZX_DCAPS_ALIGN_BUFSIZE |*/ \ - AZX_DCAPS_NO_64BIT | AZX_DCAPS_CORBRP_SELF_CLEAR |\ + (AZX_DCAPS_NO_MSI | AZX_DCAPS_CORBRP_SELF_CLEAR |\ AZX_DCAPS_SNOOP_TYPE(NVIDIA)) #define AZX_DCAPS_PRESET_CTHDA \ @@ -1716,6 +1715,10 @@ static int azx_first_init(struct azx *chip) } } + /* NVidia hardware normally only supports up to 40 bits of DMA */ + if (chip->pci->vendor == PCI_VENDOR_ID_NVIDIA) + dma_bits = 40; + /* disable 64bit DMA address on some devices */ if (chip->driver_caps & AZX_DCAPS_NO_64BIT) { dev_dbg(card->dev, "Disabling 64bit DMA\n"); diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index b58e8c76346a..2f909dd8b7b8 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -5811,8 +5811,6 @@ static const struct hda_model_fixup alc269_fixup_models[] = { #define ALC295_STANDARD_PINS \ {0x12, 0xb7a60130}, \ {0x14, 0x90170110}, \ - {0x17, 0x21014020}, \ - {0x18, 0x21a19030}, \ {0x21, 0x04211020} #define ALC298_STANDARD_PINS \ @@ -5859,11 +5857,19 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = { {0x1b, 0x02011020}, {0x21, 0x0221101f}), SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, + {0x14, 0x90170110}, + {0x1b, 0x01011020}, + {0x21, 0x0221101f}), + SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, {0x14, 0x90170130}, {0x1b, 0x01014020}, {0x21, 0x0221103f}), SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, {0x14, 0x90170130}, + {0x1b, 0x01011020}, + {0x21, 0x0221103f}), + SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, + {0x14, 0x90170130}, {0x1b, 0x02011020}, {0x21, 0x0221103f}), SND_HDA_PIN_QUIRK(0x10ec0255, 0x1028, "Dell", ALC255_FIXUP_DELL1_MIC_NO_PRESENCE, @@ -6039,7 +6045,13 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = { ALC292_STANDARD_PINS, {0x13, 0x90a60140}), SND_HDA_PIN_QUIRK(0x10ec0295, 0x1028, "Dell", ALC269_FIXUP_DELL1_MIC_NO_PRESENCE, - ALC295_STANDARD_PINS), + ALC295_STANDARD_PINS, + {0x17, 0x21014020}, + {0x18, 0x21a19030}), + SND_HDA_PIN_QUIRK(0x10ec0295, 0x1028, "Dell", ALC269_FIXUP_DELL1_MIC_NO_PRESENCE, + ALC295_STANDARD_PINS, + {0x17, 0x21014040}, + {0x18, 0x21a19050}), SND_HDA_PIN_QUIRK(0x10ec0298, 0x1028, "Dell", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE, ALC298_STANDARD_PINS, {0x17, 0x90170110}), @@ -6613,6 +6625,7 @@ enum { ALC891_FIXUP_HEADSET_MODE, ALC891_FIXUP_DELL_MIC_NO_PRESENCE, ALC662_FIXUP_ACER_VERITON, + ALC892_FIXUP_ASROCK_MOBO, }; static const struct hda_fixup alc662_fixups[] = { @@ -6889,6 +6902,16 @@ static const struct hda_fixup alc662_fixups[] = { { } } }, + [ALC892_FIXUP_ASROCK_MOBO] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x15, 0x40f000f0 }, /* disabled */ + { 0x16, 0x40f000f0 }, /* disabled */ + { 0x18, 0x01014011 }, /* LO */ + { 0x1a, 0x01014012 }, /* LO */ + { } + } + }, }; static const struct snd_pci_quirk alc662_fixup_tbl[] = { @@ -6926,6 +6949,7 @@ static const struct snd_pci_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc051, "Samsung R720", ALC662_FIXUP_IDEAPAD), SND_PCI_QUIRK(0x17aa, 0x38af, "Lenovo Ideapad Y550P", ALC662_FIXUP_IDEAPAD), SND_PCI_QUIRK(0x17aa, 0x3a0d, "Lenovo Ideapad Y550", ALC662_FIXUP_IDEAPAD), + SND_PCI_QUIRK(0x1849, 0x5892, "ASRock B150M", ALC892_FIXUP_ASROCK_MOBO), SND_PCI_QUIRK(0x19da, 0xa130, "Zotac Z68", ALC662_FIXUP_ZOTAC_Z68), SND_PCI_QUIRK(0x1b0a, 0x01b8, "ACER Veriton", ALC662_FIXUP_ACER_VERITON), SND_PCI_QUIRK(0x1b35, 0x2206, "CZC P10T", ALC662_FIXUP_CZC_P10T), diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index c60a776e815d..8a59d4782a0f 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -2907,6 +2907,23 @@ AU0828_DEVICE(0x2040, 0x7260, "Hauppauge", "HVR-950Q"), AU0828_DEVICE(0x2040, 0x7213, "Hauppauge", "HVR-950Q"), AU0828_DEVICE(0x2040, 0x7270, "Hauppauge", "HVR-950Q"), +/* Syntek STK1160 */ +{ + .match_flags = USB_DEVICE_ID_MATCH_DEVICE | + USB_DEVICE_ID_MATCH_INT_CLASS | + USB_DEVICE_ID_MATCH_INT_SUBCLASS, + .idVendor = 0x05e1, + .idProduct = 0x0408, + .bInterfaceClass = USB_CLASS_AUDIO, + .bInterfaceSubClass = USB_SUBCLASS_AUDIOCONTROL, + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .vendor_name = "Syntek", + .product_name = "STK1160", + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_AUDIO_ALIGN_TRANSFER + } +}, + /* Digidesign Mbox */ { /* Thanks to Clemens Ladisch <clemens@ladisch.de> */ diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index 4490601a9235..e8a1f699058a 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -754,7 +754,7 @@ static struct rela *find_switch_table(struct objtool_file *file, if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest && (insn->jump_dest->offset <= insn->offset || - insn->jump_dest->offset >= orig_insn->offset)) + insn->jump_dest->offset > orig_insn->offset)) break; text_rela = find_rela_by_dest_range(insn->sec, insn->offset, diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile index 877a8a4721b6..c012edbdb13b 100644 --- a/tools/virtio/ringtest/Makefile +++ b/tools/virtio/ringtest/Makefile @@ -3,8 +3,8 @@ all: all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder ptr_ring noring CFLAGS += -Wall -CFLAGS += -pthread -O2 -ggdb -LDFLAGS += -pthread -O2 -ggdb +CFLAGS += -pthread -O2 -ggdb -flto -fwhole-program +LDFLAGS += -pthread -O2 -ggdb -flto -fwhole-program main.o: main.c main.h ring.o: ring.c main.h diff --git a/tools/virtio/ringtest/main.c b/tools/virtio/ringtest/main.c index 147abb452a6c..f31353fac541 100644 --- a/tools/virtio/ringtest/main.c +++ b/tools/virtio/ringtest/main.c @@ -96,7 +96,13 @@ void set_affinity(const char *arg) assert(!ret); } -static void run_guest(void) +void poll_used(void) +{ + while (used_empty()) + busy_wait(); +} + +static void __attribute__((__flatten__)) run_guest(void) { int completed_before; int completed = 0; @@ -141,7 +147,7 @@ static void run_guest(void) assert(completed <= bufs); assert(started <= bufs); if (do_sleep) { - if (enable_call()) + if (used_empty() && enable_call()) wait_for_call(); } else { poll_used(); @@ -149,7 +155,13 @@ static void run_guest(void) } } -static void run_host(void) +void poll_avail(void) +{ + while (avail_empty()) + busy_wait(); +} + +static void __attribute__((__flatten__)) run_host(void) { int completed_before; int completed = 0; @@ -160,7 +172,7 @@ static void run_host(void) for (;;) { if (do_sleep) { - if (enable_kick()) + if (avail_empty() && enable_kick()) wait_for_kick(); } else { poll_avail(); diff --git a/tools/virtio/ringtest/main.h b/tools/virtio/ringtest/main.h index 16917acb0ade..34e63cc4c572 100644 --- a/tools/virtio/ringtest/main.h +++ b/tools/virtio/ringtest/main.h @@ -56,15 +56,15 @@ void alloc_ring(void); int add_inbuf(unsigned, void *, void *); void *get_buf(unsigned *, void **); void disable_call(); +bool used_empty(); bool enable_call(); void kick_available(); -void poll_used(); /* host side */ void disable_kick(); +bool avail_empty(); bool enable_kick(); bool use_buf(unsigned *, void **); void call_used(); -void poll_avail(); /* implemented by main */ extern bool do_sleep; diff --git a/tools/virtio/ringtest/noring.c b/tools/virtio/ringtest/noring.c index eda2f4824130..b8d1c1daac7c 100644 --- a/tools/virtio/ringtest/noring.c +++ b/tools/virtio/ringtest/noring.c @@ -24,8 +24,9 @@ void *get_buf(unsigned *lenp, void **bufp) return "Buffer"; } -void poll_used(void) +bool used_empty() { + return false; } void disable_call() @@ -54,8 +55,9 @@ bool enable_kick() assert(0); } -void poll_avail(void) +bool avail_empty() { + return false; } bool use_buf(unsigned *lenp, void **bufp) diff --git a/tools/virtio/ringtest/ptr_ring.c b/tools/virtio/ringtest/ptr_ring.c index bd2ad1d3b7a9..635b07b4fdd3 100644 --- a/tools/virtio/ringtest/ptr_ring.c +++ b/tools/virtio/ringtest/ptr_ring.c @@ -133,18 +133,9 @@ void *get_buf(unsigned *lenp, void **bufp) return datap; } -void poll_used(void) +bool used_empty() { - void *b; - - do { - if (tailcnt == headcnt || __ptr_ring_full(&array)) { - b = NULL; - barrier(); - } else { - b = "Buffer\n"; - } - } while (!b); + return (tailcnt == headcnt || __ptr_ring_full(&array)); } void disable_call() @@ -173,14 +164,9 @@ bool enable_kick() assert(0); } -void poll_avail(void) +bool avail_empty() { - void *b; - - do { - barrier(); - b = __ptr_ring_peek(&array); - } while (!b); + return !__ptr_ring_peek(&array); } bool use_buf(unsigned *lenp, void **bufp) diff --git a/tools/virtio/ringtest/ring.c b/tools/virtio/ringtest/ring.c index c25c8d248b6b..747c5dd47be8 100644 --- a/tools/virtio/ringtest/ring.c +++ b/tools/virtio/ringtest/ring.c @@ -163,12 +163,11 @@ void *get_buf(unsigned *lenp, void **bufp) return datap; } -void poll_used(void) +bool used_empty() { unsigned head = (ring_size - 1) & guest.last_used_idx; - while (ring[head].flags & DESC_HW) - busy_wait(); + return (ring[head].flags & DESC_HW); } void disable_call() @@ -180,13 +179,11 @@ void disable_call() bool enable_call() { - unsigned head = (ring_size - 1) & guest.last_used_idx; - event->call_index = guest.last_used_idx; /* Flush call index write */ /* Barrier D (for pairing) */ smp_mb(); - return ring[head].flags & DESC_HW; + return used_empty(); } void kick_available(void) @@ -213,20 +210,17 @@ void disable_kick() bool enable_kick() { - unsigned head = (ring_size - 1) & host.used_idx; - event->kick_index = host.used_idx; /* Barrier C (for pairing) */ smp_mb(); - return !(ring[head].flags & DESC_HW); + return avail_empty(); } -void poll_avail(void) +bool avail_empty() { unsigned head = (ring_size - 1) & host.used_idx; - while (!(ring[head].flags & DESC_HW)) - busy_wait(); + return !(ring[head].flags & DESC_HW); } bool use_buf(unsigned *lenp, void **bufp) diff --git a/tools/virtio/ringtest/virtio_ring_0_9.c b/tools/virtio/ringtest/virtio_ring_0_9.c index 761866212aac..bbc3043b2fb1 100644 --- a/tools/virtio/ringtest/virtio_ring_0_9.c +++ b/tools/virtio/ringtest/virtio_ring_0_9.c @@ -194,24 +194,16 @@ void *get_buf(unsigned *lenp, void **bufp) return datap; } -void poll_used(void) +bool used_empty() { + unsigned short last_used_idx = guest.last_used_idx; #ifdef RING_POLL - unsigned head = (ring_size - 1) & guest.last_used_idx; + unsigned short head = last_used_idx & (ring_size - 1); + unsigned index = ring.used->ring[head].id; - for (;;) { - unsigned index = ring.used->ring[head].id; - - if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1)) - busy_wait(); - else - break; - } + return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1); #else - unsigned head = guest.last_used_idx; - - while (ring.used->idx == head) - busy_wait(); + return ring.used->idx == last_used_idx; #endif } @@ -224,22 +216,11 @@ void disable_call() bool enable_call() { - unsigned short last_used_idx; - - vring_used_event(&ring) = (last_used_idx = guest.last_used_idx); + vring_used_event(&ring) = guest.last_used_idx; /* Flush call index write */ /* Barrier D (for pairing) */ smp_mb(); -#ifdef RING_POLL - { - unsigned short head = last_used_idx & (ring_size - 1); - unsigned index = ring.used->ring[head].id; - - return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1); - } -#else - return ring.used->idx == last_used_idx; -#endif + return used_empty(); } void kick_available(void) @@ -266,36 +247,21 @@ void disable_kick() bool enable_kick() { - unsigned head = host.used_idx; - - vring_avail_event(&ring) = head; + vring_avail_event(&ring) = host.used_idx; /* Barrier C (for pairing) */ smp_mb(); -#ifdef RING_POLL - { - unsigned index = ring.avail->ring[head & (ring_size - 1)]; - - return (index ^ head ^ 0x8000) & ~(ring_size - 1); - } -#else - return head == ring.avail->idx; -#endif + return avail_empty(); } -void poll_avail(void) +bool avail_empty() { unsigned head = host.used_idx; #ifdef RING_POLL - for (;;) { - unsigned index = ring.avail->ring[head & (ring_size - 1)]; - if ((index ^ head ^ 0x8000) & ~(ring_size - 1)) - busy_wait(); - else - break; - } + unsigned index = ring.avail->ring[head & (ring_size - 1)]; + + return ((index ^ head ^ 0x8000) & ~(ring_size - 1)); #else - while (ring.avail->idx == head) - busy_wait(); + return head == ring.avail->idx; #endif } diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index f397e9b20370..a29786dd9522 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -42,6 +42,7 @@ #ifdef CONFIG_HAVE_KVM_IRQFD +static struct workqueue_struct *irqfd_cleanup_wq; static void irqfd_inject(struct work_struct *work) @@ -167,7 +168,7 @@ irqfd_deactivate(struct kvm_kernel_irqfd *irqfd) list_del_init(&irqfd->list); - schedule_work(&irqfd->shutdown); + queue_work(irqfd_cleanup_wq, &irqfd->shutdown); } int __attribute__((weak)) kvm_arch_set_irq_inatomic( @@ -554,7 +555,7 @@ kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args) * so that we guarantee there will not be any more interrupts on this * gsi once this deassign function returns. */ - flush_work(&irqfd->shutdown); + flush_workqueue(irqfd_cleanup_wq); return 0; } @@ -591,7 +592,7 @@ kvm_irqfd_release(struct kvm *kvm) * Block until we know all outstanding shutdown jobs have completed * since we do not take a kvm* reference. */ - flush_work(&irqfd->shutdown); + flush_workqueue(irqfd_cleanup_wq); } @@ -621,8 +622,23 @@ void kvm_irq_routing_update(struct kvm *kvm) spin_unlock_irq(&kvm->irqfds.lock); } +/* + * create a host-wide workqueue for issuing deferred shutdown requests + * aggregated from all vm* instances. We need our own isolated + * queue to ease flushing work items when a VM exits. + */ +int kvm_irqfd_init(void) +{ + irqfd_cleanup_wq = alloc_workqueue("kvm-irqfd-cleanup", 0, 0); + if (!irqfd_cleanup_wq) + return -ENOMEM; + + return 0; +} + void kvm_irqfd_exit(void) { + destroy_workqueue(irqfd_cleanup_wq); } #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 28510e72618a..5c360347a1e9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1346,21 +1346,19 @@ unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *w static int get_user_page_nowait(unsigned long start, int write, struct page **page) { - int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET; + int flags = FOLL_NOWAIT | FOLL_HWPOISON; if (write) flags |= FOLL_WRITE; - return __get_user_pages(current, current->mm, start, 1, flags, page, - NULL, NULL); + return get_user_pages(start, 1, flags, page, NULL); } static inline int check_user_page_hwpoison(unsigned long addr) { - int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE; + int rc, flags = FOLL_HWPOISON | FOLL_WRITE; - rc = __get_user_pages(current, current->mm, addr, 1, - flags, NULL, NULL, NULL); + rc = get_user_pages(addr, 1, flags, NULL, NULL); return rc == -EHWPOISON; } @@ -3846,7 +3844,12 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, * kvm_arch_init makes sure there's at most one caller * for architectures that support multiple implementations, * like intel and amd on x86. + * kvm_arch_init must be called before kvm_irqfd_init to avoid creating + * conflicts in case kvm is already setup for another implementation. */ + r = kvm_irqfd_init(); + if (r) + goto out_irqfd; if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) { r = -ENOMEM; @@ -3928,6 +3931,7 @@ out_free_0a: free_cpumask_var(cpus_hardware_enabled); out_free_0: kvm_irqfd_exit(); +out_irqfd: kvm_arch_exit(); out_fail: return r; |
